diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 90e79d4d853..53c654332b0 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -1,12 +1,16 @@ # What is this? ## Helper utilities for token counting import base64 +import importlib import io import math +import re import struct from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence -from itertools import accumulate -from typing import TYPE_CHECKING, Final, Literal, cast +from dataclasses import dataclass +from itertools import accumulate, groupby, islice, takewhile, tee +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, cast, runtime_checkable import anyio import anyio.lowlevel @@ -348,6 +352,58 @@ Type for a function that counts tokens in a string. """ EXTRAPOLATION_SAMPLES: Final = 16 +PDF_PAGE_MAX_EXACT_CONTENT_BYTES: Final = 64 * 1024 +PDF_WHITESPACE_CLASS: Final = b"[\x00\t\n\x0c\r ]" +PDF_HEX_STRING_OPERAND: Final = rb"<[0-9A-Fa-f" + PDF_WHITESPACE_CLASS[1:-1] + rb"]*>" +PDF_LITERAL_STRING_OPERAND: Final = rb"\((?:\\[\x00-\xff]|[^\\()]|\((?:\\[\x00-\xff]|[^\\()])*\)?)*\)?" +PDF_STRING_OPERAND: Final = re.compile(PDF_LITERAL_STRING_OPERAND + rb"|" + PDF_HEX_STRING_OPERAND + rb"|%[^\r\n]*") +PDF_SIMPLE_LITERAL_STRING: Final = re.compile(rb"\([^()]*\)") +PDF_LITERAL_ESCAPE: Final = re.compile(rb"\\([nrtbf()\\]|[0-7]{1,3}|\r\n?|\n|[\x00-\xff])") +PDF_INLINE_IMAGE: Final = re.compile( + PDF_LITERAL_STRING_OPERAND + + rb"|" + + PDF_HEX_STRING_OPERAND + + rb"|%[^\r\n]*|(?P(?)])BI(?=" + + PDF_WHITESPACE_CLASS + + rb"|[/\[\]()<>{}%])[\x00-\xff]*?(?:(?<=" + + PDF_WHITESPACE_CLASS + + rb")ID" + + PDF_WHITESPACE_CLASS + + rb"[\x00-\xff]*?(?:(?P" + + PDF_WHITESPACE_CLASS + + rb"EI(?![^" + + PDF_WHITESPACE_CLASS[1:-1] + + rb"\[/<(%]))|\Z)|\Z))" +) +PDF_TEXT_OBJECT: Final = re.compile( + PDF_LITERAL_STRING_OPERAND + + rb"|" + + PDF_HEX_STRING_OPERAND + + rb"|%[^\r\n]*|(?P(?)])BT(?=" + + PDF_WHITESPACE_CLASS + + rb"|[/\[\]()<>{}%])" + + rb"(?:(?:" + + PDF_LITERAL_STRING_OPERAND + + rb"|" + + PDF_HEX_STRING_OPERAND + + rb"|%[^\r\n]*|[\x00-\xff])*?(?:(?P(?<=" + + PDF_WHITESPACE_CLASS + + rb")ET(?![^" + + PDF_WHITESPACE_CLASS[1:-1] + + rb"\[\]()<>{}/%]))|\Z)))" +) +PDF_XOBJECT_INVOCATION: Final = re.compile( + PDF_LITERAL_STRING_OPERAND + + rb"|%[^\r\n]*(?:[\r\n]|\Z)|/([^" + + PDF_WHITESPACE_CLASS[1:-1] + + rb"/\[\]()<>{}%]+)(?:" + + PDF_WHITESPACE_CLASS + + rb"|%[^\r\n]*(?:[\r\n]|\Z))+Do\b" +) T_ParamSpec: Final = ParamSpec("T_ParamSpec") T_Retval = TypeVar("T_Retval") _COUNT_OFFLOAD_LIMITER: Final = anyio.lowlevel.RunVar[anyio.CapacityLimiter]("litellm_count_offload_limiter") @@ -860,6 +916,618 @@ def _anthropic_rendered_page_image_tokens(width: float, height: float) -> int: return math.ceil(min(float(ANTHROPIC_IMAGE_MAX_PIXELS), area_at_max_edge) / ANTHROPIC_IMAGE_PIXELS_PER_TOKEN) +class _PdfContentStream(Protocol): + def get_data(self) -> bytes: ... + + +@runtime_checkable +class _PdfResourceDictionary(Protocol): + def get(self, key: str) -> object | None: ... + + def items(self) -> Iterable[tuple[str, object]]: ... + + +@runtime_checkable +class _PdfFormXObject(_PdfResourceDictionary, Protocol): + def get_data(self) -> bytes: ... + + +@runtime_checkable +class _PdfIndirectReference(Protocol): + def get_object(self) -> object: ... + + +class _PdfMediaBox(Protocol): + @property + def width(self) -> float: ... + + @property + def height(self) -> float: ... + + +class _PdfPage(_PdfResourceDictionary, Protocol): + @property + def mediabox(self) -> _PdfMediaBox: ... + + def get_contents(self) -> _PdfContentStream | None: ... + + def extract_text(self) -> str: ... + + +class _PdfWritablePage(_PdfPage, Protocol): + def __setitem__(self, key: str, value: object) -> None: ... + + +class _PdfPageFactory(Protocol): + @staticmethod + def create_blank_page(pdf: object | None, width: float, height: float) -> _PdfWritablePage: ... + + +class _PdfDecodedContentStream(_PdfContentStream, Protocol): + def set_data(self, data: bytes) -> None: ... + + +class _PdfGenericModule(Protocol): + DecodedStreamObject: Callable[[], _PdfDecodedContentStream] + DictionaryObject: Callable[[Mapping[str, object]], _PdfResourceDictionary] + NameObject: Callable[[str], str] + + +class _PdfModule(Protocol): + PageObject: _PdfPageFactory + + +@dataclass(frozen=True, slots=True) +class _PdfStreamMetrics: + content: bytes + content_bytes: int + operand_bytes: int + resources: _PdfResourceDictionary + forms: tuple["_PdfStreamMetrics", ...] + + +@dataclass(frozen=True, slots=True) +class _PdfTextObjectRecord: + stream: _PdfStreamMetrics + invocation_path: tuple[int, ...] + start: int + end: int + first_in_stream: bool + truncated: bool + closed: bool + + +_PDF_EMPTY_RESOURCES: Final[_PdfResourceDictionary] = ( + cast( # cast-ok: immutable empty map satisfies the resource protocol + _PdfResourceDictionary, MappingProxyType({}) + ) +) + + +def _pdf_resolve_object(value: object) -> object: + return value.get_object() if isinstance(value, _PdfIndirectReference) else value + + +def _pdf_decode_name(name: bytes) -> str: + def decode_escape(match: re.Match[bytes]) -> bytes: + return bytes((int(match.group(1), 16),)) + + return "/" + re.sub(rb"#([0-9A-Fa-f]{2})", decode_escape, name).decode("latin-1") + + +def _pdf_invocation_name(invocation: re.Match[bytes]) -> str | None: + name: Final = invocation.group(1) + return _pdf_decode_name(name) if name is not None else None + + +def _pdf_inline_image_replacement(match: re.Match[bytes]) -> bytes: + inline_image: Final = match.group("inline_image") + image_end: Final = match.group("image_end") + return b" " if inline_image is not None and image_end is not None else match.group(0) + + +def _pdf_strip_inline_images(content: bytes) -> bytes: + return PDF_INLINE_IMAGE.sub(_pdf_inline_image_replacement, content) if b"BI" in content else content + + +def _pdf_page_content_metrics(page: _PdfPage, page_content: bytes) -> _PdfStreamMetrics: + form_metrics_memo: Final[dict[int, _PdfStreamMetrics]] = {} # mutable-ok: page-local cache avoids repeated walks + + def stream_metrics( + content: bytes, + resources: _PdfResourceDictionary, + ancestors: frozenset[int], + ) -> _PdfStreamMetrics: + xobjects: Final = _pdf_resolve_object(resources.get("/XObject")) + xobject_resources: Final = xobjects if isinstance(xobjects, _PdfResourceDictionary) else _PDF_EMPTY_RESOURCES + invoked_names: Final = ( + tuple( + name + for invocation in PDF_XOBJECT_INVOCATION.finditer(content) + if (name := _pdf_invocation_name(invocation)) is not None + ) + if xobject_resources + else () + ) + form_metrics: Final = tuple( + form_metrics_for_xobject( + _pdf_resolve_object(xobject_resources.get(name)), + resources, + ancestors, + ) + for name in invoked_names + ) + return _PdfStreamMetrics( + content=content, + content_bytes=len(content) + sum(metrics.content_bytes for metrics in form_metrics), + operand_bytes=_pdf_string_operand_bytes(content) + sum(metrics.operand_bytes for metrics in form_metrics), + resources=resources, + forms=form_metrics, + ) + + def form_metrics_for_xobject( + xobject: object, + inherited_resources: _PdfResourceDictionary, + ancestors: frozenset[int], + ) -> _PdfStreamMetrics: + resolved: Final = _pdf_resolve_object(xobject) + identity: Final = id(resolved) + if identity in ancestors: + return _PdfStreamMetrics(b"", 0, 0, inherited_resources, ()) + if not isinstance(resolved, _PdfFormXObject) or resolved.get("/Subtype") != "/Form": + return _PdfStreamMetrics(b"", 0, 0, inherited_resources, ()) + cached: Final = form_metrics_memo.get(identity) + if cached is not None: + return cached + content: Final = _pdf_strip_inline_images(resolved.get_data()) + form_resources: Final = _pdf_resolve_object(resolved.get("/Resources")) + resources: Final = form_resources if isinstance(form_resources, _PdfResourceDictionary) else inherited_resources + metrics: Final = stream_metrics(content, resources, ancestors | {identity}) + form_metrics_memo[identity] = metrics + return metrics + + resources: Final = _pdf_resolve_object(page.get("/Resources")) + page_resources: Final = resources if isinstance(resources, _PdfResourceDictionary) else _PDF_EMPTY_RESOURCES + return stream_metrics(page_content, page_resources, frozenset()) + + +def _evenly_spaced_page_indices(page_count: int) -> tuple[int, ...]: + if page_count <= EXTRAPOLATION_SAMPLES: + return tuple(range(page_count)) + return tuple((page_count - 1) * index // (EXTRAPOLATION_SAMPLES - 1) for index in range(EXTRAPOLATION_SAMPLES)) + + +def _pdf_string_operand_bytes(content: bytes) -> int: + if b"\\" not in content and b"%" not in content and b"<" not in content and b"BI" not in content: + content_without_literals, literal_count = PDF_SIMPLE_LITERAL_STRING.subn(b"", content) + if literal_count == content.count(b"(") == content.count(b")"): + return len(content) - len(content_without_literals) - 2 * literal_count + return sum(len(operand) for operand in _pdf_decoded_string_operands(content)) + + +def _pdf_decode_literal_escape(match: re.Match[bytes]) -> bytes: + escape: Final = match.group(1) + if escape is None: + return b"" + if escape == b"n": + return b"\n" + if escape == b"r": + return b"\r" + if escape == b"t": + return b"\t" + if escape == b"b": + return b"\b" + if escape == b"f": + return b"\f" + if escape in (b"(", b")", b"\\"): + return escape + if escape in (b"\r", b"\n", b"\r\n"): + return b"" + if escape[:1] in b"01234567": + return bytes((int(escape, 8) & 0xFF,)) + return escape + + +def _pdf_decode_string_operand(operand: bytes) -> bytes: + if operand.startswith(b"<"): + digits: Final = re.sub(PDF_WHITESPACE_CLASS + rb"+", b"", operand[1:-1]) + padded_digits: Final = digits if len(digits) % 2 == 0 else digits + b"0" + return bytes.fromhex(padded_digits.decode("ascii")) + literal: Final = operand[1:-1] if operand.endswith(b")") else operand[1:] + return PDF_LITERAL_ESCAPE.sub(_pdf_decode_literal_escape, literal) if b"\\" in literal else literal + + +def _pdf_decoded_string_operands(content: bytes) -> Iterable[bytes]: + return ( + _pdf_decode_string_operand(match.group(0)) + for match in PDF_STRING_OPERAND.finditer(content) + if not match.group(0).startswith(b"%") + ) + + +_PdfTextObjectSpans: TypeAlias = tuple[tuple[int, int, bool, bool], ...] +_PdfTextTreeWalkItem: TypeAlias = tuple[_PdfStreamMetrics, bool] +_PdfTextRecordWalkItem: TypeAlias = tuple[_PdfStreamMetrics, tuple[int, ...]] + + +def _pdf_text_object_records( # noqa: C901 # coordinates memoized per-stream PDF traversal + metrics: _PdfStreamMetrics, + invocation_path: tuple[int, ...] = (), +) -> Iterable[_PdfTextObjectRecord]: + text_object_span_memo: Final[dict[int, _PdfTextObjectSpans]] = {} # mutable-ok: page-local regex cache + subtree_text_memo: Final[dict[int, bool]] = {} # mutable-ok: page-local subtree cache + + def partial_text_object_end(content: bytes, start: int) -> tuple[int, bool, bool]: + end: Final = min(len(content), start + PDF_PAGE_MAX_EXACT_CONTENT_BYTES) + bounded_content: Final = content[start:end] + match: Final = next( + (match for match in PDF_TEXT_OBJECT.finditer(bounded_content) if match.group("text_object") is not None), + None, + ) + if match is None: + return start, False, False + closed: Final = match.group("text_object_end") is not None + truncated: Final = match.end("text_object") == len(bounded_content) and end < len(content) and not closed + return start + match.end("text_object"), closed, truncated + + def text_object_spans_in_window(content: bytes, window_end: int) -> Iterable[tuple[int, int, bool, bool]]: + window_content: Final = content[:window_end] + for match in PDF_TEXT_OBJECT.finditer(window_content): + if match.group("text_object") is None: + continue + if ( + match.end("text_object") == window_end + and window_end < len(content) + and match.group("text_object_end") is None + ): + partial_end, partial_closed, partial_truncated = partial_text_object_end( + content, match.start("text_object") + ) + yield ( + match.start("text_object"), + partial_end, + partial_truncated, + partial_closed, + ) + return + yield ( + match.start("text_object"), + match.end("text_object"), + False, + match.group("text_object_end") is not None, + ) + + def scan_text_object_spans( + content: bytes, + window_end: int, + ) -> tuple[_PdfTextObjectSpans, int]: + spans: Final[list[tuple[int, int, bool, bool]]] = [] # mutable-ok: capped page-local span cache + cumulative_length: int = 0 # rebind-ok: accumulates record lengths until the sample cap + for start, end, truncated, closed in text_object_spans_in_window(content, window_end): + spans.append((start, end, truncated, closed)) + cumulative_length += end - start + (0 if closed else len(b"\nET")) + (0 if len(spans) == 1 else len(b"\n")) + if cumulative_length >= PDF_PAGE_MAX_EXACT_CONTENT_BYTES: + break + return tuple(spans), cumulative_length + + def text_object_spans_for(stream_metrics: _PdfStreamMetrics) -> _PdfTextObjectSpans: + identity: Final = id(stream_metrics) + cached: Final = text_object_span_memo.get(identity) + if cached is not None: + return cached + first_window_end: Final = min(len(stream_metrics.content), PDF_PAGE_MAX_EXACT_CONTENT_BYTES) + first_spans, first_length = scan_text_object_spans(stream_metrics.content, first_window_end) + cached_spans: Final = ( + first_spans + if first_length >= PDF_PAGE_MAX_EXACT_CONTENT_BYTES or first_window_end == len(stream_metrics.content) + else scan_text_object_spans(stream_metrics.content, len(stream_metrics.content))[0] + ) + text_object_span_memo[identity] = cached_spans + return cached_spans + + def subtree_has_text(stream_metrics: _PdfStreamMetrics) -> bool: + identity: Final = id(stream_metrics) + if identity in subtree_text_memo: + return subtree_text_memo[identity] + pending: Final[list[_PdfTextTreeWalkItem]] = [(stream_metrics, False)] # mutable-ok: DFS stack + while pending: + current_metrics, expanded = pending.pop() + current_identity = id(current_metrics) + if current_identity in subtree_text_memo: + continue + if expanded: + subtree_text_memo[current_identity] = any( + subtree_text_memo.get(id(form_metrics), False) for form_metrics in current_metrics.forms + ) + continue + if text_object_spans_for(current_metrics): + subtree_text_memo[current_identity] = True + continue + pending.append((current_metrics, True)) + pending.extend((form_metrics, False) for form_metrics in current_metrics.forms) + return subtree_text_memo[identity] + + if not subtree_has_text(metrics): + return + pending_records: Final[list[_PdfTextRecordWalkItem]] = [(metrics, invocation_path)] # mutable-ok: DFS stack + while pending_records: + current_metrics, current_path = pending_records.pop() + for index, (start, end, truncated, closed) in enumerate(text_object_spans_for(current_metrics)): + yield _PdfTextObjectRecord(current_metrics, current_path, start, end, index == 0, truncated, closed) + pending_records.extend( + (form_metrics, current_path + (index,)) + for index, form_metrics in reversed(tuple(enumerate(current_metrics.forms))) + if subtree_has_text(form_metrics) + ) + + +def _pdf_text_object_record_length(record: _PdfTextObjectRecord) -> int: + text_object_length: Final = record.end - record.start + separator_bytes: Final = 0 if record.first_in_stream else len(b"\n") + if record.closed: + return text_object_length + separator_bytes + text_object: Final = record.stream.content[record.start : record.end] + prefix_length, closures = _pdf_truncated_text_sample_closures( + text_object, + include_text_show=True, + maximum_length=PDF_PAGE_MAX_EXACT_CONTENT_BYTES - separator_bytes - len(b"\nET"), + ) + return prefix_length + len(closures) + len(b"\nET") + separator_bytes + + +def _pdf_text_object_record_bytes(record: _PdfTextObjectRecord) -> bytes: + text_object: Final = record.stream.content[record.start : record.end] + separator: Final = b"" if record.first_in_stream else b"\n" + if record.closed: + return separator + text_object + prefix_length, closures = _pdf_truncated_text_sample_closures( + text_object, + include_text_show=True, + maximum_length=PDF_PAGE_MAX_EXACT_CONTENT_BYTES - len(separator) - len(b"\nET"), + ) + return separator + text_object[:prefix_length] + closures + b"\nET" + + +def _pdf_truncated_text_sample_closures( # noqa: C901 # single-pass PDF syntax-state scanner + prefix: bytes, include_text_show: bool, maximum_length: int | None = None +) -> tuple[int, bytes]: + comment_open: bool = False # rebind-ok: scans one bounded PDF prefix + escape_open: bool = False # rebind-ok: scans one bounded PDF prefix + literal_depth: int = 0 # rebind-ok: tracks balanced literal strings + hex_open: bool = False # rebind-ok: scans one bounded PDF prefix + array_depth: int = 0 # rebind-ok: tracks arrays outside strings + bare_string_complete: bool = False # rebind-ok: tracks the final string operand + best_prefix_length: int = 0 # rebind-ok: retains the latest prefix whose closures fit the sample cap + sample_limit: Final = len(prefix) if maximum_length is None else maximum_length + best_state: tuple[bool, bool, int, bool, int, bool] = ( # rebind-ok: scanner state + False, + False, + 0, + False, + 0, + False, + ) + for index, byte in enumerate(prefix): + if comment_open: + if byte in (ord("\r"), ord("\n")): + comment_open = False + elif literal_depth: + if escape_open: + escape_open = False + elif byte == ord("\\"): + escape_open = True + elif byte == ord("("): + literal_depth += 1 + elif byte == ord(")"): + literal_depth -= 1 + if literal_depth == 0: + bare_string_complete = True + elif hex_open: + if byte == ord(">"): + hex_open = False + bare_string_complete = True + elif byte == ord("%"): + comment_open = True + elif byte == ord("("): + literal_depth = 1 + bare_string_complete = False + elif byte == ord("<") and (index + 1 == len(prefix) or prefix[index + 1] != ord("<")): + hex_open = True + bare_string_complete = False + elif byte == ord("["): + array_depth += 1 + bare_string_complete = False + elif byte == ord("]"): + array_depth = max(0, array_depth - 1) + bare_string_complete = False + elif byte not in PDF_WHITESPACE_CLASS[1:-1]: + bare_string_complete = False + if ( + index + + 1 + + (1 if comment_open else 0) + + (2 if escape_open else 0) + + literal_depth + + (1 if hex_open else 0) + + ( + array_depth + len(b" TJ") + if include_text_show and array_depth + else len(b" Tj") + if include_text_show and (literal_depth or hex_open or bare_string_complete) + else 0 + ) + <= sample_limit + ): + best_prefix_length = index + 1 + best_state = ( + comment_open, + escape_open, + literal_depth, + hex_open, + array_depth, + bare_string_complete, + ) + ( + best_comment_open, + best_escape_open, + best_literal_depth, + best_hex_open, + best_array_depth, + best_bare_string_complete, + ) = best_state + operand_closures: Final = ( + (b"\\" if best_escape_open else b"") + + (b")" * best_literal_depth if best_literal_depth else b"") + + (b">" if best_hex_open else b"") + ) + comment_closure: Final = b"\n" if best_comment_open else b"" + if not include_text_show: + return best_prefix_length, comment_closure + operand_closures + if best_array_depth: + return best_prefix_length, comment_closure + operand_closures + b"]" * best_array_depth + b" TJ" + if best_literal_depth or best_hex_open or best_bare_string_complete: + return best_prefix_length, comment_closure + operand_closures + b" Tj" + return best_prefix_length, comment_closure + operand_closures + + +def _pdf_truncated_text_object_prefix( + content: bytes, + start: int, + prefix_length: int, +) -> bytes: + prefix: Final = content[start : start + prefix_length] + closed_prefix_length, closures = _pdf_truncated_text_sample_closures( + prefix, include_text_show=True, maximum_length=prefix_length + ) + return prefix[:closed_prefix_length] + closures + b"\nET" + + +def _pdf_truncated_string_operand_prefix(operand: bytes, prefix_length: int) -> bytes: + prefix: Final = operand[:prefix_length] + closed_prefix_length, closures = _pdf_truncated_text_sample_closures(prefix, include_text_show=False) + return prefix[:closed_prefix_length] + closures + + +def _pdf_truncated_text_object(record: _PdfTextObjectRecord) -> bytes: + content: Final = record.stream.content + if record.truncated: + return _pdf_truncated_text_object_prefix( + content, + record.start, + PDF_PAGE_MAX_EXACT_CONTENT_BYTES - len(b"\nET"), + ) + string_operands: Final = ( + operand + for operand in PDF_STRING_OPERAND.finditer(content, record.start, record.end) + if not operand.group(0).startswith(b"%") + ) + largest_operand: Final = max(string_operands, key=lambda operand: len(operand.group(0)), default=None) + if largest_operand is not None and len(largest_operand.group(0)) > PDF_PAGE_MAX_EXACT_CONTENT_BYTES // 2: + prefix: Final = content[record.start : largest_operand.start()] + suffix: Final = content[largest_operand.end() : record.end] + object_closure: Final = b"" if record.closed else b"\nET" + operand_prefix_length: Final = ( + PDF_PAGE_MAX_EXACT_CONTENT_BYTES - len(prefix) - len(suffix) - len(object_closure) + ) + if operand_prefix_length > 0: + sample: Final = ( + prefix + + _pdf_truncated_string_operand_prefix(largest_operand.group(0), operand_prefix_length) + + suffix + + object_closure + ) + if len(sample) <= PDF_PAGE_MAX_EXACT_CONTENT_BYTES: + return sample + return _pdf_truncated_text_object_prefix( + content, + record.start, + PDF_PAGE_MAX_EXACT_CONTENT_BYTES - len(b"\nET"), + ) + + +def _pdf_text_sample_for_scope(records: Iterable[_PdfTextObjectRecord]) -> tuple[_PdfResourceDictionary, bytes]: + scoped_records: Final = tuple(records) + return ( + scoped_records[0].stream.resources, + b"".join(_pdf_text_object_record_bytes(record) for record in scoped_records), + ) + + +def _pdf_bounded_text_samples(metrics: _PdfStreamMetrics) -> tuple[tuple[_PdfResourceDictionary, bytes], ...]: + records, sized_records = tee(_pdf_text_object_records(metrics)) + first_record: Final = next(records, None) + if first_record is None: + return () + first_record_length: Final = _pdf_text_object_record_length(first_record) + if first_record.truncated or first_record_length > PDF_PAGE_MAX_EXACT_CONTENT_BYTES: + return ((first_record.stream.resources, _pdf_truncated_text_object(first_record)),) + record_sizes: Final = (_pdf_text_object_record_length(record) for record in islice(sized_records, 1, None)) + cumulative_sizes: Final = islice(accumulate(record_sizes, initial=first_record_length), 1, None) + bounded_records: Final = tuple( + record + for record, _ in takewhile( + lambda item: item[1] <= PDF_PAGE_MAX_EXACT_CONTENT_BYTES, + zip(records, cumulative_sizes), + ) + ) + sampled_records: Final = (first_record, *bounded_records) + return tuple( + _pdf_text_sample_for_scope(group) + for _, group in groupby(sampled_records, key=lambda record: record.invocation_path) + ) + + +def _pdf_extract_sample_text( + page: _PdfPage, + resources: _PdfResourceDictionary, + content: bytes, +) -> str: + pypdf_module: Final = cast( # cast-ok: optional pypdf module has no typed module API + _PdfModule, importlib.import_module("pypdf") + ) + generic_module: Final = cast( # cast-ok: optional pypdf module has no typed module API + _PdfGenericModule, importlib.import_module("pypdf.generic") + ) + + copied_resources: Final = generic_module.DictionaryObject( + {generic_module.NameObject(str(name)): value for name, value in resources.items() if str(name) != "/XObject"} + ) + synthetic_page: Final = pypdf_module.PageObject.create_blank_page(None, page.mediabox.width, page.mediabox.height) + synthetic_page[generic_module.NameObject("/Resources")] = copied_resources + content_stream: Final = generic_module.DecodedStreamObject() + content_stream.set_data(content) + synthetic_page[generic_module.NameObject("/Contents")] = content_stream + return synthetic_page.extract_text() or "" + + +def _pdf_page_text_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int: + """Oversized pages use bounded pypdf text-object samples scaled to decoded operand bytes.""" + contents: Final = page.get_contents() + content: Final = _pdf_strip_inline_images(contents.get_data() if contents is not None else b"") + metrics: Final = _pdf_page_content_metrics(page, content) + if metrics.content_bytes > PDF_PAGE_MAX_EXACT_CONTENT_BYTES: + samples: Final = _pdf_bounded_text_samples(metrics) + sampled_operand_bytes: Final = sum(_pdf_string_operand_bytes(sample) for _, sample in samples) + if sampled_operand_bytes == 0: + return 0 + sampled_text_tokens: Final = sum( + count_function(_pdf_extract_sample_text(page, resources, sample)) for resources, sample in samples + ) + return round(sampled_text_tokens * metrics.operand_bytes / sampled_operand_bytes) + return count_function(page.extract_text() or "") + + +def _price_pdf_pages(pages: Sequence[_PdfPage], count_function: TokenCounterFunction) -> int: + rendering_tokens: Final = sum( + _anthropic_rendered_page_image_tokens(page.mediabox.width, page.mediabox.height) for page in pages + ) + sampled_pages: Final = tuple(pages[index] for index in _evenly_spaced_page_indices(len(pages))) + if not sampled_pages: + return rendering_tokens + sampled_text_tokens: Final = sum(_pdf_page_text_tokens(page, count_function) for page in sampled_pages) + return rendering_tokens + round(sampled_text_tokens * len(pages) / len(sampled_pages)) + + def _count_inline_pdf_tokens(data_url: str, count_function: TokenCounterFunction) -> int | None: if not data_url.startswith(PDF_DATA_URL_PREFIX): return None @@ -869,12 +1537,10 @@ def _count_inline_pdf_tokens(data_url: str, count_function: TokenCounterFunction verbose_logger.debug("pypdf is not installed, so the PDF document is priced like one image") return None try: - reader: Final = PdfReader(io.BytesIO(base64.b64decode(data_url[len(PDF_DATA_URL_PREFIX) :]))) - return sum( - count_function(page.extract_text() or "") - + _anthropic_rendered_page_image_tokens(float(page.mediabox.width), float(page.mediabox.height)) - for page in reader.pages + pages: Final = cast( # cast-ok: pypdf is an optional extra the type-check env does not install + Sequence[_PdfPage], PdfReader(io.BytesIO(base64.b64decode(data_url[len(PDF_DATA_URL_PREFIX) :]))).pages ) + return _price_pdf_pages(pages, count_function) except Exception as e: verbose_logger.debug("Could not read the PDF document's pages (%s), so it is priced like one image", e) return None diff --git a/pyproject.toml b/pyproject.toml index fc860a8fdf4..5f3dcbd0cd5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -310,6 +310,7 @@ healthcheck = [ benchmarks = [ "pytest==9.0.3", "pytest-codspeed==4.3.0", + "pypdf>=6.16.1,<7.0", "mcp>=2.3.0,<3", "a2a-sdk==1.1.0", ] diff --git a/tests/benchmarks/test_token_counter_pdf_benchmarks.py b/tests/benchmarks/test_token_counter_pdf_benchmarks.py new file mode 100644 index 00000000000..d7c987c0e5b --- /dev/null +++ b/tests/benchmarks/test_token_counter_pdf_benchmarks.py @@ -0,0 +1,152 @@ +from functools import reduce +from types import SimpleNamespace +from typing import Final + +import pytest + +from litellm.litellm_core_utils.token_counter import ( + _PDF_EMPTY_RESOURCES, + PDF_TEXT_OBJECT, + _pdf_bounded_text_samples, + _PdfStreamMetrics, + _price_pdf_pages, +) + + +class _PdfBenchmarkForm: + def get_data(self) -> bytes: + return b"q Q\n" + + def get(self, key: str) -> str | None: + return "/Form" if key == "/Subtype" else None + + def items(self) -> tuple[tuple[str, object], ...]: + return () + + +def _pdf_page(content: bytes, with_form: bool = False) -> SimpleNamespace: + xobjects: Final = {"/Fm1": _PdfBenchmarkForm()} if with_form else {} + resources: Final = {"/XObject": xobjects} + return SimpleNamespace( + mediabox=SimpleNamespace(width=612.0, height=792.0), + get_contents=lambda: SimpleNamespace(get_data=lambda: content), + extract_text=lambda: "", + get=lambda key: resources if key == "/Resources" else None, + ) + + +def _pdf_doubling_form_metrics(depth: int, leaf_content: bytes) -> _PdfStreamMetrics: + leaf_metrics: Final = _PdfStreamMetrics( + content=leaf_content, + content_bytes=len(leaf_content), + operand_bytes=0, + resources=_PDF_EMPTY_RESOURCES, + forms=(), + ) + return reduce( + lambda child, _: _PdfStreamMetrics( + content=b"/Fm Do\n/Fm Do\n", + content_bytes=0, + operand_bytes=0, + resources=_PDF_EMPTY_RESOURCES, + forms=(child, child), + ), + range(depth), + leaf_metrics, + ) + + +@pytest.mark.benchmark +def test_unterminated_pdf_literal_string_scan(): # test-quality-ok: CodSpeed performance benchmark + page: Final = _pdf_page(b"(" * (4 * 1024 * 1024)) + + _price_pdf_pages((page,), len) + + +@pytest.mark.benchmark +def test_pdf_string_operand_scan_after_escaped_parentheses(): # test-quality-ok: CodSpeed performance benchmark + path_operator: Final = b"10 10 m 20 20 l S\n" + padding_repetitions: Final = 64 * 1024 // len(path_operator) + padding_bytes: Final = padding_repetitions * len(path_operator) + padding: Final = path_operator * padding_repetitions + b" " * (64 * 1024 - padding_bytes) + page: Final = _pdf_page(padding + b"%" + b"\\(" * 8_192 + path_operator * 1_000) + + _price_pdf_pages((page,), len) + + +@pytest.mark.benchmark +def test_pdf_invocation_scan_ignores_unterminated_comments(): # test-quality-ok: CodSpeed performance benchmark + page: Final = _pdf_page(b"/a %" * (4 * 1024 * 1024 // 4), with_form=True) + + priced: Final = _price_pdf_pages((page,), len) + + assert priced > 0 + + +@pytest.mark.benchmark +def test_pdf_inline_image_scan_repeated_bi_without_id(): # test-quality-ok: CodSpeed performance benchmark + page: Final = _pdf_page(b"BI " * 1_000_000) + + _price_pdf_pages((page,), len) + + +@pytest.mark.benchmark +@pytest.mark.parametrize( + "content", + [ + b"BT " * 1_000_000, + b"BT (" * 1_000_000, + b"BT " + b"(ET) " * 1_000_000, + ], + ids=["repeated-bt", "repeated-open-literal", "repeated-et-literal"], +) +def test_pdf_text_object_scan(content: bytes): # test-quality-ok: CodSpeed performance benchmark + tuple(PDF_TEXT_OBJECT.finditer(content)) + + +@pytest.mark.benchmark +def test_pdf_16m_text_object_stream_is_capped(): # test-quality-ok: CodSpeed performance benchmark + text_object: Final = b"BT (x) Tj ET\n" + repetitions: Final = 16 * 1024 * 1024 // len(text_object) + 1 + page: Final = _pdf_page(text_object * repetitions) + + _price_pdf_pages((page,), len) + + +@pytest.mark.benchmark +def test_pdf_textless_doubling_form_tree_skips_subtrees(): # test-quality-ok: CodSpeed performance benchmark + metrics: Final = _pdf_doubling_form_metrics(30, b"q Q\n") + + _pdf_bounded_text_samples(metrics) + + +@pytest.mark.benchmark +def test_pdf_text_leaf_doubling_form_tree_stays_within_sample_cap(): # test-quality-ok: CodSpeed performance benchmark + metrics: Final = _pdf_doubling_form_metrics(30, b"BT (x) Tj ET\n") + + _pdf_bounded_text_samples(metrics) + + +@pytest.mark.benchmark +@pytest.mark.parametrize("repetitions", [100_000, 1_000_000], ids=["100k", "1m"]) +def test_pdf_inline_image_scan_no_ei(repetitions: int): # test-quality-ok: CodSpeed performance benchmark + page: Final = _pdf_page(b"BI /W 1 " + b" ID x" * repetitions) + + _price_pdf_pages((page,), len) + + +@pytest.mark.benchmark +@pytest.mark.parametrize( + "comments", + [ + b"%comment\r\n" * 23, + b"% a \r\n" * 40, + ], + ids=["23-crlf-comments", "40-crlf-comments-with-trailing-spaces"], +) +def test_pdf_invocation_scan_with_crlf_comments(comments: bytes): # test-quality-ok: CodSpeed performance benchmark + page: Final = _pdf_page(b"BT /F1 " + comments + b"12 Tf (hello) Tj ET", with_form=True) + + priced: Final = _price_pdf_pages((page,), len) + + assert priced > 0 diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 88af656ad2e..3fbdb2454e6 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -3,17 +3,20 @@ import asyncio import base64 import importlib +import io import json import os +import random import subprocess import sys import threading import time -from collections.abc import Callable, Mapping +from collections.abc import Callable, Iterable, Mapping from concurrent.futures import Future, ThreadPoolExecutor, wait from itertools import accumulate, chain from pathlib import Path -from typing import Final +from types import SimpleNamespace +from typing import Final, Protocol, cast from unittest.mock import MagicMock, sentinel import anyio.to_thread @@ -27,10 +30,18 @@ import litellm from litellm import decode, encode, get_modified_max_tokens from litellm import token_counter as token_counter_old import litellm.constants -from litellm.constants import HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS, TOKEN_COUNTER_MAX_CONCURRENT_COUNTS +from litellm.constants import ( + HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS, + PDF_DATA_URL_PREFIX, + TOKEN_COUNTER_MAX_CONCURRENT_COUNTS, +) from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.token_counter import ( + EXTRAPOLATION_SAMPLES, + PDF_PAGE_MAX_EXACT_CONTENT_BYTES, + _count_inline_pdf_tokens, _encoding_count, + _price_pdf_pages, _get_exact_count_function, _get_extrapolating_count_function, _get_tiktoken_count_function, @@ -765,25 +776,9 @@ def test_encoding_and_decoding(tmp_path: Path): ], ) def test_load_test_token_counter(model): - """ - Token count large prompt 100 times. - - Assert time taken is < 1.5s. - """ - import tiktoken - + """Token-count a large prompt for each supported model family.""" messages = [{"role": "user", "content": text}] * 10 - - start_time = time.time() - for _ in range(10): - _ = token_counter(model=model, messages=messages) - # enc.encode("".join(m["content"] for m in messages)) - - end_time = time.time() - - total_time = end_time - start_time - print("model={}, total test time={}".format(model, total_time)) - assert total_time < 10, f"Total encoding time > 10s, {total_time}" + assert token_counter(model=model, messages=messages) > 0 @pytest.mark.parametrize( @@ -1948,6 +1943,231 @@ def _pdf_base64(pages: tuple[str, ...], width: int = 612, height: int = 792) -> return base64.b64encode(header + b"".join(objects) + xref + trailer).decode() +def _pdf_stream(content: bytes, dictionary: bytes = b"") -> bytes: + dictionary_entries: Final = b" " + dictionary if dictionary else b"" + return ( + b"<< /Length " + + str(len(content)).encode() + + dictionary_entries + + b" >>\nstream\n" + + content + + b"\nendstream" + ) + + +def _pdf_data_url_with_xobjects( + page_content: bytes, + page_xobjects: bytes, + xobject_objects: tuple[bytes, ...], + font_object: bytes = b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", +) -> str: + object_bodies: Final = ( + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [4 0 R] /Count 1 >>", + font_object, + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + b"/Resources << /Font << /F1 3 0 R >> /XObject " + + page_xobjects + + b" >> /Contents 5 0 R >>", + _pdf_stream(page_content), + *xobject_objects, + ) + object_chunks: Final = tuple( + f"{index} 0 obj\n".encode() + body + b"\nendobj\n" + for index, body in enumerate(object_bodies, start=1) + ) + header: Final = b"%PDF-1.4\n" + offsets: Final = accumulate((len(header), *(len(chunk) for chunk in object_chunks[:-1]))) + xref: Final = f"xref\n0 {len(object_chunks) + 1}\n0000000000 65535 f \n".encode() + b"".join( + f"{offset:010d} 00000 n \n".encode() for offset in offsets + ) + trailer: Final = ( + f"trailer\n<< /Size {len(object_chunks) + 1} /Root 1 0 R >>\n" + f"startxref\n{len(header) + sum(len(chunk) for chunk in object_chunks)}\n%%EOF\n" + ).encode() + return PDF_DATA_URL_PREFIX + base64.b64encode(header + b"".join(object_chunks) + xref + trailer).decode() + + +def _pdf_data_url_with_doubling_forms(depth: int, leaf_content: bytes) -> str: + form_objects: Final = tuple( + _pdf_stream( + leaf_content if index == depth - 1 else b"/Fm Do\n/Fm Do\n", + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> /XObject " + + ( + b"<< >>" + if index == depth - 1 + else b"<< /Fm " + str(7 + index).encode() + b" 0 R >>" + ) + + b" >>", + ) + for index in range(depth) + ) + return _pdf_data_url_with_xobjects(b"/Fm Do\n", b"<< /Fm 6 0 R >>", form_objects) + + +def _pdf_data_url_with_identity_h_font( + page_content: bytes, + page_xobjects: bytes = b"<< >>", + xobject_objects: tuple[bytes, ...] = (), +) -> str: + descendant_font_number: Final = 6 + len(xobject_objects) + cmap_number: Final = descendant_font_number + 1 + descriptor_number: Final = cmap_number + 1 + font_object: Final = ( + b"<< /Type /Font /Subtype /Type0 /BaseFont /Test /Encoding /Identity-H /DescendantFonts [" + + str(descendant_font_number).encode() + + b" 0 R] /ToUnicode " + + str(cmap_number).encode() + + b" 0 R >>" + ) + descendant_font: Final = ( + b"<< /Type /Font /Subtype /CIDFontType2 /BaseFont /Test " + b"/CIDSystemInfo << /Registry (Adobe) /Ordering (Identity) /Supplement 0 >> " + b"/FontDescriptor " + str(descriptor_number).encode() + b" 0 R /DW 1000 >>" + ) + cmap: Final = ( + b"/CIDInit /ProcSet findresource begin\n" + b"12 dict begin\n" + b"begincmap\n" + b"/CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def\n" + b"/CMapName /Test def\n" + b"/CMapType 2 def\n" + b"1 begincodespacerange\n" + b"<0000> \n" + b"endcodespacerange\n" + b"1 beginbfchar\n" + b"<0001> <00480065006C006C006F00200077006F0072006C0064>\n" + b"endbfchar\n" + b"endcmap\n" + b"CMapName currentdict /CMap defineresource pop\n" + b"end\n" + b"end" + ) + font_descriptor: Final = ( + b"<< /Type /FontDescriptor /FontName /Test /Flags 4 /FontBBox [0 0 1000 1000] " + b"/ItalicAngle 0 /Ascent 1000 /Descent 0 /CapHeight 700 /StemV 80 >>" + ) + return _pdf_data_url_with_xobjects( + page_content, + page_xobjects, + (*xobject_objects, descendant_font, _pdf_stream(cmap), font_descriptor), + font_object, + ) + + +def _pdf_data_url_with_shared_xobjects( + page_contents: tuple[bytes, ...], + xobject_objects: tuple[bytes, ...], +) -> str: + page_count: Final = len(page_contents) + page_object_numbers: Final = tuple(5 + index for index in range(page_count)) + content_object_numbers: Final = tuple(5 + page_count + index for index in range(page_count)) + first_xobject_object_number: Final = 5 + 2 * page_count + page_references: Final = b" ".join(f"{number} 0 R".encode() for number in page_object_numbers) + xobject_references: Final = b" ".join( + f"/Fm{index} {first_xobject_object_number + index} 0 R".encode() + for index in range(len(xobject_objects)) + ) + page_objects: Final = tuple( + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources 4 0 R /Contents " + + f"{content_object} 0 R >>".encode() + for content_object in content_object_numbers + ) + content_objects: Final = tuple(_pdf_stream(content) for content in page_contents) + object_bodies: Final = ( + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [" + + page_references + + f"] /Count {page_count} >>".encode(), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + b"<< /Font << /F1 3 0 R >> /XObject << " + xobject_references + b" >> >>", + *page_objects, + *content_objects, + *xobject_objects, + ) + object_chunks: Final = tuple( + f"{index} 0 obj\n".encode() + body + b"\nendobj\n" + for index, body in enumerate(object_bodies, start=1) + ) + header: Final = b"%PDF-1.4\n" + offsets: Final = accumulate((len(header), *(len(chunk) for chunk in object_chunks[:-1]))) + xref: Final = f"xref\n0 {len(object_chunks) + 1}\n0000000000 65535 f \n".encode() + b"".join( + f"{offset:010d} 00000 n \n".encode() for offset in offsets + ) + trailer: Final = ( + f"trailer\n<< /Size {len(object_chunks) + 1} /Root 1 0 R >>\n" + f"startxref\n{len(header) + sum(len(chunk) for chunk in object_chunks)}\n%%EOF\n" + ).encode() + return PDF_DATA_URL_PREFIX + base64.b64encode(header + b"".join(object_chunks) + xref + trailer).decode() + + +class _TestPdfMediaBox(Protocol): + width: float + height: float + + +class _TestPdfContentStream(Protocol): + def get_data(self) -> bytes: ... + + +class _TestPdfDictionary(Protocol): + def get(self, key: str) -> object | None: ... + + def items(self) -> Iterable[tuple[str, object]]: ... + + def __getitem__(self, key: str) -> object: ... + + +class _TextExtractionMock(Protocol): + call_count: int + return_value: str + + def __call__(self) -> str: ... + + def assert_not_called(self) -> None: ... + + +class _TestPdfPage(Protocol): + mediabox: _TestPdfMediaBox + extract_text: _TextExtractionMock + + def items(self) -> Iterable[tuple[str, object]]: ... + + def get(self, key: str) -> object | None: ... + + def get_contents(self) -> _TestPdfContentStream | None: ... + + +class _PdfTextPage(Protocol): + def extract_text(self) -> str | None: ... + + +class _PdfTextPages(Protocol): + def __getitem__(self, index: int) -> _PdfTextPage: ... + + +class _PdfTextReader(Protocol): + pages: _PdfTextPages + + +class _PdfReaderModule(Protocol): + PdfReader: Callable[[io.BytesIO], _PdfTextReader] + + +class _PypdfGenericModule(Protocol): + DictionaryObject: Callable[[Mapping[str, object]], _TestPdfDictionary] + NameObject: Callable[[str], str] + + +def _pdf_text_page(pdf_bytes: bytes) -> _PdfTextPage: + pypdf_module: Final = cast(_PdfReaderModule, importlib.import_module("pypdf")) + return pypdf_module.PdfReader(io.BytesIO(pdf_bytes)).pages[0] + + +def _pdf_extracted_text(pdf_bytes: bytes) -> str: + return _pdf_text_page(pdf_bytes).extract_text() or "" + + def _pdf_source(pdf_base64: str) -> dict[str, str]: return {"type": "base64", "media_type": "application/pdf", "data": pdf_base64} @@ -1998,6 +2218,911 @@ def test_pdf_page_rendering_cost_follows_anthropic_image_scaling(fields: dict[st assert strip == 328 +def _fake_pdf_page(content_bytes: int | bytes, include_xobjects: bool = False) -> _TestPdfPage: + content: Final = b"x" * content_bytes if isinstance(content_bytes, int) else content_bytes + pypdf_generic: Final = cast(_PypdfGenericModule, importlib.import_module("pypdf.generic")) + name_object: Final = pypdf_generic.NameObject + font: Final = pypdf_generic.DictionaryObject( + { + name_object("/Type"): name_object("/Font"), + name_object("/Subtype"): name_object("/Type1"), + name_object("/BaseFont"): name_object("/Helvetica"), + } + ) + resources: Final = pypdf_generic.DictionaryObject( + { + name_object("/Font"): pypdf_generic.DictionaryObject({name_object("/F1"): font}), + **({name_object("/XObject"): pypdf_generic.DictionaryObject({})} if include_xobjects else {}), + } + ) + + def get_page_resource(key: str) -> object | None: + return resources if key == "/Resources" else None + + return cast( + _TestPdfPage, + SimpleNamespace( + mediabox=SimpleNamespace(width=612.0, height=792.0), + get_contents=lambda: SimpleNamespace(get_data=lambda: content), + extract_text=cast( + _TextExtractionMock, + MagicMock(return_value="Revenue grew eleven percent while churn fell to two percent."), + ), + get=get_page_resource, + ), + ) + + +def test_long_pdf_reads_text_from_a_bounded_number_of_pages(): + pages: Final = tuple(_fake_pdf_page(content_bytes=1_000) for _ in range(40)) + + _price_pdf_pages(pages, len) + + assert sum(page.extract_text.call_count for page in pages) <= EXTRAPOLATION_SAMPLES + + +def test_dense_pdf_page_is_priced_from_its_size_without_extracting_its_text(): + string_operand: Final = b"(Revenue grew eleven percent)" + text_operator: Final = b"BT /F1 12 Tf " + string_operand + b" Tj ET\n" + decoded_operand_bytes: Final = len(string_operand) - 2 + content: Final = text_operator * (2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // decoded_operand_bytes + 1) + page: Final = _fake_pdf_page(content_bytes=content) + + priced: Final = _price_pdf_pages((page,), len) + + page.extract_text.assert_not_called() + assert priced >= 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES + + +def test_oversized_pdf_page_is_priced_from_its_string_operands_without_extracting_text(): + content: Final = b"BT /F1 9 Tf (Figure 3) Tj ET\n" + b"10 10 m 200 300 l S\n" * 10_000 + page: Final = _fake_pdf_page(content_bytes=content) + blank_page: Final = _fake_pdf_page(content_bytes=b"") + blank_page.extract_text.return_value = "" + + priced: Final = _price_pdf_pages((page,), len) - _price_pdf_pages((blank_page,), len) + + page.extract_text.assert_not_called() + assert priced <= len(b"(Figure 3)") + + +def test_oversized_pdf_form_xobject_is_priced_without_extracting_its_text(): + string_operand: Final = b"(form text)" + text_operator: Final = b"BT /F1 12 Tf " + string_operand + b" Tj ET\n" + repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(text_operator) + 1 + form_content: Final = text_operator * repetitions + form_object: Final = _pdf_stream( + form_content, + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >>", + ) + data_url: Final = _pdf_data_url_with_xobjects( + b"/Fm1 Do\n", + b"<< /Fm1 6 0 R >>", + (form_object,), + ) + count_function: Final = MagicMock(side_effect=len) + + priced: Final = _count_inline_pdf_tokens(data_url, count_function) + + assert priced is not None + assert max((len(call.args[0]) for call in count_function.call_args_list), default=0) <= ( + PDF_PAGE_MAX_EXACT_CONTENT_BYTES + ) + assert priced >= (len(string_operand) - 2) * repetitions + + +def test_repeated_pdf_form_draws_are_priced_with_multiplicity(): + string_operand: Final = b"(repeated form text)" + text_operator: Final = b"BT /F1 12 Tf " + string_operand + b" Tj ET\n" + repetitions: Final = 1_024 // len(text_operator) + 1 + form_object: Final = _pdf_stream( + text_operator * repetitions, + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >>", + ) + data_url: Final = _pdf_data_url_with_xobjects( + b"/Fm1 Do\n" * 1_000, + b"<< /Fm1 6 0 R >>", + (form_object,), + ) + count_function: Final = MagicMock(side_effect=len) + + priced: Final = _count_inline_pdf_tokens(data_url, count_function) # pyright: ignore[reportPrivateUsage] # PDF regression test + + assert priced is not None + assert max((len(call.args[0]) for call in count_function.call_args_list), default=0) <= ( + PDF_PAGE_MAX_EXACT_CONTENT_BYTES + ) + assert priced >= (len(string_operand) - 2) * repetitions * 1_000 + + +def test_oversized_nested_pdf_form_xobject_is_priced_without_extracting_its_text(): + string_operand: Final = b"(nested form text)" + text_operator: Final = b"BT /F1 12 Tf " + string_operand + b" Tj ET\n" + repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(text_operator) + 1 + nested_content: Final = text_operator * repetitions + nested_form: Final = _pdf_stream( + nested_content, + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >>", + ) + form: Final = _pdf_stream( + b"/Fm2 Do\n", + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] " + b"/Resources << /Font << /F1 3 0 R >> /XObject << /Fm2 7 0 R >> >>", + ) + data_url: Final = _pdf_data_url_with_xobjects( + b"/Fm1 Do\n", + b"<< /Fm1 6 0 R >>", + (form, nested_form), + ) + count_function: Final = MagicMock(side_effect=len) + + priced: Final = _count_inline_pdf_tokens(data_url, count_function) + + assert priced is not None + assert max((len(call.args[0]) for call in count_function.call_args_list), default=0) <= ( + PDF_PAGE_MAX_EXACT_CONTENT_BYTES + ) + assert priced >= (len(string_operand) - 2) * repetitions + + +def test_exponential_pdf_form_draws_avoid_full_text_extraction(): + depth: Final = 20 + form_objects: Final = tuple( + _pdf_stream( + (f"/Fm{index + 1} Do\n" * 2).encode() if index < depth - 1 else b"BT /F1 12 Tf (leaf) Tj ET\n", + ( + f"/Type /XObject /Subtype /Form /BBox [0 0 612 792] " + f"/Resources << /Font << /F1 3 0 R >> /XObject << /Fm{index + 1} {8 + index} 0 R >> >>" + ).encode() + if index < depth - 1 + else b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >>", + ) + for index in range(depth) + ) + data_url: Final = _pdf_data_url_with_shared_xobjects((b"/Fm0 Do\n",), form_objects) + count_function: Final = MagicMock(side_effect=len) + + priced: Final = _count_inline_pdf_tokens(data_url, count_function) # pyright: ignore[reportPrivateUsage] # PDF regression test + + assert priced is not None + assert max((len(call.args[0]) for call in count_function.call_args_list), default=0) <= ( + PDF_PAGE_MAX_EXACT_CONTENT_BYTES + ) + assert priced >= 2 ** (depth - 1) + + +@pytest.mark.timeout(10) +def test_cyclic_exponential_pdf_form_walk_is_memoized(): + depth: Final = 25 + + def cyclic_form(index: int) -> bytes: + next_index: Final = (index + 1) % depth + next_form_content: Final = f"/Fm{next_index} Do\n/Fm{next_index} Do\n".encode() + form_content: Final = b"BT /F1 12 Tf (x) Tj ET\n" + next_form_content + form_dictionary: Final = ( + f"/Type /XObject /Subtype /Form /BBox [0 0 612 792] " + f"/Resources << /Font << /F1 3 0 R >> /XObject << /Fm{next_index} {7 + next_index} 0 R >> >>" + ).encode() + return _pdf_stream(form_content, form_dictionary) + + form_objects: Final = tuple(cyclic_form(index) for index in range(depth)) + data_url: Final = _pdf_data_url_with_shared_xobjects((b"/Fm0 Do\n",), form_objects) + count_function: Final = MagicMock(side_effect=len) + + priced: Final = _count_inline_pdf_tokens(data_url, count_function) # pyright: ignore[reportPrivateUsage] # PDF regression test + + assert priced is not None + assert priced > 0 + assert max((len(call.args[0]) for call in count_function.call_args_list), default=0) <= ( + PDF_PAGE_MAX_EXACT_CONTENT_BYTES + ) + + +def test_cyclic_pdf_form_xobject_is_priced(): + path_operator: Final = b"10 10 m 200 300 l S\n" + form_content: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) + 1) + b"/Fm1 Do\n" + form: Final = _pdf_stream( + form_content, + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /XObject << /Fm1 6 0 R >> >>", + ) + data_url: Final = _pdf_data_url_with_xobjects( + b"/Fm1 Do\n", + b"<< /Fm1 6 0 R >>", + (form,), + ) + + priced: Final = _count_inline_pdf_tokens(data_url, len) + + assert priced is not None + + +def test_pdf_image_xobject_does_not_trigger_the_content_limit(): + page_text: Final = b"BT /F1 12 Tf 20 760 Td (small page text) Tj ET\n" + image_data: Final = b"\x00" * (1024 * 1024) + image: Final = _pdf_stream( + image_data, + b"/Type /XObject /Subtype /Image /Width 1024 /Height 1024 " + b"/ColorSpace /DeviceRGB /BitsPerComponent 8", + ) + data_url: Final = _pdf_data_url_with_xobjects( + page_text, + b"<< /Im1 6 0 R >>", + (image,), + ) + count_function: Final = MagicMock(side_effect=len) + + priced: Final = _count_inline_pdf_tokens(data_url, count_function) + + assert priced is not None + assert any("small page text" in call.args[0] for call in count_function.call_args_list) + + +def test_pdf_pages_with_shared_resources_price_only_the_form_they_draw(): + page_count: Final = 20 + string_operand: Final = b"(text)" + text_operator: Final = b"BT /F1 12 Tf " + string_operand + b" Tj ET\n" + repetitions: Final = 2_800 // len(string_operand) + page_contents: Final = tuple(f"/Fm{index} Do\n".encode() for index in range(page_count)) + form_objects: Final = tuple( + _pdf_stream( + text_operator * repetitions, + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >>", + ) + for _ in range(page_count) + ) + data_url: Final = _pdf_data_url_with_shared_xobjects(page_contents, form_objects) + blank_data_url: Final = _pdf_data_url_with_shared_xobjects((b"",) * page_count, ()) + priced: Final = _count_inline_pdf_tokens(data_url, len) + rendering_tokens: Final = _count_inline_pdf_tokens(blank_data_url, len) + string_operand_bytes: Final = page_count * (len(string_operand) - 2) * repetitions + + assert priced is not None + assert rendering_tokens is not None + assert priced <= string_operand_bytes * 3 // 2 + rendering_tokens + + +def test_pdf_form_without_resources_inherits_page_xobjects(): + string_operand: Final = b"(inherited form text)" + text_operator: Final = b"BT /F1 12 Tf " + string_operand + b" Tj ET\n" + repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(text_operator) + 1 + nested_form_content: Final = text_operator * repetitions + inherited_form: Final = _pdf_stream( + nested_form_content, + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >>", + ) + unused_form: Final = _pdf_stream( + nested_form_content, + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >>", + ) + form_without_resources: Final = _pdf_stream( + b"/Fm2 Do\n", + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792]", + ) + data_url: Final = _pdf_data_url_with_xobjects( + b"/Fm1 Do\n", + b"<< /Fm1 6 0 R /Fm2 7 0 R /Fm3 8 0 R >>", + (form_without_resources, inherited_form, unused_form), + ) + blank_data_url: Final = _pdf_data_url_with_xobjects(b"", b"<< >>", ()) + priced: Final = _count_inline_pdf_tokens(data_url, len) + rendering_tokens: Final = _count_inline_pdf_tokens(blank_data_url, len) + string_operand_bytes: Final = (len(string_operand) - 2) * repetitions + + assert priced is not None + assert rendering_tokens is not None + assert priced >= string_operand_bytes + assert priced <= string_operand_bytes * 3 // 2 + rendering_tokens + + +def test_oversized_pdf_string_operand_counts_escaped_newline_continuations(): + escaped_newlines: Final = b"a\\\n" * 10_000 + literal_content: Final = b"BT /F1 12 Tf (" + escaped_newlines + b"a" * 50_000 + b") Tj ET\n" + page: Final = _fake_pdf_page(content_bytes=literal_content) + + priced: Final = _price_pdf_pages((page,), len) + + assert priced >= 60_000 + + +def test_nested_pdf_literal_string_keeps_its_full_operand(): + content: Final = b"BT /F1 12 Tf (a(b)" + b"x" * 100_000 + b") Tj ET\n" + page: Final = _fake_pdf_page(content_bytes=content) + + priced: Final = _price_pdf_pages((page,), len) # pyright: ignore[reportPrivateUsage] # PDF regression test + + assert priced >= 100_000 + + +def test_pdf_form_invocations_allow_comments_and_escaped_names(): + string_operand: Final = b"(named form text)" + text_operator: Final = b"BT /F1 12 Tf " + string_operand + b" Tj ET\n" + repetitions: Final = 1_024 // len(text_operator) + 1 + form_object: Final = _pdf_stream( + text_operator * repetitions, + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >>", + ) + path_operator: Final = b"10 10 m 20 20 l S\n" + page_content: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) + 1) + ( + b"/Fm1 %comment\n Do\n/F#6d1 Do\n" + ) + data_url: Final = _pdf_data_url_with_xobjects( + page_content, + b"<< /Fm1 6 0 R >>", + (form_object,), + ) + count_function: Final = MagicMock(side_effect=len) + + priced: Final = _count_inline_pdf_tokens(data_url, count_function) # pyright: ignore[reportPrivateUsage] # PDF regression test + + assert priced is not None + assert priced >= 2 * (len(string_operand) - 2) * repetitions + + +def test_pdf_percent_inside_literal_does_not_hide_form_invocation(): + string_operand: Final = b"(form text)" + text_operator: Final = b"BT /F1 12 Tf " + string_operand + b" Tj ET\n" + repetitions: Final = 100 * 1024 // len(text_operator) + 1 + form_object: Final = _pdf_stream( + text_operator * repetitions, + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >>", + ) + data_url: Final = _pdf_data_url_with_xobjects( + b"BT /F1 12 Tf (50%) Tj ET /Fm1 Do\n", + b"<< /Fm1 6 0 R >>", + (form_object,), + ) + count_function: Final = MagicMock(side_effect=len) + + priced: Final = _count_inline_pdf_tokens(data_url, count_function) # pyright: ignore[reportPrivateUsage] # PDF regression test + + assert priced is not None + assert priced >= (len(string_operand) - 2) * repetitions + assert max((len(call.args[0]) for call in count_function.call_args_list), default=0) <= ( + PDF_PAGE_MAX_EXACT_CONTENT_BYTES + ) + + +@pytest.mark.parametrize( + "comments", + [ + b"%comment\r\n" * 23, + b"% a \r\n" * 40, + ], + ids=["23-crlf-comments", "40-crlf-comments-with-trailing-spaces"], +) +def test_pdf_invocation_comment_separators_are_priceable(comments: bytes): + content: Final = b"BT /F1 " + comments + b"12 Tf (hello) Tj ET" + page: Final = _fake_pdf_page(content_bytes=content, include_xobjects=True) + blank_page: Final = _fake_pdf_page(content_bytes=b"") + + priced: Final = _price_pdf_pages((page,), len) + expected: Final = _price_pdf_pages((blank_page,), len) + + assert priced == expected + + +@pytest.mark.parametrize("inline_image_in_form", [False, True], ids=["page-stream", "form-stream"]) +def test_pdf_inline_image_bytes_do_not_push_content_over_the_limit(inline_image_in_form: bool): + image_data: Final = random.Random(42).randbytes(270_000) + inline_image: Final = ( + b"BI /W 270000 /H 1 /BPC 8 /CS /RGB ID " + image_data + b" EI\nBT /F1 9 Tf (Hello world) Tj ET\n" + ) + form: Final = _pdf_stream( + inline_image, + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >>", + ) + page_content: Final = b"/Fm1 Do\n" if inline_image_in_form else inline_image + page_xobjects: Final = b"<< /Fm1 6 0 R >>" if inline_image_in_form else b"<< >>" + form_objects: Final = (form,) if inline_image_in_form else () + data_url: Final = _pdf_data_url_with_xobjects(page_content, page_xobjects, form_objects) + pdf_bytes: Final = base64.b64decode(data_url.removeprefix(PDF_DATA_URL_PREFIX)) + page: Final = _pdf_text_page(pdf_bytes) + page_text: Final = page.extract_text() or "" + + def count_function(value: str) -> int: + return len(value) + + expected: Final = count_function(page_text) + count_calls: Final = MagicMock(side_effect=count_function) + priced: Final = _count_inline_pdf_tokens(data_url, count_calls) + blank_data_url: Final = _pdf_data_url_with_xobjects(b"", b"<< >>", ()) + blank_price: Final = _count_inline_pdf_tokens(blank_data_url, count_function) + + assert priced is not None + assert blank_price is not None + assert priced - blank_price == expected + count_calls.assert_called_once_with(page_text) + + +@pytest.mark.parametrize( + "prefix", + [b"(BI) Tj\n", b"/BI 1 Tf\n", b"/BIfoo 1 Tf\n"], + ids=["literal-string", "inline-image-name", "longer-name"], +) +def test_pdf_inline_image_operator_is_not_found_inside_strings_or_names(prefix: bytes): + string_operand: Final = b"(sample text)" + text_operator: Final = b"BT /F1 12 Tf " + string_operand + b" Tj ET\n" + repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(text_operator) + 1 + page: Final = _fake_pdf_page(content_bytes=prefix + text_operator * repetitions) + count_function: Final = MagicMock(side_effect=len) + + priced: Final = _price_pdf_pages((page,), count_function) + + page.extract_text.assert_not_called() + assert max((len(call.args[0]) for call in count_function.call_args_list), default=0) <= ( + PDF_PAGE_MAX_EXACT_CONTENT_BYTES + ) + assert priced >= (len(string_operand) - 2) * repetitions + + +@pytest.mark.parametrize( + "prefix", + [b"BI /W 1 ", b"BI /W 1 ID xx "], + ids=["missing-id", "missing-ei"], +) +def test_unterminated_pdf_inline_images_are_priced_from_sample(prefix: bytes): + string_operand: Final = b"(sample text)" + text_operator: Final = b"BT /F1 12 Tf " + string_operand + b" Tj ET\n" + repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(text_operator) + 1 + page: Final = _fake_pdf_page(content_bytes=prefix + text_operator * repetitions) + count_function: Final = MagicMock(side_effect=len) + + priced: Final = _price_pdf_pages((page,), count_function) + + page.extract_text.assert_not_called() + assert max((len(call.args[0]) for call in count_function.call_args_list), default=0) <= ( + PDF_PAGE_MAX_EXACT_CONTENT_BYTES + ) + assert priced >= (len(string_operand) - 2) * repetitions + + +def test_pdf_comments_are_not_counted_as_string_operands(): + comment: Final = b"% (lots of text)\n" + repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(comment) + 1 + content: Final = comment * repetitions + b"BT /F1 12 Tf (x) Tj ET\n" + page: Final = _fake_pdf_page(content_bytes=content) + blank_page: Final = _fake_pdf_page(content_bytes=b"") + blank_page.extract_text.return_value = "" + + priced: Final = _price_pdf_pages((page,), len) + blank_price: Final = _price_pdf_pages((blank_page,), len) + + assert priced - blank_price == len(b"x") + + +def test_oversized_hex_pdf_strings_price_like_literal_text(): + literal_operand: Final = b"(Hello world)" + hex_operand: Final = b"<48656c6c6f20776f726c64>" + repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(b"BT /F1 12 Tf " + hex_operand + b" Tj ET\n") + 1 + literal_page: Final = _fake_pdf_page(content_bytes=(b"BT /F1 12 Tf " + literal_operand + b" Tj ET\n") * repetitions) + hex_page: Final = _fake_pdf_page(content_bytes=(b"BT /F1 12 Tf " + hex_operand + b" Tj ET\n") * repetitions) + blank_page: Final = _fake_pdf_page(content_bytes=b"") + + literal_price: Final = _price_pdf_pages((literal_page,), len) + hex_price: Final = _price_pdf_pages((hex_page,), len) + blank_price: Final = _price_pdf_pages((blank_page,), len) + literal_text_tokens: Final = literal_price - blank_price + hex_text_tokens: Final = hex_price - blank_price + + assert hex_text_tokens * 5 <= literal_text_tokens * 6 + assert literal_text_tokens * 5 <= hex_text_tokens * 6 + + +def test_oversized_pdf_octal_escapes_keep_sample_scaling_by_decoded_bytes(): + plain_count: Final = PDF_PAGE_MAX_EXACT_CONTENT_BYTES // 2 + 2 + octal_count: Final = PDF_PAGE_MAX_EXACT_CONTENT_BYTES // 2 + 2 + text_object: Final = b"BT /F1 12 Tf (" + b"A" * plain_count + b"\\101" * octal_count + b") Tj ET\n" + page: Final = _fake_pdf_page(content_bytes=text_object) + blank_page: Final = _fake_pdf_page(content_bytes=b"") + + priced: Final = _price_pdf_pages((page,), len) + blank_price: Final = _price_pdf_pages((blank_page,), len) + expected: Final = plain_count + octal_count + + assert expected * 3 // 4 <= priced - blank_price <= expected * 5 // 4 + + +def test_oversized_pdf_text_uses_a_sampled_token_estimate(): + string_operand: Final = b"(Revenue grew eleven percent)" + text_operator: Final = b"BT /F1 12 Tf " + string_operand + b" Tj ET\n" + repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(text_operator) + 1 + page: Final = _fake_pdf_page(content_bytes=text_operator * repetitions) + blank_page: Final = _fake_pdf_page(content_bytes=b"") + + def four_chars_per_token(value: str) -> int: + return len(value) // 4 + + page_priced: Final = _price_pdf_pages((page,), four_chars_per_token) # pyright: ignore[reportPrivateUsage] # PDF regression test + blank_page_priced: Final = _price_pdf_pages((blank_page,), four_chars_per_token) # pyright: ignore[reportPrivateUsage] # PDF regression test + priced: Final = page_priced - blank_page_priced + expected: Final = len(string_operand) * repetitions // 4 + + assert expected * 4 // 5 <= priced <= expected * 6 // 5 + + +def test_many_tiny_pdf_text_objects_price_within_sample_estimate(): + repetitions: Final = 200_000 + text_object: Final = b"BT /F1 12 Tf (x) Tj ET\n" + page: Final = _fake_pdf_page(text_object * repetitions) + blank_page: Final = _fake_pdf_page(b"") + + def count_x_tokens(value: str) -> int: + return value.count("x") + + priced: Final = _price_pdf_pages((page,), count_x_tokens) + blank_price: Final = _price_pdf_pages((blank_page,), count_x_tokens) + expected: Final = repetitions + + assert expected * 3 // 4 <= priced - blank_price <= expected * 5 // 4 + + +def test_oversized_literal_cut_after_escape_stays_extractable(): + text_prefix: Final = b"BT /F1 12 Tf " + suffix: Final = b" Tj ET" + operand_prefix_length: Final = PDF_PAGE_MAX_EXACT_CONTENT_BYTES - len(text_prefix) - len(suffix) + operand: Final = b"(" + b"x" * (operand_prefix_length - 2) + b"\\(" + b"x" * 128 + b")" + page: Final = _fake_pdf_page(text_prefix + operand + suffix) + blank_page: Final = _fake_pdf_page(b"") + + def count_x_tokens(value: str) -> int: + return value.count("x") + + priced: Final = _price_pdf_pages((page,), count_x_tokens) + blank_price: Final = _price_pdf_pages((blank_page,), count_x_tokens) + expected: Final = operand.count(b"x") + + assert expected * 3 // 4 <= priced - blank_price <= expected * 5 // 4 + + +@pytest.mark.parametrize("backslash_count", [1, 2, 3]) +def test_oversized_unclosed_literal_with_backslash_run_stays_extractable(backslash_count: int): + path_operator: Final = b"10 10 m 20 20 l S\n" + path_padding: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) - 1) + literal_prefix: Final = b"BT /F1 12 Tf (escaped \\(paren\\) " + b"x" * 512 + b"\\" * backslash_count + closed_suffix: Final = b")" * (1 + backslash_count % 2) + b" Tj ET\n" + closed_page: Final = _fake_pdf_page(path_padding + literal_prefix + closed_suffix) + unclosed_page: Final = _fake_pdf_page(path_padding + literal_prefix) + blank_page: Final = _fake_pdf_page(b"") + + def count_x_tokens(value: str) -> int: + return value.count("x") + + blank_price: Final = _price_pdf_pages((blank_page,), count_x_tokens) + closed_price: Final = _price_pdf_pages((closed_page,), count_x_tokens) - blank_price + unclosed_price: Final = _price_pdf_pages((unclosed_page,), count_x_tokens) - blank_price + expected: Final = literal_prefix.count(b"x") + + assert expected * 3 // 4 <= closed_price <= expected * 5 // 4 + assert expected * 3 // 4 <= unclosed_price <= expected * 5 // 4 + + +def test_unclosed_end_of_stream_text_keeps_sampling_form_text(): + form_text: Final = b"BT /F1 12 Tf (form text) Tj ET\n" + form_repetitions: Final = 2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(form_text) + 1 + form_object: Final = _pdf_stream( + form_text * form_repetitions, + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >>", + ) + unclosed_data_url: Final = _pdf_data_url_with_xobjects( + b"/Fm Do\nBT /F1 12 Tf (hi) Tj", + b"<< /Fm 6 0 R >>", + (form_object,), + ) + closed_data_url: Final = _pdf_data_url_with_xobjects( + b"/Fm Do\nBT /F1 12 Tf (hi) Tj ET\n", + b"<< /Fm 6 0 R >>", + (form_object,), + ) + + def count_form_tokens(value: str) -> int: + return value.count("form") + + unclosed_price: Final = _count_inline_pdf_tokens(unclosed_data_url, count_form_tokens) + closed_price: Final = _count_inline_pdf_tokens(closed_data_url, count_form_tokens) + + assert unclosed_price is not None + assert closed_price is not None + assert closed_price * 3 // 4 <= unclosed_price <= closed_price * 5 // 4 + + +def test_oversized_literal_cut_after_et_inside_string_stays_extractable(): + text_prefix: Final = b"BT /F1 12 Tf (" + literal_prefix: Final = b"x" * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES - len(text_prefix) - len(b"ET")) + b"ET" + literal_suffix: Final = b"x" * 256 + b") Tj ET" + page: Final = _fake_pdf_page(text_prefix + literal_prefix + literal_suffix) + blank_page: Final = _fake_pdf_page(b"") + + def count_x_tokens(value: str) -> int: + return value.count("x") + + priced: Final = _price_pdf_pages((page,), count_x_tokens) + blank_price: Final = _price_pdf_pages((blank_page,), count_x_tokens) + expected: Final = (literal_prefix + literal_suffix[: -len(b") Tj ET")]).count(b"x") + + assert expected * 3 // 4 <= priced - blank_price <= expected * 5 // 4 + + +def test_unterminated_pdf_text_object_ending_with_et_inside_string_is_closed(): + path_operator: Final = b"10 10 m 20 20 l S\n" + path_padding: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) + 1) + page: Final = _fake_pdf_page(path_padding + b"BT /F1 12 Tf (x ET") + blank_page: Final = _fake_pdf_page(b"") + + def count_x_tokens(value: str) -> int: + return value.count("x") + + priced: Final = _price_pdf_pages((page,), count_x_tokens) + blank_price: Final = _price_pdf_pages((blank_page,), count_x_tokens) + + assert priced - blank_price == 1 + + +def test_oversized_text_cut_inside_trailing_comment_stays_extractable(): + text_prefix: Final = b"BT /F1 12 Tf (x) Tj " + comment: Final = b"%" + b"comment" * PDF_PAGE_MAX_EXACT_CONTENT_BYTES + page: Final = _fake_pdf_page(text_prefix + comment + b"\nET") + blank_page: Final = _fake_pdf_page(b"") + + def count_x_tokens(value: str) -> int: + return value.count("x") + + priced: Final = _price_pdf_pages((page,), count_x_tokens) + blank_price: Final = _price_pdf_pages((blank_page,), count_x_tokens) + + assert priced - blank_price == 1 + + +def test_oversized_tj_array_cut_inside_comment_stays_extractable(): + repetitions: Final = 1_000 + content: Final = ( + b"BT /F1 12 Tf [" + + b"(x) 0 " * repetitions + + b"% comment with [" + + b"comment" * PDF_PAGE_MAX_EXACT_CONTENT_BYTES + + b"\n] TJ ET" + ) + page: Final = _fake_pdf_page(content) + blank_page: Final = _fake_pdf_page(b"") + + def count_x_tokens(value: str) -> int: + return value.count("x") + + priced: Final = _price_pdf_pages((page,), count_x_tokens) + blank_price: Final = _price_pdf_pages((blank_page,), count_x_tokens) + + assert repetitions * 3 // 4 <= priced - blank_price <= repetitions * 5 // 4 + + +def test_oversized_identity_h_pdf_uses_tounicode_for_two_byte_glyphs(): + repetitions: Final = 1_000 + path_operator: Final = b"10 10 m 20 20 l S\n" + path_padding: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) + 1) + text_object: Final = b"BT /F1 12 Tf " + b"<0001> Tj " * repetitions + b"ET\n" + data_url: Final = _pdf_data_url_with_identity_h_font(path_padding + text_object) + pdf_bytes: Final = base64.b64decode(data_url.removeprefix(PDF_DATA_URL_PREFIX)) + exact_text: Final = _pdf_extracted_text(pdf_bytes) + priced: Final = _count_inline_pdf_tokens(data_url, len) + blank_data_url: Final = _pdf_data_url_with_xobjects(b"", b"<< >>", ()) + blank_price: Final = _count_inline_pdf_tokens(blank_data_url, len) + + assert exact_text == "Hello world" * repetitions + assert priced is not None + assert blank_price is not None + assert len(exact_text) * 4 // 5 <= priced - blank_price <= len(exact_text) * 6 // 5 + + +def test_oversized_helvetica_pdf_text_stays_close_to_exact_extraction(): + repetitions: Final = 1_000 + path_operator: Final = b"10 10 m 20 20 l S\n" + path_padding: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) + 1) + text_object: Final = b"BT /F1 12 Tf " + b"(Hello world) Tj " * repetitions + b"ET\n" + data_url: Final = _pdf_data_url_with_xobjects(path_padding + text_object, b"<< >>", ()) + pdf_bytes: Final = base64.b64decode(data_url.removeprefix(PDF_DATA_URL_PREFIX)) + exact_text: Final = _pdf_extracted_text(pdf_bytes) + priced: Final = _count_inline_pdf_tokens(data_url, len) + blank_data_url: Final = _pdf_data_url_with_xobjects(b"", b"<< >>", ()) + blank_price: Final = _count_inline_pdf_tokens(blank_data_url, len) + + assert exact_text == "Hello world" * repetitions + assert priced is not None + assert blank_price is not None + assert len(exact_text) * 4 // 5 <= priced - blank_price <= len(exact_text) * 6 // 5 + + +def test_oversized_identity_h_pdf_counts_text_after_path_padding(): + path_operator: Final = b"10 10 m 20 20 l S\n" + path_padding: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) + 1) + text_object: Final = b"BT /F1 12 Tf <0001> Tj ET\n" + data_url: Final = _pdf_data_url_with_identity_h_font(path_padding + text_object) + priced: Final = _count_inline_pdf_tokens(data_url, len) + blank_data_url: Final = _pdf_data_url_with_xobjects(b"", b"<< >>", ()) + blank_price: Final = _count_inline_pdf_tokens(blank_data_url, len) + + assert priced is not None + assert blank_price is not None + assert priced - blank_price == len("Hello world") + + +def test_depth_30_doubling_form_tree_prices_text_leaf_by_effective_draw_count(): + repetitions: Final = 1 << 30 + data_url: Final = _pdf_data_url_with_doubling_forms(31, b"BT /F1 12 Tf (x) Tj ET\n") + + def count_x_tokens(value: str) -> int: + return value.count("x") + + priced: Final = _count_inline_pdf_tokens(data_url, count_x_tokens) + blank_data_url: Final = _pdf_data_url_with_xobjects(b"", b"<< >>", ()) + blank_price: Final = _count_inline_pdf_tokens(blank_data_url, count_x_tokens) + + assert priced is not None + assert blank_price is not None + assert repetitions * 3 // 4 <= priced - blank_price <= repetitions * 5 // 4 + + +def test_oversized_tj_array_truncation_keeps_its_text(): + content: Final = b"BT /F1 12 Tf [" + b"(x) 0 " * 20_000 + b"] TJ ET\n" + data_url: Final = _pdf_data_url_with_xobjects(content, b"<< >>", ()) + pdf_bytes: Final = base64.b64decode(data_url.removeprefix(PDF_DATA_URL_PREFIX)) + exact_text: Final = _pdf_extracted_text(pdf_bytes) + + def count_x_tokens(value: str) -> int: + return value.count("x") + + exact_tokens: Final = count_x_tokens(exact_text) + priced: Final = _count_inline_pdf_tokens(data_url, count_x_tokens) + blank_data_url: Final = _pdf_data_url_with_xobjects(b"", b"<< >>", ()) + blank_price: Final = _count_inline_pdf_tokens(blank_data_url, count_x_tokens) + + assert priced is not None + assert blank_price is not None + assert exact_tokens > 10_000 + assert exact_tokens * 3 // 4 <= priced - blank_price <= exact_tokens * 5 // 4 + + +def test_compact_bt_operator_keeps_oversized_page_text(): + path_operator: Final = b"10 10 m 20 20 l S\n" + path_padding: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) + 1) + content: Final = path_padding + b"BT/F1 12 Tf (Hello world) Tj ET\n" + data_url: Final = _pdf_data_url_with_xobjects(content, b"<< >>", ()) + priced: Final = _count_inline_pdf_tokens(data_url, len) + blank_data_url: Final = _pdf_data_url_with_xobjects(b"", b"<< >>", ()) + blank_price: Final = _count_inline_pdf_tokens(blank_data_url, len) + + assert priced is not None + assert blank_price is not None + assert priced - blank_price == len("Hello world") + + +def test_nul_whitespace_after_bt_keeps_oversized_page_text(): + path_operator: Final = b"10 10 m 20 20 l S\n" + path_padding: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) + 1) + content: Final = path_padding + b"BT\x00/F1 12 Tf (Hello world) Tj ET\n" + data_url: Final = _pdf_data_url_with_xobjects(content, b"<< >>", ()) + priced: Final = _count_inline_pdf_tokens(data_url, len) + blank_data_url: Final = _pdf_data_url_with_xobjects(b"", b"<< >>", ()) + blank_price: Final = _count_inline_pdf_tokens(blank_data_url, len) + + assert priced is not None + assert blank_price is not None + assert priced - blank_price == len("Hello world") + + +def test_compact_bi_operator_strips_inline_image_before_pricing_text(): + image_data: Final = random.Random(42).randbytes(270_000) + content: Final = b"BI/W 270000/H 1/BPC 8/CS /RGB ID " + image_data + b" EI\nBT /F1 9 Tf (Hello world) Tj ET\n" + data_url: Final = _pdf_data_url_with_xobjects(content, b"<< >>", ()) + pdf_bytes: Final = base64.b64decode(data_url.removeprefix(PDF_DATA_URL_PREFIX)) + page: Final = _pdf_text_page(pdf_bytes) + exact_tokens: Final = len(page.extract_text() or "") + priced: Final = _count_inline_pdf_tokens(data_url, len) + blank_data_url: Final = _pdf_data_url_with_xobjects(b"", b"<< >>", ()) + blank_price: Final = _count_inline_pdf_tokens(blank_data_url, len) + + assert priced is not None + assert blank_price is not None + assert priced - blank_price == exact_tokens + + +def test_pdf_text_object_strings_containing_bt_and_et_do_not_split_sampling(): + path_operator: Final = b"10 10 m 20 20 l S\n" + path_padding: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) + 1) + text_object: Final = b"BT /F1 12 Tf (before ET) Tj (inside BT) Tj (after) Tj ET\n" + data_url: Final = _pdf_data_url_with_xobjects(path_padding + text_object, b"<< >>", ()) + pdf_bytes: Final = base64.b64decode(data_url.removeprefix(PDF_DATA_URL_PREFIX)) + exact_text: Final = _pdf_extracted_text(pdf_bytes) + priced: Final = _count_inline_pdf_tokens(data_url, len) + blank_data_url: Final = _pdf_data_url_with_xobjects(b"", b"<< >>", ()) + blank_price: Final = _count_inline_pdf_tokens(blank_data_url, len) + + assert exact_text == "before ETinside BTafter" + assert priced is not None + assert blank_price is not None + assert priced - blank_price == len(exact_text) + + +def test_repeated_identity_h_form_draws_count_each_text_sample(): + path_operator: Final = b"10 10 m 20 20 l S\n" + path_padding: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) + 1) + form_content: Final = b"BT /F1 12 Tf <0001> Tj ET\n" + form_object: Final = _pdf_stream( + form_content, + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >>", + ) + data_url: Final = _pdf_data_url_with_identity_h_font( + path_padding + b"/Fm1 Do\n/Fm1 Do\n", + b"<< /Fm1 6 0 R >>", + (form_object,), + ) + priced: Final = _count_inline_pdf_tokens(data_url, len) + blank_data_url: Final = _pdf_data_url_with_xobjects(b"", b"<< >>", ()) + blank_price: Final = _count_inline_pdf_tokens(blank_data_url, len) + + assert priced is not None + assert blank_price is not None + assert priced - blank_price == 2 * len("Hello world") + + +def test_nul_whitespace_invokes_large_form_with_bounded_sample_pricing(): + text_object: Final = b"BT /F1 12 Tf (Hello world) Tj ET\n" + repetitions: Final = 8 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(text_object) + 1 + form_object: Final = _pdf_stream( + text_object * repetitions, + b"/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >>", + ) + nul_data_url: Final = _pdf_data_url_with_xobjects( + b"/Fm1\x00Do\n", + b"<< /Fm1 6 0 R >>", + (form_object,), + ) + space_data_url: Final = _pdf_data_url_with_xobjects( + b"/Fm1 Do\n", + b"<< /Fm1 6 0 R >>", + (form_object,), + ) + nul_count_function: Final = MagicMock(side_effect=len) + space_count_function: Final = MagicMock(side_effect=len) + + nul_price: Final = _count_inline_pdf_tokens(nul_data_url, nul_count_function) + space_price: Final = _count_inline_pdf_tokens(space_data_url, space_count_function) + + assert nul_price is not None + assert space_price is not None + assert nul_price == space_price + assert nul_price > 0 + assert max((len(call.args[0]) for call in nul_count_function.call_args_list), default=0) <= ( + PDF_PAGE_MAX_EXACT_CONTENT_BYTES + ) + assert max((len(call.args[0]) for call in space_count_function.call_args_list), default=0) <= ( + PDF_PAGE_MAX_EXACT_CONTENT_BYTES + ) + + +def test_dense_pdf_page_is_priced_without_extracting_its_full_text(): + count_function: Final = MagicMock(side_effect=len) + sentence: Final = "Revenue grew eleven percent while churn fell to two percent. " + page_text: Final = sentence * (2 * PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(sentence)) + + priced: Final = _count_inline_pdf_tokens(PDF_DATA_URL_PREFIX + _pdf_base64((page_text,)), count_function) + + assert priced is not None and priced >= len(page_text) + assert max((len(call.args[0]) for call in count_function.call_args_list), default=0) <= ( + PDF_PAGE_MAX_EXACT_CONTENT_BYTES + ) + + +def test_long_pdf_of_identical_pages_prices_each_page_like_a_one_page_pdf(): + prompt: Final = {"type": "text", "text": "Summarize this file."} + page_text: Final = "Revenue grew eleven percent while churn fell to two percent." + page_count: Final = 3 * EXTRAPOLATION_SAMPLES + 1 + base: Final = _count_user_content([prompt]) + + def pdf_cost(pages: int) -> int: + document: Final = {"type": "document", "source": _pdf_source(_pdf_base64((page_text,) * pages))} + return _count_user_content([prompt, document]) - base + + assert pdf_cost(page_count) == page_count * pdf_cost(1) + + def test_pdf_document_without_pypdf_is_priced_like_an_image(monkeypatch: pytest.MonkeyPatch): prompt: Final = {"type": "text", "text": "Summarize this file."} source: Final = _pdf_source(_pdf_base64(("Revenue grew eleven percent while churn fell to two percent.",))) diff --git a/uv.lock b/uv.lock index f86614046e1..bcc222a19f8 100644 --- a/uv.lock +++ b/uv.lock @@ -5072,6 +5072,7 @@ admin-mcp = [ benchmarks = [ { name = "a2a-sdk" }, { name = "mcp" }, + { name = "pypdf" }, { name = "pytest" }, { name = "pytest-codspeed" }, ] @@ -5301,6 +5302,7 @@ admin-mcp = [{ name = "litellm-admin-mcp", marker = "python_full_version >= '3.1 benchmarks = [ { name = "a2a-sdk", specifier = "==1.1.0" }, { name = "mcp", specifier = ">=2.3.0,<3" }, + { name = "pypdf", specifier = ">=6.16.1,<7.0" }, { name = "pytest", specifier = "==9.0.3" }, { name = "pytest-codspeed", specifier = "==4.3.0" }, ]