This commit is contained in:
Mateo Wang 2026-10-27 09:24:20 +08:00 • committed by GitHub
commit 36e2ec0147
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 1974 additions and 28 deletions

View file

@ -1,12 +1,16 @@
# What is this?
## Helper utilities for token counting
import base64
import importlib
import io
import math
import re
import struct
from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
from itertools import accumulate
from typing import TYPE_CHECKING, Final, Literal, cast
from dataclasses import dataclass
from itertools import accumulate, groupby, islice, takewhile, tee
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, cast, runtime_checkable
import anyio
import anyio.lowlevel
@ -348,6 +352,58 @@ Type for a function that counts tokens in a string.
"""
EXTRAPOLATION_SAMPLES: Final = 16
PDF_PAGE_MAX_EXACT_CONTENT_BYTES: Final = 64 * 1024
PDF_WHITESPACE_CLASS: Final = b"[\x00\t\n\x0c\r ]"
PDF_HEX_STRING_OPERAND: Final = rb"<[0-9A-Fa-f" + PDF_WHITESPACE_CLASS[1:-1] + rb"]*>"
PDF_LITERAL_STRING_OPERAND: Final = rb"\((?:\\[\x00-\xff]|[^\\()]|\((?:\\[\x00-\xff]|[^\\()])*\)?)*\)?"
PDF_STRING_OPERAND: Final = re.compile(PDF_LITERAL_STRING_OPERAND + rb"|" + PDF_HEX_STRING_OPERAND + rb"|%[^\r\n]*")
PDF_SIMPLE_LITERAL_STRING: Final = re.compile(rb"\([^()]*\)")
PDF_LITERAL_ESCAPE: Final = re.compile(rb"\\([nrtbf()\\]|[0-7]{1,3}|\r\n?|\n|[\x00-\xff])")
PDF_INLINE_IMAGE: Final = re.compile(
PDF_LITERAL_STRING_OPERAND
+ rb"|"
+ PDF_HEX_STRING_OPERAND
+ rb"|%[^\r\n]*|(?P<inline_image>(?<![^"
+ PDF_WHITESPACE_CLASS[1:-1]
+ rb"\]>)])BI(?="
+ PDF_WHITESPACE_CLASS
+ rb"|[/\[\]()<>{}%])[\x00-\xff]*?(?:(?<="
+ PDF_WHITESPACE_CLASS
+ rb")ID"
+ PDF_WHITESPACE_CLASS
+ rb"[\x00-\xff]*?(?:(?P<image_end>"
+ PDF_WHITESPACE_CLASS
+ rb"EI(?![^"
+ PDF_WHITESPACE_CLASS[1:-1]
+ rb"\[/<(%]))|\Z)|\Z))"
)
PDF_TEXT_OBJECT: Final = re.compile(
PDF_LITERAL_STRING_OPERAND
+ rb"|"
+ PDF_HEX_STRING_OPERAND
+ rb"|%[^\r\n]*|(?P<text_object>(?<![^"
+ PDF_WHITESPACE_CLASS[1:-1]
+ rb"\]>)])BT(?="
+ PDF_WHITESPACE_CLASS
+ rb"|[/\[\]()<>{}%])"
+ rb"(?:(?:"
+ PDF_LITERAL_STRING_OPERAND
+ rb"|"
+ PDF_HEX_STRING_OPERAND
+ rb"|%[^\r\n]*|[\x00-\xff])*?(?:(?P<text_object_end>(?<="
+ PDF_WHITESPACE_CLASS
+ rb")ET(?![^"
+ PDF_WHITESPACE_CLASS[1:-1]
+ rb"\[\]()<>{}/%]))|\Z)))"
)
PDF_XOBJECT_INVOCATION: Final = re.compile(
PDF_LITERAL_STRING_OPERAND
+ rb"|%[^\r\n]*(?:[\r\n]|\Z)|/([^"
+ PDF_WHITESPACE_CLASS[1:-1]
+ rb"/\[\]()<>{}%]+)(?:"
+ PDF_WHITESPACE_CLASS
+ rb"|%[^\r\n]*(?:[\r\n]|\Z))+Do\b"
)
T_ParamSpec: Final = ParamSpec("T_ParamSpec")
T_Retval = TypeVar("T_Retval")
_COUNT_OFFLOAD_LIMITER: Final = anyio.lowlevel.RunVar[anyio.CapacityLimiter]("litellm_count_offload_limiter")
@ -860,6 +916,618 @@ def _anthropic_rendered_page_image_tokens(width: float, height: float) -> int:
return math.ceil(min(float(ANTHROPIC_IMAGE_MAX_PIXELS), area_at_max_edge) / ANTHROPIC_IMAGE_PIXELS_PER_TOKEN)
class _PdfContentStream(Protocol):
def get_data(self) -> bytes: ...
@runtime_checkable
class _PdfResourceDictionary(Protocol):
def get(self, key: str) -> object | None: ...
def items(self) -> Iterable[tuple[str, object]]: ...
@runtime_checkable
class _PdfFormXObject(_PdfResourceDictionary, Protocol):
def get_data(self) -> bytes: ...
@runtime_checkable
class _PdfIndirectReference(Protocol):
def get_object(self) -> object: ...
class _PdfMediaBox(Protocol):
@property
def width(self) -> float: ...
@property
def height(self) -> float: ...
class _PdfPage(_PdfResourceDictionary, Protocol):
@property
def mediabox(self) -> _PdfMediaBox: ...
def get_contents(self) -> _PdfContentStream | None: ...
def extract_text(self) -> str: ...
class _PdfWritablePage(_PdfPage, Protocol):
def __setitem__(self, key: str, value: object) -> None: ...
class _PdfPageFactory(Protocol):
@staticmethod
def create_blank_page(pdf: object | None, width: float, height: float) -> _PdfWritablePage: ...
class _PdfDecodedContentStream(_PdfContentStream, Protocol):
def set_data(self, data: bytes) -> None: ...
class _PdfGenericModule(Protocol):
DecodedStreamObject: Callable[[], _PdfDecodedContentStream]
DictionaryObject: Callable[[Mapping[str, object]], _PdfResourceDictionary]
NameObject: Callable[[str], str]
class _PdfModule(Protocol):
PageObject: _PdfPageFactory
@dataclass(frozen=True, slots=True)
class _PdfStreamMetrics:
content: bytes
content_bytes: int
operand_bytes: int
resources: _PdfResourceDictionary
forms: tuple["_PdfStreamMetrics", ...]
@dataclass(frozen=True, slots=True)
class _PdfTextObjectRecord:
stream: _PdfStreamMetrics
invocation_path: tuple[int, ...]
start: int
end: int
first_in_stream: bool
truncated: bool
closed: bool
_PDF_EMPTY_RESOURCES: Final[_PdfResourceDictionary] = (
cast( # cast-ok: immutable empty map satisfies the resource protocol
_PdfResourceDictionary, MappingProxyType({})
)
)
def _pdf_resolve_object(value: object) -> object:
return value.get_object() if isinstance(value, _PdfIndirectReference) else value
def _pdf_decode_name(name: bytes) -> str:
def decode_escape(match: re.Match[bytes]) -> bytes:
return bytes((int(match.group(1), 16),))
return "/" + re.sub(rb"#([0-9A-Fa-f]{2})", decode_escape, name).decode("latin-1")
def _pdf_invocation_name(invocation: re.Match[bytes]) -> str | None:
name: Final = invocation.group(1)
return _pdf_decode_name(name) if name is not None else None
def _pdf_inline_image_replacement(match: re.Match[bytes]) -> bytes:
inline_image: Final = match.group("inline_image")
image_end: Final = match.group("image_end")
return b" " if inline_image is not None and image_end is not None else match.group(0)
def _pdf_strip_inline_images(content: bytes) -> bytes:
return PDF_INLINE_IMAGE.sub(_pdf_inline_image_replacement, content) if b"BI" in content else content
def _pdf_page_content_metrics(page: _PdfPage, page_content: bytes) -> _PdfStreamMetrics:
form_metrics_memo: Final[dict[int, _PdfStreamMetrics]] = {} # mutable-ok: page-local cache avoids repeated walks
def stream_metrics(
content: bytes,
resources: _PdfResourceDictionary,
ancestors: frozenset[int],
) -> _PdfStreamMetrics:
xobjects: Final = _pdf_resolve_object(resources.get("/XObject"))
xobject_resources: Final = xobjects if isinstance(xobjects, _PdfResourceDictionary) else _PDF_EMPTY_RESOURCES
invoked_names: Final = (
tuple(
name
for invocation in PDF_XOBJECT_INVOCATION.finditer(content)
if (name := _pdf_invocation_name(invocation)) is not None
)
if xobject_resources
else ()
)
form_metrics: Final = tuple(
form_metrics_for_xobject(
_pdf_resolve_object(xobject_resources.get(name)),
resources,
ancestors,
)
for name in invoked_names
)
return _PdfStreamMetrics(
content=content,
content_bytes=len(content) + sum(metrics.content_bytes for metrics in form_metrics),
operand_bytes=_pdf_string_operand_bytes(content) + sum(metrics.operand_bytes for metrics in form_metrics),
resources=resources,
forms=form_metrics,
)
def form_metrics_for_xobject(
xobject: object,
inherited_resources: _PdfResourceDictionary,
ancestors: frozenset[int],
) -> _PdfStreamMetrics:
resolved: Final = _pdf_resolve_object(xobject)
identity: Final = id(resolved)
if identity in ancestors:
return _PdfStreamMetrics(b"", 0, 0, inherited_resources, ())
if not isinstance(resolved, _PdfFormXObject) or resolved.get("/Subtype") != "/Form":
return _PdfStreamMetrics(b"", 0, 0, inherited_resources, ())
cached: Final = form_metrics_memo.get(identity)
if cached is not None:
return cached
content: Final = _pdf_strip_inline_images(resolved.get_data())
form_resources: Final = _pdf_resolve_object(resolved.get("/Resources"))
resources: Final = form_resources if isinstance(form_resources, _PdfResourceDictionary) else inherited_resources
metrics: Final = stream_metrics(content, resources, ancestors | {identity})
form_metrics_memo[identity] = metrics
return metrics
resources: Final = _pdf_resolve_object(page.get("/Resources"))
page_resources: Final = resources if isinstance(resources, _PdfResourceDictionary) else _PDF_EMPTY_RESOURCES
return stream_metrics(page_content, page_resources, frozenset())
def _evenly_spaced_page_indices(page_count: int) -> tuple[int, ...]:
if page_count <= EXTRAPOLATION_SAMPLES:
return tuple(range(page_count))
return tuple((page_count - 1) * index // (EXTRAPOLATION_SAMPLES - 1) for index in range(EXTRAPOLATION_SAMPLES))
def _pdf_string_operand_bytes(content: bytes) -> int:
if b"\\" not in content and b"%" not in content and b"<" not in content and b"BI" not in content:
content_without_literals, literal_count = PDF_SIMPLE_LITERAL_STRING.subn(b"", content)
if literal_count == content.count(b"(") == content.count(b")"):
return len(content) - len(content_without_literals) - 2 * literal_count
return sum(len(operand) for operand in _pdf_decoded_string_operands(content))
def _pdf_decode_literal_escape(match: re.Match[bytes]) -> bytes:
escape: Final = match.group(1)
if escape is None:
return b""
if escape == b"n":
return b"\n"
if escape == b"r":
return b"\r"
if escape == b"t":
return b"\t"
if escape == b"b":
return b"\b"
if escape == b"f":
return b"\f"
if escape in (b"(", b")", b"\\"):
return escape
if escape in (b"\r", b"\n", b"\r\n"):
return b""
if escape[:1] in b"01234567":
return bytes((int(escape, 8) & 0xFF,))
return escape
def _pdf_decode_string_operand(operand: bytes) -> bytes:
if operand.startswith(b"<"):
digits: Final = re.sub(PDF_WHITESPACE_CLASS + rb"+", b"", operand[1:-1])
padded_digits: Final = digits if len(digits) % 2 == 0 else digits + b"0"
return bytes.fromhex(padded_digits.decode("ascii"))
literal: Final = operand[1:-1] if operand.endswith(b")") else operand[1:]
return PDF_LITERAL_ESCAPE.sub(_pdf_decode_literal_escape, literal) if b"\\" in literal else literal
def _pdf_decoded_string_operands(content: bytes) -> Iterable[bytes]:
return (
_pdf_decode_string_operand(match.group(0))
for match in PDF_STRING_OPERAND.finditer(content)
if not match.group(0).startswith(b"%")
)
_PdfTextObjectSpans: TypeAlias = tuple[tuple[int, int, bool, bool], ...]
_PdfTextTreeWalkItem: TypeAlias = tuple[_PdfStreamMetrics, bool]
_PdfTextRecordWalkItem: TypeAlias = tuple[_PdfStreamMetrics, tuple[int, ...]]
def _pdf_text_object_records( # noqa: C901 # coordinates memoized per-stream PDF traversal
metrics: _PdfStreamMetrics,
invocation_path: tuple[int, ...] = (),
) -> Iterable[_PdfTextObjectRecord]:
text_object_span_memo: Final[dict[int, _PdfTextObjectSpans]] = {} # mutable-ok: page-local regex cache
subtree_text_memo: Final[dict[int, bool]] = {} # mutable-ok: page-local subtree cache
def partial_text_object_end(content: bytes, start: int) -> tuple[int, bool, bool]:
end: Final = min(len(content), start + PDF_PAGE_MAX_EXACT_CONTENT_BYTES)
bounded_content: Final = content[start:end]
match: Final = next(
(match for match in PDF_TEXT_OBJECT.finditer(bounded_content) if match.group("text_object") is not None),
None,
)
if match is None:
return start, False, False
closed: Final = match.group("text_object_end") is not None
truncated: Final = match.end("text_object") == len(bounded_content) and end < len(content) and not closed
return start + match.end("text_object"), closed, truncated
def text_object_spans_in_window(content: bytes, window_end: int) -> Iterable[tuple[int, int, bool, bool]]:
window_content: Final = content[:window_end]
for match in PDF_TEXT_OBJECT.finditer(window_content):
if match.group("text_object") is None:
continue
if (
match.end("text_object") == window_end
and window_end < len(content)
and match.group("text_object_end") is None
):
partial_end, partial_closed, partial_truncated = partial_text_object_end(
content, match.start("text_object")
)
yield (
match.start("text_object"),
partial_end,
partial_truncated,
partial_closed,
)
return
yield (
match.start("text_object"),
match.end("text_object"),
False,
match.group("text_object_end") is not None,
)
def scan_text_object_spans(
content: bytes,
window_end: int,
) -> tuple[_PdfTextObjectSpans, int]:
spans: Final[list[tuple[int, int, bool, bool]]] = [] # mutable-ok: capped page-local span cache
cumulative_length: int = 0 # rebind-ok: accumulates record lengths until the sample cap
for start, end, truncated, closed in text_object_spans_in_window(content, window_end):
spans.append((start, end, truncated, closed))
cumulative_length += end - start + (0 if closed else len(b"\nET")) + (0 if len(spans) == 1 else len(b"\n"))
if cumulative_length >= PDF_PAGE_MAX_EXACT_CONTENT_BYTES:
break
return tuple(spans), cumulative_length
def text_object_spans_for(stream_metrics: _PdfStreamMetrics) -> _PdfTextObjectSpans:
identity: Final = id(stream_metrics)
cached: Final = text_object_span_memo.get(identity)
if cached is not None:
return cached
first_window_end: Final = min(len(stream_metrics.content), PDF_PAGE_MAX_EXACT_CONTENT_BYTES)
first_spans, first_length = scan_text_object_spans(stream_metrics.content, first_window_end)
cached_spans: Final = (
first_spans
if first_length >= PDF_PAGE_MAX_EXACT_CONTENT_BYTES or first_window_end == len(stream_metrics.content)
else scan_text_object_spans(stream_metrics.content, len(stream_metrics.content))[0]
)
text_object_span_memo[identity] = cached_spans
return cached_spans
def subtree_has_text(stream_metrics: _PdfStreamMetrics) -> bool:
identity: Final = id(stream_metrics)
if identity in subtree_text_memo:
return subtree_text_memo[identity]
pending: Final[list[_PdfTextTreeWalkItem]] = [(stream_metrics, False)] # mutable-ok: DFS stack
while pending:
current_metrics, expanded = pending.pop()
current_identity = id(current_metrics)
if current_identity in subtree_text_memo:
continue
if expanded:
subtree_text_memo[current_identity] = any(
subtree_text_memo.get(id(form_metrics), False) for form_metrics in current_metrics.forms
)
continue
if text_object_spans_for(current_metrics):
subtree_text_memo[current_identity] = True
continue
pending.append((current_metrics, True))
pending.extend((form_metrics, False) for form_metrics in current_metrics.forms)
return subtree_text_memo[identity]
if not subtree_has_text(metrics):
return
pending_records: Final[list[_PdfTextRecordWalkItem]] = [(metrics, invocation_path)] # mutable-ok: DFS stack
while pending_records:
current_metrics, current_path = pending_records.pop()
for index, (start, end, truncated, closed) in enumerate(text_object_spans_for(current_metrics)):
yield _PdfTextObjectRecord(current_metrics, current_path, start, end, index == 0, truncated, closed)
pending_records.extend(
(form_metrics, current_path + (index,))
for index, form_metrics in reversed(tuple(enumerate(current_metrics.forms)))
if subtree_has_text(form_metrics)
)
def _pdf_text_object_record_length(record: _PdfTextObjectRecord) -> int:
text_object_length: Final = record.end - record.start
separator_bytes: Final = 0 if record.first_in_stream else len(b"\n")
if record.closed:
return text_object_length + separator_bytes
text_object: Final = record.stream.content[record.start : record.end]
prefix_length, closures = _pdf_truncated_text_sample_closures(
text_object,
include_text_show=True,
maximum_length=PDF_PAGE_MAX_EXACT_CONTENT_BYTES - separator_bytes - len(b"\nET"),
)
return prefix_length + len(closures) + len(b"\nET") + separator_bytes
def _pdf_text_object_record_bytes(record: _PdfTextObjectRecord) -> bytes:
text_object: Final = record.stream.content[record.start : record.end]
separator: Final = b"" if record.first_in_stream else b"\n"
if record.closed:
return separator + text_object
prefix_length, closures = _pdf_truncated_text_sample_closures(
text_object,
include_text_show=True,
maximum_length=PDF_PAGE_MAX_EXACT_CONTENT_BYTES - len(separator) - len(b"\nET"),
)
return separator + text_object[:prefix_length] + closures + b"\nET"
def _pdf_truncated_text_sample_closures( # noqa: C901 # single-pass PDF syntax-state scanner
prefix: bytes, include_text_show: bool, maximum_length: int | None = None
) -> tuple[int, bytes]:
comment_open: bool = False # rebind-ok: scans one bounded PDF prefix
escape_open: bool = False # rebind-ok: scans one bounded PDF prefix
literal_depth: int = 0 # rebind-ok: tracks balanced literal strings
hex_open: bool = False # rebind-ok: scans one bounded PDF prefix
array_depth: int = 0 # rebind-ok: tracks arrays outside strings
bare_string_complete: bool = False # rebind-ok: tracks the final string operand
best_prefix_length: int = 0 # rebind-ok: retains the latest prefix whose closures fit the sample cap
sample_limit: Final = len(prefix) if maximum_length is None else maximum_length
best_state: tuple[bool, bool, int, bool, int, bool] = ( # rebind-ok: scanner state
False,
False,
0,
False,
0,
False,
)
for index, byte in enumerate(prefix):
if comment_open:
if byte in (ord("\r"), ord("\n")):
comment_open = False
elif literal_depth:
if escape_open:
escape_open = False
elif byte == ord("\\"):
escape_open = True
elif byte == ord("("):
literal_depth += 1
elif byte == ord(")"):
literal_depth -= 1
if literal_depth == 0:
bare_string_complete = True
elif hex_open:
if byte == ord(">"):
hex_open = False
bare_string_complete = True
elif byte == ord("%"):
comment_open = True
elif byte == ord("("):
literal_depth = 1
bare_string_complete = False
elif byte == ord("<") and (index + 1 == len(prefix) or prefix[index + 1] != ord("<")):
hex_open = True
bare_string_complete = False
elif byte == ord("["):
array_depth += 1
bare_string_complete = False
elif byte == ord("]"):
array_depth = max(0, array_depth - 1)
bare_string_complete = False
elif byte not in PDF_WHITESPACE_CLASS[1:-1]:
bare_string_complete = False
if (
index
+ 1
+ (1 if comment_open else 0)
+ (2 if escape_open else 0)
+ literal_depth
+ (1 if hex_open else 0)
+ (
array_depth + len(b" TJ")
if include_text_show and array_depth
else len(b" Tj")
if include_text_show and (literal_depth or hex_open or bare_string_complete)
else 0
)
<= sample_limit
):
best_prefix_length = index + 1
best_state = (
comment_open,
escape_open,
literal_depth,
hex_open,
array_depth,
bare_string_complete,
)
(
best_comment_open,
best_escape_open,
best_literal_depth,
best_hex_open,
best_array_depth,
best_bare_string_complete,
) = best_state
operand_closures: Final = (
(b"\\" if best_escape_open else b"")
+ (b")" * best_literal_depth if best_literal_depth else b"")
+ (b">" if best_hex_open else b"")
)
comment_closure: Final = b"\n" if best_comment_open else b""
if not include_text_show:
return best_prefix_length, comment_closure + operand_closures
if best_array_depth:
return best_prefix_length, comment_closure + operand_closures + b"]" * best_array_depth + b" TJ"
if best_literal_depth or best_hex_open or best_bare_string_complete:
return best_prefix_length, comment_closure + operand_closures + b" Tj"
return best_prefix_length, comment_closure + operand_closures
def _pdf_truncated_text_object_prefix(
content: bytes,
start: int,
prefix_length: int,
) -> bytes:
prefix: Final = content[start : start + prefix_length]
closed_prefix_length, closures = _pdf_truncated_text_sample_closures(
prefix, include_text_show=True, maximum_length=prefix_length
)
return prefix[:closed_prefix_length] + closures + b"\nET"
def _pdf_truncated_string_operand_prefix(operand: bytes, prefix_length: int) -> bytes:
prefix: Final = operand[:prefix_length]
closed_prefix_length, closures = _pdf_truncated_text_sample_closures(prefix, include_text_show=False)
return prefix[:closed_prefix_length] + closures
def _pdf_truncated_text_object(record: _PdfTextObjectRecord) -> bytes:
content: Final = record.stream.content
if record.truncated:
return _pdf_truncated_text_object_prefix(
content,
record.start,
PDF_PAGE_MAX_EXACT_CONTENT_BYTES - len(b"\nET"),
)
string_operands: Final = (
operand
for operand in PDF_STRING_OPERAND.finditer(content, record.start, record.end)
if not operand.group(0).startswith(b"%")
)
largest_operand: Final = max(string_operands, key=lambda operand: len(operand.group(0)), default=None)
if largest_operand is not None and len(largest_operand.group(0)) > PDF_PAGE_MAX_EXACT_CONTENT_BYTES // 2:
prefix: Final = content[record.start : largest_operand.start()]
suffix: Final = content[largest_operand.end() : record.end]
object_closure: Final = b"" if record.closed else b"\nET"
operand_prefix_length: Final = (
PDF_PAGE_MAX_EXACT_CONTENT_BYTES - len(prefix) - len(suffix) - len(object_closure)
)
if operand_prefix_length > 0:
sample: Final = (
prefix
+ _pdf_truncated_string_operand_prefix(largest_operand.group(0), operand_prefix_length)
+ suffix
+ object_closure
)
if len(sample) <= PDF_PAGE_MAX_EXACT_CONTENT_BYTES:
return sample
return _pdf_truncated_text_object_prefix(
content,
record.start,
PDF_PAGE_MAX_EXACT_CONTENT_BYTES - len(b"\nET"),
)
def _pdf_text_sample_for_scope(records: Iterable[_PdfTextObjectRecord]) -> tuple[_PdfResourceDictionary, bytes]:
scoped_records: Final = tuple(records)
return (
scoped_records[0].stream.resources,
b"".join(_pdf_text_object_record_bytes(record) for record in scoped_records),
)
def _pdf_bounded_text_samples(metrics: _PdfStreamMetrics) -> tuple[tuple[_PdfResourceDictionary, bytes], ...]:
records, sized_records = tee(_pdf_text_object_records(metrics))
first_record: Final = next(records, None)
if first_record is None:
return ()
first_record_length: Final = _pdf_text_object_record_length(first_record)
if first_record.truncated or first_record_length > PDF_PAGE_MAX_EXACT_CONTENT_BYTES:
return ((first_record.stream.resources, _pdf_truncated_text_object(first_record)),)
record_sizes: Final = (_pdf_text_object_record_length(record) for record in islice(sized_records, 1, None))
cumulative_sizes: Final = islice(accumulate(record_sizes, initial=first_record_length), 1, None)
bounded_records: Final = tuple(
record
for record, _ in takewhile(
lambda item: item[1] <= PDF_PAGE_MAX_EXACT_CONTENT_BYTES,
zip(records, cumulative_sizes),
)
)
sampled_records: Final = (first_record, *bounded_records)
return tuple(
_pdf_text_sample_for_scope(group)
for _, group in groupby(sampled_records, key=lambda record: record.invocation_path)
)
def _pdf_extract_sample_text(
page: _PdfPage,
resources: _PdfResourceDictionary,
content: bytes,
) -> str:
pypdf_module: Final = cast( # cast-ok: optional pypdf module has no typed module API
_PdfModule, importlib.import_module("pypdf")
)
generic_module: Final = cast( # cast-ok: optional pypdf module has no typed module API
_PdfGenericModule, importlib.import_module("pypdf.generic")
)
copied_resources: Final = generic_module.DictionaryObject(
{generic_module.NameObject(str(name)): value for name, value in resources.items() if str(name) != "/XObject"}
)
synthetic_page: Final = pypdf_module.PageObject.create_blank_page(None, page.mediabox.width, page.mediabox.height)
synthetic_page[generic_module.NameObject("/Resources")] = copied_resources
content_stream: Final = generic_module.DecodedStreamObject()
content_stream.set_data(content)
synthetic_page[generic_module.NameObject("/Contents")] = content_stream
return synthetic_page.extract_text() or ""
def _pdf_page_text_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int:
"""Oversized pages use bounded pypdf text-object samples scaled to decoded operand bytes."""
contents: Final = page.get_contents()
content: Final = _pdf_strip_inline_images(contents.get_data() if contents is not None else b"")
metrics: Final = _pdf_page_content_metrics(page, content)
if metrics.content_bytes > PDF_PAGE_MAX_EXACT_CONTENT_BYTES:
samples: Final = _pdf_bounded_text_samples(metrics)
sampled_operand_bytes: Final = sum(_pdf_string_operand_bytes(sample) for _, sample in samples)
if sampled_operand_bytes == 0:
return 0
sampled_text_tokens: Final = sum(
count_function(_pdf_extract_sample_text(page, resources, sample)) for resources, sample in samples
)
return round(sampled_text_tokens * metrics.operand_bytes / sampled_operand_bytes)
return count_function(page.extract_text() or "")
def _price_pdf_pages(pages: Sequence[_PdfPage], count_function: TokenCounterFunction) -> int:
rendering_tokens: Final = sum(
_anthropic_rendered_page_image_tokens(page.mediabox.width, page.mediabox.height) for page in pages
)
sampled_pages: Final = tuple(pages[index] for index in _evenly_spaced_page_indices(len(pages)))
if not sampled_pages:
return rendering_tokens
sampled_text_tokens: Final = sum(_pdf_page_text_tokens(page, count_function) for page in sampled_pages)
return rendering_tokens + round(sampled_text_tokens * len(pages) / len(sampled_pages))
def _count_inline_pdf_tokens(data_url: str, count_function: TokenCounterFunction) -> int | None:
if not data_url.startswith(PDF_DATA_URL_PREFIX):
return None
@ -869,12 +1537,10 @@ def _count_inline_pdf_tokens(data_url: str, count_function: TokenCounterFunction
verbose_logger.debug("pypdf is not installed, so the PDF document is priced like one image")
return None
try:
reader: Final = PdfReader(io.BytesIO(base64.b64decode(data_url[len(PDF_DATA_URL_PREFIX) :])))
return sum(
count_function(page.extract_text() or "")
+ _anthropic_rendered_page_image_tokens(float(page.mediabox.width), float(page.mediabox.height))
for page in reader.pages
pages: Final = cast( # cast-ok: pypdf is an optional extra the type-check env does not install
Sequence[_PdfPage], PdfReader(io.BytesIO(base64.b64decode(data_url[len(PDF_DATA_URL_PREFIX) :]))).pages
)
return _price_pdf_pages(pages, count_function)
except Exception as e:
verbose_logger.debug("Could not read the PDF document's pages (%s), so it is priced like one image", e)
return None

View file

@ -310,6 +310,7 @@ healthcheck = [
benchmarks = [
"pytest==9.0.3",
"pytest-codspeed==4.3.0",
"pypdf>=6.16.1,<7.0",
"mcp>=2.3.0,<3",
"a2a-sdk==1.1.0",
]

View file

@ -0,0 +1,152 @@
from functools import reduce
from types import SimpleNamespace
from typing import Final
import pytest
from litellm.litellm_core_utils.token_counter import (
_PDF_EMPTY_RESOURCES,
PDF_TEXT_OBJECT,
_pdf_bounded_text_samples,
_PdfStreamMetrics,
_price_pdf_pages,
)
class _PdfBenchmarkForm:
def get_data(self) -> bytes:
return b"q Q\n"
def get(self, key: str) -> str | None:
return "/Form" if key == "/Subtype" else None
def items(self) -> tuple[tuple[str, object], ...]:
return ()
def _pdf_page(content: bytes, with_form: bool = False) -> SimpleNamespace:
xobjects: Final = {"/Fm1": _PdfBenchmarkForm()} if with_form else {}
resources: Final = {"/XObject": xobjects}
return SimpleNamespace(
mediabox=SimpleNamespace(width=612.0, height=792.0),
get_contents=lambda: SimpleNamespace(get_data=lambda: content),
extract_text=lambda: "",
get=lambda key: resources if key == "/Resources" else None,
)
def _pdf_doubling_form_metrics(depth: int, leaf_content: bytes) -> _PdfStreamMetrics:
leaf_metrics: Final = _PdfStreamMetrics(
content=leaf_content,
content_bytes=len(leaf_content),
operand_bytes=0,
resources=_PDF_EMPTY_RESOURCES,
forms=(),
)
return reduce(
lambda child, _: _PdfStreamMetrics(
content=b"/Fm Do\n/Fm Do\n",
content_bytes=0,
operand_bytes=0,
resources=_PDF_EMPTY_RESOURCES,
forms=(child, child),
),
range(depth),
leaf_metrics,
)
@pytest.mark.benchmark
def test_unterminated_pdf_literal_string_scan(): # test-quality-ok: CodSpeed performance benchmark
page: Final = _pdf_page(b"(" * (4 * 1024 * 1024))
_price_pdf_pages((page,), len)
@pytest.mark.benchmark
def test_pdf_string_operand_scan_after_escaped_parentheses(): # test-quality-ok: CodSpeed performance benchmark
path_operator: Final = b"10 10 m 20 20 l S\n"
padding_repetitions: Final = 64 * 1024 // len(path_operator)
padding_bytes: Final = padding_repetitions * len(path_operator)
padding: Final = path_operator * padding_repetitions + b" " * (64 * 1024 - padding_bytes)
page: Final = _pdf_page(padding + b"%" + b"\\(" * 8_192 + path_operator * 1_000)
_price_pdf_pages((page,), len)
@pytest.mark.benchmark
def test_pdf_invocation_scan_ignores_unterminated_comments(): # test-quality-ok: CodSpeed performance benchmark
page: Final = _pdf_page(b"/a %" * (4 * 1024 * 1024 // 4), with_form=True)
priced: Final = _price_pdf_pages((page,), len)
assert priced > 0
@pytest.mark.benchmark
def test_pdf_inline_image_scan_repeated_bi_without_id(): # test-quality-ok: CodSpeed performance benchmark
page: Final = _pdf_page(b"BI " * 1_000_000)
_price_pdf_pages((page,), len)
@pytest.mark.benchmark
@pytest.mark.parametrize(
"content",
[
b"BT " * 1_000_000,
b"BT (" * 1_000_000,
b"BT " + b"(ET) " * 1_000_000,
],
ids=["repeated-bt", "repeated-open-literal", "repeated-et-literal"],
)
def test_pdf_text_object_scan(content: bytes): # test-quality-ok: CodSpeed performance benchmark
tuple(PDF_TEXT_OBJECT.finditer(content))
@pytest.mark.benchmark
def test_pdf_16m_text_object_stream_is_capped(): # test-quality-ok: CodSpeed performance benchmark
text_object: Final = b"BT (x) Tj ET\n"
repetitions: Final = 16 * 1024 * 1024 // len(text_object) + 1
page: Final = _pdf_page(text_object * repetitions)
_price_pdf_pages((page,), len)
@pytest.mark.benchmark
def test_pdf_textless_doubling_form_tree_skips_subtrees(): # test-quality-ok: CodSpeed performance benchmark
metrics: Final = _pdf_doubling_form_metrics(30, b"q Q\n")
_pdf_bounded_text_samples(metrics)
@pytest.mark.benchmark
def test_pdf_text_leaf_doubling_form_tree_stays_within_sample_cap(): # test-quality-ok: CodSpeed performance benchmark
metrics: Final = _pdf_doubling_form_metrics(30, b"BT (x) Tj ET\n")
_pdf_bounded_text_samples(metrics)
@pytest.mark.benchmark
@pytest.mark.parametrize("repetitions", [100_000, 1_000_000], ids=["100k", "1m"])
def test_pdf_inline_image_scan_no_ei(repetitions: int): # test-quality-ok: CodSpeed performance benchmark
page: Final = _pdf_page(b"BI /W 1 " + b" ID x" * repetitions)
_price_pdf_pages((page,), len)
@pytest.mark.benchmark
@pytest.mark.parametrize(
"comments",
[
b"%comment\r\n" * 23,
b"% a \r\n" * 40,
],
ids=["23-crlf-comments", "40-crlf-comments-with-trailing-spaces"],
)
def test_pdf_invocation_scan_with_crlf_comments(comments: bytes): # test-quality-ok: CodSpeed performance benchmark
page: Final = _pdf_page(b"BT /F1 " + comments + b"12 Tf (hello) Tj ET", with_form=True)
priced: Final = _price_pdf_pages((page,), len)
assert priced > 0

File diff suppressed because it is too large Load diff

2
uv.lock generated
View file

@ -5072,6 +5072,7 @@ admin-mcp = [
benchmarks = [
{ name = "a2a-sdk" },
{ name = "mcp" },
{ name = "pypdf" },
{ name = "pytest" },
{ name = "pytest-codspeed" },
]
@ -5301,6 +5302,7 @@ admin-mcp = [{ name = "litellm-admin-mcp", marker = "python_full_version >= '3.1
benchmarks = [
{ name = "a2a-sdk", specifier = "==1.1.0" },
{ name = "mcp", specifier = ">=2.3.0,<3" },
{ name = "pypdf", specifier = ">=6.16.1,<7.0" },
{ name = "pytest", specifier = "==9.0.3" },
{ name = "pytest-codspeed", specifier = "==4.3.0" },
]