diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 8e91b4d32e1..1c7b5b43fa7 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -1678,7 +1678,7 @@ def _pdf_strip_inline_images(content: bytes) -> bytes: def _pdf_page_content_metrics(page: _PdfPage, page_content: bytes) -> _PdfStreamMetrics: - form_metrics_memo: Final[dict[int, _PdfStreamMetrics]] = {} # mutable-ok: page-local cache avoids repeated walks + form_metrics_memo: Final[dict[tuple[int, int], _PdfStreamMetrics]] = {} # mutable-ok: page-local walk cache def stream_metrics( content: bytes, @@ -1724,14 +1724,15 @@ def _pdf_page_content_metrics(page: _PdfPage, page_content: bytes) -> _PdfStream return _PdfStreamMetrics(b"", 0, 0, inherited_resources, (), extraction_limited=True) if not isinstance(resolved, _PdfFormXObject) or resolved.get("/Subtype") != "/Form": return _PdfStreamMetrics(b"", 0, 0, inherited_resources, ()) - cached: Final = form_metrics_memo.get(identity) + form_resources: Final = _pdf_resolve_object(resolved.get("/Resources")) + resources: Final = form_resources if isinstance(form_resources, _PdfResourceDictionary) else inherited_resources + cache_key: Final = (identity, id(resources)) + cached: Final = form_metrics_memo.get(cache_key) if cached is not None: return cached content: Final = _pdf_strip_inline_images(resolved.get_data()) - form_resources: Final = _pdf_resolve_object(resolved.get("/Resources")) - resources: Final = form_resources if isinstance(form_resources, _PdfResourceDictionary) else inherited_resources metrics: Final = stream_metrics(content, resources, ancestors | {identity}) - form_metrics_memo[identity] = metrics + form_metrics_memo[cache_key] = metrics return metrics resources: Final = _pdf_resolve_object(page.get("/Resources")) diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index c00a46a7d6b..9192cbe2600 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -3577,6 +3577,45 @@ def test_pdf_text_object_strings_containing_bt_and_et_do_not_split_sampling(): assert priced - blank_price == len(exact_text) +@pytest.mark.parametrize("pdf_pricing", ["claude", "openai"]) +@pytest.mark.parametrize(("second_text", "second_count"), [(b"second", 7), (b"", 0)]) +def test_shared_pdf_form_uses_each_parents_resources( + pdf_pricing: Literal["claude", "openai"], second_text: bytes, second_count: int +): + path: Final = b"10 10 m 20 20 l S\n" + padding: Final = path * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path) + 1) + attributes: Final = b"/Type /XObject /Subtype /Form /BBox [0 0 612 792]" + fonts: Final = b" /Resources << /Font << /F1 3 0 R >> >>" + + def document(second_leaf: int) -> str: + return _pdf_data_url_with_xobjects( + padding + b"/First Do\n/Second Do\n", + b"<< /First 6 0 R /Second 7 0 R >>", + ( + _pdf_stream( + b"/Shared Do", + attributes + b" /Resources << /XObject << /Shared 8 0 R /Leaf 9 0 R >> >>", + ), + _pdf_stream( + b"/Shared Do", + attributes + f" /Resources << /XObject << /Shared 8 0 R /Leaf {second_leaf} 0 R >> >>".encode(), + ), + _pdf_stream(b"/Leaf Do", attributes), + _pdf_stream(b"BT /F1 12 Tf (first) Tj ET", attributes + fonts), + _pdf_stream(b"BT /F1 12 Tf (" + second_text + b") Tj ET", attributes + fonts), + ), + ) + + def count_text(text: str) -> int: + return text.count("first") + 7 * text.count("second") + + shared_price: Final = _count_inline_pdf_tokens(document(9), count_text, pdf_pricing) + distinct_price: Final = _count_inline_pdf_tokens(document(10), count_text, pdf_pricing) + assert shared_price is not None + assert distinct_price is not None + assert distinct_price - shared_price == second_count - 1 + + def test_repeated_identity_h_form_draws_count_each_text_sample(): path_operator: Final = b"10 10 m 20 20 l S\n" path_padding: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) + 1)