fix(token_counter): align form resource handling

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
mateo 2026-11-08 09:22:38 +00:00
parent c44fec70c4
commit 51dcbd8284
2 changed files with 45 additions and 5 deletions

View file

@ -1678,7 +1678,7 @@ def _pdf_strip_inline_images(content: bytes) -> bytes:
def _pdf_page_content_metrics(page: _PdfPage, page_content: bytes) -> _PdfStreamMetrics:
form_metrics_memo: Final[dict[int, _PdfStreamMetrics]] = {} # mutable-ok: page-local cache avoids repeated walks
form_metrics_memo: Final[dict[tuple[int, int], _PdfStreamMetrics]] = {} # mutable-ok: page-local walk cache
def stream_metrics(
content: bytes,
@ -1724,14 +1724,15 @@ def _pdf_page_content_metrics(page: _PdfPage, page_content: bytes) -> _PdfStream
return _PdfStreamMetrics(b"", 0, 0, inherited_resources, (), extraction_limited=True)
if not isinstance(resolved, _PdfFormXObject) or resolved.get("/Subtype") != "/Form":
return _PdfStreamMetrics(b"", 0, 0, inherited_resources, ())
cached: Final = form_metrics_memo.get(identity)
form_resources: Final = _pdf_resolve_object(resolved.get("/Resources"))
resources: Final = form_resources if isinstance(form_resources, _PdfResourceDictionary) else inherited_resources
cache_key: Final = (identity, id(resources))
cached: Final = form_metrics_memo.get(cache_key)
if cached is not None:
return cached
content: Final = _pdf_strip_inline_images(resolved.get_data())
form_resources: Final = _pdf_resolve_object(resolved.get("/Resources"))
resources: Final = form_resources if isinstance(form_resources, _PdfResourceDictionary) else inherited_resources
metrics: Final = stream_metrics(content, resources, ancestors | {identity})
form_metrics_memo[identity] = metrics
form_metrics_memo[cache_key] = metrics
return metrics
resources: Final = _pdf_resolve_object(page.get("/Resources"))

View file

@ -3577,6 +3577,45 @@ def test_pdf_text_object_strings_containing_bt_and_et_do_not_split_sampling():
assert priced - blank_price == len(exact_text)
@pytest.mark.parametrize("pdf_pricing", ["claude", "openai"])
@pytest.mark.parametrize(("second_text", "second_count"), [(b"second", 7), (b"", 0)])
def test_shared_pdf_form_uses_each_parents_resources(
pdf_pricing: Literal["claude", "openai"], second_text: bytes, second_count: int
):
path: Final = b"10 10 m 20 20 l S\n"
padding: Final = path * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path) + 1)
attributes: Final = b"/Type /XObject /Subtype /Form /BBox [0 0 612 792]"
fonts: Final = b" /Resources << /Font << /F1 3 0 R >> >>"
def document(second_leaf: int) -> str:
return _pdf_data_url_with_xobjects(
padding + b"/First Do\n/Second Do\n",
b"<< /First 6 0 R /Second 7 0 R >>",
(
_pdf_stream(
b"/Shared Do",
attributes + b" /Resources << /XObject << /Shared 8 0 R /Leaf 9 0 R >> >>",
),
_pdf_stream(
b"/Shared Do",
attributes + f" /Resources << /XObject << /Shared 8 0 R /Leaf {second_leaf} 0 R >> >>".encode(),
),
_pdf_stream(b"/Leaf Do", attributes),
_pdf_stream(b"BT /F1 12 Tf (first) Tj ET", attributes + fonts),
_pdf_stream(b"BT /F1 12 Tf (" + second_text + b") Tj ET", attributes + fonts),
),
)
def count_text(text: str) -> int:
return text.count("first") + 7 * text.count("second")
shared_price: Final = _count_inline_pdf_tokens(document(9), count_text, pdf_pricing)
distinct_price: Final = _count_inline_pdf_tokens(document(10), count_text, pdf_pricing)
assert shared_price is not None
assert distinct_price is not None
assert distinct_price - shared_price == second_count - 1
def test_repeated_identity_h_form_draws_count_each_text_sample():
path_operator: Final = b"10 10 m 20 20 l S\n"
path_padding: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) + 1)