mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(token_counter): align form resource handling
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
c44fec70c4
commit
51dcbd8284
2 changed files with 45 additions and 5 deletions
|
|
@ -1678,7 +1678,7 @@ def _pdf_strip_inline_images(content: bytes) -> bytes:
|
|||
|
||||
|
||||
def _pdf_page_content_metrics(page: _PdfPage, page_content: bytes) -> _PdfStreamMetrics:
|
||||
form_metrics_memo: Final[dict[int, _PdfStreamMetrics]] = {} # mutable-ok: page-local cache avoids repeated walks
|
||||
form_metrics_memo: Final[dict[tuple[int, int], _PdfStreamMetrics]] = {} # mutable-ok: page-local walk cache
|
||||
|
||||
def stream_metrics(
|
||||
content: bytes,
|
||||
|
|
@ -1724,14 +1724,15 @@ def _pdf_page_content_metrics(page: _PdfPage, page_content: bytes) -> _PdfStream
|
|||
return _PdfStreamMetrics(b"", 0, 0, inherited_resources, (), extraction_limited=True)
|
||||
if not isinstance(resolved, _PdfFormXObject) or resolved.get("/Subtype") != "/Form":
|
||||
return _PdfStreamMetrics(b"", 0, 0, inherited_resources, ())
|
||||
cached: Final = form_metrics_memo.get(identity)
|
||||
form_resources: Final = _pdf_resolve_object(resolved.get("/Resources"))
|
||||
resources: Final = form_resources if isinstance(form_resources, _PdfResourceDictionary) else inherited_resources
|
||||
cache_key: Final = (identity, id(resources))
|
||||
cached: Final = form_metrics_memo.get(cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
content: Final = _pdf_strip_inline_images(resolved.get_data())
|
||||
form_resources: Final = _pdf_resolve_object(resolved.get("/Resources"))
|
||||
resources: Final = form_resources if isinstance(form_resources, _PdfResourceDictionary) else inherited_resources
|
||||
metrics: Final = stream_metrics(content, resources, ancestors | {identity})
|
||||
form_metrics_memo[identity] = metrics
|
||||
form_metrics_memo[cache_key] = metrics
|
||||
return metrics
|
||||
|
||||
resources: Final = _pdf_resolve_object(page.get("/Resources"))
|
||||
|
|
|
|||
|
|
@ -3577,6 +3577,45 @@ def test_pdf_text_object_strings_containing_bt_and_et_do_not_split_sampling():
|
|||
assert priced - blank_price == len(exact_text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pdf_pricing", ["claude", "openai"])
|
||||
@pytest.mark.parametrize(("second_text", "second_count"), [(b"second", 7), (b"", 0)])
|
||||
def test_shared_pdf_form_uses_each_parents_resources(
|
||||
pdf_pricing: Literal["claude", "openai"], second_text: bytes, second_count: int
|
||||
):
|
||||
path: Final = b"10 10 m 20 20 l S\n"
|
||||
padding: Final = path * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path) + 1)
|
||||
attributes: Final = b"/Type /XObject /Subtype /Form /BBox [0 0 612 792]"
|
||||
fonts: Final = b" /Resources << /Font << /F1 3 0 R >> >>"
|
||||
|
||||
def document(second_leaf: int) -> str:
|
||||
return _pdf_data_url_with_xobjects(
|
||||
padding + b"/First Do\n/Second Do\n",
|
||||
b"<< /First 6 0 R /Second 7 0 R >>",
|
||||
(
|
||||
_pdf_stream(
|
||||
b"/Shared Do",
|
||||
attributes + b" /Resources << /XObject << /Shared 8 0 R /Leaf 9 0 R >> >>",
|
||||
),
|
||||
_pdf_stream(
|
||||
b"/Shared Do",
|
||||
attributes + f" /Resources << /XObject << /Shared 8 0 R /Leaf {second_leaf} 0 R >> >>".encode(),
|
||||
),
|
||||
_pdf_stream(b"/Leaf Do", attributes),
|
||||
_pdf_stream(b"BT /F1 12 Tf (first) Tj ET", attributes + fonts),
|
||||
_pdf_stream(b"BT /F1 12 Tf (" + second_text + b") Tj ET", attributes + fonts),
|
||||
),
|
||||
)
|
||||
|
||||
def count_text(text: str) -> int:
|
||||
return text.count("first") + 7 * text.count("second")
|
||||
|
||||
shared_price: Final = _count_inline_pdf_tokens(document(9), count_text, pdf_pricing)
|
||||
distinct_price: Final = _count_inline_pdf_tokens(document(10), count_text, pdf_pricing)
|
||||
assert shared_price is not None
|
||||
assert distinct_price is not None
|
||||
assert distinct_price - shared_price == second_count - 1
|
||||
|
||||
|
||||
def test_repeated_identity_h_form_draws_count_each_text_sample():
|
||||
path_operator: Final = b"10 10 m 20 20 l S\n"
|
||||
path_padding: Final = path_operator * (PDF_PAGE_MAX_EXACT_CONTENT_BYTES // len(path_operator) + 1)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue