fix(token-counter): count each team deployment by its PDF pricing and honor base_model

Budget reservation and the TPM limiter now look up the deployments that serve the
caller's team, by base_model when set. A deployment that reads PDF pages and prices
them unlike the group name counts with its own model, any other with the group name.
A comment between an XObject name and Do now reads as whitespace, and the timing
tests moved to CodSpeed benchmarks.
This commit is contained in:
mateo-berri 2026-10-28 22:21:48 -07:00
parent f947ab62f2
commit 317c4caffc
9 changed files with 244 additions and 67 deletions

View file

@ -709,7 +709,7 @@ class _MessageCountParams:
@cached_property
def pdf_pricing(self) -> PdfPagePricing:
return _pdf_page_pricing(self._model)
return pdf_page_pricing(self._model)
def _is_openai_chat_model(model: str) -> bool:
@ -753,10 +753,15 @@ def _is_openai_or_azure_model(model: str) -> bool:
return _is_openai_chat_model(model) or _openai_model_family(model) is not None or _is_azure_openai_chat_model(model)
def _pdf_page_pricing(model: str | None) -> PdfPagePricing:
def pdf_page_pricing(model: str | None) -> PdfPagePricing:
return "openai" if _is_openai_or_azure_model(model or "") else "claude"
def reads_pdf_pages(model: str) -> bool:
entry: Final = _cost_map_entry(model) or _cost_map_entry(model.partition("/")[2])
return entry.get("supports_pdf_input") is True
def token_counter(
model="",
custom_tokenizer: dict | SelectTokenizerResponse | None = None,
@ -1262,7 +1267,7 @@ def _pdf_xobjects(container: Mapping[str, object], inherited: _PdfXObjects) -> _
_PDF_WHITESPACE: Final = rb"\x00\t\n\x0c\r "
_PDF_TOKEN_END: Final = _PDF_WHITESPACE + rb"()<>\[\]{}/%"
_PDF_XOBJECT_DRAW: Final = re.compile(
rb"/([^" + _PDF_TOKEN_END + rb"]*)[" + _PDF_WHITESPACE + rb"]+Do(?![^" + _PDF_TOKEN_END + rb"])"
rb"/([^" + _PDF_TOKEN_END + rb"]*)(?:[" + _PDF_WHITESPACE + rb"]|%[^\r\n]*[\r\n])+Do(?![^" + _PDF_TOKEN_END + rb"])"
)
_PDF_INLINE_IMAGE: Final = re.compile(rb"(?<![^" + _PDF_WHITESPACE + rb"])BI(?![^" + _PDF_WHITESPACE + rb"/])")
_PDF_NAME_ESCAPE: Final = re.compile(rb"#([0-9A-Fa-f]{2})")

View file

@ -160,10 +160,10 @@ def _resolve_model_group_alias_via_proxy_router(model: str) -> str | None:
return resolve_model_group_alias(llm_router.model_group_alias, model)
def _resolve_counted_models_via_proxy_router(model: str) -> Sequence[str]:
def _resolve_counted_models_via_proxy_router(model: str, team_id: str | None) -> Sequence[str]:
from litellm.proxy.proxy_server import llm_router
return models_counted_for(model, llm_router)
return models_counted_for(model, llm_router, team_id)
def _resolve_ptu_team_ceiling_via_proxy_router(team_id: str, model_group: str) -> PTUTeamCeiling | None:
@ -885,7 +885,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
time_provider: Callable[[], datetime] | None = None,
tag_rate_limit_resolver: TagRateLimitResolver = resolve_tag_rate_limits_from_db,
model_group_resolver: Callable[[str], str | None] = _resolve_model_group_alias_via_proxy_router,
counted_models_resolver: Callable[[str], Sequence[str]] = _resolve_counted_models_via_proxy_router,
counted_models_resolver: Callable[[str, str | None], Sequence[str]] = _resolve_counted_models_via_proxy_router,
media_model_resolver: Callable[[str, str | None], tuple[str, ...]] = _resolve_media_models_via_proxy_router,
ptu_team_ceiling_resolver: Callable[
[str, str], PTUTeamCeiling | None
@ -4082,14 +4082,18 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
call_type=call_type,
include_media=False,
)
counted_models: Final = self._counted_models_resolver(requested_model) if requested_model else (None,)
counted_models: Final = (
self._counted_models_resolver(requested_model, user_api_key_dict.team_id) if requested_model else (None,)
)
raw_estimated_input_tokens: Final = max(
[
await offload_token_count(self._estimate_precise_input_tokens)(
data=data, model=model, call_type=call_type
await asyncio.gather(
*(
offload_token_count(self._estimate_precise_input_tokens)(
data=data, model=model, call_type=call_type
)
for model in counted_models
)
for model in counted_models
]
)
)
estimated_input_tokens: Final = max(raw_estimated_input_tokens, 1)
estimated_output_tokens: Final = (

View file

@ -275,6 +275,7 @@ async def reserve_budget_for_request(
request_body=request_body,
route=route,
llm_router=llm_router,
team_id=valid_token.team_id,
raw_body=raw_body,
)
@ -1596,6 +1597,7 @@ async def count_request_input_tokens(
request_body: dict,
route: str,
llm_router: Router | None,
team_id: str | None,
raw_body: bytes | None = None,
) -> Mapping[str, int]:
"""Input-token count per candidate model, counted once per request.
@ -1604,7 +1606,9 @@ async def count_request_input_tokens(
models: Final = _get_request_models(request_body=request_body, route=route, llm_router=llm_router)
if not models:
return MappingProxyType({})
counted_models: Final = MappingProxyType({model: models_counted_for(model, llm_router) for model in models})
counted_models: Final = MappingProxyType(
{model: models_counted_for(model, llm_router, team_id) for model in models}
)
counts: Final = await count_input_tokens(
request_body=request_body,
raw_body=raw_body,

View file

@ -19,6 +19,7 @@ from typing import TYPE_CHECKING, Final
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.token_counter import pdf_page_pricing, reads_pdf_pages
from litellm.responses.litellm_completion_transformation.compaction_items import (
billable_input_since_newest_compaction,
holds_compaction_summary,
@ -133,15 +134,20 @@ def _count_input_tokens_for_models(
)
def models_counted_for(model: str, llm_router: Router | None) -> tuple[str, ...]:
"""The names a request for `model` is counted with: each deployment's model minus its provider prefix, the
name `/v1/messages/count_tokens` counts with, or `model` itself when no deployment serves it."""
deployments: Final = (llm_router.get_model_list(model_name=model) or ()) if llm_router is not None else ()
deployment_models: Final = (deployment["litellm_params"].get("model") for deployment in deployments)
served: Final = tuple(
dict.fromkeys(name.partition("/")[2] or name for name in deployment_models if isinstance(name, str) and name)
def models_counted_for(model: str, llm_router: Router | None, team_id: str | None) -> tuple[str, ...]:
"""The names a request for `model` from `team_id` is counted with: `model` itself, except for a deployment that
reads PDF pages and prices them unlike `model`, which is counted with its own model minus the provider prefix,
the name `/v1/messages/count_tokens` counts with."""
served: Final = (
llm_router.resolved_litellm_models(model, team_id, use_base_model=True) if llm_router is not None else ()
)
return served or (model,)
return tuple(dict.fromkeys(_counted_name(model, served_model) for served_model in served)) or (model,)
def _counted_name(model: str, served_model: str) -> str:
if pdf_page_pricing(served_model) == pdf_page_pricing(model) or not reads_pdf_pages(served_model):
return model
return served_model.partition("/")[2] or served_model
def count_input_tokens_for_model(request_body: dict, model: str) -> int | None:

View file

@ -0,0 +1,126 @@
"""PDF pages and model names that once took seconds to minutes to count, kept here so a slowdown shows on CodSpeed.
Each input is smaller than the one that exposed the slowdown, since the old cost grew with the same input size."""
import base64
from itertools import accumulate, chain
from typing import Final
import pytest
from litellm.litellm_core_utils.token_counter import token_counter
TEXT_LINE: Final = "Revenue grew eleven percent while churn fell to two percent."
def _pdf_base64(text: str, page_resources: str, drawing: bytes, xobjects: tuple[bytes, ...] = ()) -> str:
escaped: Final = text.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)")
stream: Final = f"BT /F1 12 Tf 72 720 Td ({escaped}) Tj ET".encode("latin-1") + drawing
bodies: Final = (
b"<< /Type /Catalog /Pages 2 0 R >>",
b"<< /Type /Pages /Count 1 /Kids [ 5 0 R ] >>",
b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
f"<< /Length {len(stream)} >>\nstream\n".encode() + stream + b"\nendstream",
f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources {page_resources} /Contents 4 0 R >>".encode(),
*xobjects,
)
header: Final = b"%PDF-1.4\n"
objects: Final = tuple(
f"{number} 0 obj\n".encode() + body + b"\nendobj\n" for number, body in enumerate(bodies, start=1)
)
offsets: Final = accumulate((len(header), *(len(obj) for obj in objects[:-1])))
xref: Final = f"xref\n0 {len(objects) + 1}\n0000000000 65535 f \n".encode() + b"".join(
f"{offset:010d} 00000 n \n".encode() for offset in offsets
)
trailer: Final = (
f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\n"
f"startxref\n{len(header) + sum(len(obj) for obj in objects)}\n%%EOF\n"
).encode()
return base64.b64encode(header + b"".join(objects) + xref + trailer).decode()
def _form(resources_number: int, drawing: bytes) -> bytes:
return (
f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Resources {resources_number} 0 R "
f"/Length {len(drawing)} >>\nstream\n".encode()
+ drawing
+ b"\nendstream"
)
def _count_pdf(pdf_base64: str) -> int:
document: Final = {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": pdf_base64}}
return token_counter(model="gpt-4o", messages=[{"role": "user", "content": [document]}])
def _page_of_drawing_operators() -> str:
return _pdf_base64("", "<< >>", b" q Q" * 50_000)
def _page_of_forms_sharing_resources() -> str:
form_count: Final = 300
shared_resources: Final = (
"<< /XObject << " + " ".join(f"/F{index} {8 + index} 0 R" for index in range(form_count)) + " >> >>"
).encode()
forms: Final = tuple(_form(7, b"/F0 Do") for _ in range(form_count))
return _pdf_base64(
TEXT_LINE,
"<< /Font << /F1 3 0 R >> /XObject << /X0 6 0 R >> >>",
b" q 10 0 0 10 0 0 cm /X0 Do Q",
(forms[0], shared_resources, *forms),
)
def _page_of_parents_sharing_a_child_form() -> str:
parent_count: Final = 100
first: Final = 6
child_number: Final = first + 2 * parent_count
child_drawing: Final = b" ".join((b"/D Do",) * 4000)
child: Final = (
f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length {len(child_drawing)} >>\nstream\n".encode()
+ child_drawing
+ b"\nendstream"
)
parents: Final = tuple(
chain.from_iterable(
(_form(first + 2 * index + 1, b"/C Do"), f"<< /XObject << /C {child_number} 0 R >> >>".encode())
for index in range(parent_count)
)
)
page_resources: Final = (
"<< /Font << /F1 3 0 R >> /XObject << "
+ " ".join(f"/P{index} {first + 2 * index} 0 R" for index in range(parent_count))
+ " >> >>"
)
page_drawing: Final = b" " + b" ".join(f"/P{index} Do".encode() for index in range(parent_count))
return _pdf_base64(TEXT_LINE, page_resources, page_drawing, (*parents, child))
PAGE_OF_DRAWING_OPERATORS: Final = _page_of_drawing_operators()
PAGE_OF_FORMS_SHARING_RESOURCES: Final = _page_of_forms_sharing_resources()
PAGE_OF_PARENTS_SHARING_A_CHILD_FORM: Final = _page_of_parents_sharing_a_child_form()
BLANK_PAGE_COUNT: Final = _count_pdf(_pdf_base64("", "<< >>", b""))
TEXT_PAGE_COUNT: Final = _count_pdf(_pdf_base64(TEXT_LINE, "<< /Font << /F1 3 0 R >> >>", b""))
LONG_UNKNOWN_MODEL_NAME: Final = "unknown-" + "a-" * 5_000
NAME_SAMPLE: Final = "Ceci est un texte en français: éèàç. " * 8
UNKNOWN_MODEL_NAME_COUNT: Final = token_counter(model="my-private-model", text=NAME_SAMPLE)
@pytest.mark.benchmark
def test_pdf_page_of_drawing_operators_and_no_image() -> None:
assert _count_pdf(PAGE_OF_DRAWING_OPERATORS) == BLANK_PAGE_COUNT
@pytest.mark.benchmark
def test_pdf_page_of_forms_sharing_one_resources_dictionary() -> None:
assert _count_pdf(PAGE_OF_FORMS_SHARING_RESOURCES) == TEXT_PAGE_COUNT
@pytest.mark.benchmark
def test_pdf_page_of_parent_forms_sharing_one_child_form() -> None:
assert _count_pdf(PAGE_OF_PARENTS_SHARING_A_CHILD_FORM) > TEXT_PAGE_COUNT
@pytest.mark.benchmark
def test_token_counter_long_unknown_model_name() -> None:
assert token_counter(model=LONG_UNKNOWN_MODEL_NAME, text=NAME_SAMPLE) == UNKNOWN_MODEL_NAME_COUNT

View file

@ -29,7 +29,11 @@ import litellm
from litellm import decode, encode, get_modified_max_tokens
from litellm import token_counter as token_counter_old
import litellm.constants
from litellm.constants import HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS, TOKEN_COUNTER_MAX_CONCURRENT_COUNTS
from litellm.constants import (
HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS,
PDF_MAX_XOBJECT_DRAWS_PER_PAGE,
TOKEN_COUNTER_MAX_CONCURRENT_COUNTS,
)
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.token_counter import (
_encoding_count,
@ -2283,13 +2287,19 @@ def test_openai_model_finds_an_image_a_shared_form_without_resources_draws_from_
) == _rendered_page_cost()
@pytest.mark.timeout(10)
def test_openai_model_prices_a_page_of_a_million_drawing_operators_promptly():
"""A page of a million drawing operators and no image took about 30 seconds to price, since every operator was
parsed to find the images it draws, while the stream compresses into a request a few kilobytes long."""
@pytest.mark.parametrize(
("drawing", "draws_image"),
[(b" /X0 %scale it first\n Do", True), (b" /X0 % Do\n", False)],
ids=["comment-before-do", "do-inside-a-comment"],
)
def test_openai_model_reads_a_comment_in_a_content_stream_as_whitespace(drawing: bytes, draws_image: bool):
"""A comment runs to the end of its line and separates tokens like whitespace, so it neither hides the Do after
it nor draws anything itself."""
model: Final = "gpt-4o"
assert _pdf_cost(model, _pdf_base64(("",), drawing=b" q Q" * 1_000_000, page_resources="<< >>")) == 0
assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=(_ONE_PIXEL_IMAGE,), drawing=drawing)) == (
_text_cost(model, _REVENUE_LINE) + (_rendered_page_cost() if draws_image else 13)
)
@pytest.mark.parametrize("escaped_name", ["/Im#231", "/#C4#E3", "/caf#C3#A9"], ids=["hex_escape", "gbk", "utf8"])
@ -2309,32 +2319,16 @@ def test_openai_model_finds_an_image_drawn_by_an_escaped_name(escaped_name: str)
) == _rendered_page_cost()
@pytest.mark.timeout(10)
def test_openai_model_reads_resources_shared_by_many_forms_once():
"""3,000 forms that all list the same /Resources took 17 seconds to price before each dictionary was read once."""
@pytest.mark.parametrize(("parent_count", "draws_too_many"), [(1, False), (2, True)])
def test_openai_model_prices_a_page_whose_forms_draw_too_many_xobjects_as_rendered(
parent_count: int, draws_too_many: bool
):
"""A page prices as if it drew an image once its forms draw more than PDF_MAX_XOBJECT_DRAWS_PER_PAGE XObjects
between them, so a page of forms nested to blow up the walk cannot stall the request."""
model: Final = "gpt-4o"
form_count: Final = 3000
shared_resources_number: Final = 7
shared_resources: Final = (
"<< /XObject << " + " ".join(f"/F{index} {8 + index} 0 R" for index in range(form_count)) + " >> >>"
).encode()
forms: Final = tuple(_form_xobject(shared_resources_number, b"/F0 Do") for _ in range(form_count))
assert _pdf_cost(
model, _pdf_base64((_REVENUE_LINE,), xobjects=(forms[0], shared_resources, *forms), drawing=_DRAW_X0)
) == (_text_cost(model, _REVENUE_LINE) + 13)
@pytest.mark.timeout(10)
def test_openai_model_prices_a_child_form_shared_by_many_parents_promptly():
"""1,000 forms that each list their own /Resources and draw one shared 4,000-operator child took about 45
seconds to price, since the child was parsed and walked again under every parent. A page now prices as if it
drew an image once its forms draw more than a fixed number of XObjects, and each form is parsed once."""
model: Final = "gpt-4o"
parent_count: Final = 1000
first: Final = 6
child_number: Final = first + 2 * parent_count
child_drawing: Final = b" ".join((b"/D Do",) * 4000)
child_drawing: Final = b" ".join((b"/D Do",) * (PDF_MAX_XOBJECT_DRAWS_PER_PAGE // 2))
child: Final = (
f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length {len(child_drawing)} >>\nstream\n".encode()
+ child_drawing
@ -2358,7 +2352,7 @@ def test_openai_model_prices_a_child_form_shared_by_many_parents_promptly():
_pdf_base64(
(_REVENUE_LINE,), xobjects=(*parents, child), drawing=page_drawing, page_resources=page_resources
),
) == _text_cost(model, _REVENUE_LINE) + _rendered_page_cost()
) == _text_cost(model, _REVENUE_LINE) + (_rendered_page_cost() if draws_too_many else 13)
_NON_LATIN_SAMPLE: Final = "Привет, как дела? これは日本語のテキストです。 Ceci est un texte en français: éèàç. " * 8
@ -2461,16 +2455,6 @@ def test_openai_fine_tunes_count_like_their_base_model(fine_tune: str, base_mode
)
@pytest.mark.timeout(10)
def test_a_very_long_unknown_model_name_counts_promptly():
"""A 200,000-word model name took minutes to count, since every dash-joined prefix of it was tried."""
long_name: Final = "unknown-" + "a-" * 200_000
assert token_counter_new(model=long_name, text=_NON_LATIN_SAMPLE) == token_counter_new(
model="my-private-model", text=_NON_LATIN_SAMPLE
)
@pytest.mark.parametrize(
"model",
[

View file

@ -9106,7 +9106,7 @@ async def test_project_itpm_reserves_a_pdf_at_the_largest_count_of_the_group_dep
cache: Final = DualCache()
handler: Final = _PROXY_MaxParallelRequestsHandler(
internal_usage_cache=InternalUsageCache(cache),
counted_models_resolver=lambda model: ("claude-haiku-5-5", "gpt-5.6-sol"),
counted_models_resolver=lambda model, team_id: ("claude-haiku-5-5", "gpt-5.6-sol"),
)
messages: Final = _one_page_pdf_message()
caller: Final = UserAPIKeyAuth(

View file

@ -440,7 +440,7 @@ async def test_reservation_counts_a_pdf_with_the_deployment_behind_a_gpt_named_g
)
counts: Final = await count_request_input_tokens(
request_body=request_body, route="/chat/completions", llm_router=router
request_body=request_body, route="/chat/completions", llm_router=router, team_id=None
)
assert counts == {"gpt-4o": count_input_tokens_for_model(request_body=request_body, model="claude-haiku-5-5")}

View file

@ -82,18 +82,66 @@ def test_a_model_group_counts_like_the_deployment_that_serves_it(group: str, dep
request_body: Final = {"model": group, "messages": _one_page_pdf_message()}
counted_with: Final = deployment_model.partition("/")[2]
assert models_counted_for(group, _router((group, deployment_model))) == (counted_with,)
assert models_counted_for(group, _router((group, deployment_model)), None) == (counted_with,)
assert count_input_tokens_for_model(request_body=request_body, model=counted_with) != count_input_tokens_for_model(
request_body=request_body, model=group
)
@pytest.mark.parametrize(
("group", "deployment_model"),
[
("my-claude", "anthropic/claude-haiku-5-5"),
("my-llama", "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo"),
("my-cohere", "cohere/command-r-plus"),
("gpt-4o", "cohere/command-r-plus"),
],
ids=["claude-alias", "llama-alias", "cohere-alias", "gpt-named-group-served-by-a-model-that-reads-no-pdf"],
)
def test_a_group_whose_deployment_prices_pdf_pages_alike_or_reads_none_counts_as_the_group(
group: str, deployment_model: str
) -> None:
assert models_counted_for(group, _router((group, deployment_model)), None) == (group,)
def test_a_custom_named_deployment_counts_with_its_base_model():
router: Final = Router(
model_list=[
{
"model_name": "reports",
"litellm_params": {"model": "azure/prod-deploy-1", "api_key": "x", "api_base": "https://x"},
"model_info": {"base_model": "azure/gpt-5.6-sol"},
},
{"model_name": "drafts", "litellm_params": {"model": "azure/prod-deploy-2", "api_key": "x"}},
]
)
assert models_counted_for("reports", router, None) == ("gpt-5.6-sol",)
assert models_counted_for("drafts", router, None) == ("drafts",)
def test_a_group_of_differently_priced_deployments_counts_with_each_and_a_name_no_deployment_serves_counts_as_is():
router: Final = _router(("mixed", "anthropic/claude-haiku-5-5"), ("mixed", "azure/gpt-5.6-sol"))
assert models_counted_for("mixed", router) == ("claude-haiku-5-5", "gpt-5.6-sol")
assert models_counted_for("not-deployed", router) == ("not-deployed",)
assert models_counted_for("gpt-4o", None) == ("gpt-4o",)
assert models_counted_for("mixed", router, None) == ("mixed", "gpt-5.6-sol")
assert models_counted_for("not-deployed", router, None) == ("not-deployed",)
assert models_counted_for("gpt-4o", None, None) == ("gpt-4o",)
def test_a_team_request_counts_with_the_deployment_its_team_is_served_by():
router: Final = Router(
model_list=[
{"model_name": "reports", "litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": "x"}},
{
"model_name": "reports-team-a",
"litellm_params": {"model": "azure/gpt-5.6-sol", "api_key": "x"},
"model_info": {"team_id": "team-a", "team_public_model_name": "reports"},
},
]
)
assert models_counted_for("reports", router, "team-a") == ("gpt-5.6-sol",)
assert models_counted_for("reports", router, "team-b") == ("reports",)
def test_count_input_tokens_for_model_counts_anthropic_auto_tool_choice() -> None: