From 317c4caffca39110e8721fdcdec1761d3eecb47f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:21:48 -0700 Subject: [PATCH] fix(token-counter): count each team deployment by its PDF pricing and honor base_model Budget reservation and the TPM limiter now look up the deployments that serve the caller's team, by base_model when set. A deployment that reads PDF pages and prices them unlike the group name counts with its own model, any other with the group name. A comment between an XObject name and Do now reads as whitespace, and the timing tests moved to CodSpeed benchmarks. --- litellm/litellm_core_utils/token_counter.py | 11 +- .../hooks/parallel_request_limiter_v3.py | 22 +-- .../spend_tracking/budget_reservation.py | 6 +- litellm/proxy/spend_tracking/input_tokens.py | 22 +-- .../test_token_counter_pdf_benchmarks.py | 126 ++++++++++++++++++ .../litellm_core_utils/test_token_counter.py | 64 ++++----- .../hooks/test_parallel_request_limiter_v3.py | 2 +- .../spend_tracking/test_budget_reservation.py | 2 +- .../proxy/spend_tracking/test_input_tokens.py | 56 +++++++- 9 files changed, 244 insertions(+), 67 deletions(-) create mode 100644 tests/benchmarks/test_token_counter_pdf_benchmarks.py diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 3472e9d2b56..69505309ef0 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -709,7 +709,7 @@ class _MessageCountParams: @cached_property def pdf_pricing(self) -> PdfPagePricing: - return _pdf_page_pricing(self._model) + return pdf_page_pricing(self._model) def _is_openai_chat_model(model: str) -> bool: @@ -753,10 +753,15 @@ def _is_openai_or_azure_model(model: str) -> bool: return _is_openai_chat_model(model) or _openai_model_family(model) is not None or _is_azure_openai_chat_model(model) -def _pdf_page_pricing(model: str | None) -> PdfPagePricing: +def pdf_page_pricing(model: str | None) -> PdfPagePricing: return "openai" if _is_openai_or_azure_model(model or "") else "claude" +def reads_pdf_pages(model: str) -> bool: + entry: Final = _cost_map_entry(model) or _cost_map_entry(model.partition("/")[2]) + return entry.get("supports_pdf_input") is True + + def token_counter( model="", custom_tokenizer: dict | SelectTokenizerResponse | None = None, @@ -1262,7 +1267,7 @@ def _pdf_xobjects(container: Mapping[str, object], inherited: _PdfXObjects) -> _ _PDF_WHITESPACE: Final = rb"\x00\t\n\x0c\r " _PDF_TOKEN_END: Final = _PDF_WHITESPACE + rb"()<>\[\]{}/%" _PDF_XOBJECT_DRAW: Final = re.compile( - rb"/([^" + _PDF_TOKEN_END + rb"]*)[" + _PDF_WHITESPACE + rb"]+Do(?![^" + _PDF_TOKEN_END + rb"])" + rb"/([^" + _PDF_TOKEN_END + rb"]*)(?:[" + _PDF_WHITESPACE + rb"]|%[^\r\n]*[\r\n])+Do(?![^" + _PDF_TOKEN_END + rb"])" ) _PDF_INLINE_IMAGE: Final = re.compile(rb"(? str | None: return resolve_model_group_alias(llm_router.model_group_alias, model) -def _resolve_counted_models_via_proxy_router(model: str) -> Sequence[str]: +def _resolve_counted_models_via_proxy_router(model: str, team_id: str | None) -> Sequence[str]: from litellm.proxy.proxy_server import llm_router - return models_counted_for(model, llm_router) + return models_counted_for(model, llm_router, team_id) def _resolve_ptu_team_ceiling_via_proxy_router(team_id: str, model_group: str) -> PTUTeamCeiling | None: @@ -885,7 +885,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): time_provider: Callable[[], datetime] | None = None, tag_rate_limit_resolver: TagRateLimitResolver = resolve_tag_rate_limits_from_db, model_group_resolver: Callable[[str], str | None] = _resolve_model_group_alias_via_proxy_router, - counted_models_resolver: Callable[[str], Sequence[str]] = _resolve_counted_models_via_proxy_router, + counted_models_resolver: Callable[[str, str | None], Sequence[str]] = _resolve_counted_models_via_proxy_router, media_model_resolver: Callable[[str, str | None], tuple[str, ...]] = _resolve_media_models_via_proxy_router, ptu_team_ceiling_resolver: Callable[ [str, str], PTUTeamCeiling | None @@ -4082,14 +4082,18 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): call_type=call_type, include_media=False, ) - counted_models: Final = self._counted_models_resolver(requested_model) if requested_model else (None,) + counted_models: Final = ( + self._counted_models_resolver(requested_model, user_api_key_dict.team_id) if requested_model else (None,) + ) raw_estimated_input_tokens: Final = max( - [ - await offload_token_count(self._estimate_precise_input_tokens)( - data=data, model=model, call_type=call_type + await asyncio.gather( + *( + offload_token_count(self._estimate_precise_input_tokens)( + data=data, model=model, call_type=call_type + ) + for model in counted_models ) - for model in counted_models - ] + ) ) estimated_input_tokens: Final = max(raw_estimated_input_tokens, 1) estimated_output_tokens: Final = ( diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 957700015dc..3f3256a059b 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -275,6 +275,7 @@ async def reserve_budget_for_request( request_body=request_body, route=route, llm_router=llm_router, + team_id=valid_token.team_id, raw_body=raw_body, ) @@ -1596,6 +1597,7 @@ async def count_request_input_tokens( request_body: dict, route: str, llm_router: Router | None, + team_id: str | None, raw_body: bytes | None = None, ) -> Mapping[str, int]: """Input-token count per candidate model, counted once per request. @@ -1604,7 +1606,9 @@ async def count_request_input_tokens( models: Final = _get_request_models(request_body=request_body, route=route, llm_router=llm_router) if not models: return MappingProxyType({}) - counted_models: Final = MappingProxyType({model: models_counted_for(model, llm_router) for model in models}) + counted_models: Final = MappingProxyType( + {model: models_counted_for(model, llm_router, team_id) for model in models} + ) counts: Final = await count_input_tokens( request_body=request_body, raw_body=raw_body, diff --git a/litellm/proxy/spend_tracking/input_tokens.py b/litellm/proxy/spend_tracking/input_tokens.py index 4c157d64640..35b0854d693 100644 --- a/litellm/proxy/spend_tracking/input_tokens.py +++ b/litellm/proxy/spend_tracking/input_tokens.py @@ -19,6 +19,7 @@ from typing import TYPE_CHECKING, Final import litellm from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.token_counter import pdf_page_pricing, reads_pdf_pages from litellm.responses.litellm_completion_transformation.compaction_items import ( billable_input_since_newest_compaction, holds_compaction_summary, @@ -133,15 +134,20 @@ def _count_input_tokens_for_models( ) -def models_counted_for(model: str, llm_router: Router | None) -> tuple[str, ...]: - """The names a request for `model` is counted with: each deployment's model minus its provider prefix, the - name `/v1/messages/count_tokens` counts with, or `model` itself when no deployment serves it.""" - deployments: Final = (llm_router.get_model_list(model_name=model) or ()) if llm_router is not None else () - deployment_models: Final = (deployment["litellm_params"].get("model") for deployment in deployments) - served: Final = tuple( - dict.fromkeys(name.partition("/")[2] or name for name in deployment_models if isinstance(name, str) and name) +def models_counted_for(model: str, llm_router: Router | None, team_id: str | None) -> tuple[str, ...]: + """The names a request for `model` from `team_id` is counted with: `model` itself, except for a deployment that + reads PDF pages and prices them unlike `model`, which is counted with its own model minus the provider prefix, + the name `/v1/messages/count_tokens` counts with.""" + served: Final = ( + llm_router.resolved_litellm_models(model, team_id, use_base_model=True) if llm_router is not None else () ) - return served or (model,) + return tuple(dict.fromkeys(_counted_name(model, served_model) for served_model in served)) or (model,) + + +def _counted_name(model: str, served_model: str) -> str: + if pdf_page_pricing(served_model) == pdf_page_pricing(model) or not reads_pdf_pages(served_model): + return model + return served_model.partition("/")[2] or served_model def count_input_tokens_for_model(request_body: dict, model: str) -> int | None: diff --git a/tests/benchmarks/test_token_counter_pdf_benchmarks.py b/tests/benchmarks/test_token_counter_pdf_benchmarks.py new file mode 100644 index 00000000000..b2a86459438 --- /dev/null +++ b/tests/benchmarks/test_token_counter_pdf_benchmarks.py @@ -0,0 +1,126 @@ +"""PDF pages and model names that once took seconds to minutes to count, kept here so a slowdown shows on CodSpeed. + +Each input is smaller than the one that exposed the slowdown, since the old cost grew with the same input size.""" + +import base64 +from itertools import accumulate, chain +from typing import Final + +import pytest + +from litellm.litellm_core_utils.token_counter import token_counter + +TEXT_LINE: Final = "Revenue grew eleven percent while churn fell to two percent." + + +def _pdf_base64(text: str, page_resources: str, drawing: bytes, xobjects: tuple[bytes, ...] = ()) -> str: + escaped: Final = text.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)") + stream: Final = f"BT /F1 12 Tf 72 720 Td ({escaped}) Tj ET".encode("latin-1") + drawing + bodies: Final = ( + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Count 1 /Kids [ 5 0 R ] >>", + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + f"<< /Length {len(stream)} >>\nstream\n".encode() + stream + b"\nendstream", + f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources {page_resources} /Contents 4 0 R >>".encode(), + *xobjects, + ) + header: Final = b"%PDF-1.4\n" + objects: Final = tuple( + f"{number} 0 obj\n".encode() + body + b"\nendobj\n" for number, body in enumerate(bodies, start=1) + ) + offsets: Final = accumulate((len(header), *(len(obj) for obj in objects[:-1]))) + xref: Final = f"xref\n0 {len(objects) + 1}\n0000000000 65535 f \n".encode() + b"".join( + f"{offset:010d} 00000 n \n".encode() for offset in offsets + ) + trailer: Final = ( + f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\n" + f"startxref\n{len(header) + sum(len(obj) for obj in objects)}\n%%EOF\n" + ).encode() + return base64.b64encode(header + b"".join(objects) + xref + trailer).decode() + + +def _form(resources_number: int, drawing: bytes) -> bytes: + return ( + f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Resources {resources_number} 0 R " + f"/Length {len(drawing)} >>\nstream\n".encode() + + drawing + + b"\nendstream" + ) + + +def _count_pdf(pdf_base64: str) -> int: + document: Final = {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": pdf_base64}} + return token_counter(model="gpt-4o", messages=[{"role": "user", "content": [document]}]) + + +def _page_of_drawing_operators() -> str: + return _pdf_base64("", "<< >>", b" q Q" * 50_000) + + +def _page_of_forms_sharing_resources() -> str: + form_count: Final = 300 + shared_resources: Final = ( + "<< /XObject << " + " ".join(f"/F{index} {8 + index} 0 R" for index in range(form_count)) + " >> >>" + ).encode() + forms: Final = tuple(_form(7, b"/F0 Do") for _ in range(form_count)) + return _pdf_base64( + TEXT_LINE, + "<< /Font << /F1 3 0 R >> /XObject << /X0 6 0 R >> >>", + b" q 10 0 0 10 0 0 cm /X0 Do Q", + (forms[0], shared_resources, *forms), + ) + + +def _page_of_parents_sharing_a_child_form() -> str: + parent_count: Final = 100 + first: Final = 6 + child_number: Final = first + 2 * parent_count + child_drawing: Final = b" ".join((b"/D Do",) * 4000) + child: Final = ( + f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length {len(child_drawing)} >>\nstream\n".encode() + + child_drawing + + b"\nendstream" + ) + parents: Final = tuple( + chain.from_iterable( + (_form(first + 2 * index + 1, b"/C Do"), f"<< /XObject << /C {child_number} 0 R >> >>".encode()) + for index in range(parent_count) + ) + ) + page_resources: Final = ( + "<< /Font << /F1 3 0 R >> /XObject << " + + " ".join(f"/P{index} {first + 2 * index} 0 R" for index in range(parent_count)) + + " >> >>" + ) + page_drawing: Final = b" " + b" ".join(f"/P{index} Do".encode() for index in range(parent_count)) + return _pdf_base64(TEXT_LINE, page_resources, page_drawing, (*parents, child)) + + +PAGE_OF_DRAWING_OPERATORS: Final = _page_of_drawing_operators() +PAGE_OF_FORMS_SHARING_RESOURCES: Final = _page_of_forms_sharing_resources() +PAGE_OF_PARENTS_SHARING_A_CHILD_FORM: Final = _page_of_parents_sharing_a_child_form() +BLANK_PAGE_COUNT: Final = _count_pdf(_pdf_base64("", "<< >>", b"")) +TEXT_PAGE_COUNT: Final = _count_pdf(_pdf_base64(TEXT_LINE, "<< /Font << /F1 3 0 R >> >>", b"")) +LONG_UNKNOWN_MODEL_NAME: Final = "unknown-" + "a-" * 5_000 +NAME_SAMPLE: Final = "Ceci est un texte en français: éèàç. " * 8 +UNKNOWN_MODEL_NAME_COUNT: Final = token_counter(model="my-private-model", text=NAME_SAMPLE) + + +@pytest.mark.benchmark +def test_pdf_page_of_drawing_operators_and_no_image() -> None: + assert _count_pdf(PAGE_OF_DRAWING_OPERATORS) == BLANK_PAGE_COUNT + + +@pytest.mark.benchmark +def test_pdf_page_of_forms_sharing_one_resources_dictionary() -> None: + assert _count_pdf(PAGE_OF_FORMS_SHARING_RESOURCES) == TEXT_PAGE_COUNT + + +@pytest.mark.benchmark +def test_pdf_page_of_parent_forms_sharing_one_child_form() -> None: + assert _count_pdf(PAGE_OF_PARENTS_SHARING_A_CHILD_FORM) > TEXT_PAGE_COUNT + + +@pytest.mark.benchmark +def test_token_counter_long_unknown_model_name() -> None: + assert token_counter(model=LONG_UNKNOWN_MODEL_NAME, text=NAME_SAMPLE) == UNKNOWN_MODEL_NAME_COUNT diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index d260d56d765..607f8240905 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -29,7 +29,11 @@ import litellm from litellm import decode, encode, get_modified_max_tokens from litellm import token_counter as token_counter_old import litellm.constants -from litellm.constants import HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS, TOKEN_COUNTER_MAX_CONCURRENT_COUNTS +from litellm.constants import ( + HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS, + PDF_MAX_XOBJECT_DRAWS_PER_PAGE, + TOKEN_COUNTER_MAX_CONCURRENT_COUNTS, +) from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.token_counter import ( _encoding_count, @@ -2283,13 +2287,19 @@ def test_openai_model_finds_an_image_a_shared_form_without_resources_draws_from_ ) == _rendered_page_cost() -@pytest.mark.timeout(10) -def test_openai_model_prices_a_page_of_a_million_drawing_operators_promptly(): - """A page of a million drawing operators and no image took about 30 seconds to price, since every operator was - parsed to find the images it draws, while the stream compresses into a request a few kilobytes long.""" +@pytest.mark.parametrize( + ("drawing", "draws_image"), + [(b" /X0 %scale it first\n Do", True), (b" /X0 % Do\n", False)], + ids=["comment-before-do", "do-inside-a-comment"], +) +def test_openai_model_reads_a_comment_in_a_content_stream_as_whitespace(drawing: bytes, draws_image: bool): + """A comment runs to the end of its line and separates tokens like whitespace, so it neither hides the Do after + it nor draws anything itself.""" model: Final = "gpt-4o" - assert _pdf_cost(model, _pdf_base64(("",), drawing=b" q Q" * 1_000_000, page_resources="<< >>")) == 0 + assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=(_ONE_PIXEL_IMAGE,), drawing=drawing)) == ( + _text_cost(model, _REVENUE_LINE) + (_rendered_page_cost() if draws_image else 13) + ) @pytest.mark.parametrize("escaped_name", ["/Im#231", "/#C4#E3", "/caf#C3#A9"], ids=["hex_escape", "gbk", "utf8"]) @@ -2309,32 +2319,16 @@ def test_openai_model_finds_an_image_drawn_by_an_escaped_name(escaped_name: str) ) == _rendered_page_cost() -@pytest.mark.timeout(10) -def test_openai_model_reads_resources_shared_by_many_forms_once(): - """3,000 forms that all list the same /Resources took 17 seconds to price before each dictionary was read once.""" +@pytest.mark.parametrize(("parent_count", "draws_too_many"), [(1, False), (2, True)]) +def test_openai_model_prices_a_page_whose_forms_draw_too_many_xobjects_as_rendered( + parent_count: int, draws_too_many: bool +): + """A page prices as if it drew an image once its forms draw more than PDF_MAX_XOBJECT_DRAWS_PER_PAGE XObjects + between them, so a page of forms nested to blow up the walk cannot stall the request.""" model: Final = "gpt-4o" - form_count: Final = 3000 - shared_resources_number: Final = 7 - shared_resources: Final = ( - "<< /XObject << " + " ".join(f"/F{index} {8 + index} 0 R" for index in range(form_count)) + " >> >>" - ).encode() - forms: Final = tuple(_form_xobject(shared_resources_number, b"/F0 Do") for _ in range(form_count)) - - assert _pdf_cost( - model, _pdf_base64((_REVENUE_LINE,), xobjects=(forms[0], shared_resources, *forms), drawing=_DRAW_X0) - ) == (_text_cost(model, _REVENUE_LINE) + 13) - - -@pytest.mark.timeout(10) -def test_openai_model_prices_a_child_form_shared_by_many_parents_promptly(): - """1,000 forms that each list their own /Resources and draw one shared 4,000-operator child took about 45 - seconds to price, since the child was parsed and walked again under every parent. A page now prices as if it - drew an image once its forms draw more than a fixed number of XObjects, and each form is parsed once.""" - model: Final = "gpt-4o" - parent_count: Final = 1000 first: Final = 6 child_number: Final = first + 2 * parent_count - child_drawing: Final = b" ".join((b"/D Do",) * 4000) + child_drawing: Final = b" ".join((b"/D Do",) * (PDF_MAX_XOBJECT_DRAWS_PER_PAGE // 2)) child: Final = ( f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length {len(child_drawing)} >>\nstream\n".encode() + child_drawing @@ -2358,7 +2352,7 @@ def test_openai_model_prices_a_child_form_shared_by_many_parents_promptly(): _pdf_base64( (_REVENUE_LINE,), xobjects=(*parents, child), drawing=page_drawing, page_resources=page_resources ), - ) == _text_cost(model, _REVENUE_LINE) + _rendered_page_cost() + ) == _text_cost(model, _REVENUE_LINE) + (_rendered_page_cost() if draws_too_many else 13) _NON_LATIN_SAMPLE: Final = "Привет, как дела? これは日本語のテキストです。 Ceci est un texte en français: éèàç. " * 8 @@ -2461,16 +2455,6 @@ def test_openai_fine_tunes_count_like_their_base_model(fine_tune: str, base_mode ) -@pytest.mark.timeout(10) -def test_a_very_long_unknown_model_name_counts_promptly(): - """A 200,000-word model name took minutes to count, since every dash-joined prefix of it was tried.""" - long_name: Final = "unknown-" + "a-" * 200_000 - - assert token_counter_new(model=long_name, text=_NON_LATIN_SAMPLE) == token_counter_new( - model="my-private-model", text=_NON_LATIN_SAMPLE - ) - - @pytest.mark.parametrize( "model", [ diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index 856e7aa7ed5..c82ecb72b09 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -9106,7 +9106,7 @@ async def test_project_itpm_reserves_a_pdf_at_the_largest_count_of_the_group_dep cache: Final = DualCache() handler: Final = _PROXY_MaxParallelRequestsHandler( internal_usage_cache=InternalUsageCache(cache), - counted_models_resolver=lambda model: ("claude-haiku-5-5", "gpt-5.6-sol"), + counted_models_resolver=lambda model, team_id: ("claude-haiku-5-5", "gpt-5.6-sol"), ) messages: Final = _one_page_pdf_message() caller: Final = UserAPIKeyAuth( diff --git a/tests/unit/proxy/spend_tracking/test_budget_reservation.py b/tests/unit/proxy/spend_tracking/test_budget_reservation.py index 08b1cf318c2..65a1969ece6 100644 --- a/tests/unit/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/unit/proxy/spend_tracking/test_budget_reservation.py @@ -440,7 +440,7 @@ async def test_reservation_counts_a_pdf_with_the_deployment_behind_a_gpt_named_g ) counts: Final = await count_request_input_tokens( - request_body=request_body, route="/chat/completions", llm_router=router + request_body=request_body, route="/chat/completions", llm_router=router, team_id=None ) assert counts == {"gpt-4o": count_input_tokens_for_model(request_body=request_body, model="claude-haiku-5-5")} diff --git a/tests/unit/proxy/spend_tracking/test_input_tokens.py b/tests/unit/proxy/spend_tracking/test_input_tokens.py index 1236fb516a5..148001be32b 100644 --- a/tests/unit/proxy/spend_tracking/test_input_tokens.py +++ b/tests/unit/proxy/spend_tracking/test_input_tokens.py @@ -82,18 +82,66 @@ def test_a_model_group_counts_like_the_deployment_that_serves_it(group: str, dep request_body: Final = {"model": group, "messages": _one_page_pdf_message()} counted_with: Final = deployment_model.partition("/")[2] - assert models_counted_for(group, _router((group, deployment_model))) == (counted_with,) + assert models_counted_for(group, _router((group, deployment_model)), None) == (counted_with,) assert count_input_tokens_for_model(request_body=request_body, model=counted_with) != count_input_tokens_for_model( request_body=request_body, model=group ) +@pytest.mark.parametrize( + ("group", "deployment_model"), + [ + ("my-claude", "anthropic/claude-haiku-5-5"), + ("my-llama", "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo"), + ("my-cohere", "cohere/command-r-plus"), + ("gpt-4o", "cohere/command-r-plus"), + ], + ids=["claude-alias", "llama-alias", "cohere-alias", "gpt-named-group-served-by-a-model-that-reads-no-pdf"], +) +def test_a_group_whose_deployment_prices_pdf_pages_alike_or_reads_none_counts_as_the_group( + group: str, deployment_model: str +) -> None: + assert models_counted_for(group, _router((group, deployment_model)), None) == (group,) + + +def test_a_custom_named_deployment_counts_with_its_base_model(): + router: Final = Router( + model_list=[ + { + "model_name": "reports", + "litellm_params": {"model": "azure/prod-deploy-1", "api_key": "x", "api_base": "https://x"}, + "model_info": {"base_model": "azure/gpt-5.6-sol"}, + }, + {"model_name": "drafts", "litellm_params": {"model": "azure/prod-deploy-2", "api_key": "x"}}, + ] + ) + + assert models_counted_for("reports", router, None) == ("gpt-5.6-sol",) + assert models_counted_for("drafts", router, None) == ("drafts",) + + def test_a_group_of_differently_priced_deployments_counts_with_each_and_a_name_no_deployment_serves_counts_as_is(): router: Final = _router(("mixed", "anthropic/claude-haiku-5-5"), ("mixed", "azure/gpt-5.6-sol")) - assert models_counted_for("mixed", router) == ("claude-haiku-5-5", "gpt-5.6-sol") - assert models_counted_for("not-deployed", router) == ("not-deployed",) - assert models_counted_for("gpt-4o", None) == ("gpt-4o",) + assert models_counted_for("mixed", router, None) == ("mixed", "gpt-5.6-sol") + assert models_counted_for("not-deployed", router, None) == ("not-deployed",) + assert models_counted_for("gpt-4o", None, None) == ("gpt-4o",) + + +def test_a_team_request_counts_with_the_deployment_its_team_is_served_by(): + router: Final = Router( + model_list=[ + {"model_name": "reports", "litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": "x"}}, + { + "model_name": "reports-team-a", + "litellm_params": {"model": "azure/gpt-5.6-sol", "api_key": "x"}, + "model_info": {"team_id": "team-a", "team_public_model_name": "reports"}, + }, + ] + ) + + assert models_counted_for("reports", router, "team-a") == ("gpt-5.6-sol",) + assert models_counted_for("reports", router, "team-b") == ("reports",) def test_count_input_tokens_for_model_counts_anthropic_auto_tool_choice() -> None: