mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(token-counter): count each team deployment by its PDF pricing and honor base_model
Budget reservation and the TPM limiter now look up the deployments that serve the caller's team, by base_model when set. A deployment that reads PDF pages and prices them unlike the group name counts with its own model, any other with the group name. A comment between an XObject name and Do now reads as whitespace, and the timing tests moved to CodSpeed benchmarks.
This commit is contained in:
parent
f947ab62f2
commit
317c4caffc
9 changed files with 244 additions and 67 deletions
|
|
@ -709,7 +709,7 @@ class _MessageCountParams:
|
|||
|
||||
@cached_property
|
||||
def pdf_pricing(self) -> PdfPagePricing:
|
||||
return _pdf_page_pricing(self._model)
|
||||
return pdf_page_pricing(self._model)
|
||||
|
||||
|
||||
def _is_openai_chat_model(model: str) -> bool:
|
||||
|
|
@ -753,10 +753,15 @@ def _is_openai_or_azure_model(model: str) -> bool:
|
|||
return _is_openai_chat_model(model) or _openai_model_family(model) is not None or _is_azure_openai_chat_model(model)
|
||||
|
||||
|
||||
def _pdf_page_pricing(model: str | None) -> PdfPagePricing:
|
||||
def pdf_page_pricing(model: str | None) -> PdfPagePricing:
|
||||
return "openai" if _is_openai_or_azure_model(model or "") else "claude"
|
||||
|
||||
|
||||
def reads_pdf_pages(model: str) -> bool:
|
||||
entry: Final = _cost_map_entry(model) or _cost_map_entry(model.partition("/")[2])
|
||||
return entry.get("supports_pdf_input") is True
|
||||
|
||||
|
||||
def token_counter(
|
||||
model="",
|
||||
custom_tokenizer: dict | SelectTokenizerResponse | None = None,
|
||||
|
|
@ -1262,7 +1267,7 @@ def _pdf_xobjects(container: Mapping[str, object], inherited: _PdfXObjects) -> _
|
|||
_PDF_WHITESPACE: Final = rb"\x00\t\n\x0c\r "
|
||||
_PDF_TOKEN_END: Final = _PDF_WHITESPACE + rb"()<>\[\]{}/%"
|
||||
_PDF_XOBJECT_DRAW: Final = re.compile(
|
||||
rb"/([^" + _PDF_TOKEN_END + rb"]*)[" + _PDF_WHITESPACE + rb"]+Do(?![^" + _PDF_TOKEN_END + rb"])"
|
||||
rb"/([^" + _PDF_TOKEN_END + rb"]*)(?:[" + _PDF_WHITESPACE + rb"]|%[^\r\n]*[\r\n])+Do(?![^" + _PDF_TOKEN_END + rb"])"
|
||||
)
|
||||
_PDF_INLINE_IMAGE: Final = re.compile(rb"(?<![^" + _PDF_WHITESPACE + rb"])BI(?![^" + _PDF_WHITESPACE + rb"/])")
|
||||
_PDF_NAME_ESCAPE: Final = re.compile(rb"#([0-9A-Fa-f]{2})")
|
||||
|
|
|
|||
|
|
@ -160,10 +160,10 @@ def _resolve_model_group_alias_via_proxy_router(model: str) -> str | None:
|
|||
return resolve_model_group_alias(llm_router.model_group_alias, model)
|
||||
|
||||
|
||||
def _resolve_counted_models_via_proxy_router(model: str) -> Sequence[str]:
|
||||
def _resolve_counted_models_via_proxy_router(model: str, team_id: str | None) -> Sequence[str]:
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
return models_counted_for(model, llm_router)
|
||||
return models_counted_for(model, llm_router, team_id)
|
||||
|
||||
|
||||
def _resolve_ptu_team_ceiling_via_proxy_router(team_id: str, model_group: str) -> PTUTeamCeiling | None:
|
||||
|
|
@ -885,7 +885,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
time_provider: Callable[[], datetime] | None = None,
|
||||
tag_rate_limit_resolver: TagRateLimitResolver = resolve_tag_rate_limits_from_db,
|
||||
model_group_resolver: Callable[[str], str | None] = _resolve_model_group_alias_via_proxy_router,
|
||||
counted_models_resolver: Callable[[str], Sequence[str]] = _resolve_counted_models_via_proxy_router,
|
||||
counted_models_resolver: Callable[[str, str | None], Sequence[str]] = _resolve_counted_models_via_proxy_router,
|
||||
media_model_resolver: Callable[[str, str | None], tuple[str, ...]] = _resolve_media_models_via_proxy_router,
|
||||
ptu_team_ceiling_resolver: Callable[
|
||||
[str, str], PTUTeamCeiling | None
|
||||
|
|
@ -4082,14 +4082,18 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
call_type=call_type,
|
||||
include_media=False,
|
||||
)
|
||||
counted_models: Final = self._counted_models_resolver(requested_model) if requested_model else (None,)
|
||||
counted_models: Final = (
|
||||
self._counted_models_resolver(requested_model, user_api_key_dict.team_id) if requested_model else (None,)
|
||||
)
|
||||
raw_estimated_input_tokens: Final = max(
|
||||
[
|
||||
await offload_token_count(self._estimate_precise_input_tokens)(
|
||||
data=data, model=model, call_type=call_type
|
||||
await asyncio.gather(
|
||||
*(
|
||||
offload_token_count(self._estimate_precise_input_tokens)(
|
||||
data=data, model=model, call_type=call_type
|
||||
)
|
||||
for model in counted_models
|
||||
)
|
||||
for model in counted_models
|
||||
]
|
||||
)
|
||||
)
|
||||
estimated_input_tokens: Final = max(raw_estimated_input_tokens, 1)
|
||||
estimated_output_tokens: Final = (
|
||||
|
|
|
|||
|
|
@ -275,6 +275,7 @@ async def reserve_budget_for_request(
|
|||
request_body=request_body,
|
||||
route=route,
|
||||
llm_router=llm_router,
|
||||
team_id=valid_token.team_id,
|
||||
raw_body=raw_body,
|
||||
)
|
||||
|
||||
|
|
@ -1596,6 +1597,7 @@ async def count_request_input_tokens(
|
|||
request_body: dict,
|
||||
route: str,
|
||||
llm_router: Router | None,
|
||||
team_id: str | None,
|
||||
raw_body: bytes | None = None,
|
||||
) -> Mapping[str, int]:
|
||||
"""Input-token count per candidate model, counted once per request.
|
||||
|
|
@ -1604,7 +1606,9 @@ async def count_request_input_tokens(
|
|||
models: Final = _get_request_models(request_body=request_body, route=route, llm_router=llm_router)
|
||||
if not models:
|
||||
return MappingProxyType({})
|
||||
counted_models: Final = MappingProxyType({model: models_counted_for(model, llm_router) for model in models})
|
||||
counted_models: Final = MappingProxyType(
|
||||
{model: models_counted_for(model, llm_router, team_id) for model in models}
|
||||
)
|
||||
counts: Final = await count_input_tokens(
|
||||
request_body=request_body,
|
||||
raw_body=raw_body,
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from typing import TYPE_CHECKING, Final
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.litellm_core_utils.token_counter import pdf_page_pricing, reads_pdf_pages
|
||||
from litellm.responses.litellm_completion_transformation.compaction_items import (
|
||||
billable_input_since_newest_compaction,
|
||||
holds_compaction_summary,
|
||||
|
|
@ -133,15 +134,20 @@ def _count_input_tokens_for_models(
|
|||
)
|
||||
|
||||
|
||||
def models_counted_for(model: str, llm_router: Router | None) -> tuple[str, ...]:
|
||||
"""The names a request for `model` is counted with: each deployment's model minus its provider prefix, the
|
||||
name `/v1/messages/count_tokens` counts with, or `model` itself when no deployment serves it."""
|
||||
deployments: Final = (llm_router.get_model_list(model_name=model) or ()) if llm_router is not None else ()
|
||||
deployment_models: Final = (deployment["litellm_params"].get("model") for deployment in deployments)
|
||||
served: Final = tuple(
|
||||
dict.fromkeys(name.partition("/")[2] or name for name in deployment_models if isinstance(name, str) and name)
|
||||
def models_counted_for(model: str, llm_router: Router | None, team_id: str | None) -> tuple[str, ...]:
|
||||
"""The names a request for `model` from `team_id` is counted with: `model` itself, except for a deployment that
|
||||
reads PDF pages and prices them unlike `model`, which is counted with its own model minus the provider prefix,
|
||||
the name `/v1/messages/count_tokens` counts with."""
|
||||
served: Final = (
|
||||
llm_router.resolved_litellm_models(model, team_id, use_base_model=True) if llm_router is not None else ()
|
||||
)
|
||||
return served or (model,)
|
||||
return tuple(dict.fromkeys(_counted_name(model, served_model) for served_model in served)) or (model,)
|
||||
|
||||
|
||||
def _counted_name(model: str, served_model: str) -> str:
|
||||
if pdf_page_pricing(served_model) == pdf_page_pricing(model) or not reads_pdf_pages(served_model):
|
||||
return model
|
||||
return served_model.partition("/")[2] or served_model
|
||||
|
||||
|
||||
def count_input_tokens_for_model(request_body: dict, model: str) -> int | None:
|
||||
|
|
|
|||
126
tests/benchmarks/test_token_counter_pdf_benchmarks.py
Normal file
126
tests/benchmarks/test_token_counter_pdf_benchmarks.py
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
"""PDF pages and model names that once took seconds to minutes to count, kept here so a slowdown shows on CodSpeed.
|
||||
|
||||
Each input is smaller than the one that exposed the slowdown, since the old cost grew with the same input size."""
|
||||
|
||||
import base64
|
||||
from itertools import accumulate, chain
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.litellm_core_utils.token_counter import token_counter
|
||||
|
||||
TEXT_LINE: Final = "Revenue grew eleven percent while churn fell to two percent."
|
||||
|
||||
|
||||
def _pdf_base64(text: str, page_resources: str, drawing: bytes, xobjects: tuple[bytes, ...] = ()) -> str:
|
||||
escaped: Final = text.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)")
|
||||
stream: Final = f"BT /F1 12 Tf 72 720 Td ({escaped}) Tj ET".encode("latin-1") + drawing
|
||||
bodies: Final = (
|
||||
b"<< /Type /Catalog /Pages 2 0 R >>",
|
||||
b"<< /Type /Pages /Count 1 /Kids [ 5 0 R ] >>",
|
||||
b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
|
||||
f"<< /Length {len(stream)} >>\nstream\n".encode() + stream + b"\nendstream",
|
||||
f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources {page_resources} /Contents 4 0 R >>".encode(),
|
||||
*xobjects,
|
||||
)
|
||||
header: Final = b"%PDF-1.4\n"
|
||||
objects: Final = tuple(
|
||||
f"{number} 0 obj\n".encode() + body + b"\nendobj\n" for number, body in enumerate(bodies, start=1)
|
||||
)
|
||||
offsets: Final = accumulate((len(header), *(len(obj) for obj in objects[:-1])))
|
||||
xref: Final = f"xref\n0 {len(objects) + 1}\n0000000000 65535 f \n".encode() + b"".join(
|
||||
f"{offset:010d} 00000 n \n".encode() for offset in offsets
|
||||
)
|
||||
trailer: Final = (
|
||||
f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\n"
|
||||
f"startxref\n{len(header) + sum(len(obj) for obj in objects)}\n%%EOF\n"
|
||||
).encode()
|
||||
return base64.b64encode(header + b"".join(objects) + xref + trailer).decode()
|
||||
|
||||
|
||||
def _form(resources_number: int, drawing: bytes) -> bytes:
|
||||
return (
|
||||
f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Resources {resources_number} 0 R "
|
||||
f"/Length {len(drawing)} >>\nstream\n".encode()
|
||||
+ drawing
|
||||
+ b"\nendstream"
|
||||
)
|
||||
|
||||
|
||||
def _count_pdf(pdf_base64: str) -> int:
|
||||
document: Final = {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": pdf_base64}}
|
||||
return token_counter(model="gpt-4o", messages=[{"role": "user", "content": [document]}])
|
||||
|
||||
|
||||
def _page_of_drawing_operators() -> str:
|
||||
return _pdf_base64("", "<< >>", b" q Q" * 50_000)
|
||||
|
||||
|
||||
def _page_of_forms_sharing_resources() -> str:
|
||||
form_count: Final = 300
|
||||
shared_resources: Final = (
|
||||
"<< /XObject << " + " ".join(f"/F{index} {8 + index} 0 R" for index in range(form_count)) + " >> >>"
|
||||
).encode()
|
||||
forms: Final = tuple(_form(7, b"/F0 Do") for _ in range(form_count))
|
||||
return _pdf_base64(
|
||||
TEXT_LINE,
|
||||
"<< /Font << /F1 3 0 R >> /XObject << /X0 6 0 R >> >>",
|
||||
b" q 10 0 0 10 0 0 cm /X0 Do Q",
|
||||
(forms[0], shared_resources, *forms),
|
||||
)
|
||||
|
||||
|
||||
def _page_of_parents_sharing_a_child_form() -> str:
|
||||
parent_count: Final = 100
|
||||
first: Final = 6
|
||||
child_number: Final = first + 2 * parent_count
|
||||
child_drawing: Final = b" ".join((b"/D Do",) * 4000)
|
||||
child: Final = (
|
||||
f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length {len(child_drawing)} >>\nstream\n".encode()
|
||||
+ child_drawing
|
||||
+ b"\nendstream"
|
||||
)
|
||||
parents: Final = tuple(
|
||||
chain.from_iterable(
|
||||
(_form(first + 2 * index + 1, b"/C Do"), f"<< /XObject << /C {child_number} 0 R >> >>".encode())
|
||||
for index in range(parent_count)
|
||||
)
|
||||
)
|
||||
page_resources: Final = (
|
||||
"<< /Font << /F1 3 0 R >> /XObject << "
|
||||
+ " ".join(f"/P{index} {first + 2 * index} 0 R" for index in range(parent_count))
|
||||
+ " >> >>"
|
||||
)
|
||||
page_drawing: Final = b" " + b" ".join(f"/P{index} Do".encode() for index in range(parent_count))
|
||||
return _pdf_base64(TEXT_LINE, page_resources, page_drawing, (*parents, child))
|
||||
|
||||
|
||||
PAGE_OF_DRAWING_OPERATORS: Final = _page_of_drawing_operators()
|
||||
PAGE_OF_FORMS_SHARING_RESOURCES: Final = _page_of_forms_sharing_resources()
|
||||
PAGE_OF_PARENTS_SHARING_A_CHILD_FORM: Final = _page_of_parents_sharing_a_child_form()
|
||||
BLANK_PAGE_COUNT: Final = _count_pdf(_pdf_base64("", "<< >>", b""))
|
||||
TEXT_PAGE_COUNT: Final = _count_pdf(_pdf_base64(TEXT_LINE, "<< /Font << /F1 3 0 R >> >>", b""))
|
||||
LONG_UNKNOWN_MODEL_NAME: Final = "unknown-" + "a-" * 5_000
|
||||
NAME_SAMPLE: Final = "Ceci est un texte en français: éèàç. " * 8
|
||||
UNKNOWN_MODEL_NAME_COUNT: Final = token_counter(model="my-private-model", text=NAME_SAMPLE)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_pdf_page_of_drawing_operators_and_no_image() -> None:
|
||||
assert _count_pdf(PAGE_OF_DRAWING_OPERATORS) == BLANK_PAGE_COUNT
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_pdf_page_of_forms_sharing_one_resources_dictionary() -> None:
|
||||
assert _count_pdf(PAGE_OF_FORMS_SHARING_RESOURCES) == TEXT_PAGE_COUNT
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_pdf_page_of_parent_forms_sharing_one_child_form() -> None:
|
||||
assert _count_pdf(PAGE_OF_PARENTS_SHARING_A_CHILD_FORM) > TEXT_PAGE_COUNT
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_token_counter_long_unknown_model_name() -> None:
|
||||
assert token_counter(model=LONG_UNKNOWN_MODEL_NAME, text=NAME_SAMPLE) == UNKNOWN_MODEL_NAME_COUNT
|
||||
|
|
@ -29,7 +29,11 @@ import litellm
|
|||
from litellm import decode, encode, get_modified_max_tokens
|
||||
from litellm import token_counter as token_counter_old
|
||||
import litellm.constants
|
||||
from litellm.constants import HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS, TOKEN_COUNTER_MAX_CONCURRENT_COUNTS
|
||||
from litellm.constants import (
|
||||
HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS,
|
||||
PDF_MAX_XOBJECT_DRAWS_PER_PAGE,
|
||||
TOKEN_COUNTER_MAX_CONCURRENT_COUNTS,
|
||||
)
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.litellm_core_utils.token_counter import (
|
||||
_encoding_count,
|
||||
|
|
@ -2283,13 +2287,19 @@ def test_openai_model_finds_an_image_a_shared_form_without_resources_draws_from_
|
|||
) == _rendered_page_cost()
|
||||
|
||||
|
||||
@pytest.mark.timeout(10)
|
||||
def test_openai_model_prices_a_page_of_a_million_drawing_operators_promptly():
|
||||
"""A page of a million drawing operators and no image took about 30 seconds to price, since every operator was
|
||||
parsed to find the images it draws, while the stream compresses into a request a few kilobytes long."""
|
||||
@pytest.mark.parametrize(
|
||||
("drawing", "draws_image"),
|
||||
[(b" /X0 %scale it first\n Do", True), (b" /X0 % Do\n", False)],
|
||||
ids=["comment-before-do", "do-inside-a-comment"],
|
||||
)
|
||||
def test_openai_model_reads_a_comment_in_a_content_stream_as_whitespace(drawing: bytes, draws_image: bool):
|
||||
"""A comment runs to the end of its line and separates tokens like whitespace, so it neither hides the Do after
|
||||
it nor draws anything itself."""
|
||||
model: Final = "gpt-4o"
|
||||
|
||||
assert _pdf_cost(model, _pdf_base64(("",), drawing=b" q Q" * 1_000_000, page_resources="<< >>")) == 0
|
||||
assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=(_ONE_PIXEL_IMAGE,), drawing=drawing)) == (
|
||||
_text_cost(model, _REVENUE_LINE) + (_rendered_page_cost() if draws_image else 13)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("escaped_name", ["/Im#231", "/#C4#E3", "/caf#C3#A9"], ids=["hex_escape", "gbk", "utf8"])
|
||||
|
|
@ -2309,32 +2319,16 @@ def test_openai_model_finds_an_image_drawn_by_an_escaped_name(escaped_name: str)
|
|||
) == _rendered_page_cost()
|
||||
|
||||
|
||||
@pytest.mark.timeout(10)
|
||||
def test_openai_model_reads_resources_shared_by_many_forms_once():
|
||||
"""3,000 forms that all list the same /Resources took 17 seconds to price before each dictionary was read once."""
|
||||
@pytest.mark.parametrize(("parent_count", "draws_too_many"), [(1, False), (2, True)])
|
||||
def test_openai_model_prices_a_page_whose_forms_draw_too_many_xobjects_as_rendered(
|
||||
parent_count: int, draws_too_many: bool
|
||||
):
|
||||
"""A page prices as if it drew an image once its forms draw more than PDF_MAX_XOBJECT_DRAWS_PER_PAGE XObjects
|
||||
between them, so a page of forms nested to blow up the walk cannot stall the request."""
|
||||
model: Final = "gpt-4o"
|
||||
form_count: Final = 3000
|
||||
shared_resources_number: Final = 7
|
||||
shared_resources: Final = (
|
||||
"<< /XObject << " + " ".join(f"/F{index} {8 + index} 0 R" for index in range(form_count)) + " >> >>"
|
||||
).encode()
|
||||
forms: Final = tuple(_form_xobject(shared_resources_number, b"/F0 Do") for _ in range(form_count))
|
||||
|
||||
assert _pdf_cost(
|
||||
model, _pdf_base64((_REVENUE_LINE,), xobjects=(forms[0], shared_resources, *forms), drawing=_DRAW_X0)
|
||||
) == (_text_cost(model, _REVENUE_LINE) + 13)
|
||||
|
||||
|
||||
@pytest.mark.timeout(10)
|
||||
def test_openai_model_prices_a_child_form_shared_by_many_parents_promptly():
|
||||
"""1,000 forms that each list their own /Resources and draw one shared 4,000-operator child took about 45
|
||||
seconds to price, since the child was parsed and walked again under every parent. A page now prices as if it
|
||||
drew an image once its forms draw more than a fixed number of XObjects, and each form is parsed once."""
|
||||
model: Final = "gpt-4o"
|
||||
parent_count: Final = 1000
|
||||
first: Final = 6
|
||||
child_number: Final = first + 2 * parent_count
|
||||
child_drawing: Final = b" ".join((b"/D Do",) * 4000)
|
||||
child_drawing: Final = b" ".join((b"/D Do",) * (PDF_MAX_XOBJECT_DRAWS_PER_PAGE // 2))
|
||||
child: Final = (
|
||||
f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length {len(child_drawing)} >>\nstream\n".encode()
|
||||
+ child_drawing
|
||||
|
|
@ -2358,7 +2352,7 @@ def test_openai_model_prices_a_child_form_shared_by_many_parents_promptly():
|
|||
_pdf_base64(
|
||||
(_REVENUE_LINE,), xobjects=(*parents, child), drawing=page_drawing, page_resources=page_resources
|
||||
),
|
||||
) == _text_cost(model, _REVENUE_LINE) + _rendered_page_cost()
|
||||
) == _text_cost(model, _REVENUE_LINE) + (_rendered_page_cost() if draws_too_many else 13)
|
||||
|
||||
|
||||
_NON_LATIN_SAMPLE: Final = "Привет, как дела? これは日本語のテキストです。 Ceci est un texte en français: éèàç. " * 8
|
||||
|
|
@ -2461,16 +2455,6 @@ def test_openai_fine_tunes_count_like_their_base_model(fine_tune: str, base_mode
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.timeout(10)
|
||||
def test_a_very_long_unknown_model_name_counts_promptly():
|
||||
"""A 200,000-word model name took minutes to count, since every dash-joined prefix of it was tried."""
|
||||
long_name: Final = "unknown-" + "a-" * 200_000
|
||||
|
||||
assert token_counter_new(model=long_name, text=_NON_LATIN_SAMPLE) == token_counter_new(
|
||||
model="my-private-model", text=_NON_LATIN_SAMPLE
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
|
|
|
|||
|
|
@ -9106,7 +9106,7 @@ async def test_project_itpm_reserves_a_pdf_at_the_largest_count_of_the_group_dep
|
|||
cache: Final = DualCache()
|
||||
handler: Final = _PROXY_MaxParallelRequestsHandler(
|
||||
internal_usage_cache=InternalUsageCache(cache),
|
||||
counted_models_resolver=lambda model: ("claude-haiku-5-5", "gpt-5.6-sol"),
|
||||
counted_models_resolver=lambda model, team_id: ("claude-haiku-5-5", "gpt-5.6-sol"),
|
||||
)
|
||||
messages: Final = _one_page_pdf_message()
|
||||
caller: Final = UserAPIKeyAuth(
|
||||
|
|
|
|||
|
|
@ -440,7 +440,7 @@ async def test_reservation_counts_a_pdf_with_the_deployment_behind_a_gpt_named_g
|
|||
)
|
||||
|
||||
counts: Final = await count_request_input_tokens(
|
||||
request_body=request_body, route="/chat/completions", llm_router=router
|
||||
request_body=request_body, route="/chat/completions", llm_router=router, team_id=None
|
||||
)
|
||||
|
||||
assert counts == {"gpt-4o": count_input_tokens_for_model(request_body=request_body, model="claude-haiku-5-5")}
|
||||
|
|
|
|||
|
|
@ -82,18 +82,66 @@ def test_a_model_group_counts_like_the_deployment_that_serves_it(group: str, dep
|
|||
request_body: Final = {"model": group, "messages": _one_page_pdf_message()}
|
||||
counted_with: Final = deployment_model.partition("/")[2]
|
||||
|
||||
assert models_counted_for(group, _router((group, deployment_model))) == (counted_with,)
|
||||
assert models_counted_for(group, _router((group, deployment_model)), None) == (counted_with,)
|
||||
assert count_input_tokens_for_model(request_body=request_body, model=counted_with) != count_input_tokens_for_model(
|
||||
request_body=request_body, model=group
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("group", "deployment_model"),
|
||||
[
|
||||
("my-claude", "anthropic/claude-haiku-5-5"),
|
||||
("my-llama", "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo"),
|
||||
("my-cohere", "cohere/command-r-plus"),
|
||||
("gpt-4o", "cohere/command-r-plus"),
|
||||
],
|
||||
ids=["claude-alias", "llama-alias", "cohere-alias", "gpt-named-group-served-by-a-model-that-reads-no-pdf"],
|
||||
)
|
||||
def test_a_group_whose_deployment_prices_pdf_pages_alike_or_reads_none_counts_as_the_group(
|
||||
group: str, deployment_model: str
|
||||
) -> None:
|
||||
assert models_counted_for(group, _router((group, deployment_model)), None) == (group,)
|
||||
|
||||
|
||||
def test_a_custom_named_deployment_counts_with_its_base_model():
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "reports",
|
||||
"litellm_params": {"model": "azure/prod-deploy-1", "api_key": "x", "api_base": "https://x"},
|
||||
"model_info": {"base_model": "azure/gpt-5.6-sol"},
|
||||
},
|
||||
{"model_name": "drafts", "litellm_params": {"model": "azure/prod-deploy-2", "api_key": "x"}},
|
||||
]
|
||||
)
|
||||
|
||||
assert models_counted_for("reports", router, None) == ("gpt-5.6-sol",)
|
||||
assert models_counted_for("drafts", router, None) == ("drafts",)
|
||||
|
||||
|
||||
def test_a_group_of_differently_priced_deployments_counts_with_each_and_a_name_no_deployment_serves_counts_as_is():
|
||||
router: Final = _router(("mixed", "anthropic/claude-haiku-5-5"), ("mixed", "azure/gpt-5.6-sol"))
|
||||
|
||||
assert models_counted_for("mixed", router) == ("claude-haiku-5-5", "gpt-5.6-sol")
|
||||
assert models_counted_for("not-deployed", router) == ("not-deployed",)
|
||||
assert models_counted_for("gpt-4o", None) == ("gpt-4o",)
|
||||
assert models_counted_for("mixed", router, None) == ("mixed", "gpt-5.6-sol")
|
||||
assert models_counted_for("not-deployed", router, None) == ("not-deployed",)
|
||||
assert models_counted_for("gpt-4o", None, None) == ("gpt-4o",)
|
||||
|
||||
|
||||
def test_a_team_request_counts_with_the_deployment_its_team_is_served_by():
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{"model_name": "reports", "litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": "x"}},
|
||||
{
|
||||
"model_name": "reports-team-a",
|
||||
"litellm_params": {"model": "azure/gpt-5.6-sol", "api_key": "x"},
|
||||
"model_info": {"team_id": "team-a", "team_public_model_name": "reports"},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
assert models_counted_for("reports", router, "team-a") == ("gpt-5.6-sol",)
|
||||
assert models_counted_for("reports", router, "team-b") == ("reports",)
|
||||
|
||||
|
||||
def test_count_input_tokens_for_model_counts_anthropic_auto_tool_choice() -> None:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue