mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): count a team's wildcard route the way routing serves it and reserve OpenAI chat's per-PDF charge
This commit is contained in:
parent
1ea5aafe7a
commit
20dcfde079
7 changed files with 158 additions and 9 deletions
|
|
@ -753,6 +753,7 @@ ANTHROPIC_IMAGE_MAX_LONG_EDGE_PX: Final = 1568
|
|||
ANTHROPIC_IMAGE_MAX_PIXELS: Final = 1_150_000
|
||||
ANTHROPIC_IMAGE_PIXELS_PER_TOKEN: Final = 750
|
||||
OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS: Final = 13
|
||||
OPENAI_CHAT_PDF_FILE_TOKENS: Final = 192
|
||||
PDF_DATA_URL_PREFIX: Final = "data:application/pdf;base64,"
|
||||
PDF_MAX_FORM_XOBJECT_DEPTH: Final = 32
|
||||
PDF_MAX_XOBJECT_DRAWS_PER_PAGE: Final = 10_000
|
||||
|
|
|
|||
|
|
@ -76,7 +76,11 @@ from litellm.proxy.hooks.batch_enqueued_tokens import (
|
|||
canonical_provider_batch_id,
|
||||
)
|
||||
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
|
||||
from litellm.proxy.spend_tracking.input_tokens import models_counted_for
|
||||
from litellm.proxy.spend_tracking.input_tokens import (
|
||||
inline_pdf_file_count,
|
||||
models_counted_for,
|
||||
openai_chat_pdf_file_tokens,
|
||||
)
|
||||
from litellm.responses.litellm_completion_transformation.compaction_items import billable_input_since_newest_compaction
|
||||
from litellm.router_utils.add_retry_fallback_headers import (
|
||||
ensure_response_additional_headers,
|
||||
|
|
@ -514,6 +518,7 @@ _TPM_FLOOR_FRACTION: Final = 4
|
|||
# check on data["input"] must exclude these call types, or a Responses call
|
||||
# gets misclassified as an embedding and skips output-token reservation/caps.
|
||||
RESPONSES_API_CALL_TYPES: Final = ("aresponses", "responses")
|
||||
CHAT_COMPLETION_API_CALL_TYPES: Final = (CallTypes.completion.value, CallTypes.acompletion.value)
|
||||
EMBEDDING_API_CALL_TYPES: Final = ("aembedding", "embedding")
|
||||
TEXT_COMPLETION_API_CALL_TYPES: Final = ("atext_completion", "text_completion")
|
||||
RERANK_API_CALL_TYPES: Final = (CallTypes.rerank.value, CallTypes.arerank.value)
|
||||
|
|
@ -4016,6 +4021,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
|
||||
audio_token_estimate: Final = self._estimate_audio_content_tokens(messages)
|
||||
countable_messages: Final = self._strip_audio_content_blocks(messages) if audio_token_estimate > 0 else messages
|
||||
chat_pdf_files: Final = (
|
||||
inline_pdf_file_count(countable_messages) if call_type in CHAT_COMPLETION_API_CALL_TYPES else 0
|
||||
)
|
||||
|
||||
try:
|
||||
estimate: Final = max(
|
||||
|
|
@ -4031,6 +4039,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
use_default_image_token_count=True,
|
||||
)
|
||||
)
|
||||
+ openai_chat_pdf_file_tokens(tokenizer_model, chat_pdf_files)
|
||||
for tokenizer_model in (self._counted_models_resolver(model, team_id) if model else ("",))
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from pydantic import ConfigDict, TypeAdapter
|
|||
import litellm
|
||||
from litellm._internal_context import with_service_target
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route
|
||||
from litellm.litellm_core_utils.duration_parser import duration_in_seconds
|
||||
from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import select_tier_for_input, tier_rate
|
||||
from litellm.proxy._types import (
|
||||
|
|
@ -43,13 +44,16 @@ from litellm.proxy.common_utils.user_api_key_cache import (
|
|||
from litellm.proxy.spend_tracking.input_tokens import (
|
||||
count_input_tokens,
|
||||
count_input_tokens_for_model,
|
||||
inline_pdf_file_count,
|
||||
models_counted_for,
|
||||
openai_chat_pdf_file_tokens,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging
|
||||
from litellm.router import Router
|
||||
from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget
|
||||
from litellm.types.router import DeploymentTypedDict
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -1614,17 +1618,25 @@ async def count_request_input_tokens(
|
|||
raw_body=raw_body,
|
||||
models=tuple(dict.fromkeys(chain.from_iterable(counted_models.values()))),
|
||||
)
|
||||
chat_pdf_files: Final = (
|
||||
inline_pdf_file_count(request_body.get("messages"))
|
||||
if CallTypes.acompletion in (get_call_types_for_route(route) or ())
|
||||
else 0
|
||||
)
|
||||
return MappingProxyType(
|
||||
{
|
||||
model: largest
|
||||
for model, names in counted_models.items()
|
||||
if (largest := _largest_count(counts, names)) is not None
|
||||
if (largest := _largest_count(counts, names, chat_pdf_files)) is not None
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _largest_count(counts: Mapping[str, int], names: Sequence[str]) -> int | None:
|
||||
return max((counts[name] for name in names if name in counts), default=None)
|
||||
def _largest_count(counts: Mapping[str, int], names: Sequence[str], chat_pdf_files: int) -> int | None:
|
||||
return max(
|
||||
(counts[name] + openai_chat_pdf_file_tokens(name, chat_pdf_files) for name in names if name in counts),
|
||||
default=None,
|
||||
)
|
||||
|
||||
|
||||
def _estimate_input_tokens(
|
||||
|
|
|
|||
|
|
@ -13,12 +13,14 @@ from __future__ import annotations
|
|||
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import OPENAI_CHAT_PDF_FILE_TOKENS, PDF_DATA_URL_PREFIX
|
||||
from litellm.litellm_core_utils.token_counter import (
|
||||
deployment_openai_model,
|
||||
pdf_page_pricing,
|
||||
|
|
@ -29,7 +31,7 @@ from litellm.responses.litellm_completion_transformation.compaction_items import
|
|||
billable_input_since_newest_compaction,
|
||||
holds_compaction_summary,
|
||||
)
|
||||
from litellm.router_utils.deployment_pricing import deployment_pricing_models
|
||||
from litellm.router_utils.deployment_pricing import deployment_pricing_models, str_keyed
|
||||
from litellm.rust_bridge import runtime
|
||||
from litellm.rust_bridge.catalog import Route, RouteContext
|
||||
from litellm.rust_bridge.token_counter import (
|
||||
|
|
@ -45,6 +47,8 @@ if TYPE_CHECKING:
|
|||
|
||||
TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS: Final = 30_000
|
||||
|
||||
_OPENAI_PREFIX: Final = "openai/"
|
||||
|
||||
_INPUT_SIZE_FIELDS: Final = ("messages", "prompt", "input", "query", "documents", "tools", "tool_choice")
|
||||
|
||||
|
||||
|
|
@ -142,19 +146,55 @@ def _count_input_tokens_for_models(
|
|||
|
||||
def models_counted_for(model: str, llm_router: Router | None, team_id: str | None) -> tuple[str, ...]:
|
||||
"""The names to count a request for `model` from `team_id` with, keeping the largest count, one per deployment
|
||||
serving it.
|
||||
routing would serve it from: the ones named `model`, else those of the one wildcard route that serves it.
|
||||
|
||||
An OpenAI deployment counts as its OpenAI model, so it gets that model's tokenizer and PDF page pricing. A
|
||||
An OpenAI deployment counts as its OpenAI model, so it gets that model's tokenizer and PDF page pricing, prefixed
|
||||
`openai/` when OpenAI itself serves it, so `openai_chat_pdf_file_tokens` can add what its chat completions bills. A
|
||||
deployment that reads PDF pages and prices them unlike `model` counts as its model minus the provider prefix, the
|
||||
name `/v1/messages/count_tokens` counts with. Any other deployment counts as the requested name with the encoding
|
||||
tiktoken gives that name as written, without loading its vendor's tokenizer. A name no deployment serves counts as
|
||||
written."""
|
||||
deployments: Final = (llm_router.get_model_list(model_name=model, team_id=team_id) or ()) if llm_router else ()
|
||||
deployments: Final = _routed_deployments(model, llm_router, team_id) if llm_router else ()
|
||||
return tuple(dict.fromkeys(_counted_name(model, deployment) for deployment in deployments)) or (model,)
|
||||
|
||||
|
||||
def _routed_deployments(model: str, llm_router: Router, team_id: str | None) -> Sequence[object]:
|
||||
named: Final = (
|
||||
*llm_router.get_all_deployments(model_name=model, team_id=team_id),
|
||||
*llm_router.get_model_list_from_model_alias(model_name=model),
|
||||
*llm_router.get_model_list_from_routing_groups(model_name=model),
|
||||
)
|
||||
return named or llm_router.wildcard_route_deployments(model, team_id)
|
||||
|
||||
|
||||
def openai_chat_pdf_file_tokens(counted_model: str, pdf_files: int) -> int:
|
||||
"""What OpenAI chat completions bills on top of the counted pages of `pdf_files` inline PDF files, when
|
||||
`counted_model`, a name from `models_counted_for`, is served by OpenAI itself."""
|
||||
return OPENAI_CHAT_PDF_FILE_TOKENS * pdf_files if counted_model.startswith(_OPENAI_PREFIX) else 0
|
||||
|
||||
|
||||
def inline_pdf_file_count(messages: object) -> int:
|
||||
return sum(1 for part in _content_parts(messages) if _is_inline_pdf_file(part))
|
||||
|
||||
|
||||
def _content_parts(messages: object) -> Iterator[Mapping[str, object]]:
|
||||
contents: Final = (_as_list(str_keyed(message).get("content")) for message in _as_list(messages))
|
||||
return (str_keyed(part) for part in chain.from_iterable(contents))
|
||||
|
||||
|
||||
def _as_list(value: object) -> Sequence[object]:
|
||||
return value if isinstance(value, list) else ()
|
||||
|
||||
|
||||
def _is_inline_pdf_file(part: Mapping[str, object]) -> bool:
|
||||
file_data: Final = str_keyed(part.get("file")).get("file_data")
|
||||
return part.get("type") == "file" and isinstance(file_data, str) and file_data.startswith(PDF_DATA_URL_PREFIX)
|
||||
|
||||
|
||||
def _counted_name(model: str, deployment: object) -> str:
|
||||
openai_model: Final = deployment_openai_model(deployment)
|
||||
if openai_model is not None and _served_by_openai(deployment):
|
||||
return _OPENAI_PREFIX + openai_model.removeprefix(_OPENAI_PREFIX)
|
||||
if openai_model is not None:
|
||||
return openai_model
|
||||
served: Final = next(iter(deployment_pricing_models(deployment)), model)
|
||||
|
|
@ -163,6 +203,21 @@ def _counted_name(model: str, deployment: object) -> str:
|
|||
return unresolved_tokenizer_model(model)
|
||||
|
||||
|
||||
def _served_by_openai(deployment: object) -> bool:
|
||||
params: Final = str_keyed(str_keyed(deployment).get("litellm_params"))
|
||||
model: Final = params.get("model")
|
||||
provider: Final = params.get("custom_llm_provider")
|
||||
if not isinstance(model, str):
|
||||
return False
|
||||
try:
|
||||
_, served_provider, _, _ = litellm.get_llm_provider(
|
||||
model=model, custom_llm_provider=provider if isinstance(provider, str) else None
|
||||
)
|
||||
except litellm.exceptions.BadRequestError:
|
||||
return False
|
||||
return served_provider == "openai"
|
||||
|
||||
|
||||
def count_input_tokens_for_model(request_body: dict, model: str) -> int | None:
|
||||
try:
|
||||
if "messages" in request_body:
|
||||
|
|
|
|||
|
|
@ -9134,6 +9134,30 @@ async def test_project_itpm_reserves_a_pdf_at_the_largest_count_of_the_group_dep
|
|||
assert stash.itpm_reserved_tokens == claude_count
|
||||
|
||||
|
||||
def test_project_itpm_adds_what_openai_chat_bills_on_top_of_a_pdf_files_pages_on_chat_completions_only() -> None:
|
||||
from litellm.constants import OPENAI_CHAT_PDF_FILE_TOKENS
|
||||
from tests.unit.proxy.spend_tracking.test_input_tokens import _one_page_pdf_message
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{"model_name": name, "litellm_params": {"model": model, "api_key": "sk-test"}}
|
||||
for name, model in (("on-openai", "openai/gpt-5.6-sol"), ("on-azure", "azure/gpt-5.6-sol"))
|
||||
]
|
||||
)
|
||||
handler: Final = _PROXY_MaxParallelRequestsHandler(
|
||||
internal_usage_cache=InternalUsageCache(DualCache()),
|
||||
counted_models_resolver=lambda model, team_id: models_counted_for(model, router, team_id),
|
||||
)
|
||||
data: Final = {"messages": _one_page_pdf_message()}
|
||||
|
||||
assert handler._estimate_precise_input_tokens(data, "on-openai", "acompletion") == (
|
||||
handler._estimate_precise_input_tokens(data, "on-azure", "acompletion") + OPENAI_CHAT_PDF_FILE_TOKENS
|
||||
)
|
||||
assert handler._estimate_precise_input_tokens(data, "on-openai", "anthropic_messages") == (
|
||||
handler._estimate_precise_input_tokens(data, "on-azure", "anthropic_messages")
|
||||
)
|
||||
|
||||
|
||||
def _extraction_request(state: str, call_id: str) -> dict[str, object]:
|
||||
return {
|
||||
"model": "invoice-reader",
|
||||
|
|
|
|||
|
|
@ -469,6 +469,33 @@ async def _input_tokens(router: Router, model: str) -> dict[str, int]:
|
|||
)
|
||||
|
||||
|
||||
async def _reserved_input_tokens(router: Router, model: str, route: str, messages: list[dict[str, object]]) -> int:
|
||||
counts: Final = await count_request_input_tokens(
|
||||
request_body={"model": model, "messages": messages}, route=route, llm_router=router, team_id=None
|
||||
)
|
||||
return counts[model]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_openai_chat_request_reserves_what_openai_chat_bills_on_top_of_each_pdf_files_pages() -> None:
|
||||
from litellm.constants import OPENAI_CHAT_PDF_FILE_TOKENS
|
||||
from tests.unit.proxy.spend_tracking.test_input_tokens import _one_page_pdf_message
|
||||
|
||||
router: Final = _router_with_groups(("on-openai", "openai/gpt-5.6-sol"), ("on-azure", "azure/gpt-5.6-sol"))
|
||||
two_pdfs: Final = [*_one_page_pdf_message(), *_one_page_pdf_message()]
|
||||
|
||||
assert await _reserved_input_tokens(router, "on-openai", "/v1/chat/completions", two_pdfs) == (
|
||||
await _reserved_input_tokens(router, "on-azure", "/v1/chat/completions", two_pdfs)
|
||||
+ 2 * OPENAI_CHAT_PDF_FILE_TOKENS
|
||||
)
|
||||
assert await _reserved_input_tokens(router, "on-openai", "/v1/messages", two_pdfs) == (
|
||||
await _reserved_input_tokens(router, "on-azure", "/v1/messages", two_pdfs)
|
||||
)
|
||||
assert await _reserved_input_tokens(router, "on-openai", "/v1/chat/completions", _MULTILINGUAL_MESSAGES) == (
|
||||
await _reserved_input_tokens(router, "on-azure", "/v1/chat/completions", _MULTILINGUAL_MESSAGES)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_alias_group_of_an_openai_model_reserves_on_that_models_token_count():
|
||||
router: Final = _router_with_groups(("gpt-6.1-sol", "openai/gpt-6.1-sol"), ("prod-gpt-6", "openai/gpt-6.1-sol"))
|
||||
|
|
|
|||
|
|
@ -80,12 +80,16 @@ def _router(*deployments: tuple[str, str]) -> Router:
|
|||
("pdf-reports", "azure/gpt-5.6-sol", "azure/gpt-5.6-sol"),
|
||||
("pdf-reports", "azure/responses/gpt-5.6-sol", "azure/responses/gpt-5.6-sol"),
|
||||
("pdf-reports", "openrouter/openai/gpt-5.6-sol", "openrouter/openai/gpt-5.6-sol"),
|
||||
("pdf-reports", "openai/gpt-5.6-sol", "openai/gpt-5.6-sol"),
|
||||
("pdf-reports", "gpt-5.6-sol", "openai/gpt-5.6-sol"),
|
||||
],
|
||||
ids=[
|
||||
"gpt-named-group-served-by-claude",
|
||||
"custom-group-served-by-azure-gpt",
|
||||
"custom-group-served-by-azure-responses-gpt",
|
||||
"custom-group-served-by-openrouter-gpt",
|
||||
"custom-group-served-by-openai-gpt",
|
||||
"custom-group-served-by-unprefixed-openai-gpt",
|
||||
],
|
||||
)
|
||||
def test_a_model_group_counts_like_the_deployment_that_serves_it(
|
||||
|
|
@ -155,6 +159,23 @@ def test_a_team_request_counts_with_the_deployment_its_team_is_served_by():
|
|||
assert models_counted_for("reports", router, "team-b") == ("reports",)
|
||||
|
||||
|
||||
def test_a_team_with_its_own_wildcard_counts_with_it_and_never_with_a_proxy_wide_wildcard():
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{"model_name": "*", "litellm_params": {"model": "anthropic/*", "api_key": "x"}},
|
||||
{
|
||||
"model_name": "team-a-gpt",
|
||||
"litellm_params": {"model": "azure/gpt-*", "api_key": "x", "api_base": "https://x"},
|
||||
"model_info": {"team_id": "team-a", "team_public_model_name": "gpt-*"},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
assert models_counted_for("gpt-5.6-sol", router, "team-a") == ("azure/gpt-5.6-sol",)
|
||||
assert models_counted_for("gpt-5.6-sol", router, "team-b") == models_counted_for("gpt-5.6-sol", router, None)
|
||||
assert "azure/gpt-5.6-sol" not in models_counted_for("gpt-5.6-sol", router, "team-b")
|
||||
|
||||
|
||||
def test_count_input_tokens_for_model_counts_anthropic_auto_tool_choice() -> None:
|
||||
model: Final = "claude-sonnet-5-5"
|
||||
body_without_tool_choice: Final = {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue