mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge 7729925ad7 into b7da9a2755
This commit is contained in:
commit
d6f220c8fc
14 changed files with 1086 additions and 32 deletions
|
|
@ -591,7 +591,7 @@ vertex_ai_safety_settings: Optional[dict] = None
|
|||
####### COMPLETION MODELS ###################
|
||||
from typing import Set
|
||||
|
||||
open_ai_chat_completion_models: Set = set()
|
||||
open_ai_chat_completion_models: Set[str] = set()
|
||||
open_ai_text_completion_models: Set = set()
|
||||
cohere_models: Set = set()
|
||||
cohere_chat_models: Set = set()
|
||||
|
|
|
|||
|
|
@ -752,7 +752,12 @@ ANTHROPIC_BATCHES_ROUTE: Final = "/v1/messages/batches"
|
|||
ANTHROPIC_IMAGE_MAX_LONG_EDGE_PX: Final = 1568
|
||||
ANTHROPIC_IMAGE_MAX_PIXELS: Final = 1_150_000
|
||||
ANTHROPIC_IMAGE_PIXELS_PER_TOKEN: Final = 750
|
||||
OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS: Final = 13
|
||||
PDF_DATA_URL_PREFIX: Final = "data:application/pdf;base64,"
|
||||
PDF_MAX_FORM_XOBJECT_DEPTH: Final = 32
|
||||
PDF_MAX_XOBJECT_DRAWS_PER_PAGE: Final = 10_000
|
||||
OPENAI_MODEL_NAME_MAX_DASHES: Final = 16
|
||||
OPENAI_CHAT_MODES: Final = frozenset({"chat", "responses"})
|
||||
VERTEX_BATCH_PREDICTION_JOBS_ROUTE: Final = "batchPredictionJobs"
|
||||
ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES: Final = {
|
||||
"low": 1,
|
||||
|
|
|
|||
|
|
@ -7,9 +7,11 @@ import re
|
|||
import struct
|
||||
from bisect import bisect_right
|
||||
from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
|
||||
from itertools import accumulate
|
||||
from dataclasses import dataclass
|
||||
from functools import cached_property
|
||||
from itertools import accumulate, chain, islice
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal, cast
|
||||
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, cast
|
||||
|
||||
import anyio
|
||||
import anyio.lowlevel
|
||||
|
|
@ -32,7 +34,12 @@ from litellm.constants import (
|
|||
MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES,
|
||||
MAX_TILE_HEIGHT,
|
||||
MAX_TILE_WIDTH,
|
||||
OPENAI_CHAT_MODES,
|
||||
OPENAI_MODEL_NAME_MAX_DASHES,
|
||||
OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS,
|
||||
PDF_DATA_URL_PREFIX,
|
||||
PDF_MAX_FORM_XOBJECT_DEPTH,
|
||||
PDF_MAX_XOBJECT_DRAWS_PER_PAGE,
|
||||
TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS,
|
||||
TOKEN_COUNTER_MAX_CONCURRENT_COUNTS,
|
||||
TOKEN_COUNTER_MAX_EXACT_CHARS,
|
||||
|
|
@ -604,6 +611,35 @@ TokenCounterFunction = Callable[[str], int]
|
|||
Type for a function that counts tokens in a string.
|
||||
"""
|
||||
|
||||
PdfPagePricing: TypeAlias = Literal["openai", "claude"]
|
||||
|
||||
|
||||
class _PdfBox(Protocol):
|
||||
@property
|
||||
def width(self) -> float: ...
|
||||
|
||||
@property
|
||||
def height(self) -> float: ...
|
||||
|
||||
|
||||
class _PdfContent(Protocol):
|
||||
def get_data(self) -> bytes: ...
|
||||
|
||||
|
||||
class _PdfPage(Protocol):
|
||||
@property
|
||||
def mediabox(self) -> _PdfBox: ...
|
||||
|
||||
def extract_text(self) -> str: ...
|
||||
|
||||
def get_contents(self) -> _PdfContent | None: ...
|
||||
|
||||
|
||||
class _PdfReader(Protocol):
|
||||
@property
|
||||
def pages(self) -> Sequence[_PdfPage]: ...
|
||||
|
||||
|
||||
EXTRAPOLATION_SAMPLES: Final = 16
|
||||
T_ParamSpec: Final = ParamSpec("T_ParamSpec")
|
||||
T_Retval = TypeVar("T_Retval")
|
||||
|
|
@ -660,7 +696,7 @@ class _MessageCountParams:
|
|||
if uses_legacy_message_accounting(model):
|
||||
self.tokens_per_message = 4 # every message follows <|start|>{role/name}\n{content}<|end|>\n
|
||||
self.tokens_per_name = -1 # if there's a name, the role is omitted
|
||||
elif actual_model in litellm.open_ai_chat_completion_models or actual_model in litellm.azure_llms:
|
||||
elif _is_openai_chat_model(actual_model):
|
||||
self.tokens_per_message = 3
|
||||
self.tokens_per_name = 1
|
||||
else:
|
||||
|
|
@ -669,6 +705,61 @@ class _MessageCountParams:
|
|||
self.tokens_per_name = 1
|
||||
self.count_function = _get_count_function(model, custom_tokenizer)
|
||||
self.tool_section = chat_template_tool_section(model)
|
||||
self._model = model
|
||||
|
||||
@cached_property
|
||||
def pdf_pricing(self) -> PdfPagePricing:
|
||||
return pdf_page_pricing(self._model)
|
||||
|
||||
|
||||
def _is_openai_chat_model(model: str) -> bool:
|
||||
return model in litellm.open_ai_chat_completion_models or model in litellm.azure_llms
|
||||
|
||||
|
||||
def _openai_deployment_name(model: str) -> str:
|
||||
name: Final = model.rpartition("/")[2]
|
||||
return name.split(":")[1] if name.startswith("ft:") else name
|
||||
|
||||
|
||||
def _openai_model_family(model: str) -> str | None:
|
||||
words: Final = _openai_deployment_name(model).split("-", OPENAI_MODEL_NAME_MAX_DASHES)
|
||||
prefixes: Final = ("-".join(words[:end]) for end in range(len(words), 0, -1))
|
||||
listed: Final = next((prefix for prefix in prefixes if _is_openai_chat_model(prefix)), None)
|
||||
return None if listed is None or _is_listed_non_chat_model(listed) else listed
|
||||
|
||||
|
||||
def _untyped_cost_map() -> object:
|
||||
return litellm.model_cost # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # loaded untyped
|
||||
|
||||
|
||||
def _cost_map_entry(model: str) -> Mapping[str, object]:
|
||||
cost_map: Final = _pdf_dictionary(_untyped_cost_map()) or {}
|
||||
return _pdf_dictionary(cost_map.get(model)) or {}
|
||||
|
||||
|
||||
def _is_azure_openai_chat_model(model: str) -> bool:
|
||||
name: Final = _openai_deployment_name(model)
|
||||
entry: Final = _cost_map_entry(f"azure/{name}")
|
||||
return (
|
||||
entry.get("litellm_provider") == "azure"
|
||||
and entry.get("mode") in OPENAI_CHAT_MODES
|
||||
and (entry.get("supports_pdf_input") is True or name.startswith("gpt-"))
|
||||
)
|
||||
|
||||
|
||||
def _is_openai_or_azure_model(model: str) -> bool:
|
||||
if _is_listed_non_chat_model(_openai_deployment_name(model)):
|
||||
return False
|
||||
return _is_openai_chat_model(model) or _openai_model_family(model) is not None or _is_azure_openai_chat_model(model)
|
||||
|
||||
|
||||
def pdf_page_pricing(model: str | None) -> PdfPagePricing:
|
||||
return "openai" if _is_openai_or_azure_model(model or "") else "claude"
|
||||
|
||||
|
||||
def reads_pdf_pages(model: str) -> bool:
|
||||
entry: Final = _cost_map_entry(model) or _cost_map_entry(model.partition("/")[2])
|
||||
return entry.get("supports_pdf_input") is True
|
||||
|
||||
|
||||
def token_counter(
|
||||
|
|
@ -840,6 +931,7 @@ def _count_messages(
|
|||
value,
|
||||
use_default_image_token_count,
|
||||
default_token_count,
|
||||
params.pdf_pricing,
|
||||
)
|
||||
elif key == "search_results" and isinstance(value, list):
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
|
|
@ -984,7 +1076,7 @@ def _get_exact_count_function(
|
|||
|
||||
return count_tokens
|
||||
if tokenizer_json["type"] == "openai_tokenizer":
|
||||
encoding: Final = openai_tokenizer_encoding(model)
|
||||
encoding: Final = openai_tokenizer_encoding(model or "")
|
||||
|
||||
def encode_length(text: str) -> int:
|
||||
return _encoding_count(encoding, text)
|
||||
|
|
@ -1004,18 +1096,30 @@ def openai_tokenizer_encoding(model: str) -> "Encoding":
|
|||
return get_encoding(openai_tokenizer_encoding_name(model))
|
||||
|
||||
|
||||
def _tiktoken_model_name(model: str) -> str:
|
||||
listed_name: Final = _fix_model_name(model)
|
||||
if listed_name != "gpt-3.5-turbo":
|
||||
return listed_name
|
||||
family: Final = _openai_model_family(model)
|
||||
if family is not None:
|
||||
return _fix_model_name(family)
|
||||
return _openai_deployment_name(model) if _is_azure_openai_chat_model(model) else listed_name
|
||||
|
||||
|
||||
def _is_listed_non_chat_model(model: str) -> bool:
|
||||
modes: Final = (_cost_map_entry(name).get("mode") for name in (model, f"azure/{model}"))
|
||||
return any(mode is not None and mode not in OPENAI_CHAT_MODES for mode in modes)
|
||||
|
||||
|
||||
def openai_tokenizer_encoding_name(model: str) -> str:
|
||||
"""The tiktoken encoding name for `model`, without loading the encoding."""
|
||||
from litellm.utils import print_verbose
|
||||
|
||||
model_to_use: Final = _fix_model_name(model)
|
||||
model_to_use: Final = _tiktoken_model_name(model)
|
||||
if "gpt-4o" in model_to_use:
|
||||
return "o200k_base"
|
||||
try:
|
||||
return tiktoken.encoding_name_for_model(model_to_use)
|
||||
except KeyError:
|
||||
print_verbose("Warning: model not found. Using cl100k_base encoding.")
|
||||
return "cl100k_base"
|
||||
return "cl100k_base" if _is_listed_non_chat_model(model_to_use) else "o200k_base"
|
||||
|
||||
|
||||
def uses_legacy_message_accounting(model: str) -> bool:
|
||||
|
|
@ -1124,7 +1228,165 @@ def _anthropic_rendered_page_image_tokens(width: float, height: float) -> int:
|
|||
return math.ceil(min(float(ANTHROPIC_IMAGE_MAX_PIXELS), area_at_max_edge) / ANTHROPIC_IMAGE_PIXELS_PER_TOKEN)
|
||||
|
||||
|
||||
def _count_inline_pdf_tokens(data_url: str, count_function: TokenCounterFunction) -> int | None:
|
||||
def _pdf_dictionary(node: object) -> Mapping[str, object] | None:
|
||||
return node if isinstance(node, Mapping) else None # pyright: ignore[reportUnknownVariableType] # read per key
|
||||
|
||||
|
||||
def _pdf_value(container: Mapping[str, object], key: str) -> object:
|
||||
return container[key] if key in container else None # noqa: SIM401 # pypdf resolves references only in []
|
||||
|
||||
|
||||
_PdfXObjects: TypeAlias = Mapping[str, object] | None
|
||||
_PdfXObjectUse: TypeAlias = tuple[Mapping[str, object], _PdfXObjects]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _PdfContentSummary:
|
||||
drawn_names: tuple[str, ...]
|
||||
draws_inline_image: bool
|
||||
|
||||
|
||||
_PdfDrawing: TypeAlias = tuple[_PdfContentSummary, _PdfXObjects]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _PdfWalk:
|
||||
frontier: tuple[_PdfDrawing, ...]
|
||||
seen: frozenset[tuple[int, int]]
|
||||
summaries: Mapping[int, _PdfContentSummary | None]
|
||||
draws: int
|
||||
|
||||
|
||||
def _pdf_xobjects(container: Mapping[str, object], inherited: _PdfXObjects) -> _PdfXObjects:
|
||||
resources: Final = _pdf_dictionary(_pdf_value(container, "/Resources"))
|
||||
if resources is None:
|
||||
return inherited
|
||||
return _pdf_dictionary(_pdf_value(resources, "/XObject"))
|
||||
|
||||
|
||||
_PDF_WHITESPACE: Final = rb"\x00\t\n\x0c\r "
|
||||
_PDF_TOKEN_END: Final = _PDF_WHITESPACE + rb"()<>\[\]{}/%"
|
||||
_PDF_XOBJECT_DRAW: Final = re.compile(
|
||||
rb"/([^" + _PDF_TOKEN_END + rb"]*)(?:[" + _PDF_WHITESPACE + rb"]|%[^\r\n]*[\r\n])+Do(?![^" + _PDF_TOKEN_END + rb"])"
|
||||
)
|
||||
_PDF_INLINE_IMAGE: Final = re.compile(rb"(?<![^" + _PDF_WHITESPACE + rb"])BI(?![^" + _PDF_WHITESPACE + rb"/])")
|
||||
_PDF_NAME_ESCAPE: Final = re.compile(rb"#([0-9A-Fa-f]{2})")
|
||||
_PDF_NAME_CHARSETS: Final = ("utf-8", "gbk")
|
||||
|
||||
|
||||
def _decoded(raw: bytes, charset: str) -> str | None:
|
||||
try:
|
||||
return raw.decode(charset)
|
||||
except UnicodeDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
def _pdf_name(escaped: bytes) -> str:
|
||||
raw: Final = _PDF_NAME_ESCAPE.sub(lambda escape: bytes.fromhex(escape[1].decode("ascii")), escaped)
|
||||
decoded: Final = (_decoded(raw, charset) for charset in _PDF_NAME_CHARSETS)
|
||||
return "/" + next((name for name in decoded if name is not None), raw.decode("latin-1"))
|
||||
|
||||
|
||||
def _pdf_content_summary(content: _PdfContent) -> _PdfContentSummary:
|
||||
data: Final = content.get_data()
|
||||
draws: Final = islice(_PDF_XOBJECT_DRAW.finditer(data), PDF_MAX_XOBJECT_DRAWS_PER_PAGE + 1)
|
||||
return _PdfContentSummary(
|
||||
drawn_names=tuple(_pdf_name(draw[1]) for draw in draws),
|
||||
draws_inline_image=_PDF_INLINE_IMAGE.search(data) is not None,
|
||||
)
|
||||
|
||||
|
||||
def _pdf_form_summary(form: Mapping[str, object]) -> _PdfContentSummary | None:
|
||||
from pypdf.generic import StreamObject
|
||||
|
||||
if not isinstance(form, StreamObject):
|
||||
return None
|
||||
return _pdf_content_summary(cast(_PdfContent, form)) # cast-ok: pypdf is untyped
|
||||
|
||||
|
||||
def _pdf_drawn_xobjects(drawing: _PdfDrawing) -> tuple[_PdfXObjectUse, ...]:
|
||||
summary, xobjects = drawing
|
||||
if xobjects is None:
|
||||
return ()
|
||||
resolved: Final = (_pdf_dictionary(_pdf_value(xobjects, name)) for name in summary.drawn_names)
|
||||
return tuple((xobject, xobjects) for xobject in resolved if xobject is not None)
|
||||
|
||||
|
||||
def _pdf_drawing_key(use: _PdfXObjectUse) -> tuple[int, int]:
|
||||
xobject, inherited = use
|
||||
return id(xobject), id(_pdf_xobjects(xobject, inherited))
|
||||
|
||||
|
||||
def _pdf_next_walk(walk: _PdfWalk) -> _PdfWalk | None:
|
||||
draws: Final = walk.draws + sum(len(summary.drawn_names) for summary, _ in walk.frontier)
|
||||
if draws > PDF_MAX_XOBJECT_DRAWS_PER_PAGE:
|
||||
return None
|
||||
drawn: Final = tuple(chain.from_iterable(map(_pdf_drawn_xobjects, walk.frontier)))
|
||||
if any(_pdf_value(xobject, "/Subtype") == "/Image" for xobject, _ in drawn):
|
||||
return None
|
||||
forms: Final = {_pdf_drawing_key(use): use for use in drawn if _pdf_value(use[0], "/Subtype") == "/Form"}
|
||||
unseen: Final = tuple(forms[key] for key in forms.keys() - walk.seen)
|
||||
unparsed: Final = {id(form): form for form, _ in unseen if id(form) not in walk.summaries}
|
||||
summaries: Final = MappingProxyType(
|
||||
{**walk.summaries, **{key: _pdf_form_summary(form) for key, form in unparsed.items()}}
|
||||
)
|
||||
drawings: Final = ((summaries[id(form)], _pdf_xobjects(form, inherited)) for form, inherited in unseen)
|
||||
return _PdfWalk(
|
||||
frontier=tuple((summary, xobjects) for summary, xobjects in drawings if summary is not None),
|
||||
seen=walk.seen | frozenset(forms),
|
||||
summaries=summaries,
|
||||
draws=draws,
|
||||
)
|
||||
|
||||
|
||||
def _pdf_page_draws_raster_image(page: _PdfPage) -> bool:
|
||||
contents: Final = page.get_contents()
|
||||
page_dictionary: Final = _pdf_dictionary(page)
|
||||
if contents is None or page_dictionary is None:
|
||||
return False
|
||||
walk: _PdfWalk = _PdfWalk( # rebind-ok: breadth-first walk, recursion is banned
|
||||
frontier=((_pdf_content_summary(contents), _pdf_xobjects(page_dictionary, None)),),
|
||||
seen=frozenset(),
|
||||
summaries=MappingProxyType({}),
|
||||
draws=0,
|
||||
)
|
||||
for _ in range(PDF_MAX_FORM_XOBJECT_DEPTH):
|
||||
if not walk.frontier:
|
||||
return False
|
||||
if any(summary.draws_inline_image for summary, _ in walk.frontier):
|
||||
return True
|
||||
if (next_walk := _pdf_next_walk(walk)) is None:
|
||||
return True
|
||||
walk = next_walk
|
||||
return bool(walk.frontier)
|
||||
|
||||
|
||||
def _openai_pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int:
|
||||
if _pdf_page_draws_raster_image(page):
|
||||
return _claude_pdf_page_tokens(page, count_function)
|
||||
text: Final = page.extract_text() or ""
|
||||
return count_function(text) + OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS if text.strip() else 0
|
||||
|
||||
|
||||
def _claude_pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int:
|
||||
return count_function(page.extract_text() or "") + _anthropic_rendered_page_image_tokens(
|
||||
float(page.mediabox.width), float(page.mediabox.height)
|
||||
)
|
||||
|
||||
|
||||
def _pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction, pdf_pricing: PdfPagePricing) -> int:
|
||||
match pdf_pricing:
|
||||
case "openai":
|
||||
return _openai_pdf_page_tokens(page, count_function)
|
||||
case "claude":
|
||||
return _claude_pdf_page_tokens(page, count_function)
|
||||
|
||||
|
||||
def _count_inline_pdf_tokens(
|
||||
data_url: str,
|
||||
count_function: TokenCounterFunction,
|
||||
pdf_pricing: PdfPagePricing = "claude",
|
||||
) -> int | None:
|
||||
if not data_url.startswith(PDF_DATA_URL_PREFIX):
|
||||
return None
|
||||
try:
|
||||
|
|
@ -1133,12 +1395,10 @@ def _count_inline_pdf_tokens(data_url: str, count_function: TokenCounterFunction
|
|||
verbose_logger.debug("pypdf is not installed, so the PDF document is priced like one image")
|
||||
return None
|
||||
try:
|
||||
reader: Final = PdfReader(io.BytesIO(base64.b64decode(data_url[len(PDF_DATA_URL_PREFIX) :])))
|
||||
return sum(
|
||||
count_function(page.extract_text() or "")
|
||||
+ _anthropic_rendered_page_image_tokens(float(page.mediabox.width), float(page.mediabox.height))
|
||||
for page in reader.pages
|
||||
reader: Final = cast( # cast-ok: pypdf's PdfReader satisfies _PdfReader, typed only where pypdf is installed
|
||||
_PdfReader, PdfReader(io.BytesIO(base64.b64decode(data_url[len(PDF_DATA_URL_PREFIX) :])))
|
||||
)
|
||||
return sum(_pdf_page_tokens(page, count_function, pdf_pricing) for page in reader.pages)
|
||||
except Exception as e:
|
||||
verbose_logger.debug("Could not read the PDF document's pages (%s), so it is priced like one image", e)
|
||||
return None
|
||||
|
|
@ -1148,8 +1408,9 @@ def _count_opaque_document_tokens(
|
|||
data_url: str,
|
||||
count_function: TokenCounterFunction,
|
||||
use_default_image_token_count: bool,
|
||||
pdf_pricing: PdfPagePricing = "claude",
|
||||
) -> int:
|
||||
pdf_tokens: Final = _count_inline_pdf_tokens(data_url, count_function)
|
||||
pdf_tokens: Final = _count_inline_pdf_tokens(data_url, count_function, pdf_pricing)
|
||||
if pdf_tokens is not None:
|
||||
return pdf_tokens
|
||||
return calculate_img_tokens(
|
||||
|
|
@ -1164,6 +1425,7 @@ def _count_document_tokens(
|
|||
count_function: TokenCounterFunction,
|
||||
use_default_image_token_count: bool,
|
||||
default_token_count: int | None,
|
||||
pdf_pricing: PdfPagePricing = "claude",
|
||||
) -> int:
|
||||
source: Final = document["source"]
|
||||
metadata_tokens: Final = sum(
|
||||
|
|
@ -1176,10 +1438,10 @@ def _count_document_tokens(
|
|||
if isinstance(content, str):
|
||||
return metadata_tokens + count_function(content)
|
||||
return metadata_tokens + _count_content_list(
|
||||
count_function, content, use_default_image_token_count, default_token_count
|
||||
count_function, content, use_default_image_token_count, default_token_count, pdf_pricing
|
||||
)
|
||||
return metadata_tokens + _count_opaque_document_tokens(
|
||||
_anthropic_image_source_data(source), count_function, use_default_image_token_count
|
||||
_anthropic_image_source_data(source), count_function, use_default_image_token_count, pdf_pricing
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1187,6 +1449,7 @@ def _count_file_tokens(
|
|||
file_value: object,
|
||||
count_function: TokenCounterFunction,
|
||||
use_default_image_token_count: bool,
|
||||
pdf_pricing: PdfPagePricing = "claude",
|
||||
) -> int:
|
||||
"""An OpenAI `file` block is the chat-completions spelling of a document, so it prices like one."""
|
||||
if not isinstance(file_value, Mapping):
|
||||
|
|
@ -1196,7 +1459,9 @@ def _count_file_tokens(
|
|||
name_tokens: Final = count_function(filename) if isinstance(filename, str) and filename else 0
|
||||
if not isinstance(file_data, str) or not file_data:
|
||||
return name_tokens
|
||||
return name_tokens + _count_opaque_document_tokens(file_data, count_function, use_default_image_token_count)
|
||||
return name_tokens + _count_opaque_document_tokens(
|
||||
file_data, count_function, use_default_image_token_count, pdf_pricing
|
||||
)
|
||||
|
||||
|
||||
def _count_anthropic_content(
|
||||
|
|
@ -1204,6 +1469,7 @@ def _count_anthropic_content(
|
|||
count_function: TokenCounterFunction,
|
||||
use_default_image_token_count: bool,
|
||||
default_token_count: int | None,
|
||||
pdf_pricing: PdfPagePricing = "claude",
|
||||
) -> int:
|
||||
"""
|
||||
Count tokens in Anthropic-specific content blocks (tool_use, tool_result, etc.).
|
||||
|
|
@ -1238,6 +1504,7 @@ def _count_anthropic_content(
|
|||
field_value,
|
||||
use_default_image_token_count,
|
||||
default_token_count,
|
||||
pdf_pricing,
|
||||
)
|
||||
elif isinstance(field_value, dict):
|
||||
tokens += count_function(str(field_value))
|
||||
|
|
@ -1264,6 +1531,7 @@ def _count_content_list(
|
|||
],
|
||||
use_default_image_token_count: bool,
|
||||
default_token_count: int | None,
|
||||
pdf_pricing: PdfPagePricing = "claude",
|
||||
) -> int:
|
||||
"""Recursively count tokens from a list of content blocks."""
|
||||
try:
|
||||
|
|
@ -1288,12 +1556,14 @@ def _count_content_list(
|
|||
count_function,
|
||||
use_default_image_token_count,
|
||||
default_token_count,
|
||||
pdf_pricing,
|
||||
)
|
||||
elif c["type"] == "file":
|
||||
num_tokens += _count_file_tokens(
|
||||
c.get("file"),
|
||||
count_function,
|
||||
use_default_image_token_count,
|
||||
pdf_pricing,
|
||||
)
|
||||
elif c["type"] in ("tool_use", "tool_result", "search_result"):
|
||||
num_tokens += _count_anthropic_content(
|
||||
|
|
@ -1301,6 +1571,7 @@ def _count_content_list(
|
|||
count_function,
|
||||
use_default_image_token_count,
|
||||
default_token_count,
|
||||
pdf_pricing,
|
||||
)
|
||||
elif c["type"] in ("thinking", "redacted_thinking"):
|
||||
# Claude extended thinking content block
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ from litellm.proxy.hooks.batch_enqueued_tokens import (
|
|||
canonical_provider_batch_id,
|
||||
)
|
||||
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
|
||||
from litellm.proxy.spend_tracking.input_tokens import models_counted_for
|
||||
from litellm.responses.litellm_completion_transformation.compaction_items import billable_input_since_newest_compaction
|
||||
from litellm.router_utils.add_retry_fallback_headers import (
|
||||
ensure_response_additional_headers,
|
||||
|
|
@ -159,6 +160,12 @@ def _resolve_model_group_alias_via_proxy_router(model: str) -> str | None:
|
|||
return resolve_model_group_alias(llm_router.model_group_alias, model)
|
||||
|
||||
|
||||
def _resolve_counted_models_via_proxy_router(model: str, team_id: str | None) -> Sequence[str]:
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
return models_counted_for(model, llm_router, team_id)
|
||||
|
||||
|
||||
def _resolve_ptu_team_ceiling_via_proxy_router(team_id: str, model_group: str) -> PTUTeamCeiling | None:
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
|
|
@ -878,6 +885,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
time_provider: Callable[[], datetime] | None = None,
|
||||
tag_rate_limit_resolver: TagRateLimitResolver = resolve_tag_rate_limits_from_db,
|
||||
model_group_resolver: Callable[[str], str | None] = _resolve_model_group_alias_via_proxy_router,
|
||||
counted_models_resolver: Callable[[str, str | None], Sequence[str]] = _resolve_counted_models_via_proxy_router,
|
||||
media_model_resolver: Callable[[str, str | None], tuple[str, ...]] = _resolve_media_models_via_proxy_router,
|
||||
ptu_team_ceiling_resolver: Callable[
|
||||
[str, str], PTUTeamCeiling | None
|
||||
|
|
@ -889,6 +897,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
self._time_provider = time_provider or datetime.now
|
||||
self._tag_rate_limit_resolver = tag_rate_limit_resolver
|
||||
self._model_group_resolver = model_group_resolver
|
||||
self._counted_models_resolver = counted_models_resolver
|
||||
self._media_model_resolver = media_model_resolver
|
||||
self._ptu_team_ceiling_resolver = ptu_team_ceiling_resolver
|
||||
self._fail_closed_resolver = fail_closed_resolver
|
||||
|
|
@ -4073,8 +4082,18 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
call_type=call_type,
|
||||
include_media=False,
|
||||
)
|
||||
raw_estimated_input_tokens: Final = await offload_token_count(self._estimate_precise_input_tokens)(
|
||||
data=data, model=requested_model, call_type=call_type
|
||||
counted_models: Final = (
|
||||
self._counted_models_resolver(requested_model, user_api_key_dict.team_id) if requested_model else (None,)
|
||||
)
|
||||
raw_estimated_input_tokens: Final = max(
|
||||
await asyncio.gather(
|
||||
*(
|
||||
offload_token_count(self._estimate_precise_input_tokens)(
|
||||
data=data, model=model, call_type=call_type
|
||||
)
|
||||
for model in counted_models
|
||||
)
|
||||
)
|
||||
)
|
||||
estimated_input_tokens: Final = max(raw_estimated_input_tokens, 1)
|
||||
estimated_output_tokens: Final = (
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import time
|
|||
from collections.abc import AsyncIterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import Final, NoReturn, SupportsFloat, SupportsIndex, SupportsInt, cast
|
||||
|
||||
|
|
@ -39,7 +40,11 @@ from litellm.proxy.common_utils.user_api_key_cache import (
|
|||
tag_cache_key,
|
||||
team_membership_reservation_cache_key,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.input_tokens import count_input_tokens, count_input_tokens_for_model
|
||||
from litellm.proxy.spend_tracking.input_tokens import (
|
||||
count_input_tokens,
|
||||
count_input_tokens_for_model,
|
||||
models_counted_for,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging
|
||||
from litellm.router import Router
|
||||
|
|
@ -270,6 +275,7 @@ async def reserve_budget_for_request(
|
|||
request_body=request_body,
|
||||
route=route,
|
||||
llm_router=llm_router,
|
||||
team_id=valid_token.team_id,
|
||||
raw_body=raw_body,
|
||||
)
|
||||
|
||||
|
|
@ -1591,6 +1597,7 @@ async def count_request_input_tokens(
|
|||
request_body: dict,
|
||||
route: str,
|
||||
llm_router: Router | None,
|
||||
team_id: str | None,
|
||||
raw_body: bytes | None = None,
|
||||
) -> Mapping[str, int]:
|
||||
"""Input-token count per candidate model, counted once per request.
|
||||
|
|
@ -1599,7 +1606,25 @@ async def count_request_input_tokens(
|
|||
models: Final = _get_request_models(request_body=request_body, route=route, llm_router=llm_router)
|
||||
if not models:
|
||||
return MappingProxyType({})
|
||||
return await count_input_tokens(request_body=request_body, raw_body=raw_body, models=models)
|
||||
counted_models: Final = MappingProxyType(
|
||||
{model: models_counted_for(model, llm_router, team_id) for model in models}
|
||||
)
|
||||
counts: Final = await count_input_tokens(
|
||||
request_body=request_body,
|
||||
raw_body=raw_body,
|
||||
models=tuple(dict.fromkeys(chain.from_iterable(counted_models.values()))),
|
||||
)
|
||||
return MappingProxyType(
|
||||
{
|
||||
model: largest
|
||||
for model, names in counted_models.items()
|
||||
if (largest := _largest_count(counts, names)) is not None
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _largest_count(counts: Mapping[str, int], names: Sequence[str]) -> int | None:
|
||||
return max((counts[name] for name in names if name in counts), default=None)
|
||||
|
||||
|
||||
def _estimate_input_tokens(
|
||||
|
|
|
|||
|
|
@ -15,10 +15,11 @@ import asyncio
|
|||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.litellm_core_utils.token_counter import pdf_page_pricing, reads_pdf_pages
|
||||
from litellm.responses.litellm_completion_transformation.compaction_items import (
|
||||
billable_input_since_newest_compaction,
|
||||
holds_compaction_summary,
|
||||
|
|
@ -33,6 +34,9 @@ from litellm.rust_bridge.token_counter import (
|
|||
rust_tokenizer,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
|
||||
TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS: Final = 30_000
|
||||
|
||||
_INPUT_SIZE_FIELDS: Final = ("messages", "prompt", "input", "query", "documents", "tools", "tool_choice")
|
||||
|
|
@ -130,6 +134,22 @@ def _count_input_tokens_for_models(
|
|||
)
|
||||
|
||||
|
||||
def models_counted_for(model: str, llm_router: Router | None, team_id: str | None) -> tuple[str, ...]:
|
||||
"""The names a request for `model` from `team_id` is counted with: `model` itself, except for a deployment that
|
||||
reads PDF pages and prices them unlike `model`, which is counted with its own model minus the provider prefix,
|
||||
the name `/v1/messages/count_tokens` counts with."""
|
||||
served: Final = (
|
||||
llm_router.resolved_litellm_models(model, team_id, use_base_model=True) if llm_router is not None else ()
|
||||
)
|
||||
return tuple(dict.fromkeys(_counted_name(model, served_model) for served_model in served)) or (model,)
|
||||
|
||||
|
||||
def _counted_name(model: str, served_model: str) -> str:
|
||||
if pdf_page_pricing(served_model) == pdf_page_pricing(model) or not reads_pdf_pages(served_model):
|
||||
return model
|
||||
return served_model.partition("/")[2] or served_model
|
||||
|
||||
|
||||
def count_input_tokens_for_model(request_body: dict, model: str) -> int | None:
|
||||
try:
|
||||
if "messages" in request_body:
|
||||
|
|
|
|||
|
|
@ -312,6 +312,7 @@ benchmarks = [
|
|||
"pytest-codspeed==4.3.0",
|
||||
"mcp>=2.3.0,<3",
|
||||
"a2a-sdk==1.1.0",
|
||||
"pypdf>=6.21.2,<7.0",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
|
|
|
|||
126
tests/benchmarks/test_token_counter_pdf_benchmarks.py
Normal file
126
tests/benchmarks/test_token_counter_pdf_benchmarks.py
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
"""PDF pages and model names that once took seconds to minutes to count, kept here so a slowdown shows on CodSpeed.
|
||||
|
||||
Each input is smaller than the one that exposed the slowdown, since the old cost grew with the same input size."""
|
||||
|
||||
import base64
|
||||
from itertools import accumulate, chain
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.litellm_core_utils.token_counter import token_counter
|
||||
|
||||
TEXT_LINE: Final = "Revenue grew eleven percent while churn fell to two percent."
|
||||
|
||||
|
||||
def _pdf_base64(text: str, page_resources: str, drawing: bytes, xobjects: tuple[bytes, ...] = ()) -> str:
|
||||
escaped: Final = text.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)")
|
||||
stream: Final = f"BT /F1 12 Tf 72 720 Td ({escaped}) Tj ET".encode("latin-1") + drawing
|
||||
bodies: Final = (
|
||||
b"<< /Type /Catalog /Pages 2 0 R >>",
|
||||
b"<< /Type /Pages /Count 1 /Kids [ 5 0 R ] >>",
|
||||
b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
|
||||
f"<< /Length {len(stream)} >>\nstream\n".encode() + stream + b"\nendstream",
|
||||
f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources {page_resources} /Contents 4 0 R >>".encode(),
|
||||
*xobjects,
|
||||
)
|
||||
header: Final = b"%PDF-1.4\n"
|
||||
objects: Final = tuple(
|
||||
f"{number} 0 obj\n".encode() + body + b"\nendobj\n" for number, body in enumerate(bodies, start=1)
|
||||
)
|
||||
offsets: Final = accumulate((len(header), *(len(obj) for obj in objects[:-1])))
|
||||
xref: Final = f"xref\n0 {len(objects) + 1}\n0000000000 65535 f \n".encode() + b"".join(
|
||||
f"{offset:010d} 00000 n \n".encode() for offset in offsets
|
||||
)
|
||||
trailer: Final = (
|
||||
f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\n"
|
||||
f"startxref\n{len(header) + sum(len(obj) for obj in objects)}\n%%EOF\n"
|
||||
).encode()
|
||||
return base64.b64encode(header + b"".join(objects) + xref + trailer).decode()
|
||||
|
||||
|
||||
def _form(resources_number: int, drawing: bytes) -> bytes:
|
||||
return (
|
||||
f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Resources {resources_number} 0 R "
|
||||
f"/Length {len(drawing)} >>\nstream\n".encode()
|
||||
+ drawing
|
||||
+ b"\nendstream"
|
||||
)
|
||||
|
||||
|
||||
def _count_pdf(pdf_base64: str) -> int:
|
||||
document: Final = {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": pdf_base64}}
|
||||
return token_counter(model="gpt-4o", messages=[{"role": "user", "content": [document]}])
|
||||
|
||||
|
||||
def _page_of_drawing_operators() -> str:
|
||||
return _pdf_base64("", "<< >>", b" q Q" * 50_000)
|
||||
|
||||
|
||||
def _page_of_forms_sharing_resources() -> str:
|
||||
form_count: Final = 300
|
||||
shared_resources: Final = (
|
||||
"<< /XObject << " + " ".join(f"/F{index} {8 + index} 0 R" for index in range(form_count)) + " >> >>"
|
||||
).encode()
|
||||
forms: Final = tuple(_form(7, b"/F0 Do") for _ in range(form_count))
|
||||
return _pdf_base64(
|
||||
TEXT_LINE,
|
||||
"<< /Font << /F1 3 0 R >> /XObject << /X0 6 0 R >> >>",
|
||||
b" q 10 0 0 10 0 0 cm /X0 Do Q",
|
||||
(forms[0], shared_resources, *forms),
|
||||
)
|
||||
|
||||
|
||||
def _page_of_parents_sharing_a_child_form() -> str:
|
||||
parent_count: Final = 100
|
||||
first: Final = 6
|
||||
child_number: Final = first + 2 * parent_count
|
||||
child_drawing: Final = b" ".join((b"/D Do",) * 4000)
|
||||
child: Final = (
|
||||
f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length {len(child_drawing)} >>\nstream\n".encode()
|
||||
+ child_drawing
|
||||
+ b"\nendstream"
|
||||
)
|
||||
parents: Final = tuple(
|
||||
chain.from_iterable(
|
||||
(_form(first + 2 * index + 1, b"/C Do"), f"<< /XObject << /C {child_number} 0 R >> >>".encode())
|
||||
for index in range(parent_count)
|
||||
)
|
||||
)
|
||||
page_resources: Final = (
|
||||
"<< /Font << /F1 3 0 R >> /XObject << "
|
||||
+ " ".join(f"/P{index} {first + 2 * index} 0 R" for index in range(parent_count))
|
||||
+ " >> >>"
|
||||
)
|
||||
page_drawing: Final = b" " + b" ".join(f"/P{index} Do".encode() for index in range(parent_count))
|
||||
return _pdf_base64(TEXT_LINE, page_resources, page_drawing, (*parents, child))
|
||||
|
||||
|
||||
PAGE_OF_DRAWING_OPERATORS: Final = _page_of_drawing_operators()
|
||||
PAGE_OF_FORMS_SHARING_RESOURCES: Final = _page_of_forms_sharing_resources()
|
||||
PAGE_OF_PARENTS_SHARING_A_CHILD_FORM: Final = _page_of_parents_sharing_a_child_form()
|
||||
BLANK_PAGE_COUNT: Final = _count_pdf(_pdf_base64("", "<< >>", b""))
|
||||
TEXT_PAGE_COUNT: Final = _count_pdf(_pdf_base64(TEXT_LINE, "<< /Font << /F1 3 0 R >> >>", b""))
|
||||
LONG_UNKNOWN_MODEL_NAME: Final = "unknown-" + "a-" * 5_000
|
||||
NAME_SAMPLE: Final = "Ceci est un texte en français: éèàç. " * 8
|
||||
UNKNOWN_MODEL_NAME_COUNT: Final = token_counter(model="my-private-model", text=NAME_SAMPLE)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_pdf_page_of_drawing_operators_and_no_image() -> None:
|
||||
assert _count_pdf(PAGE_OF_DRAWING_OPERATORS) == BLANK_PAGE_COUNT
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_pdf_page_of_forms_sharing_one_resources_dictionary() -> None:
|
||||
assert _count_pdf(PAGE_OF_FORMS_SHARING_RESOURCES) == TEXT_PAGE_COUNT
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_pdf_page_of_parent_forms_sharing_one_child_form() -> None:
|
||||
assert _count_pdf(PAGE_OF_PARENTS_SHARING_A_CHILD_FORM) > TEXT_PAGE_COUNT
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
def test_token_counter_long_unknown_model_name() -> None:
|
||||
assert token_counter(model=LONG_UNKNOWN_MODEL_NAME, text=NAME_SAMPLE) == UNKNOWN_MODEL_NAME_COUNT
|
||||
|
|
@ -29,7 +29,11 @@ import litellm
|
|||
from litellm import decode, encode, get_modified_max_tokens
|
||||
from litellm import token_counter as token_counter_old
|
||||
import litellm.constants
|
||||
from litellm.constants import HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS, TOKEN_COUNTER_MAX_CONCURRENT_COUNTS
|
||||
from litellm.constants import (
|
||||
HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS,
|
||||
PDF_MAX_XOBJECT_DRAWS_PER_PAGE,
|
||||
TOKEN_COUNTER_MAX_CONCURRENT_COUNTS,
|
||||
)
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.litellm_core_utils.token_counter import (
|
||||
_encoding_count,
|
||||
|
|
@ -1878,11 +1882,11 @@ def test_anthropic_image_block_without_source_raises():
|
|||
)
|
||||
|
||||
|
||||
def _count_user_content(content: list[dict]) -> int:
|
||||
def _count_user_content(content: list[dict], model: str = "anthropic/claude-fable-5") -> int:
|
||||
from litellm.litellm_core_utils.token_counter import token_counter
|
||||
|
||||
return token_counter(
|
||||
model="anthropic/claude-fable-5",
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": content}],
|
||||
use_default_image_token_count=True,
|
||||
)
|
||||
|
|
@ -1978,14 +1982,27 @@ def test_openai_file_block_without_inline_bytes_counts_what_it_carries():
|
|||
)
|
||||
|
||||
|
||||
def _pdf_base64(pages: tuple[str, ...], width: int = 612, height: int = 792) -> str:
|
||||
def _pdf_base64(
|
||||
pages: tuple[str, ...],
|
||||
width: int = 612,
|
||||
height: int = 792,
|
||||
xobjects: tuple[bytes, ...] = (),
|
||||
drawing: bytes = b"",
|
||||
page_resources: str | None = None,
|
||||
) -> str:
|
||||
"""`xobjects` are numbered from 4 + 2 * len(pages), every page lists the first one as its /X0 XObject, and every
|
||||
page's content stream ends with `drawing`. `page_resources` replaces every page's /Resources dictionary."""
|
||||
first_xobject: Final = 4 + 2 * len(pages)
|
||||
page_xobjects: Final = f" /XObject << /X0 {first_xobject} 0 R >>" if xobjects else ""
|
||||
resources: Final = page_resources if page_resources is not None else f"<< /Font << /F1 3 0 R >>{page_xobjects} >>"
|
||||
|
||||
def page_objects(index: int, text: str) -> tuple[bytes, bytes]:
|
||||
escaped: Final = text.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)")
|
||||
stream: Final = f"BT /F1 12 Tf 72 720 Td ({escaped}) Tj ET".encode("latin-1")
|
||||
stream: Final = f"BT /F1 12 Tf 72 720 Td ({escaped}) Tj ET".encode("latin-1") + drawing
|
||||
content: Final = f"<< /Length {len(stream)} >>\nstream\n".encode() + stream + b"\nendstream"
|
||||
page: Final = (
|
||||
f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {width} {height}] "
|
||||
f"/Resources << /Font << /F1 3 0 R >> >> /Contents {4 + 2 * index} 0 R >>"
|
||||
f"/Resources {resources} /Contents {4 + 2 * index} 0 R >>"
|
||||
).encode()
|
||||
return content, page
|
||||
|
||||
|
|
@ -1995,6 +2012,7 @@ def _pdf_base64(pages: tuple[str, ...], width: int = 612, height: int = 792) ->
|
|||
f"<< /Type /Pages /Count {len(pages)} /Kids [ {kids} ] >>".encode(),
|
||||
b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
|
||||
*chain.from_iterable(page_objects(index, text) for index, text in enumerate(pages)),
|
||||
*xobjects,
|
||||
)
|
||||
header: Final = b"%PDF-1.4\n"
|
||||
objects: Final = tuple(
|
||||
|
|
@ -2073,6 +2091,401 @@ def test_pdf_document_without_pypdf_is_priced_like_an_image(monkeypatch: pytest.
|
|||
assert priced_as_image < priced_by_page
|
||||
|
||||
|
||||
_REVENUE_LINE: Final = "Revenue grew eleven percent while churn fell to two percent."
|
||||
_HEADCOUNT_LINE: Final = "Headcount is flat and the office lease was renewed for three years."
|
||||
_ONE_PIXEL_IMAGE: Final = (
|
||||
b"<< /Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8 "
|
||||
b"/Length 1 >>\nstream\n\x00\nendstream"
|
||||
)
|
||||
|
||||
|
||||
_DRAW_X0: Final = b" q 10 0 0 10 0 0 cm /X0 Do Q"
|
||||
_DRAW_X1: Final = b"/X1 Do"
|
||||
|
||||
|
||||
def _form_xobject(resources_number: int, drawing: bytes = _DRAW_X1) -> bytes:
|
||||
return (
|
||||
f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Resources {resources_number} 0 R "
|
||||
f"/Length {len(drawing)} >>\nstream\n".encode()
|
||||
+ drawing
|
||||
+ b"\nendstream"
|
||||
)
|
||||
|
||||
|
||||
def _xobject_resources(xobject_number: int) -> bytes:
|
||||
return f"<< /XObject << /X1 {xobject_number} 0 R >> >>".encode()
|
||||
|
||||
|
||||
def _as_pdf_document(pdf_base64: str) -> Mapping[str, object]:
|
||||
return {"type": "document", "source": _pdf_source(pdf_base64)}
|
||||
|
||||
|
||||
def _as_pdf_file(pdf_base64: str) -> Mapping[str, object]:
|
||||
file_data: Final = "data:application/pdf;base64," + pdf_base64
|
||||
return {"type": "file", "file": {"filename": "report.pdf", "file_data": file_data}}
|
||||
|
||||
|
||||
def _as_pdf_tool_result(pdf_base64: str) -> Mapping[str, object]:
|
||||
return {"type": "tool_result", "tool_use_id": "t1", "content": [_as_pdf_document(pdf_base64)]}
|
||||
|
||||
|
||||
def _pdf_cost(model: str, pdf_base64: str, wrap: Callable[[str], Mapping[str, object]] = _as_pdf_document) -> int:
|
||||
return _count_user_content([wrap(pdf_base64)], model) - _count_user_content([wrap(_pdf_base64(()))], model)
|
||||
|
||||
|
||||
def _text_cost(model: str, *lines: str) -> int:
|
||||
return sum(
|
||||
_count_user_content([{"type": "text", "text": line}], model) - _count_user_content([], model) for line in lines
|
||||
)
|
||||
|
||||
|
||||
def _rendered_page_cost(width: int = 612, height: int = 792) -> int:
|
||||
return _pdf_cost("my-private-model", _pdf_base64(("",), width, height))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"wrap", [_as_pdf_document, _as_pdf_file, _as_pdf_tool_result], ids=["document", "file", "tool_result"]
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
["gpt-4o", "azure/gpt-5.6-sol", "o3-2025-04-16", "azure/gpt-41", "eu/gpt-5.6-sol", "gpt-5.6-sol-2026-07-09"],
|
||||
)
|
||||
def test_openai_model_prices_each_pdf_text_page_as_its_text_plus_framing(
|
||||
model: str, wrap: Callable[[str], Mapping[str, object]]
|
||||
):
|
||||
"""OpenAI models bill a PDF page with text as its text plus 13 tokens and a blank page as nothing, with no page
|
||||
image. Measured 2026-10-20 on Azure chat completions gpt-5.6-sol (1, 2, and 10 pages of 15 text tokens billed
|
||||
+28, +56, +280, and 1, 2, or 10 blank pages billed the same as none) and on OpenAI /v1/responses/input_tokens
|
||||
for gpt-4o, gpt-4.1, gpt-5, gpt-5.5, o3, and o4-mini.
|
||||
|
||||
Before the fix every model got Claude's pricing, so count_tokens answered about 1586 for a one-page text PDF
|
||||
that Azure gpt-5.6-sol billed at 61.
|
||||
"""
|
||||
one_page: Final = _pdf_cost(model, _pdf_base64((_REVENUE_LINE,)), wrap)
|
||||
|
||||
assert one_page == _text_cost(model, _REVENUE_LINE) + 13
|
||||
assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE, _HEADCOUNT_LINE)), wrap) == (
|
||||
_text_cost(model, _REVENUE_LINE, _HEADCOUNT_LINE) + 2 * 13
|
||||
)
|
||||
assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,) * 10), wrap) == 10 * one_page
|
||||
assert _pdf_cost(model, _pdf_base64(("",) * 5), wrap) == 0
|
||||
assert _pdf_cost(model, _pdf_base64(("", _REVENUE_LINE, "")), wrap) == one_page
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"xobjects",
|
||||
[(_ONE_PIXEL_IMAGE,), (_form_xobject(7), _xobject_resources(8), _ONE_PIXEL_IMAGE)],
|
||||
ids=["image_on_the_page", "image_inside_a_form"],
|
||||
)
|
||||
def test_openai_model_prices_a_page_that_draws_a_raster_image_as_its_text_plus_the_rendered_page(
|
||||
xobjects: tuple[bytes, ...],
|
||||
):
|
||||
"""A page embedding a raster image keeps the rendered-page estimate every model got before, so a reservation
|
||||
never undercounts it: on 2026-10-20 /v1/responses/input_tokens billed a page with one drawn image at 1620 on
|
||||
gpt-5.6-sol and gpt-6-sol, above OpenAI's own high-detail page image (765 for Letter), while a vector-drawn
|
||||
chart page billed no image.
|
||||
"""
|
||||
model: Final = "gpt-4o"
|
||||
|
||||
assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=xobjects, drawing=_DRAW_X0)) == (
|
||||
_text_cost(model, _REVENUE_LINE) + _rendered_page_cost()
|
||||
)
|
||||
assert _pdf_cost(model, _pdf_base64(("",), xobjects=xobjects, drawing=_DRAW_X0)) == _rendered_page_cost()
|
||||
assert _pdf_cost(model, _pdf_base64(("",), 300, 300, xobjects=xobjects, drawing=_DRAW_X0)) == (
|
||||
_rendered_page_cost(300, 300)
|
||||
)
|
||||
|
||||
|
||||
def test_openai_model_prices_no_page_image_for_an_image_the_page_lists_but_never_draws():
|
||||
"""Pages sharing one /Resources all list its logo, but only the page that paints it bills a page image: measured
|
||||
2026-10-20 on Azure gpt-5.6-sol, a 3-page PDF drawing a logo on page 1 billed 403, not three page images."""
|
||||
model: Final = "gpt-4o"
|
||||
|
||||
assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=(_ONE_PIXEL_IMAGE,))) == (
|
||||
_text_cost(model, _REVENUE_LINE) + 13
|
||||
)
|
||||
form_listing_an_unused_image: Final = (_form_xobject(7, b""), _xobject_resources(8), _ONE_PIXEL_IMAGE)
|
||||
assert _pdf_cost(model, _pdf_base64(("",), xobjects=form_listing_an_unused_image, drawing=_DRAW_X0)) == 0
|
||||
|
||||
|
||||
def test_openai_model_prices_no_image_for_forms_that_draw_none():
|
||||
"""A vector-drawn form XObject bills no page image, and a form that lists itself cannot loop the walk."""
|
||||
model: Final = "gpt-4o"
|
||||
self_referencing_form: Final = (_form_xobject(7), _xobject_resources(6))
|
||||
|
||||
assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=self_referencing_form, drawing=_DRAW_X0)) == (
|
||||
_text_cost(model, _REVENUE_LINE) + 13
|
||||
)
|
||||
|
||||
|
||||
_INLINE_IMAGE: Final = b" q 200 0 0 200 72 400 cm BI /W 1 /H 1 /CS /G /BPC 8 ID \x00 EI Q"
|
||||
|
||||
|
||||
def test_openai_model_prices_a_page_that_draws_an_inline_image_like_an_image_xobject():
|
||||
"""An image drawn inline in the content stream (BI ... ID ... EI) bills like an image XObject: measured
|
||||
2026-10-20, the same 200x200 pt picture billed 351 either way on Azure gpt-5.6-sol and 303 on gpt-4o."""
|
||||
model: Final = "gpt-4o"
|
||||
form_drawing_inline_image: Final = (
|
||||
f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length {len(_INLINE_IMAGE)} >>\nstream\n".encode()
|
||||
+ _INLINE_IMAGE
|
||||
+ b"\nendstream"
|
||||
)
|
||||
|
||||
assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), drawing=_INLINE_IMAGE)) == (
|
||||
_text_cost(model, _REVENUE_LINE) + _rendered_page_cost()
|
||||
)
|
||||
assert _pdf_cost(model, _pdf_base64(("",), drawing=_INLINE_IMAGE, page_resources="<< >>")) == (
|
||||
_rendered_page_cost()
|
||||
)
|
||||
assert _pdf_cost(model, _pdf_base64(("",), xobjects=(form_drawing_inline_image,), drawing=_DRAW_X0)) == (
|
||||
_rendered_page_cost()
|
||||
)
|
||||
|
||||
|
||||
def test_openai_model_keeps_the_text_of_a_page_whose_forms_nest_too_deep_to_walk():
|
||||
"""A chain of 2,000 nested form XObjects used to exhaust the stack, which dropped the whole document's text and
|
||||
priced it as one image. The walk now stops at a fixed depth and prices the page as if it drew an image."""
|
||||
model: Final = "gpt-4o"
|
||||
depth: Final = 2000
|
||||
first: Final = 6
|
||||
chain_objects: Final = tuple(
|
||||
chain.from_iterable(
|
||||
(_form_xobject(first + 2 * level + 1), _xobject_resources(first + 2 * level + 2)) for level in range(depth)
|
||||
)
|
||||
)
|
||||
|
||||
assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=chain_objects, drawing=_DRAW_X0)) == (
|
||||
_text_cost(model, _REVENUE_LINE) + _rendered_page_cost()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("drawing", [b" /A Do /B Do", b" /B Do /A Do"])
|
||||
def test_openai_model_finds_an_image_a_shared_form_without_resources_draws_from_one_parent(drawing: bytes):
|
||||
"""A form with no /Resources resolves names in the resources of whichever form draws it, so a child that two
|
||||
parents share reaches the image only through the parent whose resources list it."""
|
||||
model: Final = "gpt-4o"
|
||||
child_without_resources: Final = (
|
||||
b"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length 5 >>\nstream\n/Y Do\nendstream"
|
||||
)
|
||||
parents_and_child: Final = (
|
||||
_form_xobject(7, b"/C Do"),
|
||||
b"<< /XObject << /C 10 0 R >> >>",
|
||||
_form_xobject(9, b"/C Do"),
|
||||
b"<< /XObject << /C 10 0 R /Y 11 0 R >> >>",
|
||||
child_without_resources,
|
||||
_ONE_PIXEL_IMAGE,
|
||||
)
|
||||
|
||||
assert _pdf_cost(
|
||||
model,
|
||||
_pdf_base64(
|
||||
("",),
|
||||
xobjects=parents_and_child,
|
||||
drawing=drawing,
|
||||
page_resources="<< /XObject << /A 6 0 R /B 8 0 R >> >>",
|
||||
),
|
||||
) == _rendered_page_cost()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("drawing", "draws_image"),
|
||||
[(b" /X0 %scale it first\n Do", True), (b" /X0 % Do\n", False)],
|
||||
ids=["comment-before-do", "do-inside-a-comment"],
|
||||
)
|
||||
def test_openai_model_reads_a_comment_in_a_content_stream_as_whitespace(drawing: bytes, draws_image: bool):
|
||||
"""A comment runs to the end of its line and separates tokens like whitespace, so it neither hides the Do after
|
||||
it nor draws anything itself."""
|
||||
model: Final = "gpt-4o"
|
||||
|
||||
assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,), xobjects=(_ONE_PIXEL_IMAGE,), drawing=drawing)) == (
|
||||
_text_cost(model, _REVENUE_LINE) + (_rendered_page_cost() if draws_image else 13)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("escaped_name", ["/Im#231", "/#C4#E3", "/caf#C3#A9"], ids=["hex_escape", "gbk", "utf8"])
|
||||
def test_openai_model_finds_an_image_drawn_by_an_escaped_name(escaped_name: str):
|
||||
"""A name written with #xx escapes reads the same in the content stream and in /Resources, decoded the way pypdf
|
||||
decodes it, so an image drawn under such a name still prices the page as rendered."""
|
||||
model: Final = "gpt-4o"
|
||||
|
||||
assert _pdf_cost(
|
||||
model,
|
||||
_pdf_base64(
|
||||
("",),
|
||||
xobjects=(_ONE_PIXEL_IMAGE,),
|
||||
drawing=f" {escaped_name} Do".encode(),
|
||||
page_resources=f"<< /XObject << {escaped_name} 6 0 R >> >>",
|
||||
),
|
||||
) == _rendered_page_cost()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("parent_count", "draws_too_many"), [(1, False), (2, True)])
|
||||
def test_openai_model_prices_a_page_whose_forms_draw_too_many_xobjects_as_rendered(
|
||||
parent_count: int, draws_too_many: bool
|
||||
):
|
||||
"""A page prices as if it drew an image once its forms draw more than PDF_MAX_XOBJECT_DRAWS_PER_PAGE XObjects
|
||||
between them, so a page of forms nested to blow up the walk cannot stall the request."""
|
||||
model: Final = "gpt-4o"
|
||||
first: Final = 6
|
||||
child_number: Final = first + 2 * parent_count
|
||||
child_drawing: Final = b" ".join((b"/D Do",) * (PDF_MAX_XOBJECT_DRAWS_PER_PAGE // 2))
|
||||
child: Final = (
|
||||
f"<< /Type /XObject /Subtype /Form /BBox [0 0 1 1] /Length {len(child_drawing)} >>\nstream\n".encode()
|
||||
+ child_drawing
|
||||
+ b"\nendstream"
|
||||
)
|
||||
parents: Final = tuple(
|
||||
chain.from_iterable(
|
||||
(_form_xobject(first + 2 * index + 1, b"/C Do"), f"<< /XObject << /C {child_number} 0 R >> >>".encode())
|
||||
for index in range(parent_count)
|
||||
)
|
||||
)
|
||||
page_resources: Final = (
|
||||
"<< /Font << /F1 3 0 R >> /XObject << "
|
||||
+ " ".join(f"/P{index} {first + 2 * index} 0 R" for index in range(parent_count))
|
||||
+ " >> >>"
|
||||
)
|
||||
page_drawing: Final = b" " + b" ".join(f"/P{index} Do".encode() for index in range(parent_count))
|
||||
|
||||
assert _pdf_cost(
|
||||
model,
|
||||
_pdf_base64(
|
||||
(_REVENUE_LINE,), xobjects=(*parents, child), drawing=page_drawing, page_resources=page_resources
|
||||
),
|
||||
) == _text_cost(model, _REVENUE_LINE) + (_rendered_page_cost() if draws_too_many else 13)
|
||||
|
||||
|
||||
_NON_LATIN_SAMPLE: Final = "Привет, как дела? これは日本語のテキストです。 Ceci est un texte en français: éèàç. " * 8
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"gpt-5.6-sol",
|
||||
"gpt-5.4-mini",
|
||||
"gpt-6-sol",
|
||||
"azure/gpt-41",
|
||||
"azure/gpt-5.6-sol",
|
||||
"eu/gpt-5.6-sol",
|
||||
"gpt-5.6-sol-2026-07-09",
|
||||
],
|
||||
)
|
||||
def test_openai_models_tiktoken_does_not_know_count_with_o200k(model: str):
|
||||
"""OpenAI models newer than the installed tiktoken's map bill text with o200k_base: measured 2026-10-20 on
|
||||
/v1/responses/input_tokens, gpt-5, gpt-5.4-mini, gpt-5.5, gpt-5.6-sol, and gpt-6-sol all billed 255 for a
|
||||
non-Latin sample like this one that is 249 tokens in o200k_base and 297 in cl100k_base."""
|
||||
o200k: Final = tiktoken.get_encoding("o200k_base")
|
||||
|
||||
assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len(o200k.encode(_NON_LATIN_SAMPLE))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("deployment_name", "model"), [("azure/gpt-4", "gpt-4"), ("eu/gpt-35-turbo", "gpt-35-turbo")])
|
||||
def test_prefixed_openai_model_names_count_text_like_the_model_they_name(deployment_name: str, model: str):
|
||||
assert token_counter_new(model=deployment_name, text=_NON_LATIN_SAMPLE) == token_counter_new(
|
||||
model=model, text=_NON_LATIN_SAMPLE
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model", ["azure/gpt-chat-latest", "azure/codex-mini", "azure/gpt-5.3-chat", "azure/gpt-4.5-preview"]
|
||||
)
|
||||
def test_azure_openai_models_only_azure_lists_get_openai_pricing_and_o200k(model: str):
|
||||
assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,))) == _text_cost(model, _REVENUE_LINE) + 13
|
||||
assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len(
|
||||
tiktoken.get_encoding("o200k_base").encode(_NON_LATIN_SAMPLE)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"openrouter/openai/gpt-4o",
|
||||
"github_copilot/gpt-4o",
|
||||
"azure_ai/gpt-4o",
|
||||
"azure/responses/gpt-5.6-sol",
|
||||
"azure/o_series/o3-mini",
|
||||
],
|
||||
)
|
||||
def test_gpt_names_behind_any_provider_or_route_prefix_count_like_the_bare_name(model: str):
|
||||
"""The proxy counts a deployment with its model name minus the provider prefix, so the SDK prices and encodes
|
||||
a prefixed GPT name the same way the bare name counts. OpenRouter gpt-4o billed a text PDF page at 237 on
|
||||
2026-10-20, closer to OpenAI's pricing than to Claude's 1534."""
|
||||
bare_name: Final = model.rpartition("/")[2]
|
||||
pdf_with_a_drawn_image: Final = _pdf_base64((_REVENUE_LINE,), xobjects=(_ONE_PIXEL_IMAGE,), drawing=_DRAW_X0)
|
||||
|
||||
assert _pdf_cost(model, pdf_with_a_drawn_image) == _pdf_cost(bare_name, pdf_with_a_drawn_image)
|
||||
assert _pdf_cost(model, _pdf_base64(("",))) == 0
|
||||
assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len(
|
||||
tiktoken.get_encoding("o200k_base").encode(_NON_LATIN_SAMPLE)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"text-embedding-ada-002-v2",
|
||||
"tts-1",
|
||||
"whisper-1",
|
||||
"azure/whisper-1",
|
||||
"github_copilot/text-embedding-3-small-inference",
|
||||
"azure_text/gpt-35-turbo-instruct",
|
||||
"ft:davinci-002:my-org::abc123",
|
||||
],
|
||||
)
|
||||
def test_openai_models_that_are_not_chat_models_keep_counting_with_cl100k_and_claude_pdf_pricing(model: str):
|
||||
assert token_counter_new(model=model, text=_NON_LATIN_SAMPLE) == len(
|
||||
tiktoken.get_encoding("cl100k_base").encode(_NON_LATIN_SAMPLE)
|
||||
)
|
||||
assert _pdf_cost(model, _pdf_base64(("",))) == 1534
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("fine_tune", "base_model"),
|
||||
[
|
||||
("ft:gpt-4o-mini-2024-07-18:my-org::abc123", "gpt-4o-mini-2024-07-18"),
|
||||
("openai/ft:gpt-4.1-2025-04-14:my-org:support:def456", "gpt-4.1-2025-04-14"),
|
||||
],
|
||||
)
|
||||
def test_openai_fine_tunes_count_like_their_base_model(fine_tune: str, base_model: str):
|
||||
pdf: Final = _pdf_base64((_REVENUE_LINE,))
|
||||
|
||||
assert _pdf_cost(fine_tune, pdf) == _pdf_cost(base_model, pdf) == _text_cost(base_model, _REVENUE_LINE) + 13
|
||||
assert token_counter_new(model=fine_tune, text=_NON_LATIN_SAMPLE) == token_counter_new(
|
||||
model=base_model, text=_NON_LATIN_SAMPLE
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"",
|
||||
"anthropic/claude-fable-5",
|
||||
"DeepSeek-V4-Flash-0731",
|
||||
"my-private-model",
|
||||
"azure/mistral-large-latest",
|
||||
"command-r-plus",
|
||||
],
|
||||
)
|
||||
def test_non_openai_models_keep_pricing_pdf_pages_as_claude_renders_them(model: str):
|
||||
"""Only known OpenAI and Azure OpenAI models switch to OpenAI's PDF billing, so an empty or unknown model keeps
|
||||
Claude's text plus rendered page image, 1534 tokens for a blank Letter page."""
|
||||
assert _pdf_cost(model, _pdf_base64(("",))) == 1534
|
||||
assert _pdf_cost(model, _pdf_base64((_REVENUE_LINE,))) == _text_cost(model, _REVENUE_LINE) + 1534
|
||||
|
||||
|
||||
def test_messages_reach_token_count_prices_pdf_pages_like_token_counter():
|
||||
from litellm.litellm_core_utils.token_counter import messages_reach_token_count
|
||||
|
||||
messages: Final = [{"role": "user", "content": [_as_pdf_document(_pdf_base64((_REVENUE_LINE, "")))]}]
|
||||
total: Final = token_counter_new(model="azure/gpt-5.6-sol", messages=messages)
|
||||
|
||||
assert total == token_counter_new(model="azure/gpt-5.6-sol", messages=[{"role": "user", "content": []}]) + (
|
||||
_text_cost("azure/gpt-5.6-sol", _REVENUE_LINE) + 13
|
||||
)
|
||||
assert messages_reach_token_count(model="azure/gpt-5.6-sol", messages=messages, threshold=total) is True
|
||||
assert messages_reach_token_count(model="azure/gpt-5.6-sol", messages=messages, threshold=total + 1) is False
|
||||
|
||||
|
||||
def _png_data_url(width: int, height: int) -> str:
|
||||
ihdr = b"\x89PNG\r\n\x1a\n" + (13).to_bytes(4, "big") + b"IHDR" + width.to_bytes(4, "big") + height.to_bytes(4, "big")
|
||||
return "data:image/png;base64," + base64.b64encode(ihdr + b"\x08\x06\x00\x00\x00").decode()
|
||||
|
|
@ -2385,6 +2798,14 @@ def test_token_counter_uses_the_tokenizer_of_each_model_family_and_of_a_custom_t
|
|||
}
|
||||
|
||||
|
||||
def test_custom_openai_tokenizer_without_a_model_counts_with_the_default_encoding() -> None:
|
||||
sample: Final = "A custom tokenizer passed with no model still counts."
|
||||
|
||||
assert litellm.token_counter(model=None, custom_tokenizer=litellm.utils.select_tokenizer("gpt-4o"), text=sample) == (
|
||||
len(tiktoken.get_encoding("cl100k_base").encode(sample))
|
||||
)
|
||||
|
||||
|
||||
def test_empty_custom_tokenizer_uses_model_tokenizer() -> None:
|
||||
text_value: Final = "A tokenizer fallback should preserve the model encoding."
|
||||
custom_count: Final = _get_exact_count_function("gpt-3.5-turbo", {})(text_value)
|
||||
|
|
|
|||
|
|
@ -9099,6 +9099,36 @@ async def test_a2a_url_target_owns_invocation_fee_and_request_limit(
|
|||
assert "expensive" in str(denied.value.detail)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_project_itpm_reserves_a_pdf_at_the_largest_count_of_the_group_deployments() -> None:
|
||||
from tests.unit.proxy.spend_tracking.test_input_tokens import _one_page_pdf_message
|
||||
|
||||
cache: Final = DualCache()
|
||||
handler: Final = _PROXY_MaxParallelRequestsHandler(
|
||||
internal_usage_cache=InternalUsageCache(cache),
|
||||
counted_models_resolver=lambda model, team_id: ("claude-haiku-5-5", "gpt-5.6-sol"),
|
||||
)
|
||||
messages: Final = _one_page_pdf_message()
|
||||
caller: Final = UserAPIKeyAuth(
|
||||
api_key=hash_token("sk-pdf-itpm"),
|
||||
project_id="pdf-itpm",
|
||||
project_metadata={"model_itpm_limit": {"gpt-4o": 1_000_000}, "model_otpm_limit": {"gpt-4o": 1_000_000}},
|
||||
)
|
||||
|
||||
await handler.async_pre_call_hook(
|
||||
user_api_key_dict=caller,
|
||||
cache=cache,
|
||||
data={"model": "gpt-4o", "messages": messages, "max_tokens": 10},
|
||||
call_type="acompletion",
|
||||
)
|
||||
|
||||
stash: Final = get_request_stash()
|
||||
assert stash is not None
|
||||
claude_count: Final = handler._estimate_precise_input_tokens({"messages": messages}, "claude-haiku-5-5")
|
||||
assert claude_count > handler._estimate_precise_input_tokens({"messages": messages}, "gpt-4o")
|
||||
assert stash.itpm_reserved_tokens == claude_count
|
||||
|
||||
|
||||
def _extraction_request(state: str, call_id: str) -> dict[str, object]:
|
||||
return {
|
||||
"model": "invoice-reader",
|
||||
|
|
|
|||
|
|
@ -426,3 +426,22 @@ async def test_reservation_holds_cost_against_cached_org_and_project_budgets(spe
|
|||
)
|
||||
def test_budget_window_start_reads_json_encoded_windows(window: str, expected: datetime | None) -> None:
|
||||
assert get_budget_window_start(window) == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reservation_counts_a_pdf_with_the_deployment_behind_a_gpt_named_group() -> None:
|
||||
from litellm.proxy.spend_tracking.budget_reservation import count_request_input_tokens
|
||||
from litellm.proxy.spend_tracking.input_tokens import count_input_tokens_for_model
|
||||
from tests.unit.proxy.spend_tracking.test_input_tokens import _one_page_pdf_message
|
||||
|
||||
request_body: Final = {"model": "gpt-4o", "messages": _one_page_pdf_message()}
|
||||
router: Final = Router(
|
||||
model_list=[{"model_name": "gpt-4o", "litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": "x"}}]
|
||||
)
|
||||
|
||||
counts: Final = await count_request_input_tokens(
|
||||
request_body=request_body, route="/chat/completions", llm_router=router, team_id=None
|
||||
)
|
||||
|
||||
assert counts == {"gpt-4o": count_input_tokens_for_model(request_body=request_body, model="claude-haiku-5-5")}
|
||||
assert counts["gpt-4o"] > count_input_tokens_for_model(request_body=request_body, model="gpt-4o")
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
|
@ -12,7 +13,9 @@ from litellm.proxy.spend_tracking.input_tokens import (
|
|||
TOKENIZE_OFF_EVENT_LOOP_MIN_CHARS,
|
||||
count_input_tokens,
|
||||
count_input_tokens_for_model,
|
||||
models_counted_for,
|
||||
)
|
||||
from litellm.router import Router
|
||||
|
||||
CL100K_MODEL: Final = "gpt-4"
|
||||
|
||||
|
|
@ -30,6 +33,117 @@ async def test_large_input_is_still_counted() -> None:
|
|||
assert isinstance(counts, MappingProxyType)
|
||||
|
||||
|
||||
def _one_page_pdf_message() -> list[dict[str, object]]:
|
||||
stream: Final = b"BT /F1 12 Tf 72 720 Td (Revenue grew eleven percent while churn fell.) Tj ET"
|
||||
bodies: Final = (
|
||||
b"<< /Type /Catalog /Pages 2 0 R >>",
|
||||
b"<< /Type /Pages /Count 1 /Kids [ 4 0 R ] >>",
|
||||
b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
|
||||
b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >> "
|
||||
b"/Contents 5 0 R >>",
|
||||
f"<< /Length {len(stream)} >>\nstream\n".encode() + stream + b"\nendstream",
|
||||
)
|
||||
objects: Final = tuple(f"{n} 0 obj\n".encode() + body + b"\nendobj\n" for n, body in enumerate(bodies, start=1))
|
||||
header: Final = b"%PDF-1.4\n"
|
||||
offsets: Final = tuple(len(header) + sum(len(obj) for obj in objects[:index]) for index in range(len(objects)))
|
||||
xref: Final = f"xref\n0 {len(objects) + 1}\n0000000000 65535 f \n".encode() + b"".join(
|
||||
f"{offset:010d} 00000 n \n".encode() for offset in offsets
|
||||
)
|
||||
trailer: Final = (
|
||||
f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\nstartxref\n{offsets[-1] + len(objects[-1])}\n%%EOF\n"
|
||||
).encode()
|
||||
pdf: Final = base64.b64encode(header + b"".join(objects) + xref + trailer).decode()
|
||||
return [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "file", "file": {"filename": "report.pdf", "file_data": f"data:application/pdf;base64,{pdf}"}},
|
||||
{"type": "text", "text": "Summarize the report."},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def _router(*deployments: tuple[str, str]) -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{"model_name": name, "litellm_params": {"model": model, "api_key": "sk-test", "api_base": "https://x"}}
|
||||
for name, model in deployments
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("group", "deployment_model"),
|
||||
[("gpt-4o", "anthropic/claude-haiku-5-5"), ("pdf-reports", "azure/gpt-5.6-sol")],
|
||||
ids=["gpt-named-group-served-by-claude", "custom-group-served-by-azure-gpt"],
|
||||
)
|
||||
def test_a_model_group_counts_like_the_deployment_that_serves_it(group: str, deployment_model: str) -> None:
|
||||
request_body: Final = {"model": group, "messages": _one_page_pdf_message()}
|
||||
counted_with: Final = deployment_model.partition("/")[2]
|
||||
|
||||
assert models_counted_for(group, _router((group, deployment_model)), None) == (counted_with,)
|
||||
assert count_input_tokens_for_model(request_body=request_body, model=counted_with) != count_input_tokens_for_model(
|
||||
request_body=request_body, model=group
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("group", "deployment_model"),
|
||||
[
|
||||
("my-claude", "anthropic/claude-haiku-5-5"),
|
||||
("my-llama", "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo"),
|
||||
("my-cohere", "cohere/command-r-plus"),
|
||||
("gpt-4o", "cohere/command-r-plus"),
|
||||
],
|
||||
ids=["claude-alias", "llama-alias", "cohere-alias", "gpt-named-group-served-by-a-model-that-reads-no-pdf"],
|
||||
)
|
||||
def test_a_group_whose_deployment_prices_pdf_pages_alike_or_reads_none_counts_as_the_group(
|
||||
group: str, deployment_model: str
|
||||
) -> None:
|
||||
assert models_counted_for(group, _router((group, deployment_model)), None) == (group,)
|
||||
|
||||
|
||||
def test_a_custom_named_deployment_counts_with_its_base_model():
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "reports",
|
||||
"litellm_params": {"model": "azure/prod-deploy-1", "api_key": "x", "api_base": "https://x"},
|
||||
"model_info": {"base_model": "azure/gpt-5.6-sol"},
|
||||
},
|
||||
{"model_name": "drafts", "litellm_params": {"model": "azure/prod-deploy-2", "api_key": "x"}},
|
||||
]
|
||||
)
|
||||
|
||||
assert models_counted_for("reports", router, None) == ("gpt-5.6-sol",)
|
||||
assert models_counted_for("drafts", router, None) == ("drafts",)
|
||||
|
||||
|
||||
def test_a_group_of_differently_priced_deployments_counts_with_each_and_a_name_no_deployment_serves_counts_as_is():
|
||||
router: Final = _router(("mixed", "anthropic/claude-haiku-5-5"), ("mixed", "azure/gpt-5.6-sol"))
|
||||
|
||||
assert models_counted_for("mixed", router, None) == ("mixed", "gpt-5.6-sol")
|
||||
assert models_counted_for("not-deployed", router, None) == ("not-deployed",)
|
||||
assert models_counted_for("gpt-4o", None, None) == ("gpt-4o",)
|
||||
|
||||
|
||||
def test_a_team_request_counts_with_the_deployment_its_team_is_served_by():
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{"model_name": "reports", "litellm_params": {"model": "anthropic/claude-haiku-5-5", "api_key": "x"}},
|
||||
{
|
||||
"model_name": "reports-team-a",
|
||||
"litellm_params": {"model": "azure/gpt-5.6-sol", "api_key": "x"},
|
||||
"model_info": {"team_id": "team-a", "team_public_model_name": "reports"},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
assert models_counted_for("reports", router, "team-a") == ("gpt-5.6-sol",)
|
||||
assert models_counted_for("reports", router, "team-b") == ("reports",)
|
||||
|
||||
|
||||
def test_count_input_tokens_for_model_counts_anthropic_auto_tool_choice() -> None:
|
||||
model: Final = "claude-sonnet-5-5"
|
||||
body_without_tool_choice: Final = {
|
||||
|
|
|
|||
|
|
@ -144,9 +144,10 @@ async def test_each_tokenizer_gets_its_own_cached_counter(fake_tokenizers: None)
|
|||
("gemini/gemini-2.5-pro", "cl100k_base"),
|
||||
("mistral/mistral-large-latest", "cl100k_base"),
|
||||
("my-router-alias", "cl100k_base"),
|
||||
("azure/gpt-4o", "cl100k_base"),
|
||||
("command-r-plus", "cl100k_base"),
|
||||
("gpt-4o", "o200k_base"),
|
||||
("azure/gpt-4o", "o200k_base"),
|
||||
("eu/gpt-5.6-sol", "o200k_base"),
|
||||
("gpt-4o-mini", "o200k_base"),
|
||||
("gpt-4o-2024-08-06", "o200k_base"),
|
||||
("gpt-4.1", "o200k_base"),
|
||||
|
|
|
|||
2
uv.lock
generated
2
uv.lock
generated
|
|
@ -5118,6 +5118,7 @@ admin-mcp = [
|
|||
benchmarks = [
|
||||
{ name = "a2a-sdk" },
|
||||
{ name = "mcp" },
|
||||
{ name = "pypdf" },
|
||||
{ name = "pytest" },
|
||||
{ name = "pytest-codspeed" },
|
||||
]
|
||||
|
|
@ -5347,6 +5348,7 @@ admin-mcp = [{ name = "litellm-admin-mcp", marker = "python_full_version >= '3.1
|
|||
benchmarks = [
|
||||
{ name = "a2a-sdk", specifier = "==1.1.0" },
|
||||
{ name = "mcp", specifier = ">=2.3.0,<3" },
|
||||
{ name = "pypdf", specifier = ">=6.21.2,<7.0" },
|
||||
{ name = "pytest", specifier = "==9.0.3" },
|
||||
{ name = "pytest-codspeed", specifier = "==4.3.0" },
|
||||
]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue