scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-09 22:31:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 133

fix(bedrock): bill Marengo embeddings per request instead of per estimated token

Browse source 
AWS prices Marengo 2.7 and 3.0 text and image embeddings per request, never per
token, and their responses carry no token count. The old transform estimated
prompt tokens from the vector length, which billed a text request at 128 tokens
times the per-token rate (0.00896 instead of 0.00007). Marengo responses now
report zero tokens with query_count and image_count derived from the request
batch, and all six Marengo cost-map entries price per request (with the video
and audio per-second and per-image rates on the base entries). query_count is a
new prompt_tokens_details field wired to input_cost_per_query in the cost
calculator.
This commit is contained in:
mateo-berri 2026-09-07 18:20:28 -07:00
parent e256039077
commit 86790a7723
10 changed files with 240 additions and 83 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -780,6 +780,7 @@ class PromptTokensDetailsResult(TypedDict):
image_count: int
video_length_seconds: float
audio_length_seconds: float
query_count: int
def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
@ -828,6 +829,7 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
)
or 0.0
)
query_count: Final = _coerce_token_count(getattr(usage.prompt_tokens_details, "query_count", 0))
return PromptTokensDetailsResult(
cache_hit_tokens=cache_hit_tokens,
@ -841,6 +843,7 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
image_count=image_count,
video_length_seconds=float(video_length_seconds),
audio_length_seconds=float(audio_length_seconds),
query_count=query_count,
)
@ -978,6 +981,12 @@ def _calculate_input_cost(
prompt_tokens_details["audio_length_seconds"],
)
### QUERY COUNT COST
if prompt_tokens_details["query_count"]:
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_query", prompt_tokens_details["query_count"]
)
return prompt_cost
@ -1149,6 +1158,7 @@ def generic_cost_per_token(
image_count=0,
video_length_seconds=0.0,
audio_length_seconds=0.0,
query_count=0,
)
if usage.prompt_tokens_details:
prompt_tokens_details = parse_prompt_tokens_details(usage)

2
litellm/llms/bedrock/embed/embedding.py
View file

@ -229,7 +229,7 @@ class BedrockEmbedding(BaseAWSLLM):
returned_response = AmazonTitanG1Config()._transform_response(response_list=response_list, model=model)
elif provider == "twelvelabs":
returned_response = TwelveLabsMarengoEmbeddingConfig()._transform_response(
response_list=response_list, model=model
response_list=response_list, model=model, batch_data=batch_data
)
elif provider == "nova":
returned_response = AmazonNovaEmbeddingConfig()._transform_response(

142
litellm/llms/bedrock/embed/twelvelabs_marengo_transformation.py
View file

@ -7,14 +7,19 @@ Docs - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-mar
Marengo 3.0 docs - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-marengo-3.html
"""
from collections.abc import Mapping
from typing import Final, cast
from pydantic import BaseModel, ConfigDict, TypeAdapter
from typing_extensions import assert_never
from litellm.llms.bedrock.embed.twelvelabs_marengo_3_transformation import (
build_marengo_3_request,
is_marengo_3_model,
)
from litellm.types.llms.bedrock import (
TWELVELABS_EMBEDDING_INPUT_TYPES,
TWELVELABS_MARENGO_3_INPUT_TYPES,
TwelveLabsAsyncInvokeRequest,
TwelveLabsMarengo3EmbeddingRequest,
TwelveLabsMarengoEmbeddingRequest,
@ -22,7 +27,76 @@ from litellm.types.llms.bedrock import (
TwelveLabsS3Location,
TwelveLabsS3OutputDataConfig,
)
from litellm.types.utils import Embedding, EmbeddingResponse, Usage
from litellm.types.utils import Embedding, EmbeddingResponse, PromptTokensDetailsWrapper, Usage
class MarengoEmbeddingItem(BaseModel):
model_config = ConfigDict(extra="ignore", frozen=True)
embedding: tuple[float, ...]
class MarengoInvokeResponse(BaseModel):
model_config = ConfigDict(extra="ignore", frozen=True)
data: tuple[MarengoEmbeddingItem, ...] = ()
embedding: tuple[float, ...] | None = None
embeddings: tuple[MarengoEmbeddingItem, ...] = ()
def vectors(self) -> tuple[tuple[float, ...], ...]:
if self.data:
return tuple(item.embedding for item in self.data)
if self.embedding is not None:
return (self.embedding,)
return tuple(item.embedding for item in self.embeddings)
class MarengoBilledMultiInput(BaseModel):
model_config = ConfigDict(extra="ignore", frozen=True)
inputText: str | None = None
mediaSources: tuple[Mapping[str, object], ...] = ()
class MarengoBilledRequest(BaseModel):
model_config = ConfigDict(extra="ignore", frozen=True)
inputType: TWELVELABS_MARENGO_3_INPUT_TYPES | None = None
multi_input: MarengoBilledMultiInput | None = None
INVOKE_RESPONSES: Final = TypeAdapter(tuple[MarengoInvokeResponse, ...])
BILLED_REQUESTS: Final = TypeAdapter(tuple[MarengoBilledRequest, ...])
def _billed_units(request: MarengoBilledRequest) -> tuple[int, int]:
input_type: Final = request.inputType
match input_type:
case "text":
return (1, 0)
case "image":
return (0, 1)
case "text_image":
return (1, 1)
case "multi_input":
multi_input: Final = request.multi_input or MarengoBilledMultiInput()
return (1 if multi_input.inputText else 0, len(multi_input.mediaSources))
case "video" | "audio" | None:
return (0, 0)
case _:
assert_never(input_type)
def _billed_usage(batch_data: list[dict] | None) -> Usage:
units: Final = tuple(_billed_units(request) for request in BILLED_REQUESTS.validate_python(batch_data or ()))
query_count: Final = sum(text_requests for text_requests, _ in units)
image_count: Final = sum(images for _, images in units)
details: Final = (
PromptTokensDetailsWrapper(query_count=query_count or None, image_count=image_count or None)
if query_count or image_count
else None
)
return Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0, prompt_tokens_details=details)
class TwelveLabsMarengoEmbeddingConfig:
@ -223,62 +297,16 @@ class TwelveLabsMarengoEmbeddingConfig:
),
)
def _transform_response(self, response_list: list[dict], model: str) -> EmbeddingResponse:
"""
Transform TwelveLabs response to OpenAI format.
Handles the actual TwelveLabs response format: {"data": [{"embedding": [...]}]}
"""
embeddings: Final[list[Embedding]] = []
total_tokens = 0
for response in response_list:
# TwelveLabs response format has a "data" field containing the embeddings
if "data" in response and isinstance(response["data"], list):
for item in response["data"]:
if "embedding" in item:
# Single embedding response
embedding = Embedding(
embedding=item["embedding"],
index=len(embeddings),
object="embedding",
)
embeddings.append(embedding)
# Estimate token count (rough approximation)
if "inputTextTokenCount" in item:
total_tokens += item["inputTextTokenCount"]
else:
# Rough estimate: 1 token per 4 characters for text, or use embedding size
total_tokens += len(item["embedding"]) // 4
elif "embedding" in response:
# Direct embedding response (fallback for other formats)
embedding = Embedding(
embedding=response["embedding"],
index=len(embeddings),
object="embedding",
)
embeddings.append(embedding)
# Estimate token count (rough approximation)
if "inputTextTokenCount" in response:
total_tokens += response["inputTextTokenCount"]
else:
# Rough estimate: 1 token per 4 characters for text
total_tokens += len(response.get("inputText", "")) // 4
elif "embeddings" in response:
# Multiple embeddings response (from video/audio)
for i, emb in enumerate(response["embeddings"]):
embedding = Embedding(
embedding=emb["embedding"],
index=len(embeddings),
object="embedding",
)
embeddings.append(embedding)
total_tokens += len(emb["embedding"]) // 4 # Rough estimate
usage: Final = Usage(prompt_tokens=total_tokens, total_tokens=total_tokens)
return EmbeddingResponse(data=embeddings, model=model, usage=usage)
def _transform_response(
self, response_list: list[dict], model: str, batch_data: list[dict] | None = None
) -> EmbeddingResponse:
vectors: Final = tuple(
vector for response in INVOKE_RESPONSES.validate_python(response_list) for vector in response.vectors()
)
embeddings: Final = [
Embedding(embedding=list(vector), index=index, object="embedding") for index, vector in enumerate(vectors)
]
return EmbeddingResponse(data=embeddings, model=model, usage=_billed_usage(batch_data))
def _transform_async_invoke_response(self, response: dict, model: str) -> EmbeddingResponse:
"""

18
litellm/model_prices_and_context_window_backup.json
View file

@ -650,7 +650,10 @@
},
"twelvelabs.marengo-embed-2-7-v1:0": {
"deprecation_date": "2026-11-30",
"input_cost_per_token": 7e-05,
"input_cost_per_query": 7e-05,
"input_cost_per_video_per_second": 0.0007,
"input_cost_per_audio_per_second": 0.00014,
"input_cost_per_image": 0.0001,
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"max_tokens": 77,
@ -662,7 +665,7 @@
},
"us.twelvelabs.marengo-embed-2-7-v1:0": {
"deprecation_date": "2026-11-30",
"input_cost_per_token": 7e-05,
"input_cost_per_query": 7e-05,
"input_cost_per_video_per_second": 0.0007,
"input_cost_per_audio_per_second": 0.00014,
"input_cost_per_image": 0.0001,
@ -677,7 +680,7 @@
},
"eu.twelvelabs.marengo-embed-2-7-v1:0": {
"deprecation_date": "2026-11-30",
"input_cost_per_token": 7e-05,
"input_cost_per_query": 7e-05,
"input_cost_per_video_per_second": 0.0007,
"input_cost_per_audio_per_second": 0.00014,
"input_cost_per_image": 0.0001,
@ -691,7 +694,10 @@
"supports_image_input": true
},
"twelvelabs.marengo-embed-3-0-v1:0": {
"input_cost_per_token": 7e-05,
"input_cost_per_query": 7e-05,
"input_cost_per_video_per_second": 0.0007,
"input_cost_per_audio_per_second": 0.00014,
"input_cost_per_image": 0.0001,
"litellm_provider": "bedrock",
"max_input_tokens": 500,
"max_tokens": 500,
@ -702,7 +708,7 @@
"supports_image_input": true
},
"us.twelvelabs.marengo-embed-3-0-v1:0": {
"input_cost_per_token": 7e-05,
"input_cost_per_query": 7e-05,
"input_cost_per_video_per_second": 0.0007,
"input_cost_per_audio_per_second": 0.00014,
"input_cost_per_image": 0.0001,
@ -716,7 +722,7 @@
"supports_image_input": true
},
"eu.twelvelabs.marengo-embed-3-0-v1:0": {
"input_cost_per_token": 7e-05,
"input_cost_per_query": 7e-05,
"input_cost_per_video_per_second": 0.0007,
"input_cost_per_audio_per_second": 0.00014,
"input_cost_per_image": 0.0001,

7
litellm/types/utils.py
View file

@ -272,7 +272,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
input_cost_per_token_above_272k_tokens_flex: float | None
input_cost_per_token_above_512k_tokens: float | None # MiniMax-M3: prompts >512K priced at 2x input
input_cost_per_character_above_128k_tokens: float | None # only for vertex ai models
input_cost_per_query: float | None # only for rerank models
input_cost_per_query: float | None # per-request pricing: rerank, search, and Bedrock Marengo embeddings
input_cost_per_image: float | None # only for vertex ai models
input_cost_per_image_token: float | None # for gpt-image-1 and similar models
input_cost_per_video_token: float | None # for gemini omni models with video input
@ -1693,6 +1693,9 @@ class PromptTokensDetailsWrapper(
audio_length_seconds: float | None = None
"""Length of audio sent to the model. Used for multimodal embeddings priced per audio-second."""
query_count: int | None = None
"""Number of billable requests sent to the model. Used for embeddings priced per request, such as Bedrock Marengo."""
cache_write_tokens: int | None = None
"""Number of cache write (creation) tokens sent to the model. OpenAI naming (prompt_tokens_details.cache_write_tokens); this is the canonical field."""
@ -1734,6 +1737,8 @@ class PromptTokensDetailsWrapper(
del self.video_length_seconds
if self.audio_length_seconds is None:
del self.audio_length_seconds
if self.query_count is None:
del self.query_count
if self.web_search_requests is None:
del self.web_search_requests
if self.google_maps_grounding_requests is None:

2
litellm/utils.py
View file

@ -6033,7 +6033,7 @@ def get_model_info(
input_cost_per_character_above_128k_tokens: Optional[
float
] # only for vertex ai models
input_cost_per_query: Optional[float] # only for rerank models
input_cost_per_query: Optional[float] # per-request pricing: rerank, search, and Bedrock Marengo embeddings
input_cost_per_image: Optional[float] # only for vertex ai models
input_cost_per_audio_token: Optional[float]
input_cost_per_audio_per_second: Optional[float] # only for vertex ai models

18
model_prices_and_context_window.json
View file

@ -650,7 +650,10 @@
},
"twelvelabs.marengo-embed-2-7-v1:0": {
"deprecation_date": "2026-11-30",
"input_cost_per_token": 7e-05,
"input_cost_per_query": 7e-05,
"input_cost_per_video_per_second": 0.0007,
"input_cost_per_audio_per_second": 0.00014,
"input_cost_per_image": 0.0001,
"litellm_provider": "bedrock",
"max_input_tokens": 77,
"max_tokens": 77,
@ -662,7 +665,7 @@
},
"us.twelvelabs.marengo-embed-2-7-v1:0": {
"deprecation_date": "2026-11-30",
"input_cost_per_token": 7e-05,
"input_cost_per_query": 7e-05,
"input_cost_per_video_per_second": 0.0007,
"input_cost_per_audio_per_second": 0.00014,
"input_cost_per_image": 0.0001,
@ -677,7 +680,7 @@
},
"eu.twelvelabs.marengo-embed-2-7-v1:0": {
"deprecation_date": "2026-11-30",
"input_cost_per_token": 7e-05,
"input_cost_per_query": 7e-05,
"input_cost_per_video_per_second": 0.0007,
"input_cost_per_audio_per_second": 0.00014,
"input_cost_per_image": 0.0001,
@ -691,7 +694,10 @@
"supports_image_input": true
},
"twelvelabs.marengo-embed-3-0-v1:0": {
"input_cost_per_token": 7e-05,
"input_cost_per_query": 7e-05,
"input_cost_per_video_per_second": 0.0007,
"input_cost_per_audio_per_second": 0.00014,
"input_cost_per_image": 0.0001,
"litellm_provider": "bedrock",
"max_input_tokens": 500,
"max_tokens": 500,
@ -702,7 +708,7 @@
"supports_image_input": true
},
"us.twelvelabs.marengo-embed-3-0-v1:0": {
"input_cost_per_token": 7e-05,
"input_cost_per_query": 7e-05,
"input_cost_per_video_per_second": 0.0007,
"input_cost_per_audio_per_second": 0.00014,
"input_cost_per_image": 0.0001,
@ -716,7 +722,7 @@
"supports_image_input": true
},
"eu.twelvelabs.marengo-embed-3-0-v1:0": {
"input_cost_per_token": 7e-05,
"input_cost_per_query": 7e-05,
"input_cost_per_video_per_second": 0.0007,
"input_cost_per_audio_per_second": 0.00014,
"input_cost_per_image": 0.0001,

32
tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
View file

@ -2658,6 +2658,7 @@ def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details():
"image_count": 0,
"video_length_seconds": 0.0,
"audio_length_seconds": 0.0,
"query_count": 0,
}
model_info: ModelInfo = {}
@ -3239,6 +3240,37 @@ def test_image_count_prevents_text_tokens_fallback(_local_model_cost_map):
assert completion_cost == 0.0
def test_query_count_bills_input_cost_per_query(_local_model_cost_map):
usage = Usage(
prompt_tokens=0,
completion_tokens=0,
total_tokens=0,
prompt_tokens_details=PromptTokensDetailsWrapper(query_count=3, image_count=1),
)
prompt_cost, completion_cost = generic_cost_per_token(
model="us.twelvelabs.marengo-embed-3-0-v1:0",
usage=usage,
custom_llm_provider="bedrock",
)
assert prompt_cost == pytest.approx(3 * 7e-05 + 1e-04)
assert completion_cost == 0.0
def test_query_count_is_free_without_a_per_query_price(_local_model_cost_map):
usage = Usage(
prompt_tokens=0,
completion_tokens=0,
total_tokens=0,
prompt_tokens_details=PromptTokensDetailsWrapper(query_count=1),
)
prompt_cost, _ = generic_cost_per_token(model="text-embedding-3-small", usage=usage, custom_llm_provider="openai")
assert prompt_cost == 0.0
# ---------------------------------------------------------------------------
# Data-residency (OpenAI regional processing) tests
# ---------------------------------------------------------------------------

47
tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py
View file

@ -5,6 +5,7 @@ from unittest.mock import Mock, patch
import pytest
import litellm
from litellm.llms.bedrock.embed.twelvelabs_marengo_transformation import TwelveLabsMarengoEmbeddingConfig
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
# Mock responses for different embedding models
@ -1066,17 +1067,19 @@ MARENGO_3_DUCK = "data:image/png;base64,ZHVjaw=="
@pytest.mark.parametrize(
"model,kwargs,expected_body",
"model,kwargs,expected_body,expected_usage_details",
[
(
"bedrock/us.twelvelabs.marengo-embed-3-0-v1:0",
{"input_type": "text"},
{"inputType": "text", "text": {"inputText": "a duck on water"}},
{"query_count": 1},
),
(
"bedrock/twelvelabs.marengo-embed-3-0-v1:0",
{"input_type": "text"},
{"inputType": "text", "text": {"inputText": "a duck on water"}},
{"query_count": 1},
),
(
"bedrock/us.twelvelabs.marengo-embed-3-0-v1:0",
@ -1085,6 +1088,7 @@ MARENGO_3_DUCK = "data:image/png;base64,ZHVjaw=="
"inputType": "text_image",
"text_image": {"inputText": "a duck on water", "mediaSource": {"base64String": "ZHVjaw=="}},
},
{"query_count": 1, "image_count": 1},
),
(
"bedrock/us.twelvelabs.marengo-embed-3-0-v1:0",
@ -1096,10 +1100,13 @@ MARENGO_3_DUCK = "data:image/png;base64,ZHVjaw=="
"mediaSources": [{"name": "bird", "mediaType": "image", "base64String": "ZHVjaw=="}],
},
},
{"query_count": 1, "image_count": 1},
),
],
)
def test_marengo_3_embedding_sends_the_nested_payload_and_parses_512_dims(model, kwargs, expected_body):
def test_marengo_3_embedding_sends_the_nested_payload_and_parses_512_dims(
model, kwargs, expected_body, expected_usage_details
):
client = HTTPHandler()
with patch.object(client, "post") as mock_post:
@ -1122,7 +1129,9 @@ def test_marengo_3_embedding_sends_the_nested_payload_and_parses_512_dims(model,
assert mock_post.call_args.kwargs["url"].endswith(f"/model/{model.removeprefix('bedrock/').replace(':', '%3A')}/invoke")
assert len(response.data[0]["embedding"]) == 512
assert response.data[0]["embedding"][:2] == [0.0, 0.01]
assert response.usage.prompt_tokens == 128
assert response.usage.prompt_tokens == 0
assert response.usage.total_tokens == 0
assert response.usage.prompt_tokens_details.model_dump(exclude_none=True) == expected_usage_details
def test_marengo_3_image_embedding_sends_the_media_under_the_image_key():
@ -1150,6 +1159,8 @@ def test_marengo_3_image_embedding_sends_the_media_under_the_image_key():
}
assert len(response.data[0]["embedding"]) == 512
assert response.data[0]["embedding"][:2] == [0.0, 0.01]
assert response.usage.prompt_tokens == 0
assert response.usage.prompt_tokens_details.model_dump(exclude_none=True) == {"image_count": 1}
def test_marengo_2_7_embedding_keeps_the_flat_payload():
@ -1177,6 +1188,36 @@ def test_marengo_2_7_embedding_keeps_the_flat_payload():
"textTruncate": "end",
}
assert response.data[0]["embedding"] == [0.1, 0.2, 0.3]
assert response.usage.prompt_tokens == 0
assert response.usage.prompt_tokens_details.model_dump(exclude_none=True) == {"query_count": 1}
def test_marengo_usage_counts_text_requests_and_images_across_a_batch():
duck = {"mediaType": "image", "base64String": "ZHVjaw=="}
response = TwelveLabsMarengoEmbeddingConfig()._transform_response(
response_list=[marengo_3_embedding_response, marengo_3_embedding_response, marengo_3_embedding_response],
model="us.twelvelabs.marengo-embed-3-0-v1:0",
batch_data=[
{"inputType": "text", "text": {"inputText": "a duck"}},
{"inputType": "image", "image": {"mediaSource": {"base64String": "ZHVjaw=="}}},
{"inputType": "multi_input", "multi_input": {"mediaSources": [{"name": "a", **duck}, {"name": "b", **duck}]}},
],
)
assert [item["index"] for item in response.data] == [0, 1, 2]
assert response.usage.prompt_tokens == 0
assert response.usage.total_tokens == 0
assert response.usage.prompt_tokens_details.model_dump(exclude_none=True) == {"query_count": 1, "image_count": 3}
def test_marengo_usage_without_request_data_bills_nothing():
response = TwelveLabsMarengoEmbeddingConfig()._transform_response(
response_list=[marengo_3_embedding_response], model="us.twelvelabs.marengo-embed-3-0-v1:0"
)
assert len(response.data[0]["embedding"]) == 512
assert response.usage.prompt_tokens == 0
assert response.usage.prompt_tokens_details is None
def test_marengo_3_text_image_without_media_source_is_a_bad_request():

45
tests/test_litellm/test_bedrock_marengo_embed_3_model_metadata.py
View file

@ -6,7 +6,7 @@ import pytest
import litellm
from litellm.constants import bedrock_embedding_models
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.types.utils import Usage
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
REPO_ROOT = Path(__file__).parents[2]
MAIN_PATH = REPO_ROOT / "model_prices_and_context_window.json"
@ -15,6 +15,12 @@ BACKUP_PATH = REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.js
BASE_MODEL = "twelvelabs.marengo-embed-3-0-v1:0"
PROFILE_MODELS = ("us.twelvelabs.marengo-embed-3-0-v1:0", "eu.twelvelabs.marengo-embed-3-0-v1:0")
ALL_MODELS = (BASE_MODEL, *PROFILE_MODELS)
MARENGO_2_7_MODELS = (
"twelvelabs.marengo-embed-2-7-v1:0",
"us.twelvelabs.marengo-embed-2-7-v1:0",
"eu.twelvelabs.marengo-embed-2-7-v1:0",
)
PER_REQUEST_MODELS = (*ALL_MODELS, *MARENGO_2_7_MODELS)
TEXT_REQUEST_COST = 7e-05
IMAGE_REQUEST_COST = 0.0001
@ -34,7 +40,7 @@ def test_marengo_embed_3_specs(model):
assert info["litellm_provider"] == "bedrock"
assert info["mode"] == "embedding"
assert info["input_cost_per_token"] == TEXT_REQUEST_COST
assert info["input_cost_per_query"] == TEXT_REQUEST_COST
assert info["output_cost_per_token"] == 0.0
assert info["max_input_tokens"] == 500
assert info["max_tokens"] == 500
@ -48,9 +54,11 @@ def test_marengo_embed_3_specs(model):
assert provider == "bedrock"
@pytest.mark.parametrize("model", PROFILE_MODELS)
def test_marengo_embed_3_inference_profiles_price_image_video_and_audio(model):
@pytest.mark.parametrize("model", PER_REQUEST_MODELS)
def test_marengo_prices_are_per_request_not_per_token(model):
info = _load(MAIN_PATH)[model]
assert "input_cost_per_token" not in info
assert info["input_cost_per_query"] == TEXT_REQUEST_COST
assert info["input_cost_per_image"] == IMAGE_REQUEST_COST
assert info["input_cost_per_video_per_second"] == VIDEO_COST_PER_SECOND
assert info["input_cost_per_audio_per_second"] == AUDIO_COST_PER_SECOND
@ -64,13 +72,34 @@ def test_marengo_embed_3_is_visible_to_callers(model, local_model_cost_map):
assert info["max_input_tokens"] == 500
@pytest.mark.parametrize("model", ALL_MODELS)
def test_marengo_embed_3_text_request_is_billed(model, local_model_cost_map):
@pytest.mark.parametrize("model", PER_REQUEST_MODELS)
@pytest.mark.parametrize(
"details,expected_cost",
[
(PromptTokensDetailsWrapper(query_count=1), TEXT_REQUEST_COST),
(PromptTokensDetailsWrapper(image_count=1), IMAGE_REQUEST_COST),
(PromptTokensDetailsWrapper(query_count=1, image_count=1), TEXT_REQUEST_COST + IMAGE_REQUEST_COST),
(PromptTokensDetailsWrapper(query_count=1, image_count=2), TEXT_REQUEST_COST + 2 * IMAGE_REQUEST_COST),
(PromptTokensDetailsWrapper(video_length_seconds=10), 10 * VIDEO_COST_PER_SECOND),
(PromptTokensDetailsWrapper(audio_length_seconds=10), 10 * AUDIO_COST_PER_SECOND),
],
)
def test_marengo_requests_are_billed_per_request(model, details, expected_cost, local_model_cost_map):
usage = Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0, prompt_tokens_details=details)
prompt_cost, completion_cost = litellm.cost_per_token(
model=model, usage_object=usage, custom_llm_provider="bedrock"
)
assert prompt_cost == pytest.approx(expected_cost)
assert completion_cost == 0.0
@pytest.mark.parametrize("model", PER_REQUEST_MODELS)
def test_marengo_token_counts_bill_nothing(model, local_model_cost_map):
usage = Usage(prompt_tokens=128, completion_tokens=0, total_tokens=128)
prompt_cost, completion_cost = litellm.cost_per_token(
model=model, usage_object=usage, custom_llm_provider="bedrock"
)
assert prompt_cost == pytest.approx(128 * TEXT_REQUEST_COST)
assert prompt_cost == 0.0
assert completion_cost == 0.0
@ -78,7 +107,7 @@ def test_marengo_embed_3_is_a_known_bedrock_embedding_model():
assert BASE_MODEL in bedrock_embedding_models
@pytest.mark.parametrize("model", ALL_MODELS)
@pytest.mark.parametrize("model", PER_REQUEST_MODELS)
def test_backup_matches_main(model):
main_cost = _load(MAIN_PATH)
backup_cost = _load(BACKUP_PATH)