scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-24 00:52:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 91

test(integration): audio, image and per-unit cost cases

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
kerry 2026-09-19 21:19:29 +00:00
parent 9bd648baf5
commit c0c5cc84f8
7 changed files with 836 additions and 73 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
tests/integration/_support/client.py
View file

@ -67,6 +67,21 @@ class Gateway:
headers={"Authorization": f"Bearer {self.key if key is None else key}"},
)
def request_multipart(
self,
path: str,
fields: Mapping[str, str],
files: Mapping[str, tuple[str, bytes, str]],
*,
key: str | None = None,
) -> httpx.Response:
return self.client.post(
path,
data=fields,
files=files,
headers={"Authorization": f"Bearer {self.key if key is None else key}"},
)
def post(self, path: str, body: Mapping[str, JsonValue], *, key: str | None = None) -> dict[str, JsonValue]:
response: Final = self.request("POST", path, body, key=key)
assert response.status_code == 200, f"POST {path}: {response.status_code} {response.text}"

6
tests/integration/_support/upstream.py
View file

@ -22,6 +22,7 @@ from starlette.routing import Route
from _fake_openai_endpoint_server import chat_completions, completions, embeddings, health, moderations
from integration.cost_calculation.cost_tracking_case import (
BinaryResponse,
EventStreamResponse,
JsonResponse,
SseResponse,
@ -212,6 +213,11 @@ class Provider:
media_type=response.content_type,
status_code=response.status,
)
case BinaryResponse():
return Response(
content=b"\x00" * response.length,
media_type=response.content_type,
)
case SseResponse():
stream_body: Final = ("\n\n".join(response.frames) + "\n\n").replace(
"$REQUEST_ID", scenario_id

48
tests/integration/contracts.json
View file

@ -1320,6 +1320,54 @@
],
"tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_full_usage]": [
"quota_management.spend_tracking.scripted_wire.logs_cost"
],
"tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[whisper-next-transcriptions-per-second]": [
"quota_management.spend_tracking.cost_matrix.logs_cost"
],
"tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[whisper-verbose-next-transcriptions-duration]": [
"quota_management.spend_tracking.cost_matrix.logs_cost"
],
"tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-4o-transcribe-next-transcriptions-tokens]": [
"quota_management.spend_tracking.cost_matrix.logs_cost"
],
"tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[nova-next-transcriptions-per-second]": [
"quota_management.spend_tracking.cost_matrix.logs_cost"
],
"tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-whisper-next-transcriptions-deployment]": [
"quota_management.spend_tracking.cost_matrix.logs_cost"
],
"tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[tts-next-speech-per-character]": [
"quota_management.spend_tracking.cost_matrix.logs_cost"
],
"tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[tts-next-hd-speech-per-character]": [
"quota_management.spend_tracking.cost_matrix.logs_cost"
],
"tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-tts-next-speech-deployment]": [
"quota_management.spend_tracking.cost_matrix.logs_cost"
],
"tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dall-e-3-next-images-standard]": [
"quota_management.spend_tracking.cost_matrix.logs_cost"
],
"tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dall-e-3-next-images-hd]": [
"quota_management.spend_tracking.cost_matrix.logs_cost"
],
"tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dall-e-3-next-images-wide]": [
"quota_management.spend_tracking.cost_matrix.logs_cost"
],
"tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dall-e-3-next-images-two]": [
"quota_management.spend_tracking.cost_matrix.logs_cost"
],
"tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-image-next-images-low]": [
"quota_management.spend_tracking.cost_matrix.logs_cost"
],
"tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[imagen-next-images-one]": [
"quota_management.spend_tracking.cost_matrix.logs_cost"
],
"tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[amazon-nova-canvas-next-images-one]": [
"quota_management.spend_tracking.cost_matrix.logs_cost"
],
"tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-image-next-images-edit]": [
"quota_management.spend_tracking.cost_matrix.logs_cost"
]
},
"browser": {

11
tests/integration/cost_calculation/conftest.py
View file

@ -45,9 +45,8 @@ class CostRow(BaseModel):
metadata: CostMetadata | None = None
@property
def breakdown(self) -> CostBreakdown:
assert self.metadata is not None and self.metadata.cost_breakdown is not None
return self.metadata.cost_breakdown
def breakdown(self) -> CostBreakdown | None:
return self.metadata.cost_breakdown if self.metadata is not None else None
class FailureRow(BaseModel):
@ -65,6 +64,8 @@ def approx_equal(actual: float, expected: float) -> bool:
def assert_total_is_sum_of_components(row: CostRow, context: str) -> None:
breakdown: Final = row.breakdown
if breakdown is None:
return
total: Final = sum(
cost or 0.0
for cost in (breakdown.input_cost, breakdown.output_cost, breakdown.tool_usage_cost)
@ -83,7 +84,7 @@ def _row(value: Mapping[str, object]) -> CostRow | None:
metadata_value: Final = value.get("metadata")
metadata: Final = json.loads(metadata_value) if isinstance(metadata_value, str) else metadata_value
parsed: Final = CostRow.model_validate({**value, "metadata": metadata})
return parsed if parsed.metadata and parsed.metadata.cost_breakdown else None
return parsed
def poll_cost_row(key: str) -> CostRow:
@ -165,7 +166,7 @@ def register_scenario_deployment(
**case.litellm_params,
**(
{"vertex_credentials": _vertex_service_account_json(control_url)}
if case.rates.litellm_provider == "vertex_ai-language-models"
if case.rates.litellm_provider.startswith("vertex_ai")
else {}
),
}

54
tests/integration/cost_calculation/cost_tracking_case.py
View file

@ -43,8 +43,15 @@ class CostMapEntry(BaseModel):
cache_creation_input_token_cost_above_200k_tokens: float | None = None
output_cost_per_reasoning_token: float | None = None
input_cost_per_audio_token: float | None = None
input_cost_per_second: float | None = None
output_cost_per_second: float | None = None
input_cost_per_character: float | None = None
output_cost_per_character: float | None = None
input_cost_per_image: float | None = None
output_cost_per_image: float | None = None
output_cost_per_audio_token: float | None = None
input_cost_per_image_token: float | None = None
output_cost_per_image_token: float | None = None
input_cost_per_video_token: float | None = None
input_cost_per_token_above_200k_tokens: float | None = None
output_cost_per_token_above_200k_tokens: float | None = None
@ -66,6 +73,22 @@ class Deployment(BaseModel):
base_model: str | None = None
class WavUpload(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
kind: Literal["wav"]
seconds: float
class PngUpload(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
kind: Literal["png"]
Upload: TypeAlias = Annotated[WavUpload | PngUpload, Field(discriminator="kind")]
class JsonResponse(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
@ -95,8 +118,15 @@ class EventStreamResponse(BaseModel):
events: tuple[EventStreamEvent, ...]
class BinaryResponse(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
content_type: Literal["audio/mpeg"]
length: int
StoredResponse: TypeAlias = Annotated[
JsonResponse | SseResponse | EventStreamResponse,
JsonResponse | SseResponse | EventStreamResponse | BinaryResponse,
Field(discriminator="content_type"),
]
@ -157,8 +187,13 @@ class CostTrackingTestCase(BaseModel):
"/v1/rerank",
"/v1/completions",
"/v1/moderations",
"/v1/audio/transcriptions",
"/v1/audio/speech",
"/v1/images/generations",
"/v1/images/edits",
] = "/v1/chat/completions"
deployment: Deployment | None = None
upload: Upload | None = None
request: dict[str, JsonValue]
response: StoredResponse
expected: Expected
@ -172,7 +207,8 @@ class CostTrackingTestCase(BaseModel):
provider: Final = self.rates.litellm_provider
prefix: Final = (
"openai"
if provider == "openai" and self.rates.mode == "chat"
if provider == "openai"
and self.rates.mode in {"chat", "audio_transcription", "audio_speech", "image_generation"}
else "openai/responses"
if provider == "openai"
else _PROVIDER_PREFIXES.get(provider)
@ -206,8 +242,11 @@ class _CasesFile(BaseModel):
_PROVIDER_PREFIXES: Final[Mapping[str, str]] = MappingProxyType(
{
"anthropic": "anthropic",
"bedrock": "bedrock",
"bedrock_converse": "bedrock/converse",
"deepgram": "deepgram",
"vertex_ai-language-models": "vertex_ai",
"vertex_ai-image-models": "vertex_ai",
"gemini": "",
"together_ai": "",
"fireworks_ai": "",
@ -217,6 +256,13 @@ _PROVIDER_PREFIXES: Final[Mapping[str, str]] = MappingProxyType(
_LITELLM_PARAMS: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType(
{
"anthropic": MappingProxyType({}),
"bedrock": MappingProxyType(
{
"aws_access_key_id": "AKIASCRIPTEDPROVIDER",
"aws_secret_access_key": "scripted-secret",
"aws_region_name": "us-east-1",
}
),
"bedrock_converse": MappingProxyType(
{
"aws_access_key_id": "AKIASCRIPTEDPROVIDER",
@ -224,9 +270,13 @@ _LITELLM_PARAMS: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType(
"aws_region_name": "us-east-1",
}
),
"deepgram": MappingProxyType({}),
"vertex_ai-language-models": MappingProxyType(
{"vertex_project": "cc-scripted-project", "vertex_location": "us-central1"}
),
"vertex_ai-image-models": MappingProxyType(
{"vertex_project": "cc-scripted-project", "vertex_location": "us-central1"}
),
"gemini": MappingProxyType({}),
"together_ai": MappingProxyType({}),
"fireworks_ai": MappingProxyType({}),

589
tests/integration/cost_calculation/cost_tracking_cases.json
View file

@ -408,6 +408,89 @@
"mode": "chat",
"output_cost_per_token": 3.6e-06,
"supports_function_calling": true
},
"whisper-next": {
"litellm_provider": "openai",
"mode": "audio_transcription",
"input_cost_per_second": 0.0001
},
"whisper-verbose-next": {
"litellm_provider": "openai",
"mode": "audio_transcription",
"input_cost_per_second": 0.0002
},
"gpt-4o-transcribe-next": {
"litellm_provider": "openai",
"mode": "audio_transcription",
"input_cost_per_token": 2.11e-06,
"output_cost_per_token": 3.11e-06,
"input_cost_per_audio_token": 1e-05
},
"nova-next": {
"litellm_provider": "deepgram",
"mode": "audio_transcription",
"input_cost_per_second": 0.0003
},
"azure/whisper-next": {
"litellm_provider": "azure",
"mode": "audio_transcription",
"input_cost_per_second": 0.00011
},
"tts-next": {
"litellm_provider": "openai",
"mode": "audio_speech",
"input_cost_per_character": 1e-05
},
"tts-next-hd": {
"litellm_provider": "openai",
"mode": "audio_speech",
"input_cost_per_character": 2e-05
},
"azure/tts-next": {
"litellm_provider": "azure",
"mode": "audio_speech",
"input_cost_per_character": 1.1e-05
},
"gpt-image-next": {
"litellm_provider": "openai",
"mode": "image_generation",
"input_cost_per_token": 1.71e-06,
"output_cost_per_token": 4.3e-06,
"input_cost_per_image_token": 2.2e-06,
"output_cost_per_image_token": 5.1e-06
},
"1024-x-1024/dall-e-3-next": {
"litellm_provider": "openai",
"mode": "image_generation",
"input_cost_per_image": 0.04
},
"hd/1024-x-1024/dall-e-3-next": {
"litellm_provider": "openai",
"mode": "image_generation",
"input_cost_per_image": 0.08
},
"1792-x-1024/dall-e-3-next": {
"litellm_provider": "openai",
"mode": "image_generation",
"input_cost_per_image": 0.06
},
"low/1024-x-1024/gpt-image-next": {
"litellm_provider": "openai",
"mode": "image_generation",
"input_cost_per_token": 1.7e-06,
"output_cost_per_token": 4.3e-06,
"input_cost_per_image_token": 2.2e-06,
"output_cost_per_image_token": 5.1e-06
},
"1024-x-1024/imagen-next": {
"litellm_provider": "vertex_ai-image-models",
"mode": "image_generation",
"output_cost_per_image": 0.05
},
"amazon.nova-canvas-next": {
"litellm_provider": "bedrock",
"mode": "image_generation",
"output_cost_per_image": 0.045
}
},
"cases": [
@ -25777,6 +25860,512 @@
"prompt_tokens": 11056,
"completion_tokens": 412
}
},
{
"name": "whisper-next-transcriptions-per-second",
"covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
"model": "whisper-next",
"endpoint": "/v1/audio/transcriptions",
"upload": {
"kind": "wav",
"seconds": 3.5
},
"request": {
"language": "en",
"response_format": "json"
},
"response": {
"content_type": "application/json",
"body": {
"text": "hello"
}
},
"expected": {
"spend": 0.00035,
"input_cost": 0.00035,
"output_cost": 0,
"prompt_tokens": 0,
"completion_tokens": 0
}
},
{
"name": "whisper-verbose-next-transcriptions-duration",
"covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
"model": "whisper-verbose-next",
"endpoint": "/v1/audio/transcriptions",
"upload": {
"kind": "wav",
"seconds": 3.5
},
"request": {
"response_format": "verbose_json"
},
"response": {
"content_type": "application/json",
"body": {
"text": "hello",
"duration": 12.25
}
},
"expected": {
"spend": 0.00245,
"input_cost": 0.00245,
"output_cost": 0,
"prompt_tokens": 0,
"completion_tokens": 0
}
},
{
"name": "gpt-4o-transcribe-next-transcriptions-tokens",
"covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
"model": "gpt-4o-transcribe-next",
"endpoint": "/v1/audio/transcriptions",
"upload": {
"kind": "wav",
"seconds": 1.0
},
"request": {
"response_format": "json"
},
"response": {
"content_type": "application/json",
"body": {
"text": "hello",
"usage": {
"type": "tokens",
"input_tokens": 10,
"output_tokens": 2,
"total_tokens": 12,
"input_token_details": {
"text_tokens": 2,
"audio_tokens": 8
}
}
}
},
"expected": {
"spend": 9.044e-05,
"input_cost": 8.422e-05,
"output_cost": 6.22e-06,
"prompt_tokens": 10,
"completion_tokens": 2
}
},
{
"name": "nova-next-transcriptions-per-second",
"covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
"model": "nova-next",
"endpoint": "/v1/audio/transcriptions",
"upload": {
"kind": "wav",
"seconds": 4.0
},
"request": {},
"response": {
"content_type": "application/json",
"body": {
"results": {
"channels": [
{
"alternatives": [
{
"transcript": "hello",
"confidence": 0.9
}
]
}
]
},
"metadata": {
"duration": 4.0,
"channels": 1
}
}
},
"expected": {
"spend": 0.0012,
"input_cost": 0.0012,
"output_cost": 0,
"prompt_tokens": 0,
"completion_tokens": 0
}
},
{
"name": "azure-whisper-next-transcriptions-deployment",
"covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
"model": "azure/whisper-next",
"endpoint": "/v1/audio/transcriptions",
"deployment": {
"model": "azure/cc-whisper-deployment",
"base_model": "azure/whisper-next"
},
"upload": {
"kind": "wav",
"seconds": 3.5
},
"request": {
"response_format": "json"
},
"response": {
"content_type": "application/json",
"body": {
"text": "hello"
}
},
"expected": {
"spend": 0.000385,
"input_cost": 0.000385,
"output_cost": 0,
"prompt_tokens": 0,
"completion_tokens": 0
}
},
{
"name": "tts-next-speech-per-character",
"covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
"model": "tts-next",
"endpoint": "/v1/audio/speech",
"request": {
"input": "hello world",
"voice": "alloy",
"response_format": "mp3"
},
"response": {
"content_type": "audio/mpeg",
"length": 2048
},
"expected": {
"spend": 0.0001,
"input_cost": 0.0001,
"output_cost": 0,
"prompt_tokens": 0,
"completion_tokens": 0
}
},
{
"name": "tts-next-hd-speech-per-character",
"covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
"model": "tts-next-hd",
"endpoint": "/v1/audio/speech",
"request": {
"input": "hello world",
"voice": "alloy",
"response_format": "mp3"
},
"response": {
"content_type": "audio/mpeg",
"length": 2048
},
"expected": {
"spend": 0.0002,
"input_cost": 0.0002,
"output_cost": 0,
"prompt_tokens": 0,
"completion_tokens": 0
}
},
{
"name": "azure-tts-next-speech-deployment",
"covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
"model": "azure/tts-next",
"endpoint": "/v1/audio/speech",
"deployment": {
"model": "azure/cc-tts-deployment",
"base_model": "azure/tts-next"
},
"request": {
"input": "hello world",
"voice": "alloy",
"response_format": "mp3"
},
"response": {
"content_type": "audio/mpeg",
"length": 2048
},
"expected": {
"spend": 0.00011,
"input_cost": 0.00011,
"output_cost": 0,
"prompt_tokens": 0,
"completion_tokens": 0
}
},
{
"name": "dall-e-3-next-images-standard",
"covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
"model": "1024-x-1024/dall-e-3-next",
"endpoint": "/v1/images/generations",
"deployment": {
"model": "openai/dall-e-3-next"
},
"request": {
"prompt": "a deterministic square",
"size": "1024x1024",
"quality": "standard",
"n": 1
},
"response": {
"content_type": "application/json",
"body": {
"created": 1700000000,
"data": [
{
"url": "https://x/1.png"
}
]
}
},
"expected": {
"spend": 0.04,
"input_cost": 0.04,
"output_cost": 0,
"prompt_tokens": 0,
"completion_tokens": 0
}
},
{
"name": "dall-e-3-next-images-hd",
"covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
"model": "hd/1024-x-1024/dall-e-3-next",
"endpoint": "/v1/images/generations",
"deployment": {
"model": "openai/dall-e-3-next"
},
"request": {
"prompt": "a deterministic square",
"size": "1024x1024",
"quality": "hd",
"n": 1
},
"response": {
"content_type": "application/json",
"body": {
"created": 1700000001,
"data": [
{
"url": "https://x/1.png"
}
]
}
},
"expected": {
"spend": 0.08,
"input_cost": 0.08,
"output_cost": 0,
"prompt_tokens": 0,
"completion_tokens": 0
}
},
{
"name": "dall-e-3-next-images-wide",
"covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
"model": "1792-x-1024/dall-e-3-next",
"endpoint": "/v1/images/generations",
"deployment": {
"model": "openai/dall-e-3-next"
},
"request": {
"prompt": "a deterministic wide image",
"size": "1792x1024",
"quality": "standard",
"n": 1
},
"response": {
"content_type": "application/json",
"body": {
"created": 1700000002,
"data": [
{
"url": "https://x/1.png"
}
]
}
},
"expected": {
"spend": 0.06,
"input_cost": 0.06,
"output_cost": 0,
"prompt_tokens": 0,
"completion_tokens": 0
}
},
{
"name": "dall-e-3-next-images-two",
"covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
"model": "1024-x-1024/dall-e-3-next",
"endpoint": "/v1/images/generations",
"deployment": {
"model": "openai/dall-e-3-next"
},
"request": {
"prompt": "two deterministic squares",
"size": "1024x1024",
"quality": "standard",
"n": 2
},
"response": {
"content_type": "application/json",
"body": {
"created": 1700000003,
"data": [
{
"url": "https://x/1.png"
},
{
"url": "https://x/2.png"
}
]
}
},
"expected": {
"spend": 0.08,
"input_cost": 0.08,
"output_cost": 0,
"prompt_tokens": 0,
"completion_tokens": 0
}
},
{
"name": "gpt-image-next-images-low",
"covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
"model": "gpt-image-next",
"endpoint": "/v1/images/generations",
"deployment": {
"model": "openai/gpt-image-next"
},
"request": {
"prompt": "a deterministic generated image",
"size": "1024x1024",
"quality": "low",
"n": 1
},
"response": {
"content_type": "application/json",
"body": {
"created": 1700000004,
"data": [
{
"b64_json": "AA=="
}
],
"usage": {
"total_tokens": 30,
"input_tokens": 10,
"output_tokens": 20,
"input_tokens_details": {
"text_tokens": 10,
"image_tokens": 0
}
}
}
},
"expected": {
"spend": 0.0001191,
"input_cost": 1.71e-05,
"output_cost": 0.000102,
"prompt_tokens": 10,
"completion_tokens": 20
}
},
{
"name": "imagen-next-images-one",
"covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
"model": "1024-x-1024/imagen-next",
"endpoint": "/v1/images/generations",
"request": {
"prompt": "a deterministic vertex image",
"sampleCount": 1
},
"response": {
"content_type": "application/json",
"body": {
"predictions": [
{
"bytesBase64Encoded": "AA==",
"mimeType": "image/png"
}
]
}
},
"expected": {
"spend": 0.05,
"input_cost": 0.05,
"output_cost": 0,
"prompt_tokens": 0,
"completion_tokens": 0
}
},
{
"name": "amazon-nova-canvas-next-images-one",
"covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
"model": "amazon.nova-canvas-next",
"endpoint": "/v1/images/generations",
"deployment": {
"model": "amazon.nova-canvas-next"
},
"request": {
"prompt": "a deterministic bedrock image"
},
"response": {
"content_type": "application/json",
"body": {
"images": [
"AA=="
]
}
},
"expected": {
"spend": 0.045,
"input_cost": 0.045,
"output_cost": 0,
"prompt_tokens": 0,
"completion_tokens": 0
}
},
{
"name": "gpt-image-next-images-edit",
"covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
"model": "low/1024-x-1024/gpt-image-next",
"endpoint": "/v1/images/edits",
"deployment": {
"model": "openai/gpt-image-next"
},
"upload": {
"kind": "png"
},
"request": {
"prompt": "edit this deterministic image",
"size": "1024x1024",
"quality": "low",
"n": 1
},
"response": {
"content_type": "application/json",
"body": {
"created": 1700000005,
"data": [
{
"b64_json": "AA=="
}
],
"usage": {
"total_tokens": 30,
"input_tokens": 10,
"output_tokens": 20,
"input_tokens_details": {
"text_tokens": 10,
"image_tokens": 0
}
}
}
},
"expected": {
"spend": 0.000119,
"input_cost": 1.7e-05,
"output_cost": 0.000102,
"prompt_tokens": 10,
"completion_tokens": 20
}
}
]
}

186
tests/integration/cost_calculation/test_cost_tracking.py
View file

@ -2,9 +2,15 @@
from __future__ import annotations
import io
import json
from hashlib import sha256
import struct
from typing import Final, cast
import wave
import zlib
import httpx
import pytest
from integration._support.client import JSON_OBJECT, Gateway
@ -16,6 +22,7 @@ from integration.cost_calculation.conftest import (
register_scenario_deployment,
)
from integration.cost_calculation.cost_tracking_case import (
BinaryResponse,
CASES,
CostTrackingTestCase,
ExactExpected,
@ -34,6 +41,47 @@ _CASES: Final = tuple(
)
def _wav_bytes(seconds: float) -> bytes:
frame_count: Final = round(16000 * seconds)
output: Final = io.BytesIO()
with wave.open(output, "wb") as wav:
wav.setnchannels(1)
wav.setsampwidth(2)
wav.setframerate(16000)
wav.writeframes(b"\x00\x00" * frame_count)
return output.getvalue()
def _png_bytes() -> bytes:
def chunk(kind: bytes, payload: bytes) -> bytes:
return (
struct.pack(">I", len(payload))
+ kind
+ payload
+ struct.pack(">I", zlib.crc32(kind + payload) & 0xFFFFFFFF)
)
return (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", struct.pack(">IIBBBBB", 1, 1, 8, 6, 0, 0, 0))
+ chunk(b"IDAT", zlib.compress(b"\x00\x00\x00\x00\x00"))
+ chunk(b"IEND", b"")
)
def _multipart_request(gateway: Gateway, case: CostTrackingTestCase, model_name: str, key: str) -> httpx.Response:
assert case.upload is not None
fields: Final = {
field: value if isinstance(value, str) else json.dumps(value, separators=(",", ":"))
for field, value in {**case.request, "model": model_name}.items()
}
if case.upload.kind == "wav":
files: Final = {"file": ("audio.wav", _wav_bytes(case.upload.seconds), "audio/wav")}
else:
files = {"image": ("image.png", _png_bytes(), "image/png")}
return gateway.request_multipart(case.endpoint, fields, files, key=key)
def _assert_stream_has_no_error(response_text: str) -> None:
for line in response_text.splitlines():
if not line.startswith("data:"):
@ -51,11 +99,10 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase)
with gateway.scenario() as scenario:
key: Final = scenario.key()
model_name: Final = register_scenario_deployment(scenario, case, marker, key)
response: Final = gateway.request(
"POST",
case.endpoint,
{**case.request, "model": model_name},
key=key,
response: Final = (
_multipart_request(gateway, case, model_name, key)
if case.upload is not None
else gateway.request("POST", case.endpoint, {**case.request, "model": model_name}, key=key)
)
if isinstance(case.expected, FailureExpected):
assert response.status_code == case.expected.failure.status, (
@ -90,7 +137,13 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase)
return
expected: Final = case.expected
assert isinstance(expected, ExactExpected)
if case.response.content_type == "application/json":
if isinstance(case.response, BinaryResponse):
header: Final = response.headers.get("x-litellm-response-cost")
if header is not None:
assert approx_equal(float(header), expected.spend), (
f"{case.name}: x-litellm-response-cost {header} != expected {expected.spend}"
)
elif case.response.content_type == "application/json":
header: Final = cast(str | None, response.headers.get("x-litellm-response-cost"))
assert header is not None and approx_equal(float(header), expected.spend), (
f"{case.name}: x-litellm-response-cost {header} != expected {expected.spend}"
@ -100,68 +153,69 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase)
f"(breakdown {row.breakdown.model_dump()})"
)
breakdown: Final = row.breakdown
assert breakdown.input_cost is not None and approx_equal(breakdown.input_cost, expected.input_cost), (
f"{case.name}: input_cost {breakdown.input_cost} != expected {expected.input_cost}"
)
assert breakdown.output_cost is not None and approx_equal(breakdown.output_cost, expected.output_cost), (
f"{case.name}: output_cost {breakdown.output_cost} != expected {expected.output_cost}"
)
for field, header_name, actual_component, expected_component in (
(
"cache_read_cost",
"x-litellm-response-cost-cache-read",
breakdown.cache_read_cost,
expected.cache_read_cost,
),
(
"cache_creation_cost",
"x-litellm-response-cost-cache-creation",
breakdown.cache_creation_cost,
expected.cache_creation_cost,
),
(
"reasoning_cost",
"x-litellm-response-cost-reasoning",
breakdown.reasoning_cost,
expected.reasoning_cost,
),
(
"tool_usage_cost",
"x-litellm-response-cost-tool-usage",
breakdown.tool_usage_cost,
expected.tool_usage_cost,
),
):
if expected_component is None:
continue
assert actual_component is not None and approx_equal(actual_component, expected_component), (
f"{case.name}: {field} {actual_component} != expected {expected_component}"
if breakdown is not None:
assert breakdown.input_cost is not None and approx_equal(breakdown.input_cost, expected.input_cost), (
f"{case.name}: input_cost {breakdown.input_cost} != expected {expected.input_cost}"
)
if case.response.content_type == "application/json":
header: Final = response.headers.get(header_name)
assert header is not None and approx_equal(float(header), expected_component), (
f"{case.name}: {header_name} {header} != expected {expected_component}"
assert breakdown.output_cost is not None and approx_equal(breakdown.output_cost, expected.output_cost), (
f"{case.name}: output_cost {breakdown.output_cost} != expected {expected.output_cost}"
)
for field, header_name, actual_component, expected_component in (
(
"cache_read_cost",
"x-litellm-response-cost-cache-read",
breakdown.cache_read_cost,
expected.cache_read_cost,
),
(
"cache_creation_cost",
"x-litellm-response-cost-cache-creation",
breakdown.cache_creation_cost,
expected.cache_creation_cost,
),
(
"reasoning_cost",
"x-litellm-response-cost-reasoning",
breakdown.reasoning_cost,
expected.reasoning_cost,
),
(
"tool_usage_cost",
"x-litellm-response-cost-tool-usage",
breakdown.tool_usage_cost,
expected.tool_usage_cost,
),
):
if expected_component is None:
continue
assert actual_component is not None and approx_equal(actual_component, expected_component), (
f"{case.name}: {field} {actual_component} != expected {expected_component}"
)
if case.response.content_type == "application/json":
header: Final = response.headers.get(header_name)
assert header is not None and approx_equal(float(header), expected_component), (
f"{case.name}: {header_name} {header} != expected {expected_component}"
)
if case.response.content_type == "application/json" and any(
component is not None
for component in (
expected.cache_read_cost,
expected.cache_creation_cost,
expected.reasoning_cost,
expected.tool_usage_cost,
)
):
input_header: Final = response.headers.get("x-litellm-response-cost-input")
output_header: Final = response.headers.get("x-litellm-response-cost-output")
expected_input_header: Final = expected.input_cost - (
expected.cache_read_cost or 0.0
) - (expected.cache_creation_cost or 0.0)
assert input_header is not None and approx_equal(float(input_header), expected_input_header), (
f"{case.name}: x-litellm-response-cost-input {input_header} != expected {expected_input_header}"
)
assert output_header is not None and approx_equal(float(output_header), expected.output_cost), (
f"{case.name}: x-litellm-response-cost-output {output_header} != expected {expected.output_cost}"
)
if case.response.content_type == "application/json" and any(
component is not None
for component in (
expected.cache_read_cost,
expected.cache_creation_cost,
expected.reasoning_cost,
expected.tool_usage_cost,
)
):
input_header: Final = response.headers.get("x-litellm-response-cost-input")
output_header: Final = response.headers.get("x-litellm-response-cost-output")
expected_input_header: Final = expected.input_cost - (
expected.cache_read_cost or 0.0
) - (expected.cache_creation_cost or 0.0)
assert input_header is not None and approx_equal(float(input_header), expected_input_header), (
f"{case.name}: x-litellm-response-cost-input {input_header} != expected {expected_input_header}"
)
assert output_header is not None and approx_equal(float(output_header), expected.output_cost), (
f"{case.name}: x-litellm-response-cost-output {output_header} != expected {expected.output_cost}"
)
assert row.prompt_tokens == expected.prompt_tokens, (
f"{case.name}: prompt_tokens {row.prompt_tokens} != expected {expected.prompt_tokens}"
)