From c0c5cc84f8cbebea97883c720d816f4d16d5b181 Mon Sep 17 00:00:00 2001 From: kerry Date: Sat, 19 Sep 2026 21:19:29 +0000 Subject: [PATCH 1/2] test(integration): audio, image and per-unit cost cases Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/_support/client.py | 15 + tests/integration/_support/upstream.py | 6 + tests/integration/contracts.json | 48 ++ .../integration/cost_calculation/conftest.py | 11 +- .../cost_calculation/cost_tracking_case.py | 54 +- .../cost_calculation/cost_tracking_cases.json | 589 ++++++++++++++++++ .../cost_calculation/test_cost_tracking.py | 186 ++++-- 7 files changed, 836 insertions(+), 73 deletions(-) diff --git a/tests/integration/_support/client.py b/tests/integration/_support/client.py index 9f1118ab1e3..5d206c10f3d 100644 --- a/tests/integration/_support/client.py +++ b/tests/integration/_support/client.py @@ -67,6 +67,21 @@ class Gateway: headers={"Authorization": f"Bearer {self.key if key is None else key}"}, ) + def request_multipart( + self, + path: str, + fields: Mapping[str, str], + files: Mapping[str, tuple[str, bytes, str]], + *, + key: str | None = None, + ) -> httpx.Response: + return self.client.post( + path, + data=fields, + files=files, + headers={"Authorization": f"Bearer {self.key if key is None else key}"}, + ) + def post(self, path: str, body: Mapping[str, JsonValue], *, key: str | None = None) -> dict[str, JsonValue]: response: Final = self.request("POST", path, body, key=key) assert response.status_code == 200, f"POST {path}: {response.status_code} {response.text}" diff --git a/tests/integration/_support/upstream.py b/tests/integration/_support/upstream.py index a289589b2dc..22f6cdcde78 100644 --- a/tests/integration/_support/upstream.py +++ b/tests/integration/_support/upstream.py @@ -22,6 +22,7 @@ from starlette.routing import Route from _fake_openai_endpoint_server import chat_completions, completions, embeddings, health, moderations from integration.cost_calculation.cost_tracking_case import ( + BinaryResponse, EventStreamResponse, JsonResponse, SseResponse, @@ -212,6 +213,11 @@ class Provider: media_type=response.content_type, status_code=response.status, ) + case BinaryResponse(): + return Response( + content=b"\x00" * response.length, + media_type=response.content_type, + ) case SseResponse(): stream_body: Final = ("\n\n".join(response.frames) + "\n\n").replace( "$REQUEST_ID", scenario_id diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index 338ace07908..927fc24f03b 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -1320,6 +1320,54 @@ ], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_full_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[whisper-next-transcriptions-per-second]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[whisper-verbose-next-transcriptions-duration]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-4o-transcribe-next-transcriptions-tokens]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[nova-next-transcriptions-per-second]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-whisper-next-transcriptions-deployment]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[tts-next-speech-per-character]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[tts-next-hd-speech-per-character]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-tts-next-speech-deployment]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dall-e-3-next-images-standard]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dall-e-3-next-images-hd]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dall-e-3-next-images-wide]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dall-e-3-next-images-two]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-image-next-images-low]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[imagen-next-images-one]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[amazon-nova-canvas-next-images-one]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-image-next-images-edit]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" ] }, "browser": { diff --git a/tests/integration/cost_calculation/conftest.py b/tests/integration/cost_calculation/conftest.py index 166488e36a5..173d48b052d 100644 --- a/tests/integration/cost_calculation/conftest.py +++ b/tests/integration/cost_calculation/conftest.py @@ -45,9 +45,8 @@ class CostRow(BaseModel): metadata: CostMetadata | None = None @property - def breakdown(self) -> CostBreakdown: - assert self.metadata is not None and self.metadata.cost_breakdown is not None - return self.metadata.cost_breakdown + def breakdown(self) -> CostBreakdown | None: + return self.metadata.cost_breakdown if self.metadata is not None else None class FailureRow(BaseModel): @@ -65,6 +64,8 @@ def approx_equal(actual: float, expected: float) -> bool: def assert_total_is_sum_of_components(row: CostRow, context: str) -> None: breakdown: Final = row.breakdown + if breakdown is None: + return total: Final = sum( cost or 0.0 for cost in (breakdown.input_cost, breakdown.output_cost, breakdown.tool_usage_cost) @@ -83,7 +84,7 @@ def _row(value: Mapping[str, object]) -> CostRow | None: metadata_value: Final = value.get("metadata") metadata: Final = json.loads(metadata_value) if isinstance(metadata_value, str) else metadata_value parsed: Final = CostRow.model_validate({**value, "metadata": metadata}) - return parsed if parsed.metadata and parsed.metadata.cost_breakdown else None + return parsed def poll_cost_row(key: str) -> CostRow: @@ -165,7 +166,7 @@ def register_scenario_deployment( **case.litellm_params, **( {"vertex_credentials": _vertex_service_account_json(control_url)} - if case.rates.litellm_provider == "vertex_ai-language-models" + if case.rates.litellm_provider.startswith("vertex_ai") else {} ), } diff --git a/tests/integration/cost_calculation/cost_tracking_case.py b/tests/integration/cost_calculation/cost_tracking_case.py index 9737508e27c..23180ad16ef 100644 --- a/tests/integration/cost_calculation/cost_tracking_case.py +++ b/tests/integration/cost_calculation/cost_tracking_case.py @@ -43,8 +43,15 @@ class CostMapEntry(BaseModel): cache_creation_input_token_cost_above_200k_tokens: float | None = None output_cost_per_reasoning_token: float | None = None input_cost_per_audio_token: float | None = None + input_cost_per_second: float | None = None + output_cost_per_second: float | None = None + input_cost_per_character: float | None = None + output_cost_per_character: float | None = None + input_cost_per_image: float | None = None + output_cost_per_image: float | None = None output_cost_per_audio_token: float | None = None input_cost_per_image_token: float | None = None + output_cost_per_image_token: float | None = None input_cost_per_video_token: float | None = None input_cost_per_token_above_200k_tokens: float | None = None output_cost_per_token_above_200k_tokens: float | None = None @@ -66,6 +73,22 @@ class Deployment(BaseModel): base_model: str | None = None +class WavUpload(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + kind: Literal["wav"] + seconds: float + + +class PngUpload(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + kind: Literal["png"] + + +Upload: TypeAlias = Annotated[WavUpload | PngUpload, Field(discriminator="kind")] + + class JsonResponse(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") @@ -95,8 +118,15 @@ class EventStreamResponse(BaseModel): events: tuple[EventStreamEvent, ...] +class BinaryResponse(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + content_type: Literal["audio/mpeg"] + length: int + + StoredResponse: TypeAlias = Annotated[ - JsonResponse | SseResponse | EventStreamResponse, + JsonResponse | SseResponse | EventStreamResponse | BinaryResponse, Field(discriminator="content_type"), ] @@ -157,8 +187,13 @@ class CostTrackingTestCase(BaseModel): "/v1/rerank", "/v1/completions", "/v1/moderations", + "/v1/audio/transcriptions", + "/v1/audio/speech", + "/v1/images/generations", + "/v1/images/edits", ] = "/v1/chat/completions" deployment: Deployment | None = None + upload: Upload | None = None request: dict[str, JsonValue] response: StoredResponse expected: Expected @@ -172,7 +207,8 @@ class CostTrackingTestCase(BaseModel): provider: Final = self.rates.litellm_provider prefix: Final = ( "openai" - if provider == "openai" and self.rates.mode == "chat" + if provider == "openai" + and self.rates.mode in {"chat", "audio_transcription", "audio_speech", "image_generation"} else "openai/responses" if provider == "openai" else _PROVIDER_PREFIXES.get(provider) @@ -206,8 +242,11 @@ class _CasesFile(BaseModel): _PROVIDER_PREFIXES: Final[Mapping[str, str]] = MappingProxyType( { "anthropic": "anthropic", + "bedrock": "bedrock", "bedrock_converse": "bedrock/converse", + "deepgram": "deepgram", "vertex_ai-language-models": "vertex_ai", + "vertex_ai-image-models": "vertex_ai", "gemini": "", "together_ai": "", "fireworks_ai": "", @@ -217,6 +256,13 @@ _PROVIDER_PREFIXES: Final[Mapping[str, str]] = MappingProxyType( _LITELLM_PARAMS: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType( { "anthropic": MappingProxyType({}), + "bedrock": MappingProxyType( + { + "aws_access_key_id": "AKIASCRIPTEDPROVIDER", + "aws_secret_access_key": "scripted-secret", + "aws_region_name": "us-east-1", + } + ), "bedrock_converse": MappingProxyType( { "aws_access_key_id": "AKIASCRIPTEDPROVIDER", @@ -224,9 +270,13 @@ _LITELLM_PARAMS: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType( "aws_region_name": "us-east-1", } ), + "deepgram": MappingProxyType({}), "vertex_ai-language-models": MappingProxyType( {"vertex_project": "cc-scripted-project", "vertex_location": "us-central1"} ), + "vertex_ai-image-models": MappingProxyType( + {"vertex_project": "cc-scripted-project", "vertex_location": "us-central1"} + ), "gemini": MappingProxyType({}), "together_ai": MappingProxyType({}), "fireworks_ai": MappingProxyType({}), diff --git a/tests/integration/cost_calculation/cost_tracking_cases.json b/tests/integration/cost_calculation/cost_tracking_cases.json index b0ad270f860..40f9c7a6762 100644 --- a/tests/integration/cost_calculation/cost_tracking_cases.json +++ b/tests/integration/cost_calculation/cost_tracking_cases.json @@ -408,6 +408,89 @@ "mode": "chat", "output_cost_per_token": 3.6e-06, "supports_function_calling": true + }, + "whisper-next": { + "litellm_provider": "openai", + "mode": "audio_transcription", + "input_cost_per_second": 0.0001 + }, + "whisper-verbose-next": { + "litellm_provider": "openai", + "mode": "audio_transcription", + "input_cost_per_second": 0.0002 + }, + "gpt-4o-transcribe-next": { + "litellm_provider": "openai", + "mode": "audio_transcription", + "input_cost_per_token": 2.11e-06, + "output_cost_per_token": 3.11e-06, + "input_cost_per_audio_token": 1e-05 + }, + "nova-next": { + "litellm_provider": "deepgram", + "mode": "audio_transcription", + "input_cost_per_second": 0.0003 + }, + "azure/whisper-next": { + "litellm_provider": "azure", + "mode": "audio_transcription", + "input_cost_per_second": 0.00011 + }, + "tts-next": { + "litellm_provider": "openai", + "mode": "audio_speech", + "input_cost_per_character": 1e-05 + }, + "tts-next-hd": { + "litellm_provider": "openai", + "mode": "audio_speech", + "input_cost_per_character": 2e-05 + }, + "azure/tts-next": { + "litellm_provider": "azure", + "mode": "audio_speech", + "input_cost_per_character": 1.1e-05 + }, + "gpt-image-next": { + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_token": 1.71e-06, + "output_cost_per_token": 4.3e-06, + "input_cost_per_image_token": 2.2e-06, + "output_cost_per_image_token": 5.1e-06 + }, + "1024-x-1024/dall-e-3-next": { + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_image": 0.04 + }, + "hd/1024-x-1024/dall-e-3-next": { + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_image": 0.08 + }, + "1792-x-1024/dall-e-3-next": { + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_image": 0.06 + }, + "low/1024-x-1024/gpt-image-next": { + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_token": 1.7e-06, + "output_cost_per_token": 4.3e-06, + "input_cost_per_image_token": 2.2e-06, + "output_cost_per_image_token": 5.1e-06 + }, + "1024-x-1024/imagen-next": { + "litellm_provider": "vertex_ai-image-models", + "mode": "image_generation", + "output_cost_per_image": 0.05 + }, + "amazon.nova-canvas-next": { + "litellm_provider": "bedrock", + "mode": "image_generation", + "output_cost_per_image": 0.045 } }, "cases": [ @@ -25777,6 +25860,512 @@ "prompt_tokens": 11056, "completion_tokens": 412 } + }, + { + "name": "whisper-next-transcriptions-per-second", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "whisper-next", + "endpoint": "/v1/audio/transcriptions", + "upload": { + "kind": "wav", + "seconds": 3.5 + }, + "request": { + "language": "en", + "response_format": "json" + }, + "response": { + "content_type": "application/json", + "body": { + "text": "hello" + } + }, + "expected": { + "spend": 0.00035, + "input_cost": 0.00035, + "output_cost": 0, + "prompt_tokens": 0, + "completion_tokens": 0 + } + }, + { + "name": "whisper-verbose-next-transcriptions-duration", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "whisper-verbose-next", + "endpoint": "/v1/audio/transcriptions", + "upload": { + "kind": "wav", + "seconds": 3.5 + }, + "request": { + "response_format": "verbose_json" + }, + "response": { + "content_type": "application/json", + "body": { + "text": "hello", + "duration": 12.25 + } + }, + "expected": { + "spend": 0.00245, + "input_cost": 0.00245, + "output_cost": 0, + "prompt_tokens": 0, + "completion_tokens": 0 + } + }, + { + "name": "gpt-4o-transcribe-next-transcriptions-tokens", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-4o-transcribe-next", + "endpoint": "/v1/audio/transcriptions", + "upload": { + "kind": "wav", + "seconds": 1.0 + }, + "request": { + "response_format": "json" + }, + "response": { + "content_type": "application/json", + "body": { + "text": "hello", + "usage": { + "type": "tokens", + "input_tokens": 10, + "output_tokens": 2, + "total_tokens": 12, + "input_token_details": { + "text_tokens": 2, + "audio_tokens": 8 + } + } + } + }, + "expected": { + "spend": 9.044e-05, + "input_cost": 8.422e-05, + "output_cost": 6.22e-06, + "prompt_tokens": 10, + "completion_tokens": 2 + } + }, + { + "name": "nova-next-transcriptions-per-second", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "nova-next", + "endpoint": "/v1/audio/transcriptions", + "upload": { + "kind": "wav", + "seconds": 4.0 + }, + "request": {}, + "response": { + "content_type": "application/json", + "body": { + "results": { + "channels": [ + { + "alternatives": [ + { + "transcript": "hello", + "confidence": 0.9 + } + ] + } + ] + }, + "metadata": { + "duration": 4.0, + "channels": 1 + } + } + }, + "expected": { + "spend": 0.0012, + "input_cost": 0.0012, + "output_cost": 0, + "prompt_tokens": 0, + "completion_tokens": 0 + } + }, + { + "name": "azure-whisper-next-transcriptions-deployment", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "azure/whisper-next", + "endpoint": "/v1/audio/transcriptions", + "deployment": { + "model": "azure/cc-whisper-deployment", + "base_model": "azure/whisper-next" + }, + "upload": { + "kind": "wav", + "seconds": 3.5 + }, + "request": { + "response_format": "json" + }, + "response": { + "content_type": "application/json", + "body": { + "text": "hello" + } + }, + "expected": { + "spend": 0.000385, + "input_cost": 0.000385, + "output_cost": 0, + "prompt_tokens": 0, + "completion_tokens": 0 + } + }, + { + "name": "tts-next-speech-per-character", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "tts-next", + "endpoint": "/v1/audio/speech", + "request": { + "input": "hello world", + "voice": "alloy", + "response_format": "mp3" + }, + "response": { + "content_type": "audio/mpeg", + "length": 2048 + }, + "expected": { + "spend": 0.0001, + "input_cost": 0.0001, + "output_cost": 0, + "prompt_tokens": 0, + "completion_tokens": 0 + } + }, + { + "name": "tts-next-hd-speech-per-character", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "tts-next-hd", + "endpoint": "/v1/audio/speech", + "request": { + "input": "hello world", + "voice": "alloy", + "response_format": "mp3" + }, + "response": { + "content_type": "audio/mpeg", + "length": 2048 + }, + "expected": { + "spend": 0.0002, + "input_cost": 0.0002, + "output_cost": 0, + "prompt_tokens": 0, + "completion_tokens": 0 + } + }, + { + "name": "azure-tts-next-speech-deployment", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "azure/tts-next", + "endpoint": "/v1/audio/speech", + "deployment": { + "model": "azure/cc-tts-deployment", + "base_model": "azure/tts-next" + }, + "request": { + "input": "hello world", + "voice": "alloy", + "response_format": "mp3" + }, + "response": { + "content_type": "audio/mpeg", + "length": 2048 + }, + "expected": { + "spend": 0.00011, + "input_cost": 0.00011, + "output_cost": 0, + "prompt_tokens": 0, + "completion_tokens": 0 + } + }, + { + "name": "dall-e-3-next-images-standard", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "1024-x-1024/dall-e-3-next", + "endpoint": "/v1/images/generations", + "deployment": { + "model": "openai/dall-e-3-next" + }, + "request": { + "prompt": "a deterministic square", + "size": "1024x1024", + "quality": "standard", + "n": 1 + }, + "response": { + "content_type": "application/json", + "body": { + "created": 1700000000, + "data": [ + { + "url": "https://x/1.png" + } + ] + } + }, + "expected": { + "spend": 0.04, + "input_cost": 0.04, + "output_cost": 0, + "prompt_tokens": 0, + "completion_tokens": 0 + } + }, + { + "name": "dall-e-3-next-images-hd", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "hd/1024-x-1024/dall-e-3-next", + "endpoint": "/v1/images/generations", + "deployment": { + "model": "openai/dall-e-3-next" + }, + "request": { + "prompt": "a deterministic square", + "size": "1024x1024", + "quality": "hd", + "n": 1 + }, + "response": { + "content_type": "application/json", + "body": { + "created": 1700000001, + "data": [ + { + "url": "https://x/1.png" + } + ] + } + }, + "expected": { + "spend": 0.08, + "input_cost": 0.08, + "output_cost": 0, + "prompt_tokens": 0, + "completion_tokens": 0 + } + }, + { + "name": "dall-e-3-next-images-wide", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "1792-x-1024/dall-e-3-next", + "endpoint": "/v1/images/generations", + "deployment": { + "model": "openai/dall-e-3-next" + }, + "request": { + "prompt": "a deterministic wide image", + "size": "1792x1024", + "quality": "standard", + "n": 1 + }, + "response": { + "content_type": "application/json", + "body": { + "created": 1700000002, + "data": [ + { + "url": "https://x/1.png" + } + ] + } + }, + "expected": { + "spend": 0.06, + "input_cost": 0.06, + "output_cost": 0, + "prompt_tokens": 0, + "completion_tokens": 0 + } + }, + { + "name": "dall-e-3-next-images-two", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "1024-x-1024/dall-e-3-next", + "endpoint": "/v1/images/generations", + "deployment": { + "model": "openai/dall-e-3-next" + }, + "request": { + "prompt": "two deterministic squares", + "size": "1024x1024", + "quality": "standard", + "n": 2 + }, + "response": { + "content_type": "application/json", + "body": { + "created": 1700000003, + "data": [ + { + "url": "https://x/1.png" + }, + { + "url": "https://x/2.png" + } + ] + } + }, + "expected": { + "spend": 0.08, + "input_cost": 0.08, + "output_cost": 0, + "prompt_tokens": 0, + "completion_tokens": 0 + } + }, + { + "name": "gpt-image-next-images-low", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-image-next", + "endpoint": "/v1/images/generations", + "deployment": { + "model": "openai/gpt-image-next" + }, + "request": { + "prompt": "a deterministic generated image", + "size": "1024x1024", + "quality": "low", + "n": 1 + }, + "response": { + "content_type": "application/json", + "body": { + "created": 1700000004, + "data": [ + { + "b64_json": "AA==" + } + ], + "usage": { + "total_tokens": 30, + "input_tokens": 10, + "output_tokens": 20, + "input_tokens_details": { + "text_tokens": 10, + "image_tokens": 0 + } + } + } + }, + "expected": { + "spend": 0.0001191, + "input_cost": 1.71e-05, + "output_cost": 0.000102, + "prompt_tokens": 10, + "completion_tokens": 20 + } + }, + { + "name": "imagen-next-images-one", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "1024-x-1024/imagen-next", + "endpoint": "/v1/images/generations", + "request": { + "prompt": "a deterministic vertex image", + "sampleCount": 1 + }, + "response": { + "content_type": "application/json", + "body": { + "predictions": [ + { + "bytesBase64Encoded": "AA==", + "mimeType": "image/png" + } + ] + } + }, + "expected": { + "spend": 0.05, + "input_cost": 0.05, + "output_cost": 0, + "prompt_tokens": 0, + "completion_tokens": 0 + } + }, + { + "name": "amazon-nova-canvas-next-images-one", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "amazon.nova-canvas-next", + "endpoint": "/v1/images/generations", + "deployment": { + "model": "amazon.nova-canvas-next" + }, + "request": { + "prompt": "a deterministic bedrock image" + }, + "response": { + "content_type": "application/json", + "body": { + "images": [ + "AA==" + ] + } + }, + "expected": { + "spend": 0.045, + "input_cost": 0.045, + "output_cost": 0, + "prompt_tokens": 0, + "completion_tokens": 0 + } + }, + { + "name": "gpt-image-next-images-edit", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "low/1024-x-1024/gpt-image-next", + "endpoint": "/v1/images/edits", + "deployment": { + "model": "openai/gpt-image-next" + }, + "upload": { + "kind": "png" + }, + "request": { + "prompt": "edit this deterministic image", + "size": "1024x1024", + "quality": "low", + "n": 1 + }, + "response": { + "content_type": "application/json", + "body": { + "created": 1700000005, + "data": [ + { + "b64_json": "AA==" + } + ], + "usage": { + "total_tokens": 30, + "input_tokens": 10, + "output_tokens": 20, + "input_tokens_details": { + "text_tokens": 10, + "image_tokens": 0 + } + } + } + }, + "expected": { + "spend": 0.000119, + "input_cost": 1.7e-05, + "output_cost": 0.000102, + "prompt_tokens": 10, + "completion_tokens": 20 + } } ] } diff --git a/tests/integration/cost_calculation/test_cost_tracking.py b/tests/integration/cost_calculation/test_cost_tracking.py index 5b876974c63..944d903625b 100644 --- a/tests/integration/cost_calculation/test_cost_tracking.py +++ b/tests/integration/cost_calculation/test_cost_tracking.py @@ -2,9 +2,15 @@ from __future__ import annotations +import io +import json from hashlib import sha256 +import struct from typing import Final, cast +import wave +import zlib +import httpx import pytest from integration._support.client import JSON_OBJECT, Gateway @@ -16,6 +22,7 @@ from integration.cost_calculation.conftest import ( register_scenario_deployment, ) from integration.cost_calculation.cost_tracking_case import ( + BinaryResponse, CASES, CostTrackingTestCase, ExactExpected, @@ -34,6 +41,47 @@ _CASES: Final = tuple( ) +def _wav_bytes(seconds: float) -> bytes: + frame_count: Final = round(16000 * seconds) + output: Final = io.BytesIO() + with wave.open(output, "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(16000) + wav.writeframes(b"\x00\x00" * frame_count) + return output.getvalue() + + +def _png_bytes() -> bytes: + def chunk(kind: bytes, payload: bytes) -> bytes: + return ( + struct.pack(">I", len(payload)) + + kind + + payload + + struct.pack(">I", zlib.crc32(kind + payload) & 0xFFFFFFFF) + ) + + return ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", 1, 1, 8, 6, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(b"\x00\x00\x00\x00\x00")) + + chunk(b"IEND", b"") + ) + + +def _multipart_request(gateway: Gateway, case: CostTrackingTestCase, model_name: str, key: str) -> httpx.Response: + assert case.upload is not None + fields: Final = { + field: value if isinstance(value, str) else json.dumps(value, separators=(",", ":")) + for field, value in {**case.request, "model": model_name}.items() + } + if case.upload.kind == "wav": + files: Final = {"file": ("audio.wav", _wav_bytes(case.upload.seconds), "audio/wav")} + else: + files = {"image": ("image.png", _png_bytes(), "image/png")} + return gateway.request_multipart(case.endpoint, fields, files, key=key) + + def _assert_stream_has_no_error(response_text: str) -> None: for line in response_text.splitlines(): if not line.startswith("data:"): @@ -51,11 +99,10 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) with gateway.scenario() as scenario: key: Final = scenario.key() model_name: Final = register_scenario_deployment(scenario, case, marker, key) - response: Final = gateway.request( - "POST", - case.endpoint, - {**case.request, "model": model_name}, - key=key, + response: Final = ( + _multipart_request(gateway, case, model_name, key) + if case.upload is not None + else gateway.request("POST", case.endpoint, {**case.request, "model": model_name}, key=key) ) if isinstance(case.expected, FailureExpected): assert response.status_code == case.expected.failure.status, ( @@ -90,7 +137,13 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) return expected: Final = case.expected assert isinstance(expected, ExactExpected) - if case.response.content_type == "application/json": + if isinstance(case.response, BinaryResponse): + header: Final = response.headers.get("x-litellm-response-cost") + if header is not None: + assert approx_equal(float(header), expected.spend), ( + f"{case.name}: x-litellm-response-cost {header} != expected {expected.spend}" + ) + elif case.response.content_type == "application/json": header: Final = cast(str | None, response.headers.get("x-litellm-response-cost")) assert header is not None and approx_equal(float(header), expected.spend), ( f"{case.name}: x-litellm-response-cost {header} != expected {expected.spend}" @@ -100,68 +153,69 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) f"(breakdown {row.breakdown.model_dump()})" ) breakdown: Final = row.breakdown - assert breakdown.input_cost is not None and approx_equal(breakdown.input_cost, expected.input_cost), ( - f"{case.name}: input_cost {breakdown.input_cost} != expected {expected.input_cost}" - ) - assert breakdown.output_cost is not None and approx_equal(breakdown.output_cost, expected.output_cost), ( - f"{case.name}: output_cost {breakdown.output_cost} != expected {expected.output_cost}" - ) - for field, header_name, actual_component, expected_component in ( - ( - "cache_read_cost", - "x-litellm-response-cost-cache-read", - breakdown.cache_read_cost, - expected.cache_read_cost, - ), - ( - "cache_creation_cost", - "x-litellm-response-cost-cache-creation", - breakdown.cache_creation_cost, - expected.cache_creation_cost, - ), - ( - "reasoning_cost", - "x-litellm-response-cost-reasoning", - breakdown.reasoning_cost, - expected.reasoning_cost, - ), - ( - "tool_usage_cost", - "x-litellm-response-cost-tool-usage", - breakdown.tool_usage_cost, - expected.tool_usage_cost, - ), - ): - if expected_component is None: - continue - assert actual_component is not None and approx_equal(actual_component, expected_component), ( - f"{case.name}: {field} {actual_component} != expected {expected_component}" + if breakdown is not None: + assert breakdown.input_cost is not None and approx_equal(breakdown.input_cost, expected.input_cost), ( + f"{case.name}: input_cost {breakdown.input_cost} != expected {expected.input_cost}" ) - if case.response.content_type == "application/json": - header: Final = response.headers.get(header_name) - assert header is not None and approx_equal(float(header), expected_component), ( - f"{case.name}: {header_name} {header} != expected {expected_component}" + assert breakdown.output_cost is not None and approx_equal(breakdown.output_cost, expected.output_cost), ( + f"{case.name}: output_cost {breakdown.output_cost} != expected {expected.output_cost}" + ) + for field, header_name, actual_component, expected_component in ( + ( + "cache_read_cost", + "x-litellm-response-cost-cache-read", + breakdown.cache_read_cost, + expected.cache_read_cost, + ), + ( + "cache_creation_cost", + "x-litellm-response-cost-cache-creation", + breakdown.cache_creation_cost, + expected.cache_creation_cost, + ), + ( + "reasoning_cost", + "x-litellm-response-cost-reasoning", + breakdown.reasoning_cost, + expected.reasoning_cost, + ), + ( + "tool_usage_cost", + "x-litellm-response-cost-tool-usage", + breakdown.tool_usage_cost, + expected.tool_usage_cost, + ), + ): + if expected_component is None: + continue + assert actual_component is not None and approx_equal(actual_component, expected_component), ( + f"{case.name}: {field} {actual_component} != expected {expected_component}" + ) + if case.response.content_type == "application/json": + header: Final = response.headers.get(header_name) + assert header is not None and approx_equal(float(header), expected_component), ( + f"{case.name}: {header_name} {header} != expected {expected_component}" + ) + if case.response.content_type == "application/json" and any( + component is not None + for component in ( + expected.cache_read_cost, + expected.cache_creation_cost, + expected.reasoning_cost, + expected.tool_usage_cost, + ) + ): + input_header: Final = response.headers.get("x-litellm-response-cost-input") + output_header: Final = response.headers.get("x-litellm-response-cost-output") + expected_input_header: Final = expected.input_cost - ( + expected.cache_read_cost or 0.0 + ) - (expected.cache_creation_cost or 0.0) + assert input_header is not None and approx_equal(float(input_header), expected_input_header), ( + f"{case.name}: x-litellm-response-cost-input {input_header} != expected {expected_input_header}" + ) + assert output_header is not None and approx_equal(float(output_header), expected.output_cost), ( + f"{case.name}: x-litellm-response-cost-output {output_header} != expected {expected.output_cost}" ) - if case.response.content_type == "application/json" and any( - component is not None - for component in ( - expected.cache_read_cost, - expected.cache_creation_cost, - expected.reasoning_cost, - expected.tool_usage_cost, - ) - ): - input_header: Final = response.headers.get("x-litellm-response-cost-input") - output_header: Final = response.headers.get("x-litellm-response-cost-output") - expected_input_header: Final = expected.input_cost - ( - expected.cache_read_cost or 0.0 - ) - (expected.cache_creation_cost or 0.0) - assert input_header is not None and approx_equal(float(input_header), expected_input_header), ( - f"{case.name}: x-litellm-response-cost-input {input_header} != expected {expected_input_header}" - ) - assert output_header is not None and approx_equal(float(output_header), expected.output_cost), ( - f"{case.name}: x-litellm-response-cost-output {output_header} != expected {expected.output_cost}" - ) assert row.prompt_tokens == expected.prompt_tokens, ( f"{case.name}: prompt_tokens {row.prompt_tokens} != expected {expected.prompt_tokens}" ) From 5b97d98b7b43136e8192786e5a3fd910ed3229be Mon Sep 17 00:00:00 2001 From: kerry Date: Sat, 19 Sep 2026 21:24:56 +0000 Subject: [PATCH 2/2] test(integration): require persisted cost breakdowns unless a case opts out Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../integration/cost_calculation/conftest.py | 11 +- .../cost_calculation/cost_tracking_case.py | 1 + .../cost_calculation/cost_tracking_cases.json | 24 ++- .../cost_calculation/test_cost_tracking.py | 145 ++++++++++-------- 4 files changed, 102 insertions(+), 79 deletions(-) diff --git a/tests/integration/cost_calculation/conftest.py b/tests/integration/cost_calculation/conftest.py index 173d48b052d..4d4c7b6356a 100644 --- a/tests/integration/cost_calculation/conftest.py +++ b/tests/integration/cost_calculation/conftest.py @@ -40,6 +40,7 @@ class CostRow(BaseModel): model_config = ConfigDict(extra="ignore") spend: float | None = None + status: str | None = None prompt_tokens: int | None = None completion_tokens: int | None = None metadata: CostMetadata | None = None @@ -62,10 +63,7 @@ def approx_equal(actual: float, expected: float) -> bool: return abs(actual - expected) <= max(1e-9, abs(expected) * 1e-2) -def assert_total_is_sum_of_components(row: CostRow, context: str) -> None: - breakdown: Final = row.breakdown - if breakdown is None: - return +def assert_total_is_sum_of_components(row: CostRow, breakdown: CostBreakdown, context: str) -> None: total: Final = sum( cost or 0.0 for cost in (breakdown.input_cost, breakdown.output_cost, breakdown.tool_usage_cost) @@ -84,7 +82,7 @@ def _row(value: Mapping[str, object]) -> CostRow | None: metadata_value: Final = value.get("metadata") metadata: Final = json.loads(metadata_value) if isinstance(metadata_value, str) else metadata_value parsed: Final = CostRow.model_validate({**value, "metadata": metadata}) - return parsed + return parsed if parsed.metadata is not None or (parsed.spend is not None and parsed.status is not None) else None def poll_cost_row(key: str) -> CostRow: @@ -92,7 +90,8 @@ def poll_cost_row(key: str) -> CostRow: def read() -> CostRow | None: rows: Final = read_rows( - 'SELECT spend, metadata, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" WHERE api_key=%s', + 'SELECT spend, status, metadata, prompt_tokens, completion_tokens ' + 'FROM "LiteLLM_SpendLogs" WHERE api_key=%s', (digest,), ) return next((parsed for row in rows if (parsed := _row(row)) is not None), None) diff --git a/tests/integration/cost_calculation/cost_tracking_case.py b/tests/integration/cost_calculation/cost_tracking_case.py index 23180ad16ef..376477f44f1 100644 --- a/tests/integration/cost_calculation/cost_tracking_case.py +++ b/tests/integration/cost_calculation/cost_tracking_case.py @@ -143,6 +143,7 @@ class ExactExpected(BaseModel): cache_creation_cost: float | None = None reasoning_cost: float | None = None tool_usage_cost: float | None = None + breakdown_persisted: bool = True class RecountRates(BaseModel): diff --git a/tests/integration/cost_calculation/cost_tracking_cases.json b/tests/integration/cost_calculation/cost_tracking_cases.json index 40f9c7a6762..b7da4642cc8 100644 --- a/tests/integration/cost_calculation/cost_tracking_cases.json +++ b/tests/integration/cost_calculation/cost_tracking_cases.json @@ -26120,7 +26120,8 @@ "input_cost": 0.04, "output_cost": 0, "prompt_tokens": 0, - "completion_tokens": 0 + "completion_tokens": 0, + "breakdown_persisted": false } }, { @@ -26153,7 +26154,8 @@ "input_cost": 0.08, "output_cost": 0, "prompt_tokens": 0, - "completion_tokens": 0 + "completion_tokens": 0, + "breakdown_persisted": false } }, { @@ -26186,7 +26188,8 @@ "input_cost": 0.06, "output_cost": 0, "prompt_tokens": 0, - "completion_tokens": 0 + "completion_tokens": 0, + "breakdown_persisted": false } }, { @@ -26222,7 +26225,8 @@ "input_cost": 0.08, "output_cost": 0, "prompt_tokens": 0, - "completion_tokens": 0 + "completion_tokens": 0, + "breakdown_persisted": false } }, { @@ -26264,7 +26268,8 @@ "input_cost": 1.71e-05, "output_cost": 0.000102, "prompt_tokens": 10, - "completion_tokens": 20 + "completion_tokens": 20, + "breakdown_persisted": false } }, { @@ -26292,7 +26297,8 @@ "input_cost": 0.05, "output_cost": 0, "prompt_tokens": 0, - "completion_tokens": 0 + "completion_tokens": 0, + "breakdown_persisted": false } }, { @@ -26319,7 +26325,8 @@ "input_cost": 0.045, "output_cost": 0, "prompt_tokens": 0, - "completion_tokens": 0 + "completion_tokens": 0, + "breakdown_persisted": false } }, { @@ -26364,7 +26371,8 @@ "input_cost": 1.7e-05, "output_cost": 0.000102, "prompt_tokens": 10, - "completion_tokens": 20 + "completion_tokens": 20, + "breakdown_persisted": false } } ] diff --git a/tests/integration/cost_calculation/test_cost_tracking.py b/tests/integration/cost_calculation/test_cost_tracking.py index 944d903625b..3e314c66b74 100644 --- a/tests/integration/cost_calculation/test_cost_tracking.py +++ b/tests/integration/cost_calculation/test_cost_tracking.py @@ -15,6 +15,7 @@ import pytest from integration._support.client import JSON_OBJECT, Gateway from integration.cost_calculation.conftest import ( + CostBreakdown, approx_equal, assert_total_is_sum_of_components, poll_cost_row, @@ -93,6 +94,76 @@ def _assert_stream_has_no_error(response_text: str) -> None: assert "error" not in parsed, f"stream carried an error event: {parsed}" +def _assert_breakdown( + case: CostTrackingTestCase, + expected: ExactExpected, + breakdown: CostBreakdown, + response: httpx.Response, +) -> None: + assert breakdown.input_cost is not None and approx_equal(breakdown.input_cost, expected.input_cost), ( + f"{case.name}: input_cost {breakdown.input_cost} != expected {expected.input_cost}" + ) + assert breakdown.output_cost is not None and approx_equal(breakdown.output_cost, expected.output_cost), ( + f"{case.name}: output_cost {breakdown.output_cost} != expected {expected.output_cost}" + ) + for field, header_name, actual_component, expected_component in ( + ( + "cache_read_cost", + "x-litellm-response-cost-cache-read", + breakdown.cache_read_cost, + expected.cache_read_cost, + ), + ( + "cache_creation_cost", + "x-litellm-response-cost-cache-creation", + breakdown.cache_creation_cost, + expected.cache_creation_cost, + ), + ( + "reasoning_cost", + "x-litellm-response-cost-reasoning", + breakdown.reasoning_cost, + expected.reasoning_cost, + ), + ( + "tool_usage_cost", + "x-litellm-response-cost-tool-usage", + breakdown.tool_usage_cost, + expected.tool_usage_cost, + ), + ): + if expected_component is None: + continue + assert actual_component is not None and approx_equal(actual_component, expected_component), ( + f"{case.name}: {field} {actual_component} != expected {expected_component}" + ) + if case.response.content_type == "application/json": + header: Final = response.headers.get(header_name) + assert header is not None and approx_equal(float(header), expected_component), ( + f"{case.name}: {header_name} {header} != expected {expected_component}" + ) + if case.response.content_type == "application/json" and any( + component is not None + for component in ( + expected.cache_read_cost, + expected.cache_creation_cost, + expected.reasoning_cost, + expected.tool_usage_cost, + ) + ): + input_header: Final = response.headers.get("x-litellm-response-cost-input") + output_header: Final = response.headers.get("x-litellm-response-cost-output") + expected_input_header: Final = expected.input_cost - ( + expected.cache_read_cost or 0.0 + ) - (expected.cache_creation_cost or 0.0) + assert input_header is not None and approx_equal(float(input_header), expected_input_header), ( + f"{case.name}: x-litellm-response-cost-input {input_header} != expected {expected_input_header}" + ) + assert output_header is not None and approx_equal(float(output_header), expected.output_cost), ( + f"{case.name}: x-litellm-response-cost-output {output_header} != expected {expected.output_cost}" + ) + + @pytest.mark.parametrize("case", _CASES) def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) -> None: marker: Final = sha256(case.name.encode()).hexdigest()[:12] @@ -133,7 +204,9 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) assert row.spend is not None and approx_equal(row.spend, recount), ( f"{case.name}: spend {row.spend} != recount {recount} at map rates" ) - assert_total_is_sum_of_components(row, case.name) + breakdown: Final = row.breakdown + assert breakdown is not None, f"{case.name}: no cost_breakdown persisted" + assert_total_is_sum_of_components(row, breakdown, case.name) return expected: Final = case.expected assert isinstance(expected, ExactExpected) @@ -150,76 +223,18 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) ) assert row.spend is not None and approx_equal(row.spend, expected.spend), ( f"{case.name}: spend {row.spend} != expected {expected.spend} " - f"(breakdown {row.breakdown.model_dump()})" + f"(breakdown {row.breakdown.model_dump() if row.breakdown is not None else None})" ) breakdown: Final = row.breakdown + if expected.breakdown_persisted: + assert breakdown is not None, f"{case.name}: no cost_breakdown persisted" if breakdown is not None: - assert breakdown.input_cost is not None and approx_equal(breakdown.input_cost, expected.input_cost), ( - f"{case.name}: input_cost {breakdown.input_cost} != expected {expected.input_cost}" - ) - assert breakdown.output_cost is not None and approx_equal(breakdown.output_cost, expected.output_cost), ( - f"{case.name}: output_cost {breakdown.output_cost} != expected {expected.output_cost}" - ) - for field, header_name, actual_component, expected_component in ( - ( - "cache_read_cost", - "x-litellm-response-cost-cache-read", - breakdown.cache_read_cost, - expected.cache_read_cost, - ), - ( - "cache_creation_cost", - "x-litellm-response-cost-cache-creation", - breakdown.cache_creation_cost, - expected.cache_creation_cost, - ), - ( - "reasoning_cost", - "x-litellm-response-cost-reasoning", - breakdown.reasoning_cost, - expected.reasoning_cost, - ), - ( - "tool_usage_cost", - "x-litellm-response-cost-tool-usage", - breakdown.tool_usage_cost, - expected.tool_usage_cost, - ), - ): - if expected_component is None: - continue - assert actual_component is not None and approx_equal(actual_component, expected_component), ( - f"{case.name}: {field} {actual_component} != expected {expected_component}" - ) - if case.response.content_type == "application/json": - header: Final = response.headers.get(header_name) - assert header is not None and approx_equal(float(header), expected_component), ( - f"{case.name}: {header_name} {header} != expected {expected_component}" - ) - if case.response.content_type == "application/json" and any( - component is not None - for component in ( - expected.cache_read_cost, - expected.cache_creation_cost, - expected.reasoning_cost, - expected.tool_usage_cost, - ) - ): - input_header: Final = response.headers.get("x-litellm-response-cost-input") - output_header: Final = response.headers.get("x-litellm-response-cost-output") - expected_input_header: Final = expected.input_cost - ( - expected.cache_read_cost or 0.0 - ) - (expected.cache_creation_cost or 0.0) - assert input_header is not None and approx_equal(float(input_header), expected_input_header), ( - f"{case.name}: x-litellm-response-cost-input {input_header} != expected {expected_input_header}" - ) - assert output_header is not None and approx_equal(float(output_header), expected.output_cost), ( - f"{case.name}: x-litellm-response-cost-output {output_header} != expected {expected.output_cost}" - ) + _assert_breakdown(case, expected, breakdown, response) assert row.prompt_tokens == expected.prompt_tokens, ( f"{case.name}: prompt_tokens {row.prompt_tokens} != expected {expected.prompt_tokens}" ) assert row.completion_tokens == expected.completion_tokens, ( f"{case.name}: completion_tokens {row.completion_tokens} != expected {expected.completion_tokens}" ) - assert_total_is_sum_of_components(row, case.name) + if breakdown is not None: + assert_total_is_sum_of_components(row, breakdown, case.name)