diff --git a/tests/integration/_support/client.py b/tests/integration/_support/client.py index 9f1118ab1e3..5d206c10f3d 100644 --- a/tests/integration/_support/client.py +++ b/tests/integration/_support/client.py @@ -67,6 +67,21 @@ class Gateway: headers={"Authorization": f"Bearer {self.key if key is None else key}"}, ) + def request_multipart( + self, + path: str, + fields: Mapping[str, str], + files: Mapping[str, tuple[str, bytes, str]], + *, + key: str | None = None, + ) -> httpx.Response: + return self.client.post( + path, + data=fields, + files=files, + headers={"Authorization": f"Bearer {self.key if key is None else key}"}, + ) + def post(self, path: str, body: Mapping[str, JsonValue], *, key: str | None = None) -> dict[str, JsonValue]: response: Final = self.request("POST", path, body, key=key) assert response.status_code == 200, f"POST {path}: {response.status_code} {response.text}" diff --git a/tests/integration/_support/upstream.py b/tests/integration/_support/upstream.py index a289589b2dc..22f6cdcde78 100644 --- a/tests/integration/_support/upstream.py +++ b/tests/integration/_support/upstream.py @@ -22,6 +22,7 @@ from starlette.routing import Route from _fake_openai_endpoint_server import chat_completions, completions, embeddings, health, moderations from integration.cost_calculation.cost_tracking_case import ( + BinaryResponse, EventStreamResponse, JsonResponse, SseResponse, @@ -212,6 +213,11 @@ class Provider: media_type=response.content_type, status_code=response.status, ) + case BinaryResponse(): + return Response( + content=b"\x00" * response.length, + media_type=response.content_type, + ) case SseResponse(): stream_body: Final = ("\n\n".join(response.frames) + "\n\n").replace( "$REQUEST_ID", scenario_id diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index d04e7f97bf7..dc6b218feb4 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -1336,6 +1336,54 @@ "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_full_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[whisper-next-transcriptions-per-second]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[whisper-verbose-next-transcriptions-duration]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-4o-transcribe-next-transcriptions-tokens]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[nova-next-transcriptions-per-second]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-whisper-next-transcriptions-deployment]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[tts-next-speech-per-character]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[tts-next-hd-speech-per-character]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-tts-next-speech-deployment]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dall-e-3-next-images-standard]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dall-e-3-next-images-hd]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dall-e-3-next-images-wide]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dall-e-3-next-images-two]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-image-next-images-low]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[imagen-next-images-one]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[amazon-nova-canvas-next-images-one]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-image-next-images-edit]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-text-embeddings-4-large-deployment]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], diff --git a/tests/integration/cost_calculation/conftest.py b/tests/integration/cost_calculation/conftest.py index 414711648aa..4d4c7b6356a 100644 --- a/tests/integration/cost_calculation/conftest.py +++ b/tests/integration/cost_calculation/conftest.py @@ -40,14 +40,14 @@ class CostRow(BaseModel): model_config = ConfigDict(extra="ignore") spend: float | None = None + status: str | None = None prompt_tokens: int | None = None completion_tokens: int | None = None metadata: CostMetadata | None = None @property - def breakdown(self) -> CostBreakdown: - assert self.metadata is not None and self.metadata.cost_breakdown is not None - return self.metadata.cost_breakdown + def breakdown(self) -> CostBreakdown | None: + return self.metadata.cost_breakdown if self.metadata is not None else None class FailureRow(BaseModel): @@ -63,8 +63,7 @@ def approx_equal(actual: float, expected: float) -> bool: return abs(actual - expected) <= max(1e-9, abs(expected) * 1e-2) -def assert_total_is_sum_of_components(row: CostRow, context: str) -> None: - breakdown: Final = row.breakdown +def assert_total_is_sum_of_components(row: CostRow, breakdown: CostBreakdown, context: str) -> None: total: Final = sum( cost or 0.0 for cost in (breakdown.input_cost, breakdown.output_cost, breakdown.tool_usage_cost) @@ -83,7 +82,7 @@ def _row(value: Mapping[str, object]) -> CostRow | None: metadata_value: Final = value.get("metadata") metadata: Final = json.loads(metadata_value) if isinstance(metadata_value, str) else metadata_value parsed: Final = CostRow.model_validate({**value, "metadata": metadata}) - return parsed if parsed.metadata and parsed.metadata.cost_breakdown else None + return parsed if parsed.metadata is not None or (parsed.spend is not None and parsed.status is not None) else None def poll_cost_row(key: str) -> CostRow: @@ -91,7 +90,8 @@ def poll_cost_row(key: str) -> CostRow: def read() -> CostRow | None: rows: Final = read_rows( - 'SELECT spend, metadata, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" WHERE api_key=%s', + 'SELECT spend, status, metadata, prompt_tokens, completion_tokens ' + 'FROM "LiteLLM_SpendLogs" WHERE api_key=%s', (digest,), ) return next((parsed for row in rows if (parsed := _row(row)) is not None), None) diff --git a/tests/integration/cost_calculation/cost_tracking_case.py b/tests/integration/cost_calculation/cost_tracking_case.py index e24f5d76d0d..617b68bdce0 100644 --- a/tests/integration/cost_calculation/cost_tracking_case.py +++ b/tests/integration/cost_calculation/cost_tracking_case.py @@ -46,8 +46,15 @@ class CostMapEntry(BaseModel): cache_creation_input_token_cost_above_200k_tokens: float | None = None output_cost_per_reasoning_token: float | None = None input_cost_per_audio_token: float | None = None + input_cost_per_second: float | None = None + output_cost_per_second: float | None = None + input_cost_per_character: float | None = None + output_cost_per_character: float | None = None + input_cost_per_image: float | None = None + output_cost_per_image: float | None = None output_cost_per_audio_token: float | None = None input_cost_per_image_token: float | None = None + output_cost_per_image_token: float | None = None input_cost_per_video_token: float | None = None input_cost_per_token_above_200k_tokens: float | None = None output_cost_per_token_above_200k_tokens: float | None = None @@ -69,6 +76,22 @@ class Deployment(BaseModel): base_model: str | None = None +class WavUpload(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + kind: Literal["wav"] + seconds: float + + +class PngUpload(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + kind: Literal["png"] + + +Upload: TypeAlias = Annotated[WavUpload | PngUpload, Field(discriminator="kind")] + + class JsonResponse(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") @@ -98,8 +121,15 @@ class EventStreamResponse(BaseModel): events: tuple[EventStreamEvent, ...] +class BinaryResponse(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + content_type: Literal["audio/mpeg"] + length: int + + StoredResponse: TypeAlias = Annotated[ - JsonResponse | SseResponse | EventStreamResponse, + JsonResponse | SseResponse | EventStreamResponse | BinaryResponse, Field(discriminator="content_type"), ] @@ -116,6 +146,7 @@ class ExactExpected(BaseModel): cache_creation_cost: float | None = None reasoning_cost: float | None = None tool_usage_cost: float | None = None + breakdown_persisted: bool = True class RecountRates(BaseModel): @@ -160,8 +191,13 @@ class CostTrackingTestCase(BaseModel): "/v1/rerank", "/v1/completions", "/v1/moderations", + "/v1/audio/transcriptions", + "/v1/audio/speech", + "/v1/images/generations", + "/v1/images/edits", ] = "/v1/chat/completions" deployment: Deployment | None = None + upload: Upload | None = None request: dict[str, JsonValue] response: StoredResponse expected: Expected @@ -176,7 +212,11 @@ class CostTrackingTestCase(BaseModel): prefix: Final = ( "openai" if provider == "openai" - and (self.endpoint == "/v1/responses" or self.rates.mode in {"chat", "embedding", "moderation"}) + and ( + self.endpoint == "/v1/responses" + or self.rates.mode + in {"chat", "embedding", "moderation", "audio_transcription", "audio_speech", "image_generation"} + ) else "openai/responses" if provider == "openai" else _PROVIDER_PREFIXES.get(provider) @@ -210,11 +250,13 @@ class _CasesFile(BaseModel): _PROVIDER_PREFIXES: Final[Mapping[str, str]] = MappingProxyType( { "anthropic": "anthropic", + "bedrock": "bedrock", "bedrock_converse": "bedrock/converse", + "deepgram": "deepgram", "text-completion-openai": "text-completion-openai", "cohere": "cohere", - "bedrock": "bedrock", "vertex_ai-language-models": "vertex_ai", + "vertex_ai-image-models": "vertex_ai", "vertex_ai-embedding-models": "vertex_ai", "gemini": "", "together_ai": "", @@ -225,15 +267,6 @@ _PROVIDER_PREFIXES: Final[Mapping[str, str]] = MappingProxyType( _LITELLM_PARAMS: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType( { "anthropic": MappingProxyType({}), - "bedrock_converse": MappingProxyType( - { - "aws_access_key_id": "AKIASCRIPTEDPROVIDER", - "aws_secret_access_key": "scripted-secret", - "aws_region_name": "us-east-1", - } - ), - "text-completion-openai": MappingProxyType({}), - "cohere": MappingProxyType({}), "bedrock": MappingProxyType( { "aws_access_key_id": "AKIASCRIPTEDPROVIDER", @@ -241,9 +274,22 @@ _LITELLM_PARAMS: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType( "aws_region_name": "us-east-1", } ), + "bedrock_converse": MappingProxyType( + { + "aws_access_key_id": "AKIASCRIPTEDPROVIDER", + "aws_secret_access_key": "scripted-secret", + "aws_region_name": "us-east-1", + } + ), + "deepgram": MappingProxyType({}), + "text-completion-openai": MappingProxyType({}), + "cohere": MappingProxyType({}), "vertex_ai-language-models": MappingProxyType( {"vertex_project": "cc-scripted-project", "vertex_location": "us-central1"} ), + "vertex_ai-image-models": MappingProxyType( + {"vertex_project": "cc-scripted-project", "vertex_location": "us-central1"} + ), "vertex_ai-embedding-models": MappingProxyType( {"vertex_project": "cc-scripted-project", "vertex_location": "us-central1"} ), diff --git a/tests/integration/cost_calculation/cost_tracking_cases.json b/tests/integration/cost_calculation/cost_tracking_cases.json index 162d52648eb..3ef2f477ebd 100644 --- a/tests/integration/cost_calculation/cost_tracking_cases.json +++ b/tests/integration/cost_calculation/cost_tracking_cases.json @@ -409,6 +409,89 @@ "output_cost_per_token": 3.6e-06, "supports_function_calling": true }, + "whisper-next": { + "litellm_provider": "openai", + "mode": "audio_transcription", + "input_cost_per_second": 0.0001 + }, + "whisper-verbose-next": { + "litellm_provider": "openai", + "mode": "audio_transcription", + "input_cost_per_second": 0.0002 + }, + "gpt-4o-transcribe-next": { + "litellm_provider": "openai", + "mode": "audio_transcription", + "input_cost_per_token": 2.11e-06, + "output_cost_per_token": 3.11e-06, + "input_cost_per_audio_token": 1e-05 + }, + "nova-next": { + "litellm_provider": "deepgram", + "mode": "audio_transcription", + "input_cost_per_second": 0.0003 + }, + "azure/whisper-next": { + "litellm_provider": "azure", + "mode": "audio_transcription", + "input_cost_per_second": 0.00011 + }, + "tts-next": { + "litellm_provider": "openai", + "mode": "audio_speech", + "input_cost_per_character": 1e-05 + }, + "tts-next-hd": { + "litellm_provider": "openai", + "mode": "audio_speech", + "input_cost_per_character": 2e-05 + }, + "azure/tts-next": { + "litellm_provider": "azure", + "mode": "audio_speech", + "input_cost_per_character": 1.1e-05 + }, + "gpt-image-next": { + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_token": 1.71e-06, + "output_cost_per_token": 4.3e-06, + "input_cost_per_image_token": 2.2e-06, + "output_cost_per_image_token": 5.1e-06 + }, + "1024-x-1024/dall-e-3-next": { + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_image": 0.04 + }, + "hd/1024-x-1024/dall-e-3-next": { + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_image": 0.08 + }, + "1792-x-1024/dall-e-3-next": { + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_image": 0.06 + }, + "low/1024-x-1024/gpt-image-next": { + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_token": 1.7e-06, + "output_cost_per_token": 4.3e-06, + "input_cost_per_image_token": 2.2e-06, + "output_cost_per_image_token": 5.1e-06 + }, + "1024-x-1024/imagen-next": { + "litellm_provider": "vertex_ai-image-models", + "mode": "image_generation", + "output_cost_per_image": 0.05 + }, + "amazon.nova-canvas-next": { + "litellm_provider": "bedrock", + "mode": "image_generation", + "output_cost_per_image": 0.045 + }, "text-embedding-4-small": { "input_cost_per_token": 1.01e-06, "output_cost_per_token": 0, @@ -25870,6 +25953,520 @@ "completion_tokens": 412 } }, + { + "name": "whisper-next-transcriptions-per-second", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "whisper-next", + "endpoint": "/v1/audio/transcriptions", + "upload": { + "kind": "wav", + "seconds": 3.5 + }, + "request": { + "language": "en", + "response_format": "json" + }, + "response": { + "content_type": "application/json", + "body": { + "text": "hello" + } + }, + "expected": { + "spend": 0.00035, + "input_cost": 0.00035, + "output_cost": 0, + "prompt_tokens": 0, + "completion_tokens": 0 + } + }, + { + "name": "whisper-verbose-next-transcriptions-duration", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "whisper-verbose-next", + "endpoint": "/v1/audio/transcriptions", + "upload": { + "kind": "wav", + "seconds": 3.5 + }, + "request": { + "response_format": "verbose_json" + }, + "response": { + "content_type": "application/json", + "body": { + "text": "hello", + "duration": 12.25 + } + }, + "expected": { + "spend": 0.00245, + "input_cost": 0.00245, + "output_cost": 0, + "prompt_tokens": 0, + "completion_tokens": 0 + } + }, + { + "name": "gpt-4o-transcribe-next-transcriptions-tokens", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-4o-transcribe-next", + "endpoint": "/v1/audio/transcriptions", + "upload": { + "kind": "wav", + "seconds": 1.0 + }, + "request": { + "response_format": "json" + }, + "response": { + "content_type": "application/json", + "body": { + "text": "hello", + "usage": { + "type": "tokens", + "input_tokens": 10, + "output_tokens": 2, + "total_tokens": 12, + "input_token_details": { + "text_tokens": 2, + "audio_tokens": 8 + } + } + } + }, + "expected": { + "spend": 9.044e-05, + "input_cost": 8.422e-05, + "output_cost": 6.22e-06, + "prompt_tokens": 10, + "completion_tokens": 2 + } + }, + { + "name": "nova-next-transcriptions-per-second", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "nova-next", + "endpoint": "/v1/audio/transcriptions", + "upload": { + "kind": "wav", + "seconds": 4.0 + }, + "request": {}, + "response": { + "content_type": "application/json", + "body": { + "results": { + "channels": [ + { + "alternatives": [ + { + "transcript": "hello", + "confidence": 0.9 + } + ] + } + ] + }, + "metadata": { + "duration": 4.0, + "channels": 1 + } + } + }, + "expected": { + "spend": 0.0012, + "input_cost": 0.0012, + "output_cost": 0, + "prompt_tokens": 0, + "completion_tokens": 0 + } + }, + { + "name": "azure-whisper-next-transcriptions-deployment", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "azure/whisper-next", + "endpoint": "/v1/audio/transcriptions", + "deployment": { + "model": "azure/cc-whisper-deployment", + "base_model": "azure/whisper-next" + }, + "upload": { + "kind": "wav", + "seconds": 3.5 + }, + "request": { + "response_format": "json" + }, + "response": { + "content_type": "application/json", + "body": { + "text": "hello" + } + }, + "expected": { + "spend": 0.000385, + "input_cost": 0.000385, + "output_cost": 0, + "prompt_tokens": 0, + "completion_tokens": 0 + } + }, + { + "name": "tts-next-speech-per-character", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "tts-next", + "endpoint": "/v1/audio/speech", + "request": { + "input": "hello world", + "voice": "alloy", + "response_format": "mp3" + }, + "response": { + "content_type": "audio/mpeg", + "length": 2048 + }, + "expected": { + "spend": 0.0001, + "input_cost": 0.0001, + "output_cost": 0, + "prompt_tokens": 0, + "completion_tokens": 0 + } + }, + { + "name": "tts-next-hd-speech-per-character", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "tts-next-hd", + "endpoint": "/v1/audio/speech", + "request": { + "input": "hello world", + "voice": "alloy", + "response_format": "mp3" + }, + "response": { + "content_type": "audio/mpeg", + "length": 2048 + }, + "expected": { + "spend": 0.0002, + "input_cost": 0.0002, + "output_cost": 0, + "prompt_tokens": 0, + "completion_tokens": 0 + } + }, + { + "name": "azure-tts-next-speech-deployment", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "azure/tts-next", + "endpoint": "/v1/audio/speech", + "deployment": { + "model": "azure/cc-tts-deployment", + "base_model": "azure/tts-next" + }, + "request": { + "input": "hello world", + "voice": "alloy", + "response_format": "mp3" + }, + "response": { + "content_type": "audio/mpeg", + "length": 2048 + }, + "expected": { + "spend": 0.00011, + "input_cost": 0.00011, + "output_cost": 0, + "prompt_tokens": 0, + "completion_tokens": 0 + } + }, + { + "name": "dall-e-3-next-images-standard", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "1024-x-1024/dall-e-3-next", + "endpoint": "/v1/images/generations", + "deployment": { + "model": "openai/dall-e-3-next" + }, + "request": { + "prompt": "a deterministic square", + "size": "1024x1024", + "quality": "standard", + "n": 1 + }, + "response": { + "content_type": "application/json", + "body": { + "created": 1700000000, + "data": [ + { + "url": "https://x/1.png" + } + ] + } + }, + "expected": { + "spend": 0.04, + "input_cost": 0.04, + "output_cost": 0, + "prompt_tokens": 0, + "completion_tokens": 0, + "breakdown_persisted": false + } + }, + { + "name": "dall-e-3-next-images-hd", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "hd/1024-x-1024/dall-e-3-next", + "endpoint": "/v1/images/generations", + "deployment": { + "model": "openai/dall-e-3-next" + }, + "request": { + "prompt": "a deterministic square", + "size": "1024x1024", + "quality": "hd", + "n": 1 + }, + "response": { + "content_type": "application/json", + "body": { + "created": 1700000001, + "data": [ + { + "url": "https://x/1.png" + } + ] + } + }, + "expected": { + "spend": 0.08, + "input_cost": 0.08, + "output_cost": 0, + "prompt_tokens": 0, + "completion_tokens": 0, + "breakdown_persisted": false + } + }, + { + "name": "dall-e-3-next-images-wide", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "1792-x-1024/dall-e-3-next", + "endpoint": "/v1/images/generations", + "deployment": { + "model": "openai/dall-e-3-next" + }, + "request": { + "prompt": "a deterministic wide image", + "size": "1792x1024", + "quality": "standard", + "n": 1 + }, + "response": { + "content_type": "application/json", + "body": { + "created": 1700000002, + "data": [ + { + "url": "https://x/1.png" + } + ] + } + }, + "expected": { + "spend": 0.06, + "input_cost": 0.06, + "output_cost": 0, + "prompt_tokens": 0, + "completion_tokens": 0, + "breakdown_persisted": false + } + }, + { + "name": "dall-e-3-next-images-two", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "1024-x-1024/dall-e-3-next", + "endpoint": "/v1/images/generations", + "deployment": { + "model": "openai/dall-e-3-next" + }, + "request": { + "prompt": "two deterministic squares", + "size": "1024x1024", + "quality": "standard", + "n": 2 + }, + "response": { + "content_type": "application/json", + "body": { + "created": 1700000003, + "data": [ + { + "url": "https://x/1.png" + }, + { + "url": "https://x/2.png" + } + ] + } + }, + "expected": { + "spend": 0.08, + "input_cost": 0.08, + "output_cost": 0, + "prompt_tokens": 0, + "completion_tokens": 0, + "breakdown_persisted": false + } + }, + { + "name": "gpt-image-next-images-low", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-image-next", + "endpoint": "/v1/images/generations", + "deployment": { + "model": "openai/gpt-image-next" + }, + "request": { + "prompt": "a deterministic generated image", + "size": "1024x1024", + "quality": "low", + "n": 1 + }, + "response": { + "content_type": "application/json", + "body": { + "created": 1700000004, + "data": [ + { + "b64_json": "AA==" + } + ], + "usage": { + "total_tokens": 30, + "input_tokens": 10, + "output_tokens": 20, + "input_tokens_details": { + "text_tokens": 10, + "image_tokens": 0 + } + } + } + }, + "expected": { + "spend": 0.0001191, + "input_cost": 1.71e-05, + "output_cost": 0.000102, + "prompt_tokens": 10, + "completion_tokens": 20, + "breakdown_persisted": false + } + }, + { + "name": "imagen-next-images-one", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "1024-x-1024/imagen-next", + "endpoint": "/v1/images/generations", + "request": { + "prompt": "a deterministic vertex image", + "sampleCount": 1 + }, + "response": { + "content_type": "application/json", + "body": { + "predictions": [ + { + "bytesBase64Encoded": "AA==", + "mimeType": "image/png" + } + ] + } + }, + "expected": { + "spend": 0.05, + "input_cost": 0.05, + "output_cost": 0, + "prompt_tokens": 0, + "completion_tokens": 0, + "breakdown_persisted": false + } + }, + { + "name": "amazon-nova-canvas-next-images-one", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "amazon.nova-canvas-next", + "endpoint": "/v1/images/generations", + "deployment": { + "model": "amazon.nova-canvas-next" + }, + "request": { + "prompt": "a deterministic bedrock image" + }, + "response": { + "content_type": "application/json", + "body": { + "images": [ + "AA==" + ] + } + }, + "expected": { + "spend": 0.045, + "input_cost": 0.045, + "output_cost": 0, + "prompt_tokens": 0, + "completion_tokens": 0, + "breakdown_persisted": false + } + }, + { + "name": "gpt-image-next-images-edit", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "low/1024-x-1024/gpt-image-next", + "endpoint": "/v1/images/edits", + "deployment": { + "model": "openai/gpt-image-next" + }, + "upload": { + "kind": "png" + }, + "request": { + "prompt": "edit this deterministic image", + "size": "1024x1024", + "quality": "low", + "n": 1 + }, + "response": { + "content_type": "application/json", + "body": { + "created": 1700000005, + "data": [ + { + "b64_json": "AA==" + } + ], + "usage": { + "total_tokens": 30, + "input_tokens": 10, + "output_tokens": 20, + "input_tokens_details": { + "text_tokens": 10, + "image_tokens": 0 + } + } + } + }, + "expected": { + "spend": 0.000119, + "input_cost": 1.7e-05, + "output_cost": 0.000102, + "prompt_tokens": 10, + "completion_tokens": 20, + "breakdown_persisted": false + } + }, { "name": "text-embeddings-4-small-single", "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", diff --git a/tests/integration/cost_calculation/test_cost_tracking.py b/tests/integration/cost_calculation/test_cost_tracking.py index e31b0131b27..32ecd573363 100644 --- a/tests/integration/cost_calculation/test_cost_tracking.py +++ b/tests/integration/cost_calculation/test_cost_tracking.py @@ -2,13 +2,20 @@ from __future__ import annotations +import io +import json from hashlib import sha256 +import struct from typing import Final, cast +import wave +import zlib +import httpx import pytest from integration._support.client import JSON_OBJECT, Gateway from integration.cost_calculation.conftest import ( + CostBreakdown, approx_equal, assert_total_is_sum_of_components, poll_cost_row, @@ -16,6 +23,7 @@ from integration.cost_calculation.conftest import ( register_scenario_deployment, ) from integration.cost_calculation.cost_tracking_case import ( + BinaryResponse, CASES, CostTrackingTestCase, ExactExpected, @@ -34,6 +42,47 @@ _CASES: Final = tuple( ) +def _wav_bytes(seconds: float) -> bytes: + frame_count: Final = round(16000 * seconds) + output: Final = io.BytesIO() + with wave.open(output, "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(16000) + wav.writeframes(b"\x00\x00" * frame_count) + return output.getvalue() + + +def _png_bytes() -> bytes: + def chunk(kind: bytes, payload: bytes) -> bytes: + return ( + struct.pack(">I", len(payload)) + + kind + + payload + + struct.pack(">I", zlib.crc32(kind + payload) & 0xFFFFFFFF) + ) + + return ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", 1, 1, 8, 6, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(b"\x00\x00\x00\x00\x00")) + + chunk(b"IEND", b"") + ) + + +def _multipart_request(gateway: Gateway, case: CostTrackingTestCase, model_name: str, key: str) -> httpx.Response: + assert case.upload is not None + fields: Final = { + field: value if isinstance(value, str) else json.dumps(value, separators=(",", ":")) + for field, value in {**case.request, "model": model_name}.items() + } + if case.upload.kind == "wav": + files: Final = {"file": ("audio.wav", _wav_bytes(case.upload.seconds), "audio/wav")} + else: + files = {"image": ("image.png", _png_bytes(), "image/png")} + return gateway.request_multipart(case.endpoint, fields, files, key=key) + + def _assert_stream_has_no_error(response_text: str) -> None: for line in response_text.splitlines(): if not line.startswith("data:"): @@ -47,17 +96,86 @@ def _assert_stream_has_no_error(response_text: str) -> None: ), f"stream carried an error event: {parsed}" +def _assert_breakdown( + case: CostTrackingTestCase, + expected: ExactExpected, + breakdown: CostBreakdown, + response: httpx.Response, +) -> None: + assert breakdown.input_cost is not None and approx_equal(breakdown.input_cost, expected.input_cost), ( + f"{case.name}: input_cost {breakdown.input_cost} != expected {expected.input_cost}" + ) + assert breakdown.output_cost is not None and approx_equal(breakdown.output_cost, expected.output_cost), ( + f"{case.name}: output_cost {breakdown.output_cost} != expected {expected.output_cost}" + ) + for field, header_name, actual_component, expected_component in ( + ( + "cache_read_cost", + "x-litellm-response-cost-cache-read", + breakdown.cache_read_cost, + expected.cache_read_cost, + ), + ( + "cache_creation_cost", + "x-litellm-response-cost-cache-creation", + breakdown.cache_creation_cost, + expected.cache_creation_cost, + ), + ( + "reasoning_cost", + "x-litellm-response-cost-reasoning", + breakdown.reasoning_cost, + expected.reasoning_cost, + ), + ( + "tool_usage_cost", + "x-litellm-response-cost-tool-usage", + breakdown.tool_usage_cost, + expected.tool_usage_cost, + ), + ): + if expected_component is None: + continue + assert actual_component is not None and approx_equal(actual_component, expected_component), ( + f"{case.name}: {field} {actual_component} != expected {expected_component}" + ) + if case.response.content_type == "application/json": + header: Final = response.headers.get(header_name) + assert header is not None and approx_equal(float(header), expected_component), ( + f"{case.name}: {header_name} {header} != expected {expected_component}" + ) + if case.response.content_type == "application/json" and any( + component is not None + for component in ( + expected.cache_read_cost, + expected.cache_creation_cost, + expected.reasoning_cost, + expected.tool_usage_cost, + ) + ): + input_header: Final = response.headers.get("x-litellm-response-cost-input") + output_header: Final = response.headers.get("x-litellm-response-cost-output") + expected_input_header: Final = expected.input_cost - ( + expected.cache_read_cost or 0.0 + ) - (expected.cache_creation_cost or 0.0) + assert input_header is not None and approx_equal(float(input_header), expected_input_header), ( + f"{case.name}: x-litellm-response-cost-input {input_header} != expected {expected_input_header}" + ) + assert output_header is not None and approx_equal(float(output_header), expected.output_cost), ( + f"{case.name}: x-litellm-response-cost-output {output_header} != expected {expected.output_cost}" + ) + + @pytest.mark.parametrize("case", _CASES) def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) -> None: marker: Final = sha256(case.name.encode()).hexdigest()[:12] with gateway.scenario() as scenario: key: Final = scenario.key() model_name: Final = register_scenario_deployment(scenario, case, marker, key) - response: Final = gateway.request( - "POST", - case.endpoint, - {**case.request, "model": model_name}, - key=key, + response: Final = ( + _multipart_request(gateway, case, model_name, key) + if case.upload is not None + else gateway.request("POST", case.endpoint, {**case.request, "model": model_name}, key=key) ) if isinstance(case.expected, FailureExpected): assert response.status_code == case.expected.failure.status, ( @@ -88,11 +206,19 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) assert row.spend is not None and approx_equal(row.spend, recount), ( f"{case.name}: spend {row.spend} != recount {recount} at map rates" ) - assert_total_is_sum_of_components(row, case.name) + breakdown: Final = row.breakdown + assert breakdown is not None, f"{case.name}: no cost_breakdown persisted" + assert_total_is_sum_of_components(row, breakdown, case.name) return expected: Final = case.expected assert isinstance(expected, ExactExpected) - if case.response.content_type == "application/json": + if isinstance(case.response, BinaryResponse): + header: Final = response.headers.get("x-litellm-response-cost") + if header is not None: + assert approx_equal(float(header), expected.spend), ( + f"{case.name}: x-litellm-response-cost {header} != expected {expected.spend}" + ) + elif case.response.content_type == "application/json": header: Final = cast(str | None, response.headers.get("x-litellm-response-cost")) assert ( (header is None or approx_equal(float(header), 0.0)) @@ -103,75 +229,18 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) ) assert row.spend is not None and approx_equal(row.spend, expected.spend), ( f"{case.name}: spend {row.spend} != expected {expected.spend} " - f"(breakdown {row.breakdown.model_dump()})" + f"(breakdown {row.breakdown.model_dump() if row.breakdown is not None else None})" ) breakdown: Final = row.breakdown - assert breakdown.input_cost is not None and approx_equal(breakdown.input_cost, expected.input_cost), ( - f"{case.name}: input_cost {breakdown.input_cost} != expected {expected.input_cost}" - ) - assert breakdown.output_cost is not None and approx_equal(breakdown.output_cost, expected.output_cost), ( - f"{case.name}: output_cost {breakdown.output_cost} != expected {expected.output_cost}" - ) - for field, header_name, actual_component, expected_component in ( - ( - "cache_read_cost", - "x-litellm-response-cost-cache-read", - breakdown.cache_read_cost, - expected.cache_read_cost, - ), - ( - "cache_creation_cost", - "x-litellm-response-cost-cache-creation", - breakdown.cache_creation_cost, - expected.cache_creation_cost, - ), - ( - "reasoning_cost", - "x-litellm-response-cost-reasoning", - breakdown.reasoning_cost, - expected.reasoning_cost, - ), - ( - "tool_usage_cost", - "x-litellm-response-cost-tool-usage", - breakdown.tool_usage_cost, - expected.tool_usage_cost, - ), - ): - if expected_component is None: - continue - assert actual_component is not None and approx_equal(actual_component, expected_component), ( - f"{case.name}: {field} {actual_component} != expected {expected_component}" - ) - if case.response.content_type == "application/json": - header: Final = response.headers.get(header_name) - assert header is not None and approx_equal(float(header), expected_component), ( - f"{case.name}: {header_name} {header} != expected {expected_component}" - ) - if case.response.content_type == "application/json" and any( - component is not None - for component in ( - expected.cache_read_cost, - expected.cache_creation_cost, - expected.reasoning_cost, - expected.tool_usage_cost, - ) - ): - input_header: Final = response.headers.get("x-litellm-response-cost-input") - output_header: Final = response.headers.get("x-litellm-response-cost-output") - expected_input_header: Final = expected.input_cost - ( - expected.cache_read_cost or 0.0 - ) - (expected.cache_creation_cost or 0.0) - assert input_header is not None and approx_equal(float(input_header), expected_input_header), ( - f"{case.name}: x-litellm-response-cost-input {input_header} != expected {expected_input_header}" - ) - assert output_header is not None and approx_equal(float(output_header), expected.output_cost), ( - f"{case.name}: x-litellm-response-cost-output {output_header} != expected {expected.output_cost}" - ) + if expected.breakdown_persisted: + assert breakdown is not None, f"{case.name}: no cost_breakdown persisted" + if breakdown is not None: + _assert_breakdown(case, expected, breakdown, response) assert row.prompt_tokens == expected.prompt_tokens, ( f"{case.name}: prompt_tokens {row.prompt_tokens} != expected {expected.prompt_tokens}" ) assert row.completion_tokens == expected.completion_tokens, ( f"{case.name}: completion_tokens {row.completion_tokens} != expected {expected.completion_tokens}" ) - assert_total_is_sum_of_components(row, case.name) + if breakdown is not None: + assert_total_is_sum_of_components(row, breakdown, case.name)