From 125bda30e9356551ae37657bd7481022a51d6365 Mon Sep 17 00:00:00 2001 From: kerry Date: Sat, 19 Sep 2026 20:58:13 +0000 Subject: [PATCH 1/4] test(integration): embeddings, rerank, completions and moderations cost cases Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/contracts.json | 120 +++ .../integration/cost_calculation/conftest.py | 2 +- .../cost_calculation/cost_tracking_case.py | 22 +- .../cost_calculation/cost_tracking_cases.json | 937 ++++++++++++++++++ .../cost_calculation/test_cost_tracking.py | 2 +- 5 files changed, 1080 insertions(+), 3 deletions(-) diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index 338ace07908..d86f44cf24a 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -1320,6 +1320,126 @@ ], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_full_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-embedding-4-small-single]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-embedding-4-small-batch]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-embedding-4-small-token-array]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-embedding-4-large-dimensions]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-text-embedding-4-large-deployment]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[cohere-embed-v5]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-titan-embed-v3]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-cohere-embed-v4]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[vertex-text-embedding-006]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-embedding-002]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together-embedding-v1]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks-embedding-v1]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-completion-openai-basic]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-completion-openai-stream-usage]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-completion-openai-n-best]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together-completions]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[omni-moderation-next-single]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[omni-moderation-next-list]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-text-embeddings-4-large-deployment]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-cohere-embeddings-v4]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-cohere-rerank-v4]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-embeddings-titan-v2]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[cohere-embeddings-v5]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[cohere-rerank-v4-one]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[cohere-rerank-v4-three]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[cohere-rerank-v4-total-tokens-fallback]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks-embeddings-v1]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-embeddings-002]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[omni-moderations-next-list]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[omni-moderations-next-single]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-completions-openai-basic]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-completions-openai-n-best]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-completions-openai-stream-usage]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-embeddings-3-large-dimensions]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-embeddings-4-small-batch]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-embeddings-4-small-single]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-embeddings-4-small-token-array]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together-completions-v1]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together-embeddings-v1]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[vertex-embeddings-text-006]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" ] }, "browser": { diff --git a/tests/integration/cost_calculation/conftest.py b/tests/integration/cost_calculation/conftest.py index 166488e36a5..414711648aa 100644 --- a/tests/integration/cost_calculation/conftest.py +++ b/tests/integration/cost_calculation/conftest.py @@ -165,7 +165,7 @@ def register_scenario_deployment( **case.litellm_params, **( {"vertex_credentials": _vertex_service_account_json(control_url)} - if case.rates.litellm_provider == "vertex_ai-language-models" + if case.rates.litellm_provider.startswith("vertex_ai") else {} ), } diff --git a/tests/integration/cost_calculation/cost_tracking_case.py b/tests/integration/cost_calculation/cost_tracking_case.py index 9737508e27c..e24f5d76d0d 100644 --- a/tests/integration/cost_calculation/cost_tracking_case.py +++ b/tests/integration/cost_calculation/cost_tracking_case.py @@ -35,7 +35,10 @@ class CostMapEntry(BaseModel): max_output_tokens: int | None = None supports_function_calling: bool | None = None input_cost_per_token: float | None = None + input_cost_per_query: float | None = None output_cost_per_token: float | None = None + output_vector_size: int | None = None + input_cost_per_token_batches: float | None = None cache_read_input_token_cost: float | None = None cache_creation_input_token_cost: float | None = None cache_creation_input_token_cost_above_1hr: float | None = None @@ -172,7 +175,8 @@ class CostTrackingTestCase(BaseModel): provider: Final = self.rates.litellm_provider prefix: Final = ( "openai" - if provider == "openai" and self.rates.mode == "chat" + if provider == "openai" + and (self.endpoint == "/v1/responses" or self.rates.mode in {"chat", "embedding", "moderation"}) else "openai/responses" if provider == "openai" else _PROVIDER_PREFIXES.get(provider) @@ -207,7 +211,11 @@ _PROVIDER_PREFIXES: Final[Mapping[str, str]] = MappingProxyType( { "anthropic": "anthropic", "bedrock_converse": "bedrock/converse", + "text-completion-openai": "text-completion-openai", + "cohere": "cohere", + "bedrock": "bedrock", "vertex_ai-language-models": "vertex_ai", + "vertex_ai-embedding-models": "vertex_ai", "gemini": "", "together_ai": "", "fireworks_ai": "", @@ -224,9 +232,21 @@ _LITELLM_PARAMS: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType( "aws_region_name": "us-east-1", } ), + "text-completion-openai": MappingProxyType({}), + "cohere": MappingProxyType({}), + "bedrock": MappingProxyType( + { + "aws_access_key_id": "AKIASCRIPTEDPROVIDER", + "aws_secret_access_key": "scripted-secret", + "aws_region_name": "us-east-1", + } + ), "vertex_ai-language-models": MappingProxyType( {"vertex_project": "cc-scripted-project", "vertex_location": "us-central1"} ), + "vertex_ai-embedding-models": MappingProxyType( + {"vertex_project": "cc-scripted-project", "vertex_location": "us-central1"} + ), "gemini": MappingProxyType({}), "together_ai": MappingProxyType({}), "fireworks_ai": MappingProxyType({}), diff --git a/tests/integration/cost_calculation/cost_tracking_cases.json b/tests/integration/cost_calculation/cost_tracking_cases.json index b0ad270f860..a076381c4e3 100644 --- a/tests/integration/cost_calculation/cost_tracking_cases.json +++ b/tests/integration/cost_calculation/cost_tracking_cases.json @@ -408,6 +408,98 @@ "mode": "chat", "output_cost_per_token": 3.6e-06, "supports_function_calling": true + }, + "text-embedding-4-small": { + "input_cost_per_token": 1.01e-06, + "output_cost_per_token": 0, + "litellm_provider": "openai", + "mode": "embedding" + }, + "text-embedding-3-large-next": { + "input_cost_per_token": 1.02e-06, + "output_cost_per_token": 0, + "litellm_provider": "openai", + "mode": "embedding" + }, + "azure/text-embedding-4-large": { + "input_cost_per_token": 1.03e-06, + "output_cost_per_token": 0, + "litellm_provider": "azure", + "mode": "embedding" + }, + "embed-v5": { + "input_cost_per_token": 1.04e-06, + "output_cost_per_token": 0, + "litellm_provider": "cohere", + "mode": "embedding" + }, + "amazon.titan-embed-text-v2:0": { + "input_cost_per_token": 1.05e-06, + "output_cost_per_token": 0, + "litellm_provider": "bedrock", + "mode": "embedding" + }, + "cohere.embed-english-v4": { + "input_cost_per_token": 1.06e-06, + "output_cost_per_token": 0, + "litellm_provider": "bedrock", + "mode": "embedding" + }, + "text-embedding-006": { + "input_cost_per_token": 1.07e-06, + "output_cost_per_token": 0, + "litellm_provider": "vertex_ai-embedding-models", + "mode": "embedding" + }, + "gemini/gemini-embedding-002": { + "input_cost_per_token": 1.08e-06, + "output_cost_per_token": 0, + "litellm_provider": "gemini", + "mode": "embedding" + }, + "together_ai/together-embed-v1": { + "input_cost_per_token": 1.09e-06, + "output_cost_per_token": 0, + "litellm_provider": "together_ai", + "mode": "embedding" + }, + "fireworks_ai/fireworks-embed-v1": { + "input_cost_per_token": 1.1e-06, + "output_cost_per_token": 0, + "litellm_provider": "fireworks_ai", + "mode": "embedding" + }, + "rerank-v4": { + "input_cost_per_token": 1.11e-06, + "output_cost_per_token": 0, + "input_cost_per_query": 0.0021, + "litellm_provider": "cohere", + "mode": "rerank" + }, + "cohere.rerank-v4:0": { + "input_cost_per_token": 1.12e-06, + "output_cost_per_token": 0, + "input_cost_per_query": 0.0022, + "litellm_provider": "bedrock", + "mode": "rerank" + }, + "gpt-3.5-turbo-instruct-next": { + "input_cost_per_token": 1.14e-06, + "output_cost_per_token": 2.14e-06, + "litellm_provider": "text-completion-openai", + "mode": "completion" + }, + "omni-moderation-next": { + "input_cost_per_token": null, + "output_cost_per_token": 0, + "litellm_provider": "openai", + "mode": "moderation" + }, + "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo": { + "input_cost_per_token": 1.16e-06, + "output_cost_per_token": 2.16e-06, + "litellm_provider": "together_ai", + "mode": "completion" } }, "cases": [ @@ -25777,6 +25869,851 @@ "prompt_tokens": 11056, "completion_tokens": 412 } + }, + { + "name": "text-embeddings-4-small-single", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "text-embedding-4-small", + "endpoint": "/v1/embeddings", + "request": { + "model": "$MODEL", + "input": "one embedding" + }, + "response": { + "content_type": "application/json", + "body": { + "object": "list", + "data": [ + { + "object": "embedding", + "embedding": [ + 0.1, + 0.2, + 0.3 + ], + "index": 0 + } + ], + "model": "text-embedding-4-small", + "usage": { + "prompt_tokens": 7, + "total_tokens": 7 + } + } + }, + "expected": { + "spend": 7.07e-06, + "input_cost": 7.07e-06, + "output_cost": 0.0, + "prompt_tokens": 7, + "completion_tokens": 0 + } + }, + { + "name": "text-embeddings-4-small-batch", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "text-embedding-4-small", + "endpoint": "/v1/embeddings", + "request": { + "model": "$MODEL", + "input": [ + "one", + "two", + "three" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "object": "list", + "data": [ + { + "object": "embedding", + "embedding": [ + 0.1, + 0.2, + 0.3 + ], + "index": 0 + }, + { + "object": "embedding", + "embedding": [ + 0.1, + 0.2, + 0.3 + ], + "index": 1 + }, + { + "object": "embedding", + "embedding": [ + 0.1, + 0.2, + 0.3 + ], + "index": 2 + } + ], + "model": "text-embedding-4-small", + "usage": { + "prompt_tokens": 21, + "total_tokens": 21 + } + } + }, + "expected": { + "spend": 2.1210000000000002e-05, + "input_cost": 2.1210000000000002e-05, + "output_cost": 0.0, + "prompt_tokens": 21, + "completion_tokens": 0 + } + }, + { + "name": "text-embeddings-4-small-token-array", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "text-embedding-4-small", + "endpoint": "/v1/embeddings", + "request": { + "model": "$MODEL", + "input": [ + 1, + 2, + 3, + 4 + ] + }, + "response": { + "content_type": "application/json", + "body": { + "object": "list", + "data": [ + { + "object": "embedding", + "embedding": [ + 0.1, + 0.2, + 0.3 + ], + "index": 0 + } + ], + "model": "text-embedding-4-small", + "usage": { + "prompt_tokens": 9, + "total_tokens": 9 + } + } + }, + "expected": { + "spend": 9.090000000000001e-06, + "input_cost": 9.090000000000001e-06, + "output_cost": 0.0, + "prompt_tokens": 9, + "completion_tokens": 0 + } + }, + { + "name": "text-embeddings-3-large-dimensions", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "text-embedding-3-large-next", + "endpoint": "/v1/embeddings", + "request": { + "model": "$MODEL", + "input": "large embedding", + "dimensions": 3 + }, + "response": { + "content_type": "application/json", + "body": { + "object": "list", + "data": [ + { + "object": "embedding", + "embedding": [ + 0.1, + 0.2, + 0.3 + ], + "index": 0 + } + ], + "model": "text-embedding-3-large-next", + "usage": { + "prompt_tokens": 8, + "total_tokens": 8 + } + } + }, + "expected": { + "spend": 8.16e-06, + "input_cost": 8.16e-06, + "output_cost": 0.0, + "prompt_tokens": 8, + "completion_tokens": 0 + } + }, + { + "name": "azure-text-embeddings-4-large-deployment", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "azure/text-embedding-4-large", + "endpoint": "/v1/embeddings", + "request": { + "model": "$MODEL", + "input": "azure embedding" + }, + "response": { + "content_type": "application/json", + "body": { + "object": "list", + "data": [ + { + "object": "embedding", + "embedding": [ + 0.1, + 0.2, + 0.3 + ], + "index": 0 + } + ], + "model": "azure/text-embedding-4-large", + "usage": { + "prompt_tokens": 8, + "total_tokens": 8 + } + } + }, + "expected": { + "spend": 8.24e-06, + "input_cost": 8.24e-06, + "output_cost": 0.0, + "prompt_tokens": 8, + "completion_tokens": 0 + }, + "deployment": { + "model": "azure/cc-pinned-embedding-deployment", + "base_model": "azure/text-embedding-4-large" + } + }, + { + "name": "cohere-embeddings-v5", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "embed-v5", + "endpoint": "/v1/embeddings", + "request": { + "model": "$MODEL", + "input": "cohere embedding", + "input_type": "search_query" + }, + "response": { + "content_type": "application/json", + "body": { + "id": "emb-1", + "embeddings": { + "float": [ + [ + 0.1, + 0.2, + 0.3 + ] + ] + }, + "meta": { + "billed_units": { + "input_tokens": 11 + } + } + } + }, + "expected": { + "spend": 1.144e-05, + "input_cost": 1.144e-05, + "output_cost": 0.0, + "prompt_tokens": 11, + "completion_tokens": 0 + } + }, + { + "name": "bedrock-embeddings-titan-v2", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "amazon.titan-embed-text-v2:0", + "endpoint": "/v1/embeddings", + "request": { + "model": "$MODEL", + "input": "titan embedding" + }, + "response": { + "content_type": "application/json", + "body": { + "embedding": [ + 0.1, + 0.2, + 0.3 + ], + "inputTextTokenCount": 10 + } + }, + "expected": { + "spend": 1.05e-05, + "input_cost": 1.05e-05, + "output_cost": 0.0, + "prompt_tokens": 10, + "completion_tokens": 0 + } + }, + { + "name": "bedrock-cohere-embeddings-v4", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "cohere.embed-english-v4", + "endpoint": "/v1/embeddings", + "request": { + "model": "$MODEL", + "input": "bedrock cohere embedding" + }, + "response": { + "content_type": "application/json", + "body": { + "embeddings": [ + [ + 0.1, + 0.2, + 0.3 + ] + ], + "id": "emb-bedrock-cohere-1", + "response_type": "embeddings_floats", + "texts": [ + "bedrock cohere embedding" + ] + } + }, + "expected": { + "spend": 5.3e-06, + "input_cost": 5.3e-06, + "output_cost": 0.0, + "prompt_tokens": 5, + "completion_tokens": 0 + } + }, + { + "name": "vertex-embeddings-text-006", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "text-embedding-006", + "endpoint": "/v1/embeddings", + "request": { + "model": "$MODEL", + "input": "vertex embedding" + }, + "response": { + "content_type": "application/json", + "body": { + "predictions": [ + { + "embeddings": { + "values": [ + 0.1, + 0.2, + 0.3 + ], + "statistics": { + "token_count": 7, + "truncated": false + } + } + } + ] + } + }, + "expected": { + "spend": 7.4899999999999994e-06, + "input_cost": 7.4899999999999994e-06, + "output_cost": 0.0, + "prompt_tokens": 7, + "completion_tokens": 0 + } + }, + { + "name": "gemini-embeddings-002", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini/gemini-embedding-002", + "endpoint": "/v1/embeddings", + "request": { + "model": "$MODEL", + "input": "gemini embedding" + }, + "response": { + "content_type": "application/json", + "body": { + "embeddings": [ + { + "values": [ + 0.1, + 0.2, + 0.3 + ] + } + ], + "usageMetadata": { + "promptTokenCount": 7, + "totalTokenCount": 7 + } + } + }, + "expected": { + "spend": 3.24e-06, + "input_cost": 3.24e-06, + "output_cost": 0.0, + "prompt_tokens": 3, + "completion_tokens": 0 + } + }, + { + "name": "together-embeddings-v1", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "together_ai/together-embed-v1", + "endpoint": "/v1/embeddings", + "request": { + "model": "$MODEL", + "input": "together embedding" + }, + "response": { + "content_type": "application/json", + "body": { + "object": "list", + "data": [ + { + "object": "embedding", + "embedding": [ + 0.1, + 0.2, + 0.3 + ], + "index": 0 + } + ], + "model": "together-embed-v1", + "usage": { + "prompt_tokens": 7, + "total_tokens": 7 + } + } + }, + "expected": { + "spend": 7.63e-06, + "input_cost": 7.63e-06, + "output_cost": 0.0, + "prompt_tokens": 7, + "completion_tokens": 0 + } + }, + { + "name": "fireworks-embeddings-v1", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "fireworks_ai/fireworks-embed-v1", + "endpoint": "/v1/embeddings", + "request": { + "model": "$MODEL", + "input": "fireworks embedding" + }, + "response": { + "content_type": "application/json", + "body": { + "object": "list", + "data": [ + { + "object": "embedding", + "embedding": [ + 0.1, + 0.2, + 0.3 + ], + "index": 0 + } + ], + "model": "fireworks-embed-v1", + "usage": { + "prompt_tokens": 7, + "total_tokens": 7 + } + } + }, + "expected": { + "spend": 7.7e-06, + "input_cost": 7.7e-06, + "output_cost": 0.0, + "prompt_tokens": 7, + "completion_tokens": 0 + } + }, + { + "name": "cohere-rerank-v4-one", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "rerank-v4", + "endpoint": "/v1/rerank", + "request": { + "model": "$MODEL", + "query": "rank this", + "documents": [ + "a", + "b" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "rr-$REQUEST_ID", + "results": [ + { + "index": 0, + "relevance_score": 0.9 + } + ], + "meta": { + "api_version": { + "version": "2" + }, + "billed_units": { + "search_units": 1 + } + } + } + }, + "expected": { + "spend": 0.0021, + "input_cost": 0.0021, + "output_cost": 0.0, + "prompt_tokens": 0, + "completion_tokens": 0 + } + }, + { + "name": "cohere-rerank-v4-three", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "rerank-v4", + "endpoint": "/v1/rerank", + "request": { + "model": "$MODEL", + "query": "rank this", + "documents": [ + "a long document", + "another long document", + "third long document" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "rr-three-$REQUEST_ID", + "results": [ + { + "index": 0, + "relevance_score": 0.9 + } + ], + "meta": { + "api_version": { + "version": "2" + }, + "billed_units": { + "search_units": 3 + } + } + } + }, + "expected": { + "spend": 0.0063, + "input_cost": 0.0063, + "output_cost": 0.0, + "prompt_tokens": 0, + "completion_tokens": 0 + } + }, + { + "name": "cohere-rerank-v4-total-tokens-fallback", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "rerank-v4", + "endpoint": "/v1/rerank", + "request": { + "model": "$MODEL", + "query": "rank this", + "documents": [ + "fallback a", + "fallback b" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "rr-fallback-$REQUEST_ID", + "results": [ + { + "index": 0, + "relevance_score": 0.8 + } + ], + "meta": { + "billed_units": { + "total_tokens": 99 + } + } + } + }, + "expected": { + "spend": 0.0, + "input_cost": 0.0, + "output_cost": 0.0, + "prompt_tokens": 0, + "completion_tokens": 0 + } + }, + { + "name": "bedrock-cohere-rerank-v4", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "cohere.rerank-v4:0", + "endpoint": "/v1/rerank", + "request": { + "model": "$MODEL", + "query": "rank this", + "documents": [ + "a", + "b" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "results": [ + { + "index": 0, + "relevanceScore": 0.9 + } + ], + "response_id": "rr-3", + "token_count": 1 + } + }, + "expected": { + "spend": 0.0022, + "input_cost": 0.0022, + "output_cost": 0.0, + "prompt_tokens": 0, + "completion_tokens": 0 + } + }, + { + "name": "text-completions-openai-basic", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-3.5-turbo-instruct-next", + "endpoint": "/v1/completions", + "request": { + "model": "$MODEL", + "prompt": "complete this" + }, + "response": { + "content_type": "application/json", + "body": { + "id": "cmpl-basic-$REQUEST_ID", + "object": "text_completion", + "choices": [ + { + "text": "done", + "index": 0, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 9, + "completion_tokens": 4, + "total_tokens": 13 + } + } + }, + "expected": { + "spend": 1.882e-05, + "input_cost": 1.026e-05, + "output_cost": 8.56e-06, + "prompt_tokens": 9, + "completion_tokens": 4 + } + }, + { + "name": "text-completions-openai-stream-usage", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-3.5-turbo-instruct-next", + "endpoint": "/v1/completions", + "request": { + "model": "$MODEL", + "prompt": "complete this", + "stream": true, + "stream_options": { + "include_usage": true + } + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"cmpl-$REQUEST_ID\", \"object\": \"text_completion\", \"created\": 1789789000, \"model\": \"gpt-3.5-turbo-instruct-next\", \"choices\": [{\"text\": \"done\", \"index\": 0, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"cmpl-$REQUEST_ID\", \"object\": \"text_completion\", \"created\": 1789789000, \"model\": \"gpt-3.5-turbo-instruct-next\", \"choices\": [], \"usage\": {\"prompt_tokens\": 9, \"completion_tokens\": 4, \"total_tokens\": 13}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 1.882e-05, + "input_cost": 1.026e-05, + "output_cost": 8.56e-06, + "prompt_tokens": 9, + "completion_tokens": 4 + } + }, + { + "name": "text-completions-openai-n-best", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-3.5-turbo-instruct-next", + "endpoint": "/v1/completions", + "request": { + "model": "$MODEL", + "prompt": "complete this twice", + "n": 2 + }, + "response": { + "content_type": "application/json", + "body": { + "id": "cmpl-n-best-$REQUEST_ID", + "object": "text_completion", + "choices": [ + { + "text": "done", + "index": 0, + "finish_reason": "stop" + }, + { + "text": "also done", + "index": 1, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 9, + "completion_tokens": 8, + "total_tokens": 17 + } + } + }, + "expected": { + "spend": 2.738e-05, + "input_cost": 1.026e-05, + "output_cost": 1.712e-05, + "prompt_tokens": 9, + "completion_tokens": 8 + } + }, + { + "name": "together-completions-v1", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo", + "endpoint": "/v1/completions", + "request": { + "model": "$MODEL", + "prompt": "together complete" + }, + "response": { + "content_type": "application/json", + "body": { + "id": "cmpl-together-$REQUEST_ID", + "object": "text_completion", + "choices": [ + { + "text": "done", + "index": 0, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 9, + "completion_tokens": 4, + "total_tokens": 13 + } + } + }, + "expected": { + "spend": 1.908e-05, + "input_cost": 1.0439999999999998e-05, + "output_cost": 8.64e-06, + "prompt_tokens": 9, + "completion_tokens": 4 + } + }, + { + "name": "omni-moderations-next-single", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "omni-moderation-next", + "endpoint": "/v1/moderations", + "request": { + "model": "$MODEL", + "input": "safe text" + }, + "response": { + "content_type": "application/json", + "body": { + "id": "modr-single-$REQUEST_ID", + "model": "omni-moderation-next", + "results": [ + { + "flagged": false, + "categories": {}, + "category_scores": {} + } + ] + } + }, + "expected": { + "spend": 0.0, + "input_cost": 0.0, + "output_cost": 0.0, + "prompt_tokens": 0, + "completion_tokens": 0 + } + }, + { + "name": "omni-moderations-next-list", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "omni-moderation-next", + "endpoint": "/v1/moderations", + "request": { + "model": "$MODEL", + "input": [ + "safe text", + "more safe text" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "modr-list-$REQUEST_ID", + "model": "omni-moderation-next", + "results": [ + { + "flagged": false, + "categories": {}, + "category_scores": {} + }, + { + "flagged": false, + "categories": {}, + "category_scores": {} + } + ] + } + }, + "expected": { + "spend": 0.0, + "input_cost": 0.0, + "output_cost": 0.0, + "prompt_tokens": 0, + "completion_tokens": 0 + } } ] } diff --git a/tests/integration/cost_calculation/test_cost_tracking.py b/tests/integration/cost_calculation/test_cost_tracking.py index 5b876974c63..6c1ac166196 100644 --- a/tests/integration/cost_calculation/test_cost_tracking.py +++ b/tests/integration/cost_calculation/test_cost_tracking.py @@ -92,7 +92,7 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) assert isinstance(expected, ExactExpected) if case.response.content_type == "application/json": header: Final = cast(str | None, response.headers.get("x-litellm-response-cost")) - assert header is not None and approx_equal(float(header), expected.spend), ( + assert expected.spend == 0 or (header is not None and approx_equal(float(header), expected.spend)), ( f"{case.name}: x-litellm-response-cost {header} != expected {expected.spend}" ) assert row.spend is not None and approx_equal(row.spend, expected.spend), ( From 8137d878a00c640de1db098d7993be52e81bd252 Mon Sep 17 00:00:00 2001 From: kerry Date: Sat, 19 Sep 2026 20:59:05 +0000 Subject: [PATCH 2/4] test(integration): drop contract nodes left behind by case renames Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/contracts.json | 54 -------------------------------- 1 file changed, 54 deletions(-) diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index d86f44cf24a..f6353bfe0da 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -1321,60 +1321,6 @@ "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_full_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-embedding-4-small-single]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-embedding-4-small-batch]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-embedding-4-small-token-array]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-embedding-4-large-dimensions]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-text-embedding-4-large-deployment]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[cohere-embed-v5]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-titan-embed-v3]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[bedrock-cohere-embed-v4]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[vertex-text-embedding-006]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-embedding-002]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together-embedding-v1]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks-embedding-v1]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-completion-openai-basic]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-completion-openai-stream-usage]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[text-completion-openai-n-best]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together-completions]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[omni-moderation-next-single]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[omni-moderation-next-list]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-text-embeddings-4-large-deployment]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], From 23f5df05f2b55f88aa39b6e58c6d57b3b46a3d95 Mon Sep 17 00:00:00 2001 From: kerry Date: Sat, 19 Sep 2026 21:03:51 +0000 Subject: [PATCH 3/4] test(integration): require a zero cost header to read zero when the case bills nothing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/cost_calculation/test_cost_tracking.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/integration/cost_calculation/test_cost_tracking.py b/tests/integration/cost_calculation/test_cost_tracking.py index 6c1ac166196..3822b5cb3c6 100644 --- a/tests/integration/cost_calculation/test_cost_tracking.py +++ b/tests/integration/cost_calculation/test_cost_tracking.py @@ -92,7 +92,11 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) assert isinstance(expected, ExactExpected) if case.response.content_type == "application/json": header: Final = cast(str | None, response.headers.get("x-litellm-response-cost")) - assert expected.spend == 0 or (header is not None and approx_equal(float(header), expected.spend)), ( + assert ( + (header is None or approx_equal(float(header), 0.0)) + if expected.spend == 0 + else (header is not None and approx_equal(float(header), expected.spend)) + ), ( f"{case.name}: x-litellm-response-cost {header} != expected {expected.spend}" ) assert row.spend is not None and approx_equal(row.spend, expected.spend), ( From 19e6ad5b1b18fbd56dac7263d4e9d4e8b470c22b Mon Sep 17 00:00:00 2001 From: kerry Date: Mon, 21 Sep 2026 20:48:40 +0000 Subject: [PATCH 4/4] test(integration): give perplexity sonar-next a unique input rate after merge Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/cost_calculation/cost_tracking_cases.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/cost_calculation/cost_tracking_cases.json b/tests/integration/cost_calculation/cost_tracking_cases.json index 42f69980a81..d9ebf238cd5 100644 --- a/tests/integration/cost_calculation/cost_tracking_cases.json +++ b/tests/integration/cost_calculation/cost_tracking_cases.json @@ -40,7 +40,7 @@ "perplexity/sonar-next": { "litellm_provider": "perplexity", "mode": "chat", - "input_cost_per_token": 1.05e-06, + "input_cost_per_token": 1.13e-06, "output_cost_per_token": 1.05e-06, "search_context_cost_per_query": { "search_context_size_low": 0.005, @@ -29077,7 +29077,7 @@ "usage": {"prompt_tokens": 1840, "completion_tokens": 412, "total_tokens": 2252} } }, - "expected": {"spend": 0.0023646, "input_cost": 0.001932, "output_cost": 0.0004326, "prompt_tokens": 1840, "completion_tokens": 412} + "expected": {"spend": 0.0025118, "input_cost": 0.0020792, "output_cost": 0.0004326, "prompt_tokens": 1840, "completion_tokens": 412} }, { "name": "deepseek-deepseek-v4-chat-prompt_cache_hit",