From 72b007a4aeeb9d9279b232d833ce89a5f139c23e Mon Sep 17 00:00:00 2001 From: kerry Date: Sat, 19 Sep 2026 22:46:43 +0000 Subject: [PATCH 1/4] test(integration): pricing dimension and provider cost cases Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/contracts.json | 48 ++ .../cost_calculation/cost_tracking_case.py | 27 +- .../cost_calculation/cost_tracking_cases.json | 491 ++++++++++++++++++ 3 files changed, 564 insertions(+), 2 deletions(-) diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index e86536e13b5..0a74171da29 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -1449,6 +1449,54 @@ ], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-passthrough-converse_stream]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dashscope-qwen4-max-tiered_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dashscope-qwen4-max-tiered_boundary_stays_lower_tier]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dashscope-qwen4-max-tiered_second_tier]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[dashscope-qwen4-max-tiered_above_top_range]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-lite-input_below_128k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-lite-input_above_128k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-cache_creation_1h_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[openrouter-anthropic-claude-sonnet-5-provider_reported_cost]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[openrouter-anthropic-claude-sonnet-5-token_priced]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[perplexity-sonar-next-search_queries_and_citations]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[perplexity-sonar-next-no_search]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[deepseek-deepseek-v4-chat-prompt_cache_hit]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[deepseek-deepseek-v4-chat-no_cache_fields_bills_zero_cache]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[xai-grok-5-reasoning_folded_into_completion]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[xai-grok-5-live_search]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[xai-grok-5-provider_reported_cost]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" ] }, "browser": { diff --git a/tests/integration/cost_calculation/cost_tracking_case.py b/tests/integration/cost_calculation/cost_tracking_case.py index ba64cab81ed..2da0e969458 100644 --- a/tests/integration/cost_calculation/cost_tracking_case.py +++ b/tests/integration/cost_calculation/cost_tracking_case.py @@ -25,6 +25,14 @@ class ProviderSpecificEntry(BaseModel): us: float | None = None +class TieredPrice(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + range: tuple[float, float] + input_cost_per_token: float + output_cost_per_token: float + + class CostMapEntry(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") @@ -36,11 +44,18 @@ class CostMapEntry(BaseModel): supports_function_calling: bool | None = None input_cost_per_token: float | None = None output_cost_per_token: float | None = None + input_cost_per_token_above_128k_tokens: float | None = None + output_cost_per_token_above_128k_tokens: float | None = None cache_read_input_token_cost: float | None = None cache_creation_input_token_cost: float | None = None cache_creation_input_token_cost_above_1hr: float | None = None + cache_creation_input_token_cost_above_1hr_above_200k_tokens: float | None = None cache_read_input_token_cost_above_200k_tokens: float | None = None cache_creation_input_token_cost_above_200k_tokens: float | None = None + input_cost_per_token_above_200k_tokens: float | None = None + output_cost_per_token_above_200k_tokens: float | None = None + citation_cost_per_token: float | None = None + tiered_pricing: tuple[TieredPrice, ...] | None = None output_cost_per_reasoning_token: float | None = None input_cost_per_audio_token: float | None = None input_cost_per_second: float | None = None @@ -53,8 +68,6 @@ class CostMapEntry(BaseModel): input_cost_per_image_token: float | None = None output_cost_per_image_token: float | None = None input_cost_per_video_token: float | None = None - input_cost_per_token_above_200k_tokens: float | None = None - output_cost_per_token_above_200k_tokens: float | None = None input_cost_per_token_flex: float | None = None output_cost_per_token_flex: float | None = None input_cost_per_token_priority: float | None = None @@ -270,6 +283,11 @@ _PROVIDER_PREFIXES: Final[Mapping[str, str]] = MappingProxyType( "together_ai": "", "fireworks_ai": "", "azure": "", + "dashscope": "", + "openrouter": "", + "perplexity": "", + "deepseek": "", + "xai": "", } ) _LITELLM_PARAMS: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType( @@ -301,6 +319,11 @@ _LITELLM_PARAMS: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType( "fireworks_ai": MappingProxyType({}), "azure": MappingProxyType({"api_version": "2025-04-01-preview"}), "openai": MappingProxyType({}), + "dashscope": MappingProxyType({}), + "openrouter": MappingProxyType({}), + "perplexity": MappingProxyType({}), + "deepseek": MappingProxyType({}), + "xai": MappingProxyType({}), } ) diff --git a/tests/integration/cost_calculation/cost_tracking_cases.json b/tests/integration/cost_calculation/cost_tracking_cases.json index 1179dd47592..f6183f79d25 100644 --- a/tests/integration/cost_calculation/cost_tracking_cases.json +++ b/tests/integration/cost_calculation/cost_tracking_cases.json @@ -1,5 +1,74 @@ { "cost_map": { + "dashscope/qwen4-max": { + "litellm_provider": "dashscope", + "mode": "chat", + "max_input_tokens": 252000, + "max_output_tokens": 65536, + "tiered_pricing": [ + { + "range": [0, 32000], + "input_cost_per_token": 1.3e-06, + "output_cost_per_token": 6.5e-06 + }, + { + "range": [32000, 128000], + "input_cost_per_token": 2.6e-06, + "output_cost_per_token": 1.3e-05 + }, + { + "range": [128000, 252000], + "input_cost_per_token": 3.1e-06, + "output_cost_per_token": 1.55e-05 + } + ] + }, + "gemini/gemini-3.8-flash-lite": { + "litellm_provider": "gemini", + "mode": "chat", + "input_cost_per_token": 1.1e-07, + "output_cost_per_token": 4.4e-07, + "input_cost_per_token_above_128k_tokens": 2.2e-07, + "output_cost_per_token_above_128k_tokens": 8.8e-07 + }, + "openrouter/anthropic/claude-sonnet-5": { + "litellm_provider": "openrouter", + "mode": "chat", + "input_cost_per_token": 3.2e-06, + "output_cost_per_token": 1.6e-05 + }, + "perplexity/sonar-next": { + "litellm_provider": "perplexity", + "mode": "chat", + "input_cost_per_token": 1.05e-06, + "output_cost_per_token": 1.05e-06, + "citation_cost_per_token": 2e-06, + "search_context_cost_per_query": { + "search_context_size_low": 0.005, + "search_context_size_medium": 0.008, + "search_context_size_high": 0.012 + } + }, + "deepseek/deepseek-v4-chat": { + "litellm_provider": "deepseek", + "mode": "chat", + "input_cost_per_token": 2.9e-07, + "output_cost_per_token": 4.3e-07, + "cache_read_input_token_cost": 2.9e-08, + "cache_creation_input_token_cost": 0.0 + }, + "xai/grok-5": { + "litellm_provider": "xai", + "mode": "chat", + "input_cost_per_token": 1.35e-06, + "output_cost_per_token": 2.7e-06, + "cache_read_input_token_cost": 2.1e-07, + "search_context_cost_per_query": { + "search_context_size_low": 0.005, + "search_context_size_medium": 0.005, + "search_context_size_high": 0.005 + } + }, "gpt-5.6": { "cache_read_input_token_cost": 1.75e-07, "input_cost_per_audio_token": 4e-05, @@ -165,6 +234,7 @@ "claude-sonnet-5": { "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, + "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05, "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, "cache_read_input_token_cost": 3e-07, "cache_read_input_token_cost_above_200k_tokens": 6e-07, @@ -27834,5 +27904,426 @@ }, "endpoint": "/bedrock/model/$MODEL/converse-stream" } + , + { + "name": "dashscope-qwen4-max-tiered_input", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "dashscope/qwen4-max", + "request": { + "model": "$MODEL", + "messages": [{"role": "user", "content": "tiered input"}], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "dashscope-tiered-input", + "object": "chat.completion", + "model": "qwen4-max", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1840, "completion_tokens": 412, "total_tokens": 2252} + } + }, + "expected": {"spend": 0.00507, "input_cost": 0.002392, "output_cost": 0.002678, "prompt_tokens": 1840, "completion_tokens": 412} + }, + { + "name": "dashscope-qwen4-max-tiered_boundary_stays_lower_tier", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "dashscope/qwen4-max", + "request": { + "model": "$MODEL", + "messages": [{"role": "user", "content": "tier boundary"}], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "dashscope-tiered-boundary", + "object": "chat.completion", + "model": "qwen4-max", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 32000, "completion_tokens": 412, "total_tokens": 32412} + } + }, + "expected": {"spend": 0.044278, "input_cost": 0.0416, "output_cost": 0.002678, "prompt_tokens": 32000, "completion_tokens": 412} + }, + { + "name": "dashscope-qwen4-max-tiered_second_tier", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "dashscope/qwen4-max", + "request": { + "model": "$MODEL", + "messages": [{"role": "user", "content": "tier two"}], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "dashscope-tiered-second", + "object": "chat.completion", + "model": "qwen4-max", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 40000, "completion_tokens": 412, "total_tokens": 40412} + } + }, + "expected": {"spend": 0.109356, "input_cost": 0.104, "output_cost": 0.005356, "prompt_tokens": 40000, "completion_tokens": 412} + }, + { + "name": "dashscope-qwen4-max-tiered_above_top_range", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "dashscope/qwen4-max", + "request": { + "model": "$MODEL", + "messages": [{"role": "user", "content": "top tier"}], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "dashscope-tiered-top", + "object": "chat.completion", + "model": "qwen4-max", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 300000, "completion_tokens": 412, "total_tokens": 300412} + } + }, + "expected": {"spend": 0.936386, "input_cost": 0.93, "output_cost": 0.006386, "prompt_tokens": 300000, "completion_tokens": 412} + }, + { + "name": "gemini-gemini-3.8-flash-lite-input_below_128k", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini/gemini-3.8-flash-lite", + "request": { + "model": "$MODEL", + "messages": [{"role": "user", "content": "base pricing"}], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [{"content": {"parts": [{"text": "ok"}], "role": "model"}, "finishReason": "STOP", "index": 0}], + "usageMetadata": {"promptTokenCount": 1840, "candidatesTokenCount": 412, "totalTokenCount": 2252}, + "modelVersion": "gemini-3.8-flash-lite" + } + }, + "expected": {"spend": 0.00038368, "input_cost": 0.0002024, "output_cost": 0.00018128, "prompt_tokens": 1840, "completion_tokens": 412} + }, + { + "name": "gemini-gemini-3.8-flash-lite-input_above_128k", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini/gemini-3.8-flash-lite", + "request": { + "model": "$MODEL", + "messages": [{"role": "user", "content": "above threshold"}], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [{"content": {"parts": [{"text": "ok"}], "role": "model"}, "finishReason": "STOP", "index": 0}], + "usageMetadata": {"promptTokenCount": 130000, "candidatesTokenCount": 412, "totalTokenCount": 130412}, + "modelVersion": "gemini-3.8-flash-lite" + } + }, + "expected": {"spend": 0.02896256, "input_cost": 0.0286, "output_cost": 0.00036256, "prompt_tokens": 130000, "completion_tokens": 412} + }, + { + "name": "claude-sonnet-5-cache_creation_1h_above_200k", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-sonnet-5", + "request": { + "model": "$MODEL", + "messages": [{"role": "user", "content": "one hour cache"}], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg-cache-1h", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 150000, + "cache_creation_input_tokens": 60000, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 60000}, + "cache_read_input_tokens": 0, + "output_tokens": 412 + } + } + }, + "expected": { + "spend": 1.62927, + "input_cost": 1.62, + "output_cost": 0.00927, + "cache_creation_cost": 0.72, + "prompt_tokens": 210000, + "completion_tokens": 412 + } + }, + { + "name": "openrouter-anthropic-claude-sonnet-5-provider_reported_cost", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "openrouter/anthropic/claude-sonnet-5", + "request": { + "model": "$MODEL", + "messages": [{"role": "user", "content": "reported cost"}], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "or-reported", + "object": "chat.completion", + "model": "anthropic/claude-sonnet-5", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1840, "completion_tokens": 412, "total_tokens": 2252, "cost": 0.0421} + } + }, + "expected": {"spend": 0.0421, "input_cost": 0.0, "output_cost": 0.0421, "prompt_tokens": 1840, "completion_tokens": 412} + }, + { + "name": "openrouter-anthropic-claude-sonnet-5-token_priced", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "openrouter/anthropic/claude-sonnet-5", + "request": { + "model": "$MODEL", + "messages": [{"role": "user", "content": "token pricing"}], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "or-token", + "object": "chat.completion", + "model": "anthropic/claude-sonnet-5", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1840, "completion_tokens": 412, "total_tokens": 2252} + } + }, + "expected": {"spend": 0.01248, "input_cost": 0.005888, "output_cost": 0.006592, "prompt_tokens": 1840, "completion_tokens": 412} + }, + { + "name": "perplexity-sonar-next-search_queries_and_citations", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "perplexity/sonar-next", + "request": { + "model": "$MODEL", + "messages": [{"role": "user", "content": "search"}], + "web_search_options": {"search_context_size": "high"}, + "stream": false, + "allowed_openai_params": ["web_search_options"] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "pplx-search", + "object": "chat.completion", + "model": "sonar-next", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "citations": [ + "https://e.co/aaaaaaaaaaaaaaaaaaaaaaaaaaa", + "https://e.co/bbbbbbbbbbbbbbbbbbbbbbbbbbb", + "https://e.co/ccccccccccccccccccccccccccc", + "https://e.co/ddddddddddddddddddddddddddd" + ], + "usage": {"prompt_tokens": 1840, "completion_tokens": 412, "total_tokens": 2252, "num_search_queries": 3} + } + }, + "expected": { + "spend": 0.0174446, + "input_cost": 0.002012, + "output_cost": 0.0004326, + "tool_usage_cost": 0.015, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "perplexity-sonar-next-no_search", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "perplexity/sonar-next", + "request": { + "model": "$MODEL", + "messages": [{"role": "user", "content": "no search"}], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "pplx-no-search", + "object": "chat.completion", + "model": "sonar-next", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1840, "completion_tokens": 412, "total_tokens": 2252} + } + }, + "expected": {"spend": 0.0023646, "input_cost": 0.001932, "output_cost": 0.0004326, "prompt_tokens": 1840, "completion_tokens": 412} + }, + { + "name": "deepseek-deepseek-v4-chat-prompt_cache_hit", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "deepseek/deepseek-v4-chat", + "request": { + "model": "$MODEL", + "messages": [{"role": "user", "content": "cache hit"}], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "deepseek-cache", + "object": "chat.completion", + "model": "deepseek-v4-chat", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252, + "prompt_cache_hit_tokens": 1200, + "prompt_cache_miss_tokens": 640, + "prompt_tokens_details": {"cached_tokens": 1200} + } + } + }, + "expected": { + "spend": 0.00039756, + "input_cost": 0.0002204, + "output_cost": 0.00017716, + "cache_read_cost": 0.0000348, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "deepseek-deepseek-v4-chat-no_cache_fields_bills_zero_cache", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "deepseek/deepseek-v4-chat", + "request": { + "model": "$MODEL", + "messages": [{"role": "user", "content": "no cache"}], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "deepseek-no-cache", + "object": "chat.completion", + "model": "deepseek-v4-chat", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1840, "completion_tokens": 412, "total_tokens": 2252} + } + }, + "expected": { + "spend": 0.00071076, + "input_cost": 0.0005336, + "output_cost": 0.00017716, + "cache_read_cost": 0.0, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "xai-grok-5-reasoning_folded_into_completion", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "xai/grok-5", + "request": { + "model": "$MODEL", + "messages": [{"role": "user", "content": "reasoning"}], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "xai-reasoning", + "object": "chat.completion", + "model": "grok-5", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2552, + "completion_tokens_details": {"reasoning_tokens": 300} + } + } + }, + "expected": {"spend": 0.0044064, "input_cost": 0.002484, "output_cost": 0.0019224, "prompt_tokens": 1840, "completion_tokens": 712} + }, + { + "name": "xai-grok-5-live_search", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "xai/grok-5", + "request": { + "model": "$MODEL", + "messages": [{"role": "user", "content": "live search"}], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "xai-search", + "object": "chat.completion", + "model": "grok-5", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252, + "server_side_tool_usage_details": {"web_search_calls": 2} + } + } + }, + "expected": { + "spend": 0.0135964, + "input_cost": 0.002484, + "output_cost": 0.0011224, + "tool_usage_cost": 0.01, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "xai-grok-5-provider_reported_cost", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "xai/grok-5", + "request": { + "model": "$MODEL", + "messages": [{"role": "user", "content": "reported xai cost"}], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "xai-reported", + "object": "chat.completion", + "model": "grok-5", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252, + "cost": 0.0421 + } + } + }, + "expected": {"spend": 0.0421, "input_cost": 0.0, "output_cost": 0.0421, "prompt_tokens": 1840, "completion_tokens": 412} + } ] } From c7113f043afe69582265d055d3b99795f073d555 Mon Sep 17 00:00:00 2001 From: kerry Date: Sat, 19 Sep 2026 23:01:25 +0000 Subject: [PATCH 2/4] test(integration): use per-run request ids, drop unbillable perplexity search case Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/contracts.json | 3 - .../cost_calculation/cost_tracking_case.py | 9 ++- .../cost_calculation/cost_tracking_cases.json | 72 ++++++------------- .../cost_calculation/test_cost_tracking.py | 6 +- 4 files changed, 33 insertions(+), 57 deletions(-) diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index 0a74171da29..15ddaf14fd4 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -1477,9 +1477,6 @@ "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[openrouter-anthropic-claude-sonnet-5-token_priced]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[perplexity-sonar-next-search_queries_and_citations]": [ - "quota_management.spend_tracking.cost_matrix.logs_cost" - ], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[perplexity-sonar-next-no_search]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], diff --git a/tests/integration/cost_calculation/cost_tracking_case.py b/tests/integration/cost_calculation/cost_tracking_case.py index 2da0e969458..269e1edb9eb 100644 --- a/tests/integration/cost_calculation/cost_tracking_case.py +++ b/tests/integration/cost_calculation/cost_tracking_case.py @@ -54,7 +54,6 @@ class CostMapEntry(BaseModel): cache_creation_input_token_cost_above_200k_tokens: float | None = None input_cost_per_token_above_200k_tokens: float | None = None output_cost_per_token_above_200k_tokens: float | None = None - citation_cost_per_token: float | None = None tiered_pricing: tuple[TieredPrice, ...] | None = None output_cost_per_reasoning_token: float | None = None input_cost_per_audio_token: float | None = None @@ -263,6 +262,13 @@ class CostTrackingTestCase(BaseModel): return "bedrock" return None + @property + def reports_provider_cost(self) -> bool: + if not isinstance(self.response, JsonResponse): + return False + usage: Final = self.response.body.get("usage") + return isinstance(usage, dict) and isinstance(usage.get("cost"), (int, float)) + class _CasesFile(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") @@ -409,6 +415,7 @@ def data_errors() -> tuple[str, ...]: not case.expected.breakdown_persisted and case.passthrough_provider is None and case.rates.mode != "image_generation" + and not case.reports_provider_cost ) or (not case.expected.cost_header and case.passthrough_provider is None) ) diff --git a/tests/integration/cost_calculation/cost_tracking_cases.json b/tests/integration/cost_calculation/cost_tracking_cases.json index f6183f79d25..ead2435f2ac 100644 --- a/tests/integration/cost_calculation/cost_tracking_cases.json +++ b/tests/integration/cost_calculation/cost_tracking_cases.json @@ -42,7 +42,6 @@ "mode": "chat", "input_cost_per_token": 1.05e-06, "output_cost_per_token": 1.05e-06, - "citation_cost_per_token": 2e-06, "search_context_cost_per_query": { "search_context_size_low": 0.005, "search_context_size_medium": 0.008, @@ -27918,7 +27917,7 @@ "response": { "content_type": "application/json", "body": { - "id": "dashscope-tiered-input", + "id": "$REQUEST_ID", "object": "chat.completion", "model": "qwen4-max", "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], @@ -27940,7 +27939,7 @@ "response": { "content_type": "application/json", "body": { - "id": "dashscope-tiered-boundary", + "id": "$REQUEST_ID", "object": "chat.completion", "model": "qwen4-max", "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], @@ -27962,7 +27961,7 @@ "response": { "content_type": "application/json", "body": { - "id": "dashscope-tiered-second", + "id": "$REQUEST_ID", "object": "chat.completion", "model": "qwen4-max", "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], @@ -27984,7 +27983,7 @@ "response": { "content_type": "application/json", "body": { - "id": "dashscope-tiered-top", + "id": "$REQUEST_ID", "object": "chat.completion", "model": "qwen4-max", "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], @@ -28046,7 +28045,7 @@ "response": { "content_type": "application/json", "body": { - "id": "msg-cache-1h", + "id": "msg_$REQUEST_ID", "type": "message", "role": "assistant", "model": "claude-sonnet-5", @@ -28083,14 +28082,21 @@ "response": { "content_type": "application/json", "body": { - "id": "or-reported", + "id": "chatcmpl-$REQUEST_ID", "object": "chat.completion", "model": "anthropic/claude-sonnet-5", "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 1840, "completion_tokens": 412, "total_tokens": 2252, "cost": 0.0421} } }, - "expected": {"spend": 0.0421, "input_cost": 0.0, "output_cost": 0.0421, "prompt_tokens": 1840, "completion_tokens": 412} + "expected": { + "spend": 0.0421, + "input_cost": 0.0, + "output_cost": 0.0421, + "prompt_tokens": 1840, + "completion_tokens": 412, + "breakdown_persisted": false + } }, { "name": "openrouter-anthropic-claude-sonnet-5-token_priced", @@ -28105,7 +28111,7 @@ "response": { "content_type": "application/json", "body": { - "id": "or-token", + "id": "chatcmpl-$REQUEST_ID", "object": "chat.completion", "model": "anthropic/claude-sonnet-5", "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], @@ -28114,42 +28120,6 @@ }, "expected": {"spend": 0.01248, "input_cost": 0.005888, "output_cost": 0.006592, "prompt_tokens": 1840, "completion_tokens": 412} }, - { - "name": "perplexity-sonar-next-search_queries_and_citations", - "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", - "model": "perplexity/sonar-next", - "request": { - "model": "$MODEL", - "messages": [{"role": "user", "content": "search"}], - "web_search_options": {"search_context_size": "high"}, - "stream": false, - "allowed_openai_params": ["web_search_options"] - }, - "response": { - "content_type": "application/json", - "body": { - "id": "pplx-search", - "object": "chat.completion", - "model": "sonar-next", - "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], - "citations": [ - "https://e.co/aaaaaaaaaaaaaaaaaaaaaaaaaaa", - "https://e.co/bbbbbbbbbbbbbbbbbbbbbbbbbbb", - "https://e.co/ccccccccccccccccccccccccccc", - "https://e.co/ddddddddddddddddddddddddddd" - ], - "usage": {"prompt_tokens": 1840, "completion_tokens": 412, "total_tokens": 2252, "num_search_queries": 3} - } - }, - "expected": { - "spend": 0.0174446, - "input_cost": 0.002012, - "output_cost": 0.0004326, - "tool_usage_cost": 0.015, - "prompt_tokens": 1840, - "completion_tokens": 412 - } - }, { "name": "perplexity-sonar-next-no_search", "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", @@ -28163,7 +28133,7 @@ "response": { "content_type": "application/json", "body": { - "id": "pplx-no-search", + "id": "chatcmpl-$REQUEST_ID", "object": "chat.completion", "model": "sonar-next", "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], @@ -28185,7 +28155,7 @@ "response": { "content_type": "application/json", "body": { - "id": "deepseek-cache", + "id": "chatcmpl-$REQUEST_ID", "object": "chat.completion", "model": "deepseek-v4-chat", "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], @@ -28221,7 +28191,7 @@ "response": { "content_type": "application/json", "body": { - "id": "deepseek-no-cache", + "id": "chatcmpl-$REQUEST_ID", "object": "chat.completion", "model": "deepseek-v4-chat", "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], @@ -28250,7 +28220,7 @@ "response": { "content_type": "application/json", "body": { - "id": "xai-reasoning", + "id": "chatcmpl-$REQUEST_ID", "object": "chat.completion", "model": "grok-5", "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], @@ -28277,7 +28247,7 @@ "response": { "content_type": "application/json", "body": { - "id": "xai-search", + "id": "chatcmpl-$REQUEST_ID", "object": "chat.completion", "model": "grok-5", "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], @@ -28311,7 +28281,7 @@ "response": { "content_type": "application/json", "body": { - "id": "xai-reported", + "id": "chatcmpl-$REQUEST_ID", "object": "chat.completion", "model": "grok-5", "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], diff --git a/tests/integration/cost_calculation/test_cost_tracking.py b/tests/integration/cost_calculation/test_cost_tracking.py index c2ac6e77a3f..d97bfe642a2 100644 --- a/tests/integration/cost_calculation/test_cost_tracking.py +++ b/tests/integration/cost_calculation/test_cost_tracking.py @@ -148,12 +148,14 @@ def _assert_breakdown( ): if expected_component is None: continue - assert actual_component is not None and approx_equal(actual_component, expected_component), ( + actual_value: Final = actual_component or 0.0 + assert approx_equal(actual_value, expected_component), ( f"{case.name}: {field} {actual_component} != expected {expected_component}" ) if expected.cost_header and case.response.content_type == "application/json": header: Final = response.headers.get(header_name) - assert header is not None and approx_equal(float(header), expected_component), ( + header_value: Final = float(header) if header is not None else 0.0 + assert approx_equal(header_value, expected_component), ( f"{case.name}: {header_name} {header} != expected {expected_component}" ) if expected.cost_header and case.response.content_type == "application/json" and any( From 789f0c61bf31a885ca8fd7f0f951e6078b37b78c Mon Sep 17 00:00:00 2001 From: kerry Date: Sat, 19 Sep 2026 23:12:43 +0000 Subject: [PATCH 3/4] test(integration): only accept omitted breakdown components when the case expects zero, fix xai output cost Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../cost_calculation/cost_tracking_cases.json | 2 +- .../cost_calculation/test_cost_tracking.py | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/integration/cost_calculation/cost_tracking_cases.json b/tests/integration/cost_calculation/cost_tracking_cases.json index ead2435f2ac..3a52dfee93a 100644 --- a/tests/integration/cost_calculation/cost_tracking_cases.json +++ b/tests/integration/cost_calculation/cost_tracking_cases.json @@ -28262,7 +28262,7 @@ "expected": { "spend": 0.0135964, "input_cost": 0.002484, - "output_cost": 0.0011224, + "output_cost": 0.0011124, "tool_usage_cost": 0.01, "prompt_tokens": 1840, "completion_tokens": 412 diff --git a/tests/integration/cost_calculation/test_cost_tracking.py b/tests/integration/cost_calculation/test_cost_tracking.py index d97bfe642a2..8c5f306dd9d 100644 --- a/tests/integration/cost_calculation/test_cost_tracking.py +++ b/tests/integration/cost_calculation/test_cost_tracking.py @@ -148,16 +148,15 @@ def _assert_breakdown( ): if expected_component is None: continue - actual_value: Final = actual_component or 0.0 - assert approx_equal(actual_value, expected_component), ( - f"{case.name}: {field} {actual_component} != expected {expected_component}" - ) + omitted_component_allowed: Final = expected_component == 0.0 + assert (actual_component is None and omitted_component_allowed) or ( + actual_component is not None and approx_equal(actual_component, expected_component) + ), f"{case.name}: {field} {actual_component} != expected {expected_component}" if expected.cost_header and case.response.content_type == "application/json": header: Final = response.headers.get(header_name) - header_value: Final = float(header) if header is not None else 0.0 - assert approx_equal(header_value, expected_component), ( - f"{case.name}: {header_name} {header} != expected {expected_component}" - ) + assert (header is None and omitted_component_allowed) or ( + header is not None and approx_equal(float(header), expected_component) + ), f"{case.name}: {header_name} {header} != expected {expected_component}" if expected.cost_header and case.response.content_type == "application/json" and any( component is not None for component in ( From 19e6ad5b1b18fbd56dac7263d4e9d4e8b470c22b Mon Sep 17 00:00:00 2001 From: kerry Date: Mon, 21 Sep 2026 20:48:40 +0000 Subject: [PATCH 4/4] test(integration): give perplexity sonar-next a unique input rate after merge Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/cost_calculation/cost_tracking_cases.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/cost_calculation/cost_tracking_cases.json b/tests/integration/cost_calculation/cost_tracking_cases.json index 42f69980a81..d9ebf238cd5 100644 --- a/tests/integration/cost_calculation/cost_tracking_cases.json +++ b/tests/integration/cost_calculation/cost_tracking_cases.json @@ -40,7 +40,7 @@ "perplexity/sonar-next": { "litellm_provider": "perplexity", "mode": "chat", - "input_cost_per_token": 1.05e-06, + "input_cost_per_token": 1.13e-06, "output_cost_per_token": 1.05e-06, "search_context_cost_per_query": { "search_context_size_low": 0.005, @@ -29077,7 +29077,7 @@ "usage": {"prompt_tokens": 1840, "completion_tokens": 412, "total_tokens": 2252} } }, - "expected": {"spend": 0.0023646, "input_cost": 0.001932, "output_cost": 0.0004326, "prompt_tokens": 1840, "completion_tokens": 412} + "expected": {"spend": 0.0025118, "input_cost": 0.0020792, "output_cost": 0.0004326, "prompt_tokens": 1840, "completion_tokens": 412} }, { "name": "deepseek-deepseek-v4-chat-prompt_cache_hit",