From 724f1c835e88f8dd805c77173bbd4fedb5de6402 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:53:59 -0700 Subject: [PATCH] test(cost): add azure_ai claude-haiku-5-5 cases and gpt-6 service tier cache prices (#46694) * test(cost): add Azure AI Claude Haiku 5.5 cases Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(cost): declare service tier cache price fields on CostMapEntry Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(cost): drop CostMapEntry reorder now declared on main Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../cost_calculation/cases/misc.py | 156 +++++++++++++++++- .../integration/cost_calculation/cost_map.py | 60 +++++++ 2 files changed, 215 insertions(+), 1 deletion(-) diff --git a/tests/integration/cost_calculation/cases/misc.py b/tests/integration/cost_calculation/cases/misc.py index 70e109491aa..76b06f5441b 100644 --- a/tests/integration/cost_calculation/cases/misc.py +++ b/tests/integration/cost_calculation/cases/misc.py @@ -17,7 +17,12 @@ from integration.cost_calculation.cost_tracking_case import ( WavUpload, ) from integration.cost_calculation.stream_parity.case import StreamParityTestCase, sse_frames -from integration.cost_calculation.cases.anthropic import CLAUDE_FABLE_5_1_INPUT_TEXT, CLAUDE_FABLE_5_1_STREAM +from integration.cost_calculation.cases.anthropic import ( + CLAUDE_FABLE_5_1_INPUT_TEXT, + CLAUDE_FABLE_5_1_STREAM, + CLAUDE_HAIKU_5_5_INPUT_TEXT, + CLAUDE_HAIKU_5_5_STREAM, +) # nova-next @@ -1547,6 +1552,150 @@ AZURE_AI_CLAUDE_FABLE_5_1_STREAM: Final = CLAUDE_FABLE_5_1_STREAM.model_copy( } ) +AZURE_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT: Final = CLAUDE_HAIKU_5_5_INPUT_TEXT.model_copy( + update={ + "name": "azure_ai/claude-haiku-5-5-input_text", + "model": "azure_ai/claude-haiku-5-5", + "response": JsonResponse( + content_type="application/json", + body={ + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-haiku-5-5", + "content": [ + {"type": "thinking", "thinking": "scripted thought", "signature": "scripted signature"}, + {"type": "text", "text": "scripted answer 10673f3a1db3"}, + ], + "container": None, + "stop_reason": "end_turn", + "stop_sequence": None, + "stop_details": None, + "usage": { + "input_tokens": 1840, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 412, + "output_tokens_details": {"thinking_tokens": 0}, + "service_tier": "standard", + "inference_geo": "global", + }, + "diagnostics": None, + }, + ), + "expected": ExactExpected( + spend=0.000741, input_cost=0.0003496, output_cost=0.0003914, prompt_tokens=1840, completion_tokens=412 + ), + } +) + +AZURE_AI_CLAUDE_HAIKU_5_5_STREAM: Final = CLAUDE_HAIKU_5_5_STREAM.model_copy( + update={ + "name": "azure_ai/claude-haiku-5-5-stream", + "model": "azure_ai/claude-haiku-5-5", + "response": SseResponse( + content_type="text/event-stream", + frames=sse_frames( + *( + { + "type": "message_start", + "message": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-haiku-5-5", + "content": [], + "stop_reason": None, + "stop_sequence": None, + "stop_details": None, + "container": None, + "diagnostics": None, + "usage": { + "input_tokens": 1840, + "output_tokens": 1, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": { + "ephemeral_5m_input_tokens": 0, + "ephemeral_1h_input_tokens": 0, + }, + "service_tier": "standard", + "inference_geo": "global", + }, + }, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "thinking", "thinking": "", "signature": ""}, + }, + {"type": "ping"}, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "thinking_delta", "thinking": ""}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "signature_delta", "signature": "scripted signature"}, + }, + {"type": "content_block_stop", "index": 0}, + { + "type": "content_block_start", + "index": 1, + "content_block": {"type": "text", "text": ""}, + }, + *( + { + "type": "content_block_delta", + "index": 1, + "delta": {"type": "text_delta", "text": text}, + } + for text in ( + "scripted ", + "ans", + "wer ", + "2", + "b", + "f", + "9", + "b", + "c", + "b", + "a", + "4", + "444", + ) + ), + {"type": "content_block_stop", "index": 1}, + { + "type": "message_delta", + "delta": { + "container": None, + "stop_details": None, + "stop_reason": "end_turn", + "stop_sequence": None, + }, + "usage": { + "input_tokens": 1840, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "output_tokens": 412, + "output_tokens_details": {"thinking_tokens": 0}, + }, + }, + {"type": "message_stop"}, + ) + ), + ), + "expected": ExactExpected( + spend=0.000741, input_cost=0.0003496, output_cost=0.0003914, prompt_tokens=1840, completion_tokens=412 + ), + } +) + CASES: Final[tuple[CostTrackingTestCase, ...]] = ( NOVA_NEXT_TRANSCRIPTIONS_PER_SECOND, @@ -1598,6 +1747,8 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( XAI_GROK_4_7_INPUT_TEXT, AZURE_AI_CLAUDE_FABLE_5_1_INPUT_TEXT, AZURE_AI_CLAUDE_FABLE_5_1_STREAM, + AZURE_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT, + AZURE_AI_CLAUDE_HAIKU_5_5_STREAM, ) PARITY: Final[tuple[StreamParityTestCase, ...]] = ( @@ -1605,4 +1756,7 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = ( StreamParityTestCase( plain=AZURE_AI_CLAUDE_FABLE_5_1_INPUT_TEXT, streamed=AZURE_AI_CLAUDE_FABLE_5_1_STREAM ), + StreamParityTestCase( + plain=AZURE_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT, streamed=AZURE_AI_CLAUDE_HAIKU_5_5_STREAM + ), ) diff --git a/tests/integration/cost_calculation/cost_map.py b/tests/integration/cost_calculation/cost_map.py index 6f61f9f2c5f..ff877ee2299 100644 --- a/tests/integration/cost_calculation/cost_map.py +++ b/tests/integration/cost_calculation/cost_map.py @@ -37,12 +37,20 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "input_cost_per_token_priority": 2e-07, "output_cost_per_token_flex": 2.5e-07, "input_cost_per_token_flex": 5e-08, + "cache_creation_input_token_cost_flex": 6.25e-08, + "cache_creation_input_token_cost_priority": 2.5e-07, "cache_creation_input_token_cost": 1.25e-07, "cache_creation_input_token_cost_decisions": 1e-07, + "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-07, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, "cache_creation_input_token_cost_above_272k_tokens_decisions": 2e-07, + "cache_read_input_token_cost_flex": 5e-09, + "cache_read_input_token_cost_priority": 2e-08, "cache_read_input_token_cost": 1e-08, "cache_read_input_token_cost_decisions": 1e-07, + "cache_read_input_token_cost_above_272k_tokens_flex": 1e-08, + "cache_read_input_token_cost_above_272k_tokens_priority": 4e-08, "cache_read_input_token_cost_above_272k_tokens": 2e-08, "cache_read_input_token_cost_above_272k_tokens_decisions": 2e-07, "input_cost_per_token": 1e-07, @@ -645,6 +653,26 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "max_output_tokens": 128000, "max_tokens": 128000, }, + "azure_ai/claude-haiku-5-5": { + "litellm_provider": "azure_ai", + "mode": "chat", + "input_cost_per_token": 1.9e-07, + "output_cost_per_token": 9.5e-07, + "input_cost_per_token_above_100k_tokens": 9.5e-07, + "output_cost_per_token_above_100k_tokens": 4.75e-06, + "cache_creation_input_token_cost": 2.375e-07, + "cache_creation_input_token_cost_above_1hr": 3.8e-07, + "cache_creation_input_token_cost_above_100k_tokens": 1.1875e-06, + "cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.9e-06, + "cache_read_input_token_cost": 1.9e-08, + "cache_read_input_token_cost_above_100k_tokens": 9.5e-08, + "supports_function_calling": True, + "supports_prompt_caching": True, + "supports_reasoning": True, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + }, "vertex_ai/claude-sonnet-5-5": { "litellm_provider": "vertex_ai-anthropic_models", "mode": "chat", @@ -985,10 +1013,26 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "input_cost_per_token_above_272k_tokens": 4e-06, "input_cost_per_token_above_272k_tokens_flex": 2e-06, "input_cost_per_token_above_272k_tokens_priority": 8e-06, + "input_cost_per_token_above_272k_tokens_ultrafast": 2.4e-05, "output_cost_per_token_above_272k_tokens": 2e-05, "output_cost_per_token_above_272k_tokens_flex": 1e-05, "output_cost_per_token_above_272k_tokens_priority": 4e-05, + "output_cost_per_token_above_272k_tokens_ultrafast": 0.00012, "cache_read_input_token_cost_above_272k_tokens": 2e-07, + "cache_read_input_token_cost_flex": 5e-08, + "cache_read_input_token_cost_priority": 2e-07, + "cache_read_input_token_cost_ultrafast": 6e-07, + "cache_read_input_token_cost_above_272k_tokens_flex": 1e-07, + "cache_read_input_token_cost_above_272k_tokens_priority": 4e-07, + "cache_read_input_token_cost_above_272k_tokens_ultrafast": 1.2e-06, + "cache_creation_input_token_cost_flex": 1.25e-06, + "cache_creation_input_token_cost_priority": 5e-06, + "cache_creation_input_token_cost_ultrafast": 1.5e-05, + "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, + "cache_creation_input_token_cost_above_272k_tokens_ultrafast": 3e-05, + "input_cost_per_token_ultrafast": 1.2e-05, + "output_cost_per_token_ultrafast": 6e-05, }, "claude-sonnet-5-5": { "cache_creation_input_token_cost": 4.125e-06, @@ -1156,6 +1200,12 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "cache_read_input_token_cost": 1.2e-06, "cache_creation_input_token_cost": 1.5e-05, "cache_creation_input_token_cost_above_272k_tokens": 3e-05, + "cache_creation_input_token_cost_flex": 7.5e-06, + "cache_creation_input_token_cost_priority": 3e-05, + "cache_creation_input_token_cost_ultrafast": 9e-05, + "cache_creation_input_token_cost_above_272k_tokens_flex": 1.5e-05, + "cache_creation_input_token_cost_above_272k_tokens_priority": 6e-05, + "cache_creation_input_token_cost_above_272k_tokens_ultrafast": 0.00018, "input_cost_per_token": 1.2e-05, "input_cost_per_token_flex": 6e-06, "input_cost_per_token_priority": 2.4e-05, @@ -1167,6 +1217,8 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "output_cost_per_token": 6e-05, "output_cost_per_token_flex": 3e-05, "output_cost_per_token_priority": 0.00012, + "input_cost_per_token_ultrafast": 7.2e-05, + "output_cost_per_token_ultrafast": 0.00036, "supports_function_calling": True, "input_cost_per_token_above_272k_tokens": 2.4e-05, "input_cost_per_token_above_272k_tokens_flex": 1.2e-05, @@ -1174,7 +1226,15 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "output_cost_per_token_above_272k_tokens": 0.00012, "output_cost_per_token_above_272k_tokens_flex": 6e-05, "output_cost_per_token_above_272k_tokens_priority": 0.00024, + "input_cost_per_token_above_272k_tokens_ultrafast": 0.000144, + "output_cost_per_token_above_272k_tokens_ultrafast": 0.00072, "cache_read_input_token_cost_above_272k_tokens": 2.4e-06, + "cache_read_input_token_cost_flex": 6e-07, + "cache_read_input_token_cost_priority": 2.4e-06, + "cache_read_input_token_cost_ultrafast": 7.2e-06, + "cache_read_input_token_cost_above_272k_tokens_flex": 1.2e-06, + "cache_read_input_token_cost_above_272k_tokens_priority": 4.8e-06, + "cache_read_input_token_cost_above_272k_tokens_ultrafast": 1.44e-05, }, "azure/gpt-6-sol": { "cache_read_input_token_cost": 2.3e-07,