diff --git a/tests/integration/cost_calculation/cases/azure.py b/tests/integration/cost_calculation/cases/azure.py index 5b4210f3869..96185896b21 100644 --- a/tests/integration/cost_calculation/cases/azure.py +++ b/tests/integration/cost_calculation/cases/azure.py @@ -5,6 +5,7 @@ PARITY pairs the streamed case with its plain twin where both bill the same row. from typing import Final from integration.cost_calculation.cost_tracking_case import ( + with_response_usage, with_usage, BinaryResponse, CostTrackingTestCase, @@ -6116,6 +6117,90 @@ AZURE_GPT_6_1_SOL_TIERED_PRIORITY_ABOVE_272K: Final = with_usage( ExactExpected(spend=1.449888, input_cost=1.44, output_cost=0.009888, prompt_tokens=300000, completion_tokens=412), ) +AZURE_GPT_6_1_SOL_CACHE_WRITE_5M: Final = with_response_usage( + AZURE_GPT_6_1_SOL_CACHE_READ, + "azure/gpt-6.1-sol-cache_write_5m", + { + "completion_tokens": 380, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + }, + "latency_checkpoint": { + "engine_tbt_ms": 15, + "engine_ttft_ms": 1118, + "engine_ttlt_ms": 1221, + "pre_inference_ms": 172, + "service_tbt_ms": 15, + "service_ttft_ms": 1562, + "service_ttlt_ms": 1652, + "user_visible_ttft_ms": 1390, + }, + "prompt_tokens": 12928, + "prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 12288, "cached_tokens": 0}, + "total_tokens": 13308, + }, + ExactExpected(spend=0.04296, input_cost=0.0384, output_cost=0.00456, prompt_tokens=12928, completion_tokens=380), +) +AZURE_GPT_6_LUNA_CACHE_WRITE_5M: Final = with_response_usage( + AZURE_GPT_6_LUNA_CACHE_READ, + "azure/gpt-6-luna-cache_write_5m", + { + "completion_tokens": 380, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + }, + "latency_checkpoint": { + "engine_tbt_ms": 7, + "engine_ttft_ms": 143, + "engine_ttlt_ms": 309, + "pre_inference_ms": 132, + "service_tbt_ms": 8, + "service_ttft_ms": 566, + "service_ttlt_ms": 748, + "user_visible_ttft_ms": 434, + }, + "prompt_tokens": 12928, + "prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 12288, "cached_tokens": 0}, + "total_tokens": 13308, + }, + ExactExpected(spend=0.002327, input_cost=0.00208, output_cost=0.000247, prompt_tokens=12928, completion_tokens=380), + choice_updates={"content_filter_result": {"error": {"code": "content_filter_error", "message": "The contents are not filtered"}}}, +) +AZURE_GPT_6_ASTRA_CACHE_WRITE_5M: Final = with_response_usage( + AZURE_GPT_6_ASTRA_CACHE_READ, + "azure/gpt-6-astra-cache_write_5m", + { + "completion_tokens": 380, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0, + }, + "latency_checkpoint": { + "engine_tbt_ms": 36, + "engine_ttft_ms": 537, + "engine_ttlt_ms": 751, + "pre_inference_ms": 356, + "service_tbt_ms": 41, + "service_ttft_ms": 1166, + "service_ttlt_ms": 1371, + "user_visible_ttft_ms": 810, + }, + "prompt_tokens": 12928, + "prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 12288, "cached_tokens": 0}, + "total_tokens": 13308, + }, + ExactExpected(spend=0.22375, input_cost=0.2, output_cost=0.02375, prompt_tokens=12928, completion_tokens=380), +) + + CASES: Final[tuple[CostTrackingTestCase, ...]] = ( AZURE_GPT_6_SOL_MESSAGES_BRIDGED_JSON, AZURE_GPT_6_LUNA_MESSAGES_BRIDGED_JSON, @@ -6214,6 +6299,9 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( AZURE_GPT_6_1_SOL_CACHE_READ, AZURE_GPT_6_LUNA_CACHE_READ, AZURE_GPT_6_ASTRA_CACHE_READ, + AZURE_GPT_6_1_SOL_CACHE_WRITE_5M, + AZURE_GPT_6_LUNA_CACHE_WRITE_5M, + AZURE_GPT_6_ASTRA_CACHE_WRITE_5M, ) PARITY: Final[tuple[StreamParityTestCase, ...]] = ( diff --git a/tests/integration/cost_calculation/cases/misc.py b/tests/integration/cost_calculation/cases/misc.py index d17ab7a3d31..70e109491aa 100644 --- a/tests/integration/cost_calculation/cases/misc.py +++ b/tests/integration/cost_calculation/cases/misc.py @@ -17,6 +17,7 @@ from integration.cost_calculation.cost_tracking_case import ( WavUpload, ) from integration.cost_calculation.stream_parity.case import StreamParityTestCase, sse_frames +from integration.cost_calculation.cases.anthropic import CLAUDE_FABLE_5_1_INPUT_TEXT, CLAUDE_FABLE_5_1_STREAM # nova-next @@ -1420,6 +1421,133 @@ XAI_GROK_4_7_INPUT_TEXT: Final = CostTrackingTestCase( ) +AZURE_AI_CLAUDE_FABLE_5_1_INPUT_TEXT: Final = CLAUDE_FABLE_5_1_INPUT_TEXT.model_copy( + update={ + "name": "azure_ai/claude-fable-5-1-input_text", + "model": "azure_ai/claude-fable-5-1", + "response": JsonResponse( + content_type="application/json", + body={ + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-fable-5-1", + "content": [ + {"type": "thinking", "thinking": "scripted thought", "signature": "scripted signature"}, + {"type": "text", "text": "scripted answer 3d584fa07ae1"}, + ], + "stop_reason": "end_turn", + "stop_sequence": None, + "stop_details": None, + "container": None, + "diagnostics": None, + "usage": { + "input_tokens": 1840, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 412, + "output_tokens_details": {"thinking_tokens": 0}, + "service_tier": "standard", + "inference_geo": "not_available", + }, + }, + ), + "expected": ExactExpected( + spend=0.0663, input_cost=0.03128, output_cost=0.03502, prompt_tokens=1840, completion_tokens=412 + ), + } +) + +AZURE_AI_CLAUDE_FABLE_5_1_STREAM: Final = CLAUDE_FABLE_5_1_STREAM.model_copy( + update={ + "name": "azure_ai/claude-fable-5-1-stream", + "model": "azure_ai/claude-fable-5-1", + "response": SseResponse( + content_type="text/event-stream", + frames=sse_frames( + *( + { + "type": "message_start", + "message": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-fable-5-1", + "content": [], + "stop_reason": None, + "stop_sequence": None, + "stop_details": None, + "container": None, + "diagnostics": None, + "usage": { + "input_tokens": 1840, + "output_tokens": 1, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "service_tier": "standard", + "inference_geo": "not_available", + }, + }, + }, + {"type": "content_block_start", "index": 0, "content_block": {"type": "thinking", "thinking": "", "signature": ""}}, + {"type": "ping"}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "thinking_delta", "thinking": ""}}, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "signature_delta", "signature": "scripted signature"}, + }, + {"type": "content_block_stop", "index": 0}, + {"type": "content_block_start", "index": 1, "content_block": {"type": "text", "text": ""}}, + *( + { + "type": "content_block_delta", + "index": 1, + "delta": {"type": "text_delta", "text": text}, + } + for text in ( + "scripted ", + "answer ", + "3d", + "58", + "4f", + "a0", + "7a", + "e1", + "", + "", + ) + ), + {"type": "content_block_stop", "index": 1}, + { + "type": "message_delta", + "delta": { + "container": None, + "stop_details": None, + "stop_reason": "end_turn", + "stop_sequence": None, + }, + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "output_tokens_details": {"thinking_tokens": 0}, + }, + }, + {"type": "message_stop"}, + ) + ), + ), + "expected": ExactExpected( + spend=0.0663, input_cost=0.03128, output_cost=0.03502, prompt_tokens=1840, completion_tokens=412 + ), + } +) + + CASES: Final[tuple[CostTrackingTestCase, ...]] = ( NOVA_NEXT_TRANSCRIPTIONS_PER_SECOND, AMAZON_NOVA_CANVAS_NEXT_IMAGES_ONE, @@ -1468,8 +1596,13 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( COHERE_COMMAND_A_V2_TOKENS, MISTRAL_MEDIUM_2604_JSON, XAI_GROK_4_7_INPUT_TEXT, + AZURE_AI_CLAUDE_FABLE_5_1_INPUT_TEXT, + AZURE_AI_CLAUDE_FABLE_5_1_STREAM, ) PARITY: Final[tuple[StreamParityTestCase, ...]] = ( StreamParityTestCase(plain=TEXT_COMPLETIONS_OPENAI_BASIC, streamed=TEXT_COMPLETIONS_OPENAI_STREAM_USAGE), + StreamParityTestCase( + plain=AZURE_AI_CLAUDE_FABLE_5_1_INPUT_TEXT, streamed=AZURE_AI_CLAUDE_FABLE_5_1_STREAM + ), ) diff --git a/tests/integration/cost_calculation/cases/openai.py b/tests/integration/cost_calculation/cases/openai.py index e50a82ceaa5..c4f56d4be5b 100644 --- a/tests/integration/cost_calculation/cases/openai.py +++ b/tests/integration/cost_calculation/cases/openai.py @@ -5,6 +5,7 @@ PARITY pairs the streamed case with its plain twin where both bill the same row. from typing import Final, cast from integration.cost_calculation.cost_tracking_case import ( + with_response_usage, with_usage, COST_MAP, BinaryResponse, @@ -11104,6 +11105,39 @@ GPT_6_1_SOL_TIERED_PRIORITY_ABOVE_272K: Final = with_usage( ExactExpected(spend=2.41648, input_cost=2.4, output_cost=0.01648, prompt_tokens=300000, completion_tokens=412), ) +_GPT_6_CACHE_WRITE_USAGE: Final = { + "prompt_tokens": 12928, + "completion_tokens": 380, + "total_tokens": 13308, + "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 12288, "audio_tokens": 0}, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0, + }, +} + +GPT_6_1_SOL_CACHE_WRITE_5M: Final = with_response_usage( + GPT_6_1_SOL_CACHE_READ, + "gpt-6.1-sol-cache_write_5m", + _GPT_6_CACHE_WRITE_USAGE, + ExactExpected(spend=0.0358, input_cost=0.032, output_cost=0.0038, prompt_tokens=12928, completion_tokens=380), +) +GPT_6_LUNA_CACHE_WRITE_5M: Final = with_response_usage( + GPT_6_LUNA_CACHE_READ, + "gpt-6-luna-cache_write_5m", + _GPT_6_CACHE_WRITE_USAGE, + ExactExpected(spend=0.00179, input_cost=0.0016, output_cost=0.00019, prompt_tokens=12928, completion_tokens=380), +) +GPT_6_ASTRA_CACHE_WRITE_5M: Final = with_response_usage( + GPT_6_ASTRA_CACHE_READ, + "gpt-6-astra-cache_write_5m", + _GPT_6_CACHE_WRITE_USAGE, + ExactExpected(spend=0.2148, input_cost=0.192, output_cost=0.0228, prompt_tokens=12928, completion_tokens=380), +) + + CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GPT_6_SOL_RESPONSES_DEPLOYMENT_JSON, GPT_6_SOL_CHAT_BRIDGED_JSON, @@ -11309,6 +11343,9 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GPT_6_1_SOL_CACHE_READ, GPT_6_LUNA_CACHE_READ, GPT_6_ASTRA_CACHE_READ, + GPT_6_1_SOL_CACHE_WRITE_5M, + GPT_6_LUNA_CACHE_WRITE_5M, + GPT_6_ASTRA_CACHE_WRITE_5M, ) PARITY: Final[tuple[StreamParityTestCase, ...]] = ( diff --git a/tests/integration/cost_calculation/cases/vertex_ai.py b/tests/integration/cost_calculation/cases/vertex_ai.py index 1b3e936bef5..526703bdb24 100644 --- a/tests/integration/cost_calculation/cases/vertex_ai.py +++ b/tests/integration/cost_calculation/cases/vertex_ai.py @@ -4,6 +4,14 @@ PARITY pairs the streamed case with its plain twin where both bill the same row. from typing import Final +from integration.cost_calculation.cases.anthropic import ( + CLAUDE_FABLE_5_1_INPUT_TEXT, + CLAUDE_FABLE_5_1_STREAM, + CLAUDE_OPUS_5_5_INPUT_TEXT, + CLAUDE_OPUS_5_5_STREAM, + CLAUDE_SONNET_5_5_INPUT_TEXT, + CLAUDE_SONNET_5_5_STREAM, +) from integration.cost_calculation.cost_tracking_case import ( CostTrackingTestCase, ExactExpected, @@ -15,6 +23,76 @@ from integration.cost_calculation.cost_tracking_case import ( from integration.cost_calculation.stream_parity.case import StreamParityTestCase, sse_frames +def _vertex_claude_stream_response(model: str, stop_reason: str, text_deltas: tuple[str, ...]) -> SseResponse: + text_frames: Final = ( + ( + {"type": "content_block_start", "index": 1, "content_block": {"type": "text", "text": ""}}, + *( + { + "type": "content_block_delta", + "index": 1, + "delta": {"type": "text_delta", "text": text}, + } + for text in text_deltas + ), + {"type": "content_block_stop", "index": 1}, + ) + if text_deltas + else () + ) + return SseResponse( + content_type="text/event-stream", + frames=sse_frames( + *( + { + "type": "message_start", + "message": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": model, + "content": [], + "container": None, + "stop_reason": None, + "stop_sequence": None, + "stop_details": None, + "usage": { + "input_tokens": 1840, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 0, + }, + }, + }, + {"type": "ping"}, + {"type": "content_block_start", "index": 0, "content_block": {"type": "thinking", "thinking": "", "signature": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "thinking_delta", "thinking": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "signature_delta", "signature": "scripted signature"}}, + {"type": "content_block_stop", "index": 0}, + *text_frames, + { + "type": "message_delta", + "delta": { + "stop_reason": stop_reason, + "stop_sequence": None, + "stop_details": None, + "container": None, + }, + "usage": { + "input_tokens": 1840, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "output_tokens": 412, + "output_tokens_details": {"thinking_tokens": 0}, + }, + }, + {"type": "message_stop"}, + ) + ), + ) + + # gemini-3.1-pro GEMINI_3_1_PRO_INPUT_TEXT: Final = CostTrackingTestCase( name="gemini-3.1-pro-input_text", @@ -2877,6 +2955,132 @@ VERTEX_EMBEDDINGS_TEXT_006: Final = CostTrackingTestCase( ) +VERTEX_AI_CLAUDE_SONNET_5_5_INPUT_TEXT: Final = CLAUDE_SONNET_5_5_INPUT_TEXT.model_copy( + update={ + "name": "vertex_ai/claude-sonnet-5-5-input_text", + "model": "vertex_ai/claude-sonnet-5-5", + "response": JsonResponse( + content_type="application/json", + body={ + "model": "claude-sonnet-5-5", + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "content": [{"type": "thinking", "thinking": "scripted thought", "signature": "scripted signature"}], + "container": None, + "stop_reason": "max_tokens", + "stop_sequence": None, + "stop_details": None, + "usage": { + "input_tokens": 1840, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 412, + "output_tokens_details": {"thinking_tokens": 0}, + }, + }, + ), + "expected": ExactExpected( + spend=0.01326, input_cost=0.006256, output_cost=0.007004, prompt_tokens=1840, completion_tokens=412 + ), + } +) +VERTEX_AI_CLAUDE_SONNET_5_5_STREAM: Final = CLAUDE_SONNET_5_5_STREAM.model_copy( + update={ + "name": "vertex_ai/claude-sonnet-5-5-stream", + "model": "vertex_ai/claude-sonnet-5-5", + "response": _vertex_claude_stream_response("claude-sonnet-5-5", "max_tokens", ()), + "expected": VERTEX_AI_CLAUDE_SONNET_5_5_INPUT_TEXT.expected, + } +) +VERTEX_AI_CLAUDE_OPUS_5_5_INPUT_TEXT: Final = CLAUDE_OPUS_5_5_INPUT_TEXT.model_copy( + update={ + "name": "vertex_ai/claude-opus-5-5-input_text", + "model": "vertex_ai/claude-opus-5-5", + "response": JsonResponse( + content_type="application/json", + body={ + "model": "claude-opus-5-5", + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "scripted thought", "signature": "scripted signature"}, + {"type": "text", "text": "scripted answer for cost fixtures"}, + ], + "container": None, + "stop_reason": "end_turn", + "stop_sequence": None, + "stop_details": None, + "usage": { + "input_tokens": 1840, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 412, + "output_tokens_details": {"thinking_tokens": 0}, + }, + }, + ), + "expected": ExactExpected( + spend=0.02808, input_cost=0.013248, output_cost=0.014832, prompt_tokens=1840, completion_tokens=412 + ), + } +) +VERTEX_AI_CLAUDE_OPUS_5_5_STREAM: Final = CLAUDE_OPUS_5_5_STREAM.model_copy( + update={ + "name": "vertex_ai/claude-opus-5-5-stream", + "model": "vertex_ai/claude-opus-5-5", + "response": _vertex_claude_stream_response( + "claude-opus-5-5", + "end_turn", + ("scripted ", "answer ", "for ", "cost ", "fixtures", "", "", "", ""), + ), + "expected": VERTEX_AI_CLAUDE_OPUS_5_5_INPUT_TEXT.expected, + } +) +VERTEX_AI_CLAUDE_FABLE_5_1_INPUT_TEXT: Final = CLAUDE_FABLE_5_1_INPUT_TEXT.model_copy( + update={ + "name": "vertex_ai/claude-fable-5-1-input_text", + "model": "vertex_ai/claude-fable-5-1", + "response": JsonResponse( + content_type="application/json", + body={ + "model": "claude-fable-5-1", + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "content": [{"type": "thinking", "thinking": "scripted thought", "signature": "scripted signature"}], + "container": None, + "stop_reason": "max_tokens", + "stop_sequence": None, + "stop_details": None, + "usage": { + "input_tokens": 1840, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + "output_tokens": 412, + "output_tokens_details": {"thinking_tokens": 0}, + }, + }, + ), + "expected": ExactExpected( + spend=0.0741, input_cost=0.03496, output_cost=0.03914, prompt_tokens=1840, completion_tokens=412 + ), + } +) +VERTEX_AI_CLAUDE_FABLE_5_1_STREAM: Final = CLAUDE_FABLE_5_1_STREAM.model_copy( + update={ + "name": "vertex_ai/claude-fable-5-1-stream", + "model": "vertex_ai/claude-fable-5-1", + "response": _vertex_claude_stream_response("claude-fable-5-1", "max_tokens", ()), + "expected": VERTEX_AI_CLAUDE_FABLE_5_1_INPUT_TEXT.expected, + } +) + + CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GEMINI_3_1_PRO_INPUT_TEXT, GEMINI_3_1_PRO_CACHE_READ, @@ -2926,6 +3130,12 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( IMAGEN_NEXT_IMAGES_ONE, VERTEX_EMBEDDINGS_TEXT_006, GEMINI_3_1_PRO_PASSTHROUGH_STREAM_GENERATE_CONTENT_PRICED_VIA_VERTEX_KEY, + VERTEX_AI_CLAUDE_SONNET_5_5_INPUT_TEXT, + VERTEX_AI_CLAUDE_SONNET_5_5_STREAM, + VERTEX_AI_CLAUDE_OPUS_5_5_INPUT_TEXT, + VERTEX_AI_CLAUDE_OPUS_5_5_STREAM, + VERTEX_AI_CLAUDE_FABLE_5_1_INPUT_TEXT, + VERTEX_AI_CLAUDE_FABLE_5_1_STREAM, ) PARITY: Final[tuple[StreamParityTestCase, ...]] = ( @@ -2941,4 +3151,9 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = ( plain=GEMINI_3_8_FLASH_RESPONSE_MODEL_OVERRIDE, streamed=GEMINI_3_8_FLASH_STREAM_RESPONSE_MODEL_OVERRIDE ), StreamParityTestCase(plain=GEMINI_3_8_FLASH_TOOL_CALL, streamed=GEMINI_3_8_FLASH_STREAM_TOOL_CALL), + StreamParityTestCase( + plain=VERTEX_AI_CLAUDE_SONNET_5_5_INPUT_TEXT, streamed=VERTEX_AI_CLAUDE_SONNET_5_5_STREAM + ), + StreamParityTestCase(plain=VERTEX_AI_CLAUDE_OPUS_5_5_INPUT_TEXT, streamed=VERTEX_AI_CLAUDE_OPUS_5_5_STREAM), + StreamParityTestCase(plain=VERTEX_AI_CLAUDE_FABLE_5_1_INPUT_TEXT, streamed=VERTEX_AI_CLAUDE_FABLE_5_1_STREAM), ) diff --git a/tests/integration/cost_calculation/cost_map.py b/tests/integration/cost_calculation/cost_map.py index 2a036ff23d2..6f61f9f2c5f 100644 --- a/tests/integration/cost_calculation/cost_map.py +++ b/tests/integration/cost_calculation/cost_map.py @@ -630,6 +630,87 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, }, + "azure_ai/claude-fable-5-1": { + "litellm_provider": "azure_ai", + "mode": "chat", + "input_cost_per_token": 1.7e-05, + "output_cost_per_token": 8.5e-05, + "cache_creation_input_token_cost": 2.125e-05, + "cache_creation_input_token_cost_above_1hr": 3.4e-05, + "cache_read_input_token_cost": 4.25e-07, + "supports_function_calling": True, + "supports_prompt_caching": True, + "supports_reasoning": True, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + }, + "vertex_ai/claude-sonnet-5-5": { + "litellm_provider": "vertex_ai-anthropic_models", + "mode": "chat", + "input_cost_per_token": 3.4e-06, + "input_cost_per_token_batches": 1.7e-06, + "output_cost_per_token": 1.7e-05, + "output_cost_per_token_batches": 8.5e-06, + "cache_creation_input_token_cost": 4.25e-06, + "cache_creation_input_token_cost_above_1hr": 6.8e-06, + "cache_read_input_token_cost": 1.7e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.017, + "search_context_size_low": 0.017, + "search_context_size_medium": 0.017, + }, + "supports_function_calling": True, + "supports_prompt_caching": True, + "supports_reasoning": True, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + }, + "vertex_ai/claude-opus-5-5": { + "litellm_provider": "vertex_ai-anthropic_models", + "mode": "chat", + "input_cost_per_token": 7.2e-06, + "input_cost_per_token_batches": 4.5e-06, + "output_cost_per_token": 3.6e-05, + "output_cost_per_token_batches": 2.25e-05, + "cache_creation_input_token_cost": 9e-06, + "cache_creation_input_token_cost_above_1hr": 1.44e-05, + "cache_read_input_token_cost": 3.6e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.018, + "search_context_size_low": 0.018, + "search_context_size_medium": 0.018, + }, + "supports_function_calling": True, + "supports_prompt_caching": True, + "supports_reasoning": True, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + }, + "vertex_ai/claude-fable-5-1": { + "litellm_provider": "vertex_ai-anthropic_models", + "mode": "chat", + "input_cost_per_token": 1.9e-05, + "input_cost_per_token_batches": 9.5e-06, + "output_cost_per_token": 9.5e-05, + "output_cost_per_token_batches": 4.75e-05, + "cache_creation_input_token_cost": 2.375e-05, + "cache_creation_input_token_cost_above_1hr": 3.8e-05, + "cache_read_input_token_cost": 4.75e-07, + "search_context_cost_per_query": { + "search_context_size_high": 0.019, + "search_context_size_low": 0.019, + "search_context_size_medium": 0.019, + }, + "supports_function_calling": True, + "supports_prompt_caching": True, + "supports_reasoning": True, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + }, "groq/qwen/qwen3.8-27b": { "litellm_provider": "groq", "mode": "chat", diff --git a/tests/integration/cost_calculation/cost_tracking_case.py b/tests/integration/cost_calculation/cost_tracking_case.py index c71174948ac..8f28b875d89 100644 --- a/tests/integration/cost_calculation/cost_tracking_case.py +++ b/tests/integration/cost_calculation/cost_tracking_case.py @@ -532,6 +532,7 @@ _PROVIDER_PREFIXES: Final[Mapping[str, str]] = MappingProxyType( "cohere": "cohere", "vertex_ai-language-models": "vertex_ai", "vertex_ai-image-models": "vertex_ai", + "vertex_ai-anthropic_models": "vertex_ai", "vertex_ai-embedding-models": "vertex_ai", "gemini": "", "together_ai": "", @@ -584,6 +585,9 @@ _LITELLM_PARAMS: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType( "vertex_ai-image-models": MappingProxyType( {"vertex_project": "cc-scripted-project", "vertex_location": "us-central1"} ), + "vertex_ai-anthropic_models": MappingProxyType( + {"vertex_project": "cc-scripted-project", "vertex_location": "us-central1"} + ), "vertex_ai-embedding-models": MappingProxyType( {"vertex_project": "cc-scripted-project", "vertex_location": "us-central1"} ), @@ -629,3 +633,34 @@ def with_usage( "expected": expected, } ) + + +def with_response_usage( + base: CostTrackingTestCase, + name: str, + usage: Mapping[str, JsonValue], + expected: Expected, + choice_updates: Mapping[str, JsonValue] | None = None, +) -> CostTrackingTestCase: + if not isinstance(base.response, JsonResponse): + raise TypeError(f"{base.name}: with_response_usage needs a JsonResponse base") + body: Final = {**base.response.body, "usage": dict(usage)} + response_body: Final = ( + _with_choice_updates(body, choice_updates) if choice_updates is not None else body + ) + return base.model_copy( + update={ + "name": name, + "response": base.response.model_copy(update={"body": response_body}), + "expected": expected, + } + ) + + +def _with_choice_updates( + body: Mapping[str, JsonValue], choice_updates: Mapping[str, JsonValue] +) -> dict[str, JsonValue]: + choices: Final = body.get("choices") + if not isinstance(choices, list) or not choices or not isinstance(choices[0], dict): + raise TypeError("stored body has no choice object") + return {**body, "choices": [{**choices[0], **choice_updates}, *choices[1:]]}