From cd20ecace18848e474fff822d0295c173b81beb7 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 17 Oct 2026 11:11:46 -0700 Subject: [PATCH] test(cost): add tier 1 cost tracking cells and sync usage shapes for 2026-10-17 (#47141) * test(cost): add tier 1 cost tracking cells and sync usage shapes for 2026-10-17 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(cost): restore tiered cost cases and revert out of scope usage edits Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(cost): add remaining tier 1 endpoint cells for 2026-10-17 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../cost_calculation/cases/anthropic.py | 45 ++ .../cost_calculation/cases/azure.py | 148 +++++++ .../cases/bedrock_converse.py | 125 ++++++ .../cost_calculation/cases/bedrock_mantle.py | 259 ++++++++++- .../cost_calculation/cases/gemini.py | 166 ++++++- .../cost_calculation/cases/misc.py | 112 ++++- .../cost_calculation/cases/openai.py | 360 ++++++++++++++++ .../cost_calculation/cases/vertex_ai.py | 404 +++++++++++++++++- .../integration/cost_calculation/cost_map.py | 20 + 9 files changed, 1601 insertions(+), 38 deletions(-) diff --git a/tests/integration/cost_calculation/cases/anthropic.py b/tests/integration/cost_calculation/cases/anthropic.py index a0c81d8f06d..5ad4afa9530 100644 --- a/tests/integration/cost_calculation/cases/anthropic.py +++ b/tests/integration/cost_calculation/cases/anthropic.py @@ -6899,6 +6899,35 @@ CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_JSON: Final = CLAUDE_OPUS_5_5_MESSAGES_INPUT_T } ) +def _responses_bridged_stream_case( + sibling: CostTrackingTestCase, stream_sibling: CostTrackingTestCase, name: str +) -> CostTrackingTestCase: + return sibling.model_copy( + update={ + "name": name, + "request": {**sibling.request, "stream": True}, + "response": stream_sibling.response, + } + ) + + +CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_STREAM: Final = _responses_bridged_stream_case( + CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_JSON, CLAUDE_FABLE_5_1_MESSAGES_STREAM, + "claude-fable-5-1-responses_bridged_stream", +) +CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_STREAM: Final = _responses_bridged_stream_case( + CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_JSON, CLAUDE_HAIKU_5_5_MESSAGES_STREAM, + "claude-haiku-5-5-responses_bridged_stream", +) +CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_STREAM: Final = _responses_bridged_stream_case( + CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_JSON, CLAUDE_OPUS_5_5_MESSAGES_STREAM, + "claude-opus-5-5-responses_bridged_stream", +) +CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_STREAM: Final = _responses_bridged_stream_case( + CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_JSON, CLAUDE_SONNET_5_5_MESSAGES_STREAM, + "claude-sonnet-5-5-responses_bridged_stream", +) + CLAUDE_HAIKU_5_5_TIERED_INPUT_ABOVE_100K: Final = with_usage( CLAUDE_HAIKU_5_5_INPUT_TEXT, "claude-haiku-5-5-tiered_input_above_100k", @@ -6994,6 +7023,10 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( CLAUDE_SONNET_5_RESPONSES_BRIDGED_JSON, CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_JSON, CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_JSON, + CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_STREAM, + CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_STREAM, + CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_STREAM, + CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_STREAM, CLAUDE_FABLE_5_RESPONSES_BRIDGED_JSON, CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_JSON, CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_JSON, @@ -7119,6 +7152,18 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( ) PARITY: Final[tuple[StreamParityTestCase, ...]] = ( + StreamParityTestCase( + plain=CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_JSON, streamed=CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_STREAM + ), + StreamParityTestCase( + plain=CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_JSON, streamed=CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_STREAM + ), + StreamParityTestCase( + plain=CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_JSON, streamed=CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_STREAM + ), + StreamParityTestCase( + plain=CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_JSON, streamed=CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_STREAM + ), StreamParityTestCase(plain=CLAUDE_HAIKU_4_5_INPUT_TEXT, streamed=CLAUDE_HAIKU_4_5_STREAM), StreamParityTestCase( plain=CLAUDE_HAIKU_4_5_RESPONSE_MODEL_OVERRIDE, streamed=CLAUDE_HAIKU_4_5_STREAM_RESPONSE_MODEL_OVERRIDE diff --git a/tests/integration/cost_calculation/cases/azure.py b/tests/integration/cost_calculation/cases/azure.py index 1360f65b802..d9f17c12bda 100644 --- a/tests/integration/cost_calculation/cases/azure.py +++ b/tests/integration/cost_calculation/cases/azure.py @@ -21,6 +21,8 @@ from integration.cost_calculation.cost_tracking_case import ( ) from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs from integration.cost_calculation.stream_parity.case import StreamParityTestCase, sse_frames +from integration.cost_calculation.cases.openai import _responses_json_body, _responses_stream_case +from pydantic import JsonValue def _response_for_model( @@ -47,6 +49,37 @@ def _stream_for_model(response: SseResponse, model: str) -> SseResponse: ) +def _azure_responses_json_case( + base: CostTrackingTestCase, name: str, response_model: str, content_filters: JsonValue +) -> CostTrackingTestCase: + response_case: Final = bridge_pairs((base,), ("/v1/responses",))[0].bridged + return response_case.model_copy( + update={ + "name": name, + "response": JsonResponse( + content_type="application/json", + body=_responses_json_body( + response_model, + include_content_filters=True, + content_filters=content_filters, + ), + ), + } + ) + + +def _azure_messages_stream_case( + messages_json: CostTrackingTestCase, stream_case: CostTrackingTestCase, name: str +) -> CostTrackingTestCase: + return messages_json.model_copy( + update={ + "name": name, + "request": {**messages_json.request, "stream": True}, + "response": stream_case.response, + } + ) + + # azure/gpt-5.4-mini AZURE_GPT_5_4_MINI_INPUT_TEXT: Final = CostTrackingTestCase( name="azure-gpt-5.4-mini-input_text", @@ -5812,6 +5845,85 @@ AZURE_GPT_6_ASTRA_CACHE_WRITE_5M: Final = with_response_usage( ExactExpected(spend=0.22375, input_cost=0.2, output_cost=0.02375, prompt_tokens=12928, completion_tokens=380), ) +AZURE_GPT_5_6_TERRA_TIERED_INPUT_ABOVE_272K: Final = with_usage( + AZURE_GPT_5_6_TERRA_INPUT_TEXT, + "azure/gpt-5.6-terra-tiered_input_above_272k", + { + "prompt_tokens": 300000, + "total_tokens": 300412, + "prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 0}, + }, + ExactExpected( + spend=1.3281576, input_cost=1.32, output_cost=0.0081576, prompt_tokens=300000, completion_tokens=412 + ), +) +AZURE_GPT_5_6_TERRA_TIERED_CACHE_READ_ABOVE_272K: Final = with_usage( + AZURE_GPT_5_6_TERRA_INPUT_TEXT, + "azure/gpt-5.6-terra-tiered_cache_read_above_272k", + { + "prompt_tokens": 300000, + "total_tokens": 300412, + "prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 200000}, + }, + ExactExpected( + spend=0.5361576, input_cost=0.528, output_cost=0.0081576, prompt_tokens=300000, completion_tokens=412 + ), +) + + +AZURE_GPT_6_1_SOL_RESPONSES_JSON: Final = _azure_responses_json_case( + AZURE_GPT_6_1_SOL_INPUT_TEXT, "azure-gpt-6.1-sol-responses_json", "gpt-6.1-sol", None +) +AZURE_GPT_6_LUNA_RESPONSES_JSON: Final = _azure_responses_json_case( + AZURE_GPT_6_LUNA_INPUT_TEXT, "azure-gpt-6-luna-responses_json", "gpt-6-luna", None +) +AZURE_GPT_6_ASTRA_RESPONSES_JSON: Final = _azure_responses_json_case( + AZURE_GPT_6_ASTRA_INPUT_TEXT, "azure-gpt-6-astra-responses_json", "gpt-6-astra", [] +) +AZURE_GPT_5_6_TERRA_RESPONSES_JSON: Final = _azure_responses_json_case( + AZURE_GPT_5_6_TERRA_INPUT_TEXT, "azure-gpt-5.6-terra-responses_json", "gpt-5.6-terra", None +) +AZURE_GPT_6_1_SOL_RESPONSES_STREAM: Final = _responses_stream_case( + AZURE_GPT_6_1_SOL_RESPONSES_JSON, + "azure-gpt-6.1-sol-responses_stream", + "gpt-6.1-sol", + 14, + include_content_filters=True, +) +AZURE_GPT_6_LUNA_RESPONSES_STREAM: Final = _responses_stream_case( + AZURE_GPT_6_LUNA_RESPONSES_JSON, + "azure-gpt-6-luna-responses_stream", + "gpt-6-luna", + 25, + include_content_filters=True, +) +AZURE_GPT_6_ASTRA_RESPONSES_STREAM: Final = _responses_stream_case( + AZURE_GPT_6_ASTRA_RESPONSES_JSON, + "azure-gpt-6-astra-responses_stream", + "gpt-6-astra", + 16, + include_content_filters=True, + content_filters=[], +) +AZURE_GPT_5_6_TERRA_RESPONSES_STREAM: Final = _responses_stream_case( + AZURE_GPT_5_6_TERRA_RESPONSES_JSON, + "azure-gpt-5.6-terra-responses_stream", + "gpt-5.6-terra", + 15, + include_content_filters=True, +) +AZURE_GPT_6_1_SOL_MESSAGES_BRIDGED_STREAM: Final = _azure_messages_stream_case( + AZURE_GPT_6_1_SOL_MESSAGES_BRIDGED_JSON, AZURE_GPT_6_1_SOL_STREAM, "azure-gpt-6.1-sol-messages_bridged_stream" +) +AZURE_GPT_6_LUNA_MESSAGES_BRIDGED_STREAM: Final = _azure_messages_stream_case( + AZURE_GPT_6_LUNA_MESSAGES_BRIDGED_JSON, AZURE_GPT_6_LUNA_STREAM, "azure-gpt-6-luna-messages_bridged_stream" +) +AZURE_GPT_6_ASTRA_MESSAGES_BRIDGED_STREAM: Final = _azure_messages_stream_case( + AZURE_GPT_6_ASTRA_MESSAGES_BRIDGED_JSON, AZURE_GPT_6_ASTRA_STREAM, "azure-gpt-6-astra-messages_bridged_stream" +) +AZURE_GPT_5_6_TERRA_MESSAGES_BRIDGED_STREAM: Final = _azure_messages_stream_case( + AZURE_GPT_5_6_TERRA_MESSAGES_BRIDGED_JSON, AZURE_GPT_5_6_TERRA_STREAM, "azure-gpt-5.6-terra-messages_bridged_stream" +) CASES: Final[tuple[CostTrackingTestCase, ...]] = ( AZURE_GPT_6_SOL_MESSAGES_BRIDGED_JSON, @@ -5903,6 +6015,8 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( AZURE_GPT_6_LUNA_CACHE_WRITE_5M, AZURE_GPT_6_ASTRA_CACHE_WRITE_5M, AZURE_GPT_5_6_TERRA_INPUT_TEXT, + AZURE_GPT_5_6_TERRA_TIERED_INPUT_ABOVE_272K, + AZURE_GPT_5_6_TERRA_TIERED_CACHE_READ_ABOVE_272K, AZURE_GPT_5_6_TERRA_STREAM, AZURE_GPT_5_6_TERRA_SERVICE_TIER_FLEX, AZURE_GPT_5_6_TERRA_SERVICE_TIER_PRIORITY, @@ -5911,6 +6025,18 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( AZURE_GPT_5_6_TERRA_CACHE_WRITE_5M, AZURE_GPT_5_6_TERRA_REASONING, AZURE_GPT_5_6_TERRA_TOOL_CALL, + AZURE_GPT_6_1_SOL_RESPONSES_JSON, + AZURE_GPT_6_1_SOL_RESPONSES_STREAM, + AZURE_GPT_6_LUNA_RESPONSES_JSON, + AZURE_GPT_6_LUNA_RESPONSES_STREAM, + AZURE_GPT_6_ASTRA_RESPONSES_JSON, + AZURE_GPT_6_ASTRA_RESPONSES_STREAM, + AZURE_GPT_5_6_TERRA_RESPONSES_JSON, + AZURE_GPT_5_6_TERRA_RESPONSES_STREAM, + AZURE_GPT_6_1_SOL_MESSAGES_BRIDGED_STREAM, + AZURE_GPT_6_LUNA_MESSAGES_BRIDGED_STREAM, + AZURE_GPT_6_ASTRA_MESSAGES_BRIDGED_STREAM, + AZURE_GPT_5_6_TERRA_MESSAGES_BRIDGED_STREAM, ) PARITY: Final[tuple[StreamParityTestCase, ...]] = ( @@ -5924,6 +6050,22 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = ( StreamParityTestCase(plain=AZURE_GPT_6_ASTRA_INPUT_TEXT, streamed=AZURE_GPT_6_ASTRA_STREAM), StreamParityTestCase(plain=AZURE_GPT_6_1_SOL_INPUT_TEXT, streamed=AZURE_GPT_6_1_SOL_STREAM), StreamParityTestCase(plain=AZURE_GPT_5_6_TERRA_INPUT_TEXT, streamed=AZURE_GPT_5_6_TERRA_STREAM), + StreamParityTestCase(plain=AZURE_GPT_6_1_SOL_RESPONSES_JSON, streamed=AZURE_GPT_6_1_SOL_RESPONSES_STREAM), + StreamParityTestCase(plain=AZURE_GPT_6_LUNA_RESPONSES_JSON, streamed=AZURE_GPT_6_LUNA_RESPONSES_STREAM), + StreamParityTestCase(plain=AZURE_GPT_6_ASTRA_RESPONSES_JSON, streamed=AZURE_GPT_6_ASTRA_RESPONSES_STREAM), + StreamParityTestCase(plain=AZURE_GPT_5_6_TERRA_RESPONSES_JSON, streamed=AZURE_GPT_5_6_TERRA_RESPONSES_STREAM), + StreamParityTestCase( + plain=AZURE_GPT_6_1_SOL_MESSAGES_BRIDGED_JSON, streamed=AZURE_GPT_6_1_SOL_MESSAGES_BRIDGED_STREAM + ), + StreamParityTestCase( + plain=AZURE_GPT_6_LUNA_MESSAGES_BRIDGED_JSON, streamed=AZURE_GPT_6_LUNA_MESSAGES_BRIDGED_STREAM + ), + StreamParityTestCase( + plain=AZURE_GPT_6_ASTRA_MESSAGES_BRIDGED_JSON, streamed=AZURE_GPT_6_ASTRA_MESSAGES_BRIDGED_STREAM + ), + StreamParityTestCase( + plain=AZURE_GPT_5_6_TERRA_MESSAGES_BRIDGED_JSON, streamed=AZURE_GPT_5_6_TERRA_MESSAGES_BRIDGED_STREAM + ), ) BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = ( @@ -5938,4 +6080,10 @@ BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = ( BridgeCostParityTestCase( native=AZURE_GPT_5_6_TERRA_INPUT_TEXT, bridged=AZURE_GPT_5_6_TERRA_MESSAGES_BRIDGED_JSON ), + BridgeCostParityTestCase(native=AZURE_GPT_6_1_SOL_STREAM, bridged=AZURE_GPT_6_1_SOL_MESSAGES_BRIDGED_STREAM), + BridgeCostParityTestCase(native=AZURE_GPT_6_LUNA_STREAM, bridged=AZURE_GPT_6_LUNA_MESSAGES_BRIDGED_STREAM), + BridgeCostParityTestCase(native=AZURE_GPT_6_ASTRA_STREAM, bridged=AZURE_GPT_6_ASTRA_MESSAGES_BRIDGED_STREAM), + BridgeCostParityTestCase( + native=AZURE_GPT_5_6_TERRA_STREAM, bridged=AZURE_GPT_5_6_TERRA_MESSAGES_BRIDGED_STREAM + ), ) diff --git a/tests/integration/cost_calculation/cases/bedrock_converse.py b/tests/integration/cost_calculation/cases/bedrock_converse.py index f78bdfecfa3..858db06cafc 100644 --- a/tests/integration/cost_calculation/cases/bedrock_converse.py +++ b/tests/integration/cost_calculation/cases/bedrock_converse.py @@ -6088,6 +6088,71 @@ US_ANTHROPIC_CLAUDE_HAIKU_5_5_TIERED_CACHE_WRITE_1H_ABOVE_100K: Final = with_usa ), ) +def _us_anthropic_messages_bridge_pairs( + input_case: CostTrackingTestCase, stream_case: CostTrackingTestCase +) -> tuple[BridgeCostParityTestCase, ...]: + return tuple( + BridgeCostParityTestCase( + native=pair.native, + bridged=pair.bridged.model_copy( + update={ + "name": ( + f"us.anthropic." + f"{pair.native.name.removeprefix('us.anthropic.').removesuffix('-input_text').removesuffix('-stream')}" + f"-messages_bridged_{'stream' if pair.native.request.get('stream') is True else 'json'}" + ) + } + ), + ) + for pair in bridge_pairs((input_case, stream_case), ("/v1/messages",)) + ) + + +def _us_anthropic_responses_stream_case( + json_case: CostTrackingTestCase, stream_case: CostTrackingTestCase, name: str +) -> CostTrackingTestCase: + return json_case.model_copy( + update={ + "name": name, + "request": {**json_case.request, "stream": True}, + "response": stream_case.response, + } + ) + + +US_ANTHROPIC_CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_STREAM: Final = _us_anthropic_responses_stream_case( + US_ANTHROPIC_CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_JSON, + US_ANTHROPIC_CLAUDE_FABLE_5_1_STREAM, + "us.anthropic.claude-fable-5-1-responses_bridged_stream", +) +US_ANTHROPIC_CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_STREAM: Final = _us_anthropic_responses_stream_case( + US_ANTHROPIC_CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_JSON, + US_ANTHROPIC_CLAUDE_HAIKU_5_5_STREAM, + "us.anthropic.claude-haiku-5-5-responses_bridged_stream", +) +US_ANTHROPIC_CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_STREAM: Final = _us_anthropic_responses_stream_case( + US_ANTHROPIC_CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_JSON, + US_ANTHROPIC_CLAUDE_OPUS_5_5_STREAM, + "us.anthropic.claude-opus-5-5-responses_bridged_stream", +) +US_ANTHROPIC_CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_STREAM: Final = _us_anthropic_responses_stream_case( + US_ANTHROPIC_CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_JSON, + US_ANTHROPIC_CLAUDE_SONNET_5_5_STREAM, + "us.anthropic.claude-sonnet-5-5-responses_bridged_stream", +) +_US_ANTHROPIC_FABLE_5_1_MESSAGES_PAIRS: Final = _us_anthropic_messages_bridge_pairs( + US_ANTHROPIC_CLAUDE_FABLE_5_1_INPUT_TEXT, US_ANTHROPIC_CLAUDE_FABLE_5_1_STREAM +) +_US_ANTHROPIC_HAIKU_5_5_MESSAGES_PAIRS: Final = _us_anthropic_messages_bridge_pairs( + US_ANTHROPIC_CLAUDE_HAIKU_5_5_INPUT_TEXT, US_ANTHROPIC_CLAUDE_HAIKU_5_5_STREAM +) +_US_ANTHROPIC_OPUS_5_5_MESSAGES_PAIRS: Final = _us_anthropic_messages_bridge_pairs( + US_ANTHROPIC_CLAUDE_OPUS_5_5_INPUT_TEXT, US_ANTHROPIC_CLAUDE_OPUS_5_5_STREAM +) +_US_ANTHROPIC_SONNET_5_5_MESSAGES_PAIRS: Final = _us_anthropic_messages_bridge_pairs( + US_ANTHROPIC_CLAUDE_SONNET_5_5_INPUT_TEXT, US_ANTHROPIC_CLAUDE_SONNET_5_5_STREAM +) + CASES: Final[tuple[CostTrackingTestCase, ...]] = ( US_ANTHROPIC_CLAUDE_SONNET_5_RESPONSES_BRIDGED_JSON, US_ANTHROPIC_CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_JSON, @@ -6210,6 +6275,14 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( US_ANTHROPIC_CLAUDE_FABLE_5_1_THINKING, US_ANTHROPIC_CLAUDE_OPUS_5_5_UPSTREAM_400_ZERO_SPEND, US_ANTHROPIC_CLAUDE_OPUS_5_5_INPUT_TEXT_ROLLUPS, + US_ANTHROPIC_CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_STREAM, + US_ANTHROPIC_CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_STREAM, + US_ANTHROPIC_CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_STREAM, + US_ANTHROPIC_CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_STREAM, + *tuple(pair.bridged for pair in _US_ANTHROPIC_FABLE_5_1_MESSAGES_PAIRS), + *tuple(pair.bridged for pair in _US_ANTHROPIC_HAIKU_5_5_MESSAGES_PAIRS), + *tuple(pair.bridged for pair in _US_ANTHROPIC_OPUS_5_5_MESSAGES_PAIRS), + *tuple(pair.bridged for pair in _US_ANTHROPIC_SONNET_5_5_MESSAGES_PAIRS), ) PARITY: Final[tuple[StreamParityTestCase, ...]] = ( @@ -6235,6 +6308,38 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = ( StreamParityTestCase(plain=US_ANTHROPIC_CLAUDE_HAIKU_5_5_INPUT_TEXT, streamed=US_ANTHROPIC_CLAUDE_HAIKU_5_5_STREAM), StreamParityTestCase(plain=US_ANTHROPIC_CLAUDE_FABLE_5_INPUT_TEXT, streamed=US_ANTHROPIC_CLAUDE_FABLE_5_STREAM), StreamParityTestCase(plain=US_ANTHROPIC_CLAUDE_FABLE_5_1_INPUT_TEXT, streamed=US_ANTHROPIC_CLAUDE_FABLE_5_1_STREAM), + StreamParityTestCase( + plain=US_ANTHROPIC_CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_JSON, + streamed=US_ANTHROPIC_CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_STREAM, + ), + StreamParityTestCase( + plain=US_ANTHROPIC_CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_JSON, + streamed=US_ANTHROPIC_CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_STREAM, + ), + StreamParityTestCase( + plain=US_ANTHROPIC_CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_JSON, + streamed=US_ANTHROPIC_CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_STREAM, + ), + StreamParityTestCase( + plain=US_ANTHROPIC_CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_JSON, + streamed=US_ANTHROPIC_CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_STREAM, + ), + StreamParityTestCase( + plain=_US_ANTHROPIC_FABLE_5_1_MESSAGES_PAIRS[0].bridged, + streamed=_US_ANTHROPIC_FABLE_5_1_MESSAGES_PAIRS[1].bridged, + ), + StreamParityTestCase( + plain=_US_ANTHROPIC_HAIKU_5_5_MESSAGES_PAIRS[0].bridged, + streamed=_US_ANTHROPIC_HAIKU_5_5_MESSAGES_PAIRS[1].bridged, + ), + StreamParityTestCase( + plain=_US_ANTHROPIC_OPUS_5_5_MESSAGES_PAIRS[0].bridged, + streamed=_US_ANTHROPIC_OPUS_5_5_MESSAGES_PAIRS[1].bridged, + ), + StreamParityTestCase( + plain=_US_ANTHROPIC_SONNET_5_5_MESSAGES_PAIRS[0].bridged, + streamed=_US_ANTHROPIC_SONNET_5_5_MESSAGES_PAIRS[1].bridged, + ), ) BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = ( @@ -6267,4 +6372,24 @@ BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = ( BridgeCostParityTestCase( native=US_ANTHROPIC_CLAUDE_FABLE_5_1_INPUT_TEXT, bridged=US_ANTHROPIC_CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_JSON ), + BridgeCostParityTestCase( + native=US_ANTHROPIC_CLAUDE_FABLE_5_1_STREAM, + bridged=US_ANTHROPIC_CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_STREAM, + ), + BridgeCostParityTestCase( + native=US_ANTHROPIC_CLAUDE_HAIKU_5_5_STREAM, + bridged=US_ANTHROPIC_CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_STREAM, + ), + BridgeCostParityTestCase( + native=US_ANTHROPIC_CLAUDE_OPUS_5_5_STREAM, + bridged=US_ANTHROPIC_CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_STREAM, + ), + BridgeCostParityTestCase( + native=US_ANTHROPIC_CLAUDE_SONNET_5_5_STREAM, + bridged=US_ANTHROPIC_CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_STREAM, + ), + *_US_ANTHROPIC_FABLE_5_1_MESSAGES_PAIRS, + *_US_ANTHROPIC_HAIKU_5_5_MESSAGES_PAIRS, + *_US_ANTHROPIC_OPUS_5_5_MESSAGES_PAIRS, + *_US_ANTHROPIC_SONNET_5_5_MESSAGES_PAIRS, ) diff --git a/tests/integration/cost_calculation/cases/bedrock_mantle.py b/tests/integration/cost_calculation/cases/bedrock_mantle.py index 2fa940d66f4..3cd9150033a 100644 --- a/tests/integration/cost_calculation/cases/bedrock_mantle.py +++ b/tests/integration/cost_calculation/cases/bedrock_mantle.py @@ -18,7 +18,7 @@ from integration.cost_calculation.cases.anthropic import ( CLAUDE_OPUS_5_5_THINKING, CLAUDE_OPUS_5_5_TOOL_CALL, ) -from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase +from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs from integration.cost_calculation.cost_tracking_case import ( CostTrackingTestCase, Deployment, @@ -535,14 +535,40 @@ _MANTLE_GPT_CHAT_REQUEST: Final = { "messages": [{"role": "user", "content": "cost tracking fixture"}], "max_completion_tokens": 64, } +_MANTLE_GPT_TOOL_REQUEST: Final = { + **_MANTLE_GPT_RESPONSES_REQUEST, + "tools": [ + { + "type": "function", + "name": "get_weather", + "description": "Get the weather for a city", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + } + ], + "tool_choice": "required", +} +_MANTLE_GPT_CACHE_WRITE_REQUEST: Final = { + **_MANTLE_GPT_RESPONSES_REQUEST, + "input": "cacheable pricing-harness prefix " * 1000, + "prompt_cache_key": "cost-cases-20261017", + "max_output_tokens": 64, +} def _mantle_gpt_usage( - input_tokens: int, output_tokens: int, reasoning_tokens: int, cached_tokens: int = 0 + input_tokens: int, + output_tokens: int, + reasoning_tokens: int, + cached_tokens: int = 0, + cache_write_tokens: int = 0, ) -> dict[str, JsonValue]: return { "input_tokens": input_tokens, - "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": cached_tokens}, + "input_tokens_details": {"cache_write_tokens": cache_write_tokens, "cached_tokens": cached_tokens}, "output_tokens": output_tokens, "output_tokens_details": {"reasoning_tokens": reasoning_tokens}, "total_tokens": input_tokens + output_tokens, @@ -628,6 +654,20 @@ def _mantle_gpt_cases( response_expected, _MANTLE_GPT_CHAT_REQUEST, ) + chat_stream_case: Final = chat_case.model_copy( + update={ + "name": f"bedrock_mantle-{provider_model}-chat_stream", + "request": {**chat_case.request, "stream": True}, + "response": stream_case.response, + } + ) + messages_pairs: Final = bridge_pairs((response_case, stream_case), ("/v1/messages",)) + messages_json: Final = messages_pairs[0].bridged.model_copy( + update={"name": f"bedrock_mantle-{provider_model}-messages_bridged_json"} + ) + messages_stream: Final = messages_pairs[1].bridged.model_copy( + update={"name": f"bedrock_mantle-{provider_model}-messages_bridged_stream"} + ) cache_case: Final = _mantle_gpt_json_case( BEDROCK_MANTLE_OPENAI_GPT_6_SOL_RESPONSES_JSON, provider_model, @@ -660,7 +700,73 @@ def _mantle_gpt_cases( tiered_boundary_expected, {**_MANTLE_GPT_RESPONSES_REQUEST, "max_output_tokens": 412}, ) - return (response_case, stream_case, reasoning_case, chat_case, cache_case, tiered_input, tiered_cache, tiered_boundary) + return ( + response_case, + stream_case, + reasoning_case, + chat_case, + cache_case, + tiered_input, + tiered_cache, + tiered_boundary, + chat_stream_case, + messages_json, + messages_stream, + ) + + +def _mantle_gpt_audit3_cases( + provider_model: str, + tool_input_tokens: int, + tool_output_tokens: int, + tool_reasoning_tokens: int, + tool_expected: ExactExpected, + cache_output_tokens: int, + cache_reasoning_tokens: int, + cache_expected: ExactExpected, +) -> tuple[CostTrackingTestCase, ...]: + tool_case: Final = _mantle_gpt_json_case( + BEDROCK_MANTLE_OPENAI_GPT_6_SOL_RESPONSES_JSON, + provider_model, + "tool_call", + _mantle_gpt_usage(tool_input_tokens, tool_output_tokens, tool_reasoning_tokens), + tool_expected, + _MANTLE_GPT_TOOL_REQUEST, + ) + if not isinstance(tool_case.response, JsonResponse): + raise TypeError(f"{tool_case.name}: expected a JSON response fixture") + tool_response: Final = tool_case.response.model_copy( + update={ + "body": { + **tool_case.response.body, + "output": [ + { + "arguments": '{"city":"Paris"}', + "call_id": "call_$REQUEST_ID", + "id": "fc_$REQUEST_ID", + "name": "get_weather", + "status": "completed", + "type": "function_call", + } + ], + } + } + ) + tool_case_with_response: Final = tool_case.model_copy(update={"response": tool_response}) + cache_case: Final = _mantle_gpt_json_case( + BEDROCK_MANTLE_OPENAI_GPT_6_SOL_RESPONSES_JSON, + provider_model, + "cache_write_5m", + _mantle_gpt_usage( + 1667, + cache_output_tokens, + cache_reasoning_tokens, + cache_write_tokens=1665, + ), + cache_expected, + _MANTLE_GPT_CACHE_WRITE_REQUEST, + ) + return (tool_case_with_response, cache_case) def _mantle_gpt_bridge(case: CostTrackingTestCase) -> BridgeCostParityTestCase: @@ -723,6 +829,83 @@ _MANTLE_GPT_56_TERRA_CASES: Final = _mantle_gpt_cases( ExactExpected(spend=0.84537376, input_cost=0.83776, output_cost=0.00761376, prompt_tokens=272000, completion_tokens=412), ) +_MANTLE_GPT_AUDIT3_CASES: Final = ( + *_mantle_gpt_audit3_cases( + "openai.gpt-6.1-sol", + 50, + 20, + 0, + ExactExpected(spend=0.000363, input_cost=0.000121, output_cost=0.000242, prompt_tokens=50, completion_tokens=20), + 28, + 0, + ExactExpected( + spend=0.005381765, + input_cost=0.005042965, + output_cost=0.0003388, + prompt_tokens=1667, + completion_tokens=28, + cache_creation_cost=0.005038125, + ), + ), + *_mantle_gpt_audit3_cases( + "openai.gpt-6-luna", + 51, + 18, + 0, + ExactExpected( + spend=0.000018612, input_cost=0.000006732, output_cost=0.00001188, prompt_tokens=51, completion_tokens=18 + ), + 64, + 40, + ExactExpected( + spend=0.000317229, + input_cost=0.000274989, + output_cost=0.00004224, + prompt_tokens=1667, + completion_tokens=64, + cache_creation_cost=0.000274725, + ), + ), + *_mantle_gpt_audit3_cases( + "openai.gpt-6-astra", + 51, + 18, + 0, + ExactExpected( + spend=0.0020163, input_cost=0.0007293, output_cost=0.001287, prompt_tokens=51, completion_tokens=18 + ), + 27, + 0, + ExactExpected( + spend=0.031720975, + input_cost=0.029790475, + output_cost=0.0019305, + prompt_tokens=1667, + completion_tokens=27, + cache_creation_cost=0.029761875, + ), + ), + *_mantle_gpt_audit3_cases( + "openai.gpt-5.6-terra", + 51, + 28, + 8, + ExactExpected( + spend=0.00067452, input_cost=0.00015708, output_cost=0.00051744, prompt_tokens=51, completion_tokens=28 + ), + 28, + 0, + ExactExpected( + spend=0.00693385, + input_cost=0.00641641, + output_cost=0.00051744, + prompt_tokens=1667, + completion_tokens=28, + cache_creation_cost=0.00641025, + ), + ), +) + _MANTLE_CLAUDE_OPUS_MODEL: Final = "bedrock_mantle/anthropic.claude-opus-5-5" _ADAPTIVE_USAGE: Final = { "input_tokens": 24, @@ -737,7 +920,25 @@ _ADAPTIVE_EXPECTED: Final = ExactExpected( spend=0.0007524, input_cost=0.0001584, output_cost=0.000594, prompt_tokens=24, completion_tokens=18 ) -BEDROCK_MANTLE_CLAUDE_OPUS_5_5_INPUT_TEXT: Final = with_usage( + +def _mantle_without_usage_fields(case: CostTrackingTestCase) -> CostTrackingTestCase: + if not isinstance(case.response, JsonResponse): + raise TypeError(f"{case.name}: expected a JSON response fixture") + usage: Final = case.response.body.get("usage") + if not isinstance(usage, dict): + raise TypeError(f"{case.name}: expected a usage object") + response: Final = case.response.model_copy( + update={ + "body": { + **case.response.body, + "usage": {key: value for key, value in usage.items() if key not in {"inference_geo", "speed"}}, + } + } + ) + return case.model_copy(update={"response": response}) + + +BEDROCK_MANTLE_CLAUDE_OPUS_5_5_INPUT_TEXT: Final = _mantle_without_usage_fields(with_usage( CLAUDE_OPUS_5_5_INPUT_TEXT, "bedrock_mantle-anthropic.claude-opus-5-5-input_text", _ADAPTIVE_USAGE, @@ -751,7 +952,7 @@ BEDROCK_MANTLE_CLAUDE_OPUS_5_5_INPUT_TEXT: Final = with_usage( "allowed_openai_params": ["reasoning_effort"], }, } -) +)) _MANTLE_CLAUDE_STREAM_RESPONSE: Final = SseResponse( content_type="text/event-stream", @@ -841,7 +1042,7 @@ BEDROCK_MANTLE_CLAUDE_OPUS_5_5_MESSAGES_STREAM: Final = CLAUDE_OPUS_5_5_MESSAGES } ) -BEDROCK_MANTLE_CLAUDE_OPUS_5_5_CACHE_READ: Final = with_usage( +BEDROCK_MANTLE_CLAUDE_OPUS_5_5_CACHE_READ: Final = _mantle_without_usage_fields(with_usage( CLAUDE_OPUS_5_5_CACHE_READ, "bedrock_mantle-anthropic.claude-opus-5-5-cache_read", { @@ -856,7 +1057,7 @@ BEDROCK_MANTLE_CLAUDE_OPUS_5_5_CACHE_READ: Final = with_usage( ExactExpected( spend=0.00308319, input_cost=0.00153219, output_cost=0.001551, prompt_tokens=4225, completion_tokens=47 ), -).model_copy(update={"model": _MANTLE_CLAUDE_OPUS_MODEL}) +).model_copy(update={"model": _MANTLE_CLAUDE_OPUS_MODEL})) BEDROCK_MANTLE_CLAUDE_OPUS_5_5_CACHE_WRITE_5M: Final = with_usage( CLAUDE_OPUS_5_5_CACHE_WRITE_5M, @@ -892,12 +1093,12 @@ BEDROCK_MANTLE_CLAUDE_OPUS_5_5_CACHE_WRITE_1H: Final = with_usage( ), ).model_copy(update={"model": _MANTLE_CLAUDE_OPUS_MODEL}) -BEDROCK_MANTLE_CLAUDE_OPUS_5_5_THINKING: Final = with_usage( +BEDROCK_MANTLE_CLAUDE_OPUS_5_5_THINKING: Final = _mantle_without_usage_fields(with_usage( CLAUDE_OPUS_5_5_THINKING, "bedrock_mantle-anthropic.claude-opus-5-5-thinking", _ADAPTIVE_USAGE, _ADAPTIVE_EXPECTED, -).model_copy(update={"model": _MANTLE_CLAUDE_OPUS_MODEL}) +).model_copy(update={"model": _MANTLE_CLAUDE_OPUS_MODEL})) BEDROCK_MANTLE_CLAUDE_OPUS_5_5_TOOL_CALL: Final = with_usage( CLAUDE_OPUS_5_5_TOOL_CALL, @@ -921,6 +1122,21 @@ BEDROCK_MANTLE_CLAUDE_OPUS_5_5_CHAT_BRIDGED_JSON: Final = BEDROCK_MANTLE_CLAUDE_ } ) +BEDROCK_MANTLE_CLAUDE_OPUS_5_5_RESPONSES_JSON: Final = BEDROCK_MANTLE_CLAUDE_OPUS_5_5_INPUT_TEXT.model_copy( + update={ + "name": "bedrock_mantle-anthropic.claude-opus-5-5-responses_json", + "endpoint": "/v1/responses", + "request": {"model": "$MODEL", "input": "summarize the attached material in one line"}, + } +) +BEDROCK_MANTLE_CLAUDE_OPUS_5_5_RESPONSES_STREAM: Final = BEDROCK_MANTLE_CLAUDE_OPUS_5_5_STREAM.model_copy( + update={ + "name": "bedrock_mantle-anthropic.claude-opus-5-5-responses_stream", + "endpoint": "/v1/responses", + "request": {"model": "$MODEL", "input": "summarize the attached material in one line", "stream": True}, + } +) + BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = ( _mantle_gpt_bridge(_MANTLE_GPT_61_SOL_CASES[0]), _mantle_gpt_bridge(_MANTLE_GPT_6_LUNA_CASES[0]), @@ -930,6 +1146,14 @@ BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = ( native=BEDROCK_MANTLE_CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT, bridged=BEDROCK_MANTLE_CLAUDE_OPUS_5_5_CHAT_BRIDGED_JSON, ), + *tuple( + BridgeCostParityTestCase(native=cases[0], bridged=cases[9]) + for cases in (_MANTLE_GPT_61_SOL_CASES, _MANTLE_GPT_6_LUNA_CASES, _MANTLE_GPT_6_ASTRA_CASES, _MANTLE_GPT_56_TERRA_CASES) + ), + *tuple( + BridgeCostParityTestCase(native=cases[1], bridged=cases[10]) + for cases in (_MANTLE_GPT_61_SOL_CASES, _MANTLE_GPT_6_LUNA_CASES, _MANTLE_GPT_6_ASTRA_CASES, _MANTLE_GPT_56_TERRA_CASES) + ), ) CASES: Final[tuple[CostTrackingTestCase, ...]] = ( @@ -945,11 +1169,14 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( *_MANTLE_GPT_6_LUNA_CASES, *_MANTLE_GPT_6_ASTRA_CASES, *_MANTLE_GPT_56_TERRA_CASES, + *_MANTLE_GPT_AUDIT3_CASES, BEDROCK_MANTLE_CLAUDE_OPUS_5_5_INPUT_TEXT, BEDROCK_MANTLE_CLAUDE_OPUS_5_5_STREAM, BEDROCK_MANTLE_CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT, BEDROCK_MANTLE_CLAUDE_OPUS_5_5_MESSAGES_STREAM, BEDROCK_MANTLE_CLAUDE_OPUS_5_5_CHAT_BRIDGED_JSON, + BEDROCK_MANTLE_CLAUDE_OPUS_5_5_RESPONSES_JSON, + BEDROCK_MANTLE_CLAUDE_OPUS_5_5_RESPONSES_STREAM, BEDROCK_MANTLE_CLAUDE_OPUS_5_5_CACHE_READ, BEDROCK_MANTLE_CLAUDE_OPUS_5_5_CACHE_WRITE_5M, BEDROCK_MANTLE_CLAUDE_OPUS_5_5_CACHE_WRITE_1H, @@ -972,4 +1199,16 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = ( plain=BEDROCK_MANTLE_CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT, streamed=BEDROCK_MANTLE_CLAUDE_OPUS_5_5_MESSAGES_STREAM, ), + StreamParityTestCase( + plain=BEDROCK_MANTLE_CLAUDE_OPUS_5_5_RESPONSES_JSON, + streamed=BEDROCK_MANTLE_CLAUDE_OPUS_5_5_RESPONSES_STREAM, + ), + *tuple( + StreamParityTestCase(plain=cases[3], streamed=cases[8]) + for cases in (_MANTLE_GPT_61_SOL_CASES, _MANTLE_GPT_6_LUNA_CASES, _MANTLE_GPT_6_ASTRA_CASES, _MANTLE_GPT_56_TERRA_CASES) + ), + *tuple( + StreamParityTestCase(plain=cases[9], streamed=cases[10]) + for cases in (_MANTLE_GPT_61_SOL_CASES, _MANTLE_GPT_6_LUNA_CASES, _MANTLE_GPT_6_ASTRA_CASES, _MANTLE_GPT_56_TERRA_CASES) + ), ) diff --git a/tests/integration/cost_calculation/cases/gemini.py b/tests/integration/cost_calculation/cases/gemini.py index bd64f4aeb11..ceca816c300 100644 --- a/tests/integration/cost_calculation/cases/gemini.py +++ b/tests/integration/cost_calculation/cases/gemini.py @@ -2,8 +2,10 @@ PARITY pairs the streamed case with its plain twin where both bill the same row.""" +import json from typing import Final, cast +from pydantic import JsonValue from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs from integration.cost_calculation.cost_tracking_case import ( CostTrackingTestCase, @@ -17,26 +19,74 @@ from integration.cost_calculation.cost_tracking_case import ( from integration.cost_calculation.stream_parity.case import StreamParityTestCase, sse_frames -def _gemini_case_response(response: JsonResponse | SseResponse, model_version: str) -> JsonResponse | SseResponse: +def _gemini_case_response( + response: JsonResponse | SseResponse, + model_version: str, + usage_fields: dict[str, JsonValue] | None = None, +) -> JsonResponse | SseResponse: match response: case JsonResponse(): - return response.model_copy(update={"body": {**response.body, "modelVersion": model_version}}) + usage: Final = response.body.get("usageMetadata") + body: Final = {**response.body, "modelVersion": model_version} + return response.model_copy( + update={ + "body": { + **body, + **( + { + "usageMetadata": { + **cast(dict[str, JsonValue], usage), + **usage_fields, + } + } + if usage_fields is not None and isinstance(usage, dict) + else {} + ), + } + } + ) case SseResponse(): return response.model_copy( update={ - "frames": tuple(frame.replace("gemini-3.8-flash", model_version) for frame in response.frames) + "frames": tuple( + _gemini_case_frame(frame, model_version, usage_fields) for frame in response.frames + ) } ) +def _gemini_case_frame( + frame: str, model_version: str, usage_fields: dict[str, JsonValue] | None +) -> str: + replaced: Final = frame.replace("gemini-3.8-flash", model_version) + if usage_fields is None: + return replaced + prefix, separator, payload = replaced.partition("data: ") + if not separator or payload == "[DONE]": + return replaced + event: Final = cast(dict[str, JsonValue], json.loads(payload)) + usage: Final = event.get("usageMetadata") + if not isinstance(usage, dict): + return replaced + return f"{prefix}data: {json.dumps({**event, 'usageMetadata': {**usage, **usage_fields}})}" + + def _gemini_case( - sibling: CostTrackingTestCase, name: str, model: str, model_version: str, expected: ExactExpected + sibling: CostTrackingTestCase, + name: str, + model: str, + model_version: str, + expected: ExactExpected, + *, + usage_fields: dict[str, JsonValue] | None = None, ) -> CostTrackingTestCase: return sibling.model_copy( update={ "name": name, "model": model, - "response": _gemini_case_response(cast(JsonResponse | SseResponse, sibling.response), model_version), + "response": _gemini_case_response( + cast(JsonResponse | SseResponse, sibling.response), model_version, usage_fields + ), "expected": expected, } ) @@ -1540,6 +1590,7 @@ GEMINI_GEMINI_3_8_FLASH_INPUT_TEXT: Final = CostTrackingTestCase( "candidatesTokenCount": 412, "totalTokenCount": 2252, "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 1840}], + "serviceTier": "standard", }, "modelVersion": "gemini-3.8-flash", }, @@ -1594,6 +1645,7 @@ GEMINI_GEMINI_3_8_FLASH_CACHE_READ: Final = CostTrackingTestCase( "totalTokenCount": 13308, "cachedContentTokenCount": 12288, "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 12928}], + "serviceTier": "standard", }, "modelVersion": "gemini-3.8-flash", }, @@ -1653,6 +1705,7 @@ GEMINI_GEMINI_3_8_FLASH_REASONING: Final = CostTrackingTestCase( "thoughtsTokenCount": 3480, "totalTokenCount": 5280, "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 1240}], + "serviceTier": "standard", }, "modelVersion": "gemini-3.8-flash", }, @@ -1934,6 +1987,7 @@ GEMINI_GEMINI_3_8_FLASH_STREAM: Final = CostTrackingTestCase( "candidatesTokenCount": 412, "totalTokenCount": 2252, "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 1840}], + "serviceTier": "standard", }, "modelVersion": "gemini-3.8-flash", }, @@ -3275,19 +3329,46 @@ GEMINI_GEMINI_3_1_PRO_CACHE_CREATION: Final = CostTrackingTestCase( GEMINI_GEMINI_4_ARGON_INPUT_TEXT: Final = _gemini_case( GEMINI_GEMINI_3_8_FLASH_INPUT_TEXT, "gemini-gemini-4-argon-input_text", "gemini/gemini-4-argon", - "gemini-4-argon", ExactExpected(spend=0.00858, input_cost=0.004048, output_cost=0.004532, prompt_tokens=1840, completion_tokens=412) + "gemini-4-argon", ExactExpected(spend=0.00858, input_cost=0.004048, output_cost=0.004532, prompt_tokens=1840, completion_tokens=412), + usage_fields={"candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 412}]}, ) GEMINI_GEMINI_4_ARGON_STREAM: Final = _gemini_case( GEMINI_GEMINI_3_8_FLASH_STREAM, "gemini-gemini-4-argon-stream", "gemini/gemini-4-argon", - "gemini-4-argon", ExactExpected(spend=0.00858, input_cost=0.004048, output_cost=0.004532, prompt_tokens=1840, completion_tokens=412) + "gemini-4-argon", ExactExpected(spend=0.00858, input_cost=0.004048, output_cost=0.004532, prompt_tokens=1840, completion_tokens=412), + usage_fields={"candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 412}]}, ) GEMINI_GEMINI_4_ARGON_CACHE_READ: Final = _gemini_case( GEMINI_GEMINI_3_8_FLASH_CACHE_READ, "gemini-gemini-4-argon-cache_read", "gemini/gemini-4-argon", - "gemini-4-argon", ExactExpected(spend=0.00693968, input_cost=0.00275968, output_cost=0.00418, prompt_tokens=12928, completion_tokens=380) + "gemini-4-argon", ExactExpected(spend=0.00693968, input_cost=0.00275968, output_cost=0.00418, prompt_tokens=12928, completion_tokens=380), + usage_fields={"candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 380}]}, ) GEMINI_GEMINI_4_ARGON_REASONING: Final = _gemini_case( GEMINI_GEMINI_3_8_FLASH_REASONING, "gemini-gemini-4-argon-reasoning", "gemini/gemini-4-argon", - "gemini-4-argon", ExactExpected(spend=0.047168, input_cost=0.002728, output_cost=0.04444, prompt_tokens=1240, completion_tokens=4040) + "gemini-4-argon", ExactExpected(spend=0.047168, input_cost=0.002728, output_cost=0.04444, prompt_tokens=1240, completion_tokens=4040), + usage_fields={"candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 560}]}, +) +GEMINI_GEMINI_4_ARGON_SERVICE_TIER_FLEX: Final = _gemini_case( + GEMINI_GEMINI_3_8_FLASH_SERVICE_TIER_FLEX, + "gemini-gemini-4-argon-service_tier_flex", + "gemini/gemini-4-argon", + "gemini-4-argon", + ExactExpected(spend=0.00429, input_cost=0.002024, output_cost=0.002266, prompt_tokens=1840, completion_tokens=412), +) +GEMINI_GEMINI_4_ARGON_SERVICE_TIER_PRIORITY: Final = _gemini_case( + GEMINI_GEMINI_3_8_FLASH_SERVICE_TIER_PRIORITY, + "gemini-gemini-4-argon-service_tier_priority", + "gemini/gemini-4-argon", + "gemini-4-argon", + ExactExpected(spend=0.015444, input_cost=0.0072864, output_cost=0.0081576, prompt_tokens=1840, completion_tokens=412), +) +GEMINI_GEMINI_3_1_PRO_PREVIEW_SERVICE_TIER_PRIORITY: Final = _gemini_case( + GEMINI_GEMINI_3_8_FLASH_SERVICE_TIER_PRIORITY, + "gemini-gemini-3.1-pro-preview-service_tier_priority", + "gemini/gemini-3.1-pro-preview", + "gemini-3.1-pro-preview", + ExactExpected( + spend=0.02018016, input_cost=0.0086112, output_cost=0.01156896, prompt_tokens=1840, completion_tokens=412 + ), ) GEMINI_GEMINI_4_ARGON_TOOL_CALL: Final = _gemini_case( GEMINI_GEMINI_3_8_FLASH_TOOL_CALL, "gemini-gemini-4-argon-tool_call", "gemini/gemini-4-argon", @@ -3296,28 +3377,60 @@ GEMINI_GEMINI_4_ARGON_TOOL_CALL: Final = _gemini_case( GEMINI_GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT: Final = _gemini_case( GEMINI_GEMINI_3_8_FLASH_INPUT_TEXT, "gemini-gemini-3.1-pro-preview-input_text", "gemini/gemini-3.1-pro-preview", "gemini-3.1-pro-preview", - ExactExpected(spend=0.0112112, input_cost=0.004784, output_cost=0.0064272, prompt_tokens=1840, completion_tokens=412) + ExactExpected(spend=0.0112112, input_cost=0.004784, output_cost=0.0064272, prompt_tokens=1840, completion_tokens=412), + usage_fields={"serviceTier": "standard"}, ) GEMINI_GEMINI_3_1_PRO_PREVIEW_STREAM: Final = _gemini_case( GEMINI_GEMINI_3_8_FLASH_STREAM, "gemini-gemini-3.1-pro-preview-stream", "gemini/gemini-3.1-pro-preview", "gemini-3.1-pro-preview", - ExactExpected(spend=0.0112112, input_cost=0.004784, output_cost=0.0064272, prompt_tokens=1840, completion_tokens=412) + ExactExpected(spend=0.0112112, input_cost=0.004784, output_cost=0.0064272, prompt_tokens=1840, completion_tokens=412), + usage_fields={"serviceTier": "standard"}, ) GEMINI_GEMINI_3_1_PRO_PREVIEW_CACHE_READ: Final = _gemini_case( GEMINI_GEMINI_3_8_FLASH_CACHE_READ, "gemini-gemini-3.1-pro-preview-cache_read", "gemini/gemini-3.1-pro-preview", "gemini-3.1-pro-preview", - ExactExpected(spend=0.01078688, input_cost=0.00485888, output_cost=0.005928, prompt_tokens=12928, completion_tokens=380) + ExactExpected(spend=0.01078688, input_cost=0.00485888, output_cost=0.005928, prompt_tokens=12928, completion_tokens=380), + usage_fields={"serviceTier": "standard"}, ) GEMINI_GEMINI_3_1_PRO_PREVIEW_REASONING: Final = _gemini_case( GEMINI_GEMINI_3_8_FLASH_REASONING, "gemini-gemini-3.1-pro-preview-reasoning", "gemini/gemini-3.1-pro-preview", "gemini-3.1-pro-preview", - ExactExpected(spend=0.066248, input_cost=0.003224, output_cost=0.063024, prompt_tokens=1240, completion_tokens=4040) + ExactExpected(spend=0.066248, input_cost=0.003224, output_cost=0.063024, prompt_tokens=1240, completion_tokens=4040), + usage_fields={"serviceTier": "standard"}, ) GEMINI_GEMINI_3_1_PRO_PREVIEW_TOOL_CALL: Final = _gemini_case( GEMINI_GEMINI_3_8_FLASH_TOOL_CALL, "gemini-gemini-3.1-pro-preview-tool_call", "gemini/gemini-3.1-pro-preview", "gemini-3.1-pro-preview", ExactExpected(spend=0.0112112, input_cost=0.004784, output_cost=0.0064272, prompt_tokens=1840, completion_tokens=412) ) +GEMINI_GEMINI_3_1_PRO_PREVIEW_TIERED_INPUT_ABOVE_200K: Final = _gemini_case( + GEMINI_GEMINI_3_1_PRO_TIERED_INPUT_ABOVE_200K, + "gemini-gemini-3.1-pro-preview-tiered_input_above_200k", + "gemini/gemini-3.1-pro-preview", + "gemini-3.1-pro-preview", + ExactExpected( + spend=1.079468, input_cost=1.06496, output_cost=0.014508, prompt_tokens=204800, completion_tokens=620 + ), + usage_fields={"serviceTier": "standard"}, +) +GEMINI_GEMINI_3_1_PRO_PREVIEW_TIERED_CACHE_READ_ABOVE_200K: Final = _gemini_case( + GEMINI_GEMINI_3_1_PRO_TIERED_CACHE_READ_ABOVE_200K, + "gemini-gemini-3.1-pro-preview-tiered_cache_read_above_200k", + "gemini/gemini-3.1-pro-preview", + "gemini-3.1-pro-preview", + ExactExpected( + spend=0.13742976, input_cost=0.12619776, output_cost=0.011232, prompt_tokens=205824, completion_tokens=480 + ), + usage_fields={"serviceTier": "standard"}, +) +GEMINI_GEMINI_4_ARGON_ROUTE_BRIDGES: Final = bridge_pairs( + (GEMINI_GEMINI_4_ARGON_INPUT_TEXT, GEMINI_GEMINI_4_ARGON_STREAM), + ("/v1/responses", "/v1/messages"), +) +GEMINI_GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES: Final = bridge_pairs( + (GEMINI_GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT, GEMINI_GEMINI_3_1_PRO_PREVIEW_STREAM), + ("/v1/responses", "/v1/messages"), +) CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GEMINI_GEMINI_3_1_PRO_INPUT_TEXT, @@ -3379,12 +3492,19 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GEMINI_GEMINI_4_ARGON_STREAM, GEMINI_GEMINI_4_ARGON_CACHE_READ, GEMINI_GEMINI_4_ARGON_REASONING, + GEMINI_GEMINI_4_ARGON_SERVICE_TIER_FLEX, + GEMINI_GEMINI_4_ARGON_SERVICE_TIER_PRIORITY, GEMINI_GEMINI_4_ARGON_TOOL_CALL, GEMINI_GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT, GEMINI_GEMINI_3_1_PRO_PREVIEW_STREAM, GEMINI_GEMINI_3_1_PRO_PREVIEW_CACHE_READ, + GEMINI_GEMINI_3_1_PRO_PREVIEW_TIERED_INPUT_ABOVE_200K, + GEMINI_GEMINI_3_1_PRO_PREVIEW_TIERED_CACHE_READ_ABOVE_200K, GEMINI_GEMINI_3_1_PRO_PREVIEW_REASONING, GEMINI_GEMINI_3_1_PRO_PREVIEW_TOOL_CALL, + GEMINI_GEMINI_3_1_PRO_PREVIEW_SERVICE_TIER_PRIORITY, + *(pair.bridged for pair in GEMINI_GEMINI_4_ARGON_ROUTE_BRIDGES), + *(pair.bridged for pair in GEMINI_GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES), ) PARITY: Final[tuple[StreamParityTestCase, ...]] = ( @@ -3410,6 +3530,22 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = ( StreamParityTestCase( plain=GEMINI_GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT, streamed=GEMINI_GEMINI_3_1_PRO_PREVIEW_STREAM ), + StreamParityTestCase( + plain=GEMINI_GEMINI_4_ARGON_ROUTE_BRIDGES[0].bridged, + streamed=GEMINI_GEMINI_4_ARGON_ROUTE_BRIDGES[2].bridged, + ), + StreamParityTestCase( + plain=GEMINI_GEMINI_4_ARGON_ROUTE_BRIDGES[1].bridged, + streamed=GEMINI_GEMINI_4_ARGON_ROUTE_BRIDGES[3].bridged, + ), + StreamParityTestCase( + plain=GEMINI_GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES[0].bridged, + streamed=GEMINI_GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES[2].bridged, + ), + StreamParityTestCase( + plain=GEMINI_GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES[1].bridged, + streamed=GEMINI_GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES[3].bridged, + ), ) BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = ( @@ -3422,6 +3558,6 @@ BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = ( ), ("/v1/responses", "/v1/messages"), ), - *bridge_pairs((GEMINI_GEMINI_4_ARGON_INPUT_TEXT,), ("/v1/responses",)), - *bridge_pairs((GEMINI_GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT,), ("/v1/responses",)), + *GEMINI_GEMINI_4_ARGON_ROUTE_BRIDGES, + *GEMINI_GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES, ) diff --git a/tests/integration/cost_calculation/cases/misc.py b/tests/integration/cost_calculation/cases/misc.py index 1a8b2fcb66b..760f77f91cc 100644 --- a/tests/integration/cost_calculation/cases/misc.py +++ b/tests/integration/cost_calculation/cases/misc.py @@ -19,13 +19,17 @@ from integration.cost_calculation.cost_tracking_case import ( RecountRates, SseResponse, WavUpload, + with_usage, ) from integration.cost_calculation.stream_parity.case import StreamParityTestCase, sse_frames +from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs from integration.cost_calculation.cases.anthropic import ( CLAUDE_FABLE_5_1_CACHE_READ, CLAUDE_FABLE_5_1_CACHE_WRITE_1H, CLAUDE_FABLE_5_1_CACHE_WRITE_5M, CLAUDE_FABLE_5_1_INPUT_TEXT, + CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT, + CLAUDE_FABLE_5_1_MESSAGES_STREAM, CLAUDE_FABLE_5_1_STREAM, CLAUDE_FABLE_5_1_THINKING, CLAUDE_FABLE_5_1_TOOL_CALL, @@ -33,12 +37,30 @@ from integration.cost_calculation.cases.anthropic import ( CLAUDE_HAIKU_5_5_CACHE_WRITE_1H, CLAUDE_HAIKU_5_5_CACHE_WRITE_5M, CLAUDE_HAIKU_5_5_INPUT_TEXT, + CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT, + CLAUDE_HAIKU_5_5_MESSAGES_STREAM, CLAUDE_HAIKU_5_5_STREAM, CLAUDE_HAIKU_5_5_THINKING, CLAUDE_HAIKU_5_5_TOOL_CALL, ) +def _azure_ai_claude_messages_case( + response_case: CostTrackingTestCase, + request_case: CostTrackingTestCase, + name: str, + model: str, +) -> CostTrackingTestCase: + return response_case.model_copy( + update={ + "name": name, + "model": model, + "endpoint": "/v1/messages", + "request": request_case.request, + } + ) + + def _azure_ai_claude_usage( input_tokens: int, output_tokens: int, @@ -2027,7 +2049,66 @@ AZURE_AI_CLAUDE_HAIKU_5_5_TOOL_CALL: Final = _azure_ai_claude_shape_case( ), ) - +AZURE_AI_CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT: Final = _azure_ai_claude_messages_case( + AZURE_AI_CLAUDE_FABLE_5_1_INPUT_TEXT, + CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT, + "azure_ai/claude-fable-5-1-messages_input_text", + "azure_ai/claude-fable-5-1", +) +AZURE_AI_CLAUDE_FABLE_5_1_MESSAGES_STREAM: Final = _azure_ai_claude_messages_case( + AZURE_AI_CLAUDE_FABLE_5_1_STREAM, + CLAUDE_FABLE_5_1_MESSAGES_STREAM, + "azure_ai/claude-fable-5-1-messages_stream", + "azure_ai/claude-fable-5-1", +) +AZURE_AI_CLAUDE_FABLE_5_1_ROUTE_BRIDGES: Final = bridge_pairs( + (AZURE_AI_CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT, AZURE_AI_CLAUDE_FABLE_5_1_MESSAGES_STREAM), + ("/v1/responses",), +) +AZURE_AI_CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT: Final = _azure_ai_claude_messages_case( + AZURE_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT, + CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT, + "azure_ai/claude-haiku-5-5-messages_input_text", + "azure_ai/claude-haiku-5-5", +) +AZURE_AI_CLAUDE_HAIKU_5_5_MESSAGES_STREAM: Final = _azure_ai_claude_messages_case( + AZURE_AI_CLAUDE_HAIKU_5_5_STREAM, + CLAUDE_HAIKU_5_5_MESSAGES_STREAM, + "azure_ai/claude-haiku-5-5-messages_stream", + "azure_ai/claude-haiku-5-5", +) +AZURE_AI_CLAUDE_HAIKU_5_5_ROUTE_BRIDGES: Final = bridge_pairs( + (AZURE_AI_CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT, AZURE_AI_CLAUDE_HAIKU_5_5_MESSAGES_STREAM), + ("/v1/responses",), +) +AZURE_AI_CLAUDE_HAIKU_5_5_TIERED_INPUT_ABOVE_100K: Final = with_usage( + AZURE_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT, + "azure_ai/claude-haiku-5-5-tiered_input_above_100k", + { + "input_tokens": 150000, + "output_tokens": 480, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + }, + ExactExpected( + spend=0.14478, input_cost=0.1425, output_cost=0.00228, prompt_tokens=150000, completion_tokens=480 + ), +) +AZURE_AI_CLAUDE_HAIKU_5_5_TIERED_CACHE_READ_ABOVE_100K: Final = with_usage( + AZURE_AI_CLAUDE_HAIKU_5_5_CACHE_READ, + "azure_ai/claude-haiku-5-5-tiered_cache_read_above_100k", + { + "input_tokens": 4096, + "output_tokens": 480, + "cache_read_input_tokens": 110592, + "cache_creation_input_tokens": 0, + "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0}, + }, + ExactExpected( + spend=0.01667744, input_cost=0.01439744, output_cost=0.00228, prompt_tokens=114688, completion_tokens=480 + ), +) CASES: Final[tuple[CostTrackingTestCase, ...]] = ( PERPLEXITY_SONAR_NEXT_OFF_PEAK, NOVA_NEXT_TRANSCRIPTIONS_PER_SECOND, @@ -2088,13 +2169,21 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( AZURE_AI_CLAUDE_FABLE_5_1_CACHE_WRITE_1H, AZURE_AI_CLAUDE_FABLE_5_1_THINKING, AZURE_AI_CLAUDE_FABLE_5_1_TOOL_CALL, + AZURE_AI_CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT, + AZURE_AI_CLAUDE_FABLE_5_1_MESSAGES_STREAM, + *(pair.bridged for pair in AZURE_AI_CLAUDE_FABLE_5_1_ROUTE_BRIDGES), AZURE_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT, AZURE_AI_CLAUDE_HAIKU_5_5_STREAM, AZURE_AI_CLAUDE_HAIKU_5_5_CACHE_READ, + AZURE_AI_CLAUDE_HAIKU_5_5_TIERED_INPUT_ABOVE_100K, + AZURE_AI_CLAUDE_HAIKU_5_5_TIERED_CACHE_READ_ABOVE_100K, AZURE_AI_CLAUDE_HAIKU_5_5_CACHE_WRITE_5M, AZURE_AI_CLAUDE_HAIKU_5_5_CACHE_WRITE_1H, AZURE_AI_CLAUDE_HAIKU_5_5_THINKING, AZURE_AI_CLAUDE_HAIKU_5_5_TOOL_CALL, + AZURE_AI_CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT, + AZURE_AI_CLAUDE_HAIKU_5_5_MESSAGES_STREAM, + *(pair.bridged for pair in AZURE_AI_CLAUDE_HAIKU_5_5_ROUTE_BRIDGES), ) PARITY: Final[tuple[StreamParityTestCase, ...]] = ( @@ -2105,4 +2194,25 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = ( StreamParityTestCase( plain=AZURE_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT, streamed=AZURE_AI_CLAUDE_HAIKU_5_5_STREAM ), + StreamParityTestCase( + plain=AZURE_AI_CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT, + streamed=AZURE_AI_CLAUDE_FABLE_5_1_MESSAGES_STREAM, + ), + StreamParityTestCase( + plain=AZURE_AI_CLAUDE_FABLE_5_1_ROUTE_BRIDGES[0].bridged, + streamed=AZURE_AI_CLAUDE_FABLE_5_1_ROUTE_BRIDGES[1].bridged, + ), + StreamParityTestCase( + plain=AZURE_AI_CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT, + streamed=AZURE_AI_CLAUDE_HAIKU_5_5_MESSAGES_STREAM, + ), + StreamParityTestCase( + plain=AZURE_AI_CLAUDE_HAIKU_5_5_ROUTE_BRIDGES[0].bridged, + streamed=AZURE_AI_CLAUDE_HAIKU_5_5_ROUTE_BRIDGES[1].bridged, + ), +) + +BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = ( + *AZURE_AI_CLAUDE_FABLE_5_1_ROUTE_BRIDGES, + *AZURE_AI_CLAUDE_HAIKU_5_5_ROUTE_BRIDGES, ) diff --git a/tests/integration/cost_calculation/cases/openai.py b/tests/integration/cost_calculation/cases/openai.py index 85ac05dd69d..f610fd9591a 100644 --- a/tests/integration/cost_calculation/cases/openai.py +++ b/tests/integration/cost_calculation/cases/openai.py @@ -34,6 +34,256 @@ def _stream_for_model(response: SseResponse, model: str) -> SseResponse: return response.model_copy(update={"frames": tuple(frame.replace("gpt-6-luna", model) for frame in response.frames)}) +def _responses_json_body( + model: str, + *, + include_content_filters: bool = False, + content_filters: JsonValue = None, + include_reasoning: bool = True, +) -> dict[str, JsonValue]: + reasoning_item: Final[dict[str, JsonValue]] = { + "content": [], + "encrypted_content": "$ENCRYPTED_CONTENT", + "id": "rs_$REQUEST_ID", + "summary": [], + "type": "reasoning", + } + message_item: Final[dict[str, JsonValue]] = { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "scripted response", + "type": "output_text", + } + ], + "id": "msg_$REQUEST_ID", + "phase": "final_answer", + "role": "assistant", + "status": "completed", + "type": "message", + } + output: Final = [reasoning_item, message_item] if include_reasoning else [message_item] + body: Final[dict[str, JsonValue]] = { + "access_programs": [], + "background": False, + "completed_at": 1791855453, + "created_at": 1791855452, + "error": None, + "frequency_penalty": None, + "id": "resp_$REQUEST_ID", + "incomplete_details": None, + "instructions": None, + "max_output_tokens": None, + "max_tool_calls": None, + "metadata": {}, + "moderation": None, + "model": model, + "object": "response", + "output": output, + "parallel_tool_calls": True, + "presence_penalty": None, + "previous_response_id": None, + "prompt_cache_key": None, + "prompt_cache_retention": None, + "reasoning": {"effort": "medium"}, + "safety_identifier": None, + "service_tier": "default", + "status": "completed", + "store": True, + "temperature": 1.0, + "text": None, + "tool_choice": "auto", + "tool_usage": None, + "tools": [], + "top_logprobs": None, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 1840, + "input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0}, + "output_tokens": 412, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 2252, + }, + "user": None, + } + return {**body, "content_filters": content_filters} if include_content_filters else body + + +def _responses_stream_response( + model: str, + delta_count: int, + *, + include_content_filters: bool = False, + content_filters: JsonValue = None, + include_reasoning: bool = True, +) -> SseResponse: + response_body: Final = _responses_json_body( + model, + include_content_filters=include_content_filters, + content_filters=content_filters, + include_reasoning=include_reasoning, + ) + in_progress_body: Final = { + **response_body, + "output": [], + "status": "in_progress", + "usage": None, + } + reasoning_item: Final = { + "content": [], + "encrypted_content": "$ENCRYPTED_CONTENT", + "id": "rs_$REQUEST_ID", + "summary": [], + "type": "reasoning", + } + message_item: Final = { + "content": [], + "id": "msg_$REQUEST_ID", + "role": "assistant", + "status": "in_progress", + "type": "message", + } + delta_events: Final = tuple( + { + "content_index": 0, + "delta": "scripted response" if index == 0 else "", + "item_id": "msg_$REQUEST_ID", + "logprobs": [], + "obfuscation": None, + "output_index": 1 if include_reasoning else 0, + "sequence_number": (6 if include_reasoning else 4) + index, + "type": "response.output_text.delta", + } + for index in range(delta_count) + ) + delta_start: Final = 6 if include_reasoning else 4 + message_output_index: Final = 1 if include_reasoning else 0 + reasoning_events: Final = ( + ( + { + "item": reasoning_item, + "output_index": 0, + "sequence_number": 2, + "type": "response.output_item.added", + }, + { + "item": reasoning_item, + "output_index": 0, + "sequence_number": 3, + "type": "response.output_item.done", + }, + ) + if include_reasoning + else () + ) + completed_message_item: Final = { + "content": [ + { + "annotations": [], + "logprobs": [], + "text": "scripted response", + "type": "output_text", + } + ], + "id": "msg_$REQUEST_ID", + "phase": "final_answer", + "role": "assistant", + "status": "completed", + "type": "message", + } + return SseResponse( + content_type="text/event-stream", + frames=sse_frames( + { + "response": in_progress_body, + "sequence_number": 0, + "type": "response.created", + }, + { + "response": in_progress_body, + "sequence_number": 1, + "type": "response.in_progress", + }, + *reasoning_events, + { + "item": message_item, + "output_index": message_output_index, + "sequence_number": 4 if include_reasoning else 2, + "type": "response.output_item.added", + }, + { + "content_index": 0, + "item_id": "msg_$REQUEST_ID", + "output_index": message_output_index, + "part": {"annotations": [], "logprobs": [], "text": "", "type": "output_text"}, + "sequence_number": 5 if include_reasoning else 3, + "type": "response.content_part.added", + }, + *delta_events, + { + "content_index": 0, + "item_id": "msg_$REQUEST_ID", + "logprobs": [], + "output_index": message_output_index, + "sequence_number": delta_start + delta_count, + "text": "scripted response", + "type": "response.output_text.done", + }, + { + "content_index": 0, + "item_id": "msg_$REQUEST_ID", + "output_index": message_output_index, + "part": { + "annotations": [], + "logprobs": [], + "text": "scripted response", + "type": "output_text", + }, + "sequence_number": delta_start + delta_count + 1, + "type": "response.content_part.done", + }, + { + "item": completed_message_item, + "output_index": message_output_index, + "sequence_number": delta_start + delta_count + 2, + "type": "response.output_item.done", + }, + { + "response": response_body, + "sequence_number": delta_start + delta_count + 3, + "type": "response.completed", + }, + ), + ) + + +def _responses_stream_case( + base: CostTrackingTestCase, + name: str, + response_model: str, + delta_count: int, + *, + include_content_filters: bool = False, + content_filters: JsonValue = None, + include_reasoning: bool = True, +) -> CostTrackingTestCase: + return base.model_copy( + update={ + "name": name, + "request": {**base.request, "stream": True}, + "response": _responses_stream_response( + response_model, + delta_count, + include_content_filters=include_content_filters, + content_filters=content_filters, + include_reasoning=include_reasoning, + ), + } + ) + + # gpt-5.3-codex GPT_5_3_CODEX_INPUT_TEXT: Final = CostTrackingTestCase( name="gpt-5.3-codex-input_text", @@ -11356,6 +11606,31 @@ GPT_5_6_TERRA_TOOL_CALL: Final = GPT_6_LUNA_TOOL_CALL.model_copy( } ) +GPT_5_6_TERRA_TIERED_INPUT_ABOVE_272K: Final = with_usage( + GPT_5_6_TERRA_INPUT_TEXT, + "gpt-5.6-terra-tiered_input_above_272k", + { + "prompt_tokens": 300000, + "total_tokens": 300412, + "prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 0}, + }, + ExactExpected( + spend=1.5696408, input_cost=1.56, output_cost=0.0096408, prompt_tokens=300000, completion_tokens=412 + ), +) +GPT_5_6_TERRA_TIERED_CACHE_READ_ABOVE_272K: Final = with_usage( + GPT_5_6_TERRA_INPUT_TEXT, + "gpt-5.6-terra-tiered_cache_read_above_272k", + { + "prompt_tokens": 300000, + "total_tokens": 300412, + "prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 200000}, + }, + ExactExpected( + spend=0.6336408, input_cost=0.624, output_cost=0.0096408, prompt_tokens=300000, completion_tokens=412 + ), +) + GPT_5_6_SLASH_ALIAS: Final = GPT_5_6_INPUT_TEXT.model_copy( update={"name": "gpt-5.6-slash_alias", "alias": "openai/custom/gpt-5.6-$MARKER"} @@ -11446,6 +11721,63 @@ GPT_5_6_RESPONSES_IMAGE_GENERATION_TOOL_TOKENS: Final = CostTrackingTestCase( ), ) +GPT_6_1_SOL_RESPONSES_NATIVE_STREAM: Final = _responses_stream_case( + GPT_6_1_SOL_RESPONSES_NATIVE_JSON, + "gpt-6.1-sol-responses_native_stream", + "gpt-6.1-sol", + 1, + include_reasoning=False, +) +GPT_6_LUNA_RESPONSES_NATIVE_STREAM: Final = _responses_stream_case( + GPT_6_LUNA_RESPONSES_NATIVE_JSON, + "gpt-6-luna-responses_native_stream", + "gpt-6-luna", + 45, +) +GPT_6_ASTRA_RESPONSES_NATIVE_STREAM: Final = _responses_stream_case( + GPT_6_ASTRA_RESPONSES_NATIVE_JSON, + "gpt-6-astra-responses_native_stream", + "gpt-6-astra", + 36, +) +GPT_5_6_TERRA_RESPONSES_NATIVE_STREAM: Final = _responses_stream_case( + GPT_5_6_TERRA_RESPONSES_NATIVE_JSON, + "gpt-5.6-terra-responses_native_stream", + "gpt-5.6-terra", + 95, +) +GPT_5_6_TERRA_MESSAGES_BRIDGED_JSON: Final = bridge_pairs( + (GPT_5_6_TERRA_RESPONSES_NATIVE_JSON,), ("/v1/messages",) +)[0].bridged.model_copy(update={"name": "gpt-5.6-terra-messages_bridged_json"}) +GPT_6_1_SOL_MESSAGES_BRIDGED_STREAM: Final = GPT_6_1_SOL_MESSAGES_BRIDGED_JSON.model_copy( + update={ + "name": "gpt-6.1-sol-messages_bridged_stream", + "request": {**GPT_6_1_SOL_MESSAGES_BRIDGED_JSON.request, "stream": True}, + "response": GPT_6_1_SOL_RESPONSES_NATIVE_STREAM.response, + } +) +GPT_6_LUNA_MESSAGES_BRIDGED_STREAM: Final = GPT_6_LUNA_MESSAGES_BRIDGED_JSON.model_copy( + update={ + "name": "gpt-6-luna-messages_bridged_stream", + "request": {**GPT_6_LUNA_MESSAGES_BRIDGED_JSON.request, "stream": True}, + "response": GPT_6_LUNA_RESPONSES_NATIVE_STREAM.response, + } +) +GPT_6_ASTRA_MESSAGES_BRIDGED_STREAM: Final = GPT_6_ASTRA_MESSAGES_BRIDGED_JSON.model_copy( + update={ + "name": "gpt-6-astra-messages_bridged_stream", + "request": {**GPT_6_ASTRA_MESSAGES_BRIDGED_JSON.request, "stream": True}, + "response": GPT_6_ASTRA_RESPONSES_NATIVE_STREAM.response, + } +) +GPT_5_6_TERRA_MESSAGES_BRIDGED_STREAM: Final = GPT_5_6_TERRA_MESSAGES_BRIDGED_JSON.model_copy( + update={ + "name": "gpt-5.6-terra-messages_bridged_stream", + "request": {**GPT_5_6_TERRA_MESSAGES_BRIDGED_JSON.request, "stream": True}, + "response": GPT_5_6_TERRA_RESPONSES_NATIVE_STREAM.response, + } +) + CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GPT_5_6_SLASH_ALIAS, GPT_5_6_SLASH_ALIAS_STREAM, @@ -11659,6 +11991,8 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GPT_6_LUNA_CACHE_WRITE_5M, GPT_6_ASTRA_CACHE_WRITE_5M, GPT_5_6_TERRA_INPUT_TEXT, + GPT_5_6_TERRA_TIERED_INPUT_ABOVE_272K, + GPT_5_6_TERRA_TIERED_CACHE_READ_ABOVE_272K, GPT_5_6_TERRA_STREAM, GPT_5_6_TERRA_RESPONSES_NATIVE_JSON, GPT_5_6_TERRA_RESPONSES_DEPLOYMENT_JSON, @@ -11669,6 +12003,15 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GPT_5_6_TERRA_SERVICE_TIER_FLEX, GPT_5_6_TERRA_SERVICE_TIER_PRIORITY, GPT_5_6_TERRA_TOOL_CALL, + GPT_6_1_SOL_RESPONSES_NATIVE_STREAM, + GPT_6_LUNA_RESPONSES_NATIVE_STREAM, + GPT_6_ASTRA_RESPONSES_NATIVE_STREAM, + GPT_5_6_TERRA_RESPONSES_NATIVE_STREAM, + GPT_5_6_TERRA_MESSAGES_BRIDGED_JSON, + GPT_6_1_SOL_MESSAGES_BRIDGED_STREAM, + GPT_6_LUNA_MESSAGES_BRIDGED_STREAM, + GPT_6_ASTRA_MESSAGES_BRIDGED_STREAM, + GPT_5_6_TERRA_MESSAGES_BRIDGED_STREAM, ) PARITY: Final[tuple[StreamParityTestCase, ...]] = ( @@ -11698,6 +12041,16 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = ( StreamParityTestCase(plain=GPT_6_ASTRA_INPUT_TEXT, streamed=GPT_6_ASTRA_STREAM), StreamParityTestCase(plain=GPT_6_1_SOL_INPUT_TEXT, streamed=GPT_6_1_SOL_STREAM), StreamParityTestCase(plain=GPT_5_6_TERRA_INPUT_TEXT, streamed=GPT_5_6_TERRA_STREAM), + StreamParityTestCase(plain=GPT_6_1_SOL_RESPONSES_NATIVE_JSON, streamed=GPT_6_1_SOL_RESPONSES_NATIVE_STREAM), + StreamParityTestCase(plain=GPT_6_LUNA_RESPONSES_NATIVE_JSON, streamed=GPT_6_LUNA_RESPONSES_NATIVE_STREAM), + StreamParityTestCase(plain=GPT_6_ASTRA_RESPONSES_NATIVE_JSON, streamed=GPT_6_ASTRA_RESPONSES_NATIVE_STREAM), + StreamParityTestCase(plain=GPT_5_6_TERRA_RESPONSES_NATIVE_JSON, streamed=GPT_5_6_TERRA_RESPONSES_NATIVE_STREAM), + StreamParityTestCase(plain=GPT_6_1_SOL_MESSAGES_BRIDGED_JSON, streamed=GPT_6_1_SOL_MESSAGES_BRIDGED_STREAM), + StreamParityTestCase(plain=GPT_6_LUNA_MESSAGES_BRIDGED_JSON, streamed=GPT_6_LUNA_MESSAGES_BRIDGED_STREAM), + StreamParityTestCase(plain=GPT_6_ASTRA_MESSAGES_BRIDGED_JSON, streamed=GPT_6_ASTRA_MESSAGES_BRIDGED_STREAM), + StreamParityTestCase( + plain=GPT_5_6_TERRA_MESSAGES_BRIDGED_JSON, streamed=GPT_5_6_TERRA_MESSAGES_BRIDGED_STREAM + ), ) BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = ( @@ -11726,6 +12079,13 @@ BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = ( BridgeCostParityTestCase(native=GPT_6_LUNA_RESPONSES_NATIVE_JSON, bridged=GPT_6_LUNA_MESSAGES_BRIDGED_JSON), BridgeCostParityTestCase(native=GPT_6_ASTRA_RESPONSES_NATIVE_JSON, bridged=GPT_6_ASTRA_MESSAGES_BRIDGED_JSON), BridgeCostParityTestCase(native=GPT_6_1_SOL_RESPONSES_NATIVE_JSON, bridged=GPT_6_1_SOL_MESSAGES_BRIDGED_JSON), + BridgeCostParityTestCase(native=GPT_5_6_TERRA_RESPONSES_NATIVE_JSON, bridged=GPT_5_6_TERRA_MESSAGES_BRIDGED_JSON), + BridgeCostParityTestCase(native=GPT_6_1_SOL_RESPONSES_NATIVE_STREAM, bridged=GPT_6_1_SOL_MESSAGES_BRIDGED_STREAM), + BridgeCostParityTestCase(native=GPT_6_LUNA_RESPONSES_NATIVE_STREAM, bridged=GPT_6_LUNA_MESSAGES_BRIDGED_STREAM), + BridgeCostParityTestCase(native=GPT_6_ASTRA_RESPONSES_NATIVE_STREAM, bridged=GPT_6_ASTRA_MESSAGES_BRIDGED_STREAM), + BridgeCostParityTestCase( + native=GPT_5_6_TERRA_RESPONSES_NATIVE_STREAM, bridged=GPT_5_6_TERRA_MESSAGES_BRIDGED_STREAM + ), BridgeCostParityTestCase(native=GPT_6_SOL_RESPONSES_DEPLOYMENT_JSON, bridged=GPT_6_SOL_CHAT_BRIDGED_JSON), BridgeCostParityTestCase(native=GPT_6_LUNA_RESPONSES_DEPLOYMENT_JSON, bridged=GPT_6_LUNA_CHAT_BRIDGED_JSON), BridgeCostParityTestCase(native=GPT_6_ASTRA_RESPONSES_DEPLOYMENT_JSON, bridged=GPT_6_ASTRA_CHAT_BRIDGED_JSON), diff --git a/tests/integration/cost_calculation/cases/vertex_ai.py b/tests/integration/cost_calculation/cases/vertex_ai.py index 761078ef593..0e95eb865bf 100644 --- a/tests/integration/cost_calculation/cases/vertex_ai.py +++ b/tests/integration/cost_calculation/cases/vertex_ai.py @@ -2,6 +2,7 @@ PARITY pairs the streamed case with its plain twin where both bill the same row.""" +import json from typing import Final, cast from pydantic import JsonValue @@ -11,13 +12,19 @@ from integration.cost_calculation.cases.anthropic import ( CLAUDE_FABLE_5_1_CACHE_WRITE_1H, CLAUDE_FABLE_5_1_CACHE_WRITE_5M, CLAUDE_FABLE_5_1_INPUT_TEXT, + CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT, + CLAUDE_FABLE_5_1_MESSAGES_STREAM, CLAUDE_FABLE_5_1_STREAM, CLAUDE_FABLE_5_1_THINKING, CLAUDE_FABLE_5_1_TOOL_CALL, + CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT, + CLAUDE_HAIKU_5_5_MESSAGES_STREAM, CLAUDE_OPUS_5_5_CACHE_READ, CLAUDE_OPUS_5_5_CACHE_WRITE_1H, CLAUDE_OPUS_5_5_CACHE_WRITE_5M, CLAUDE_OPUS_5_5_INPUT_TEXT, + CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT, + CLAUDE_OPUS_5_5_MESSAGES_STREAM, CLAUDE_OPUS_5_5_STREAM, CLAUDE_OPUS_5_5_THINKING, CLAUDE_OPUS_5_5_TOOL_CALL, @@ -25,6 +32,8 @@ from integration.cost_calculation.cases.anthropic import ( CLAUDE_SONNET_5_5_CACHE_WRITE_1H, CLAUDE_SONNET_5_5_CACHE_WRITE_5M, CLAUDE_SONNET_5_5_INPUT_TEXT, + CLAUDE_SONNET_5_5_MESSAGES_INPUT_TEXT, + CLAUDE_SONNET_5_5_MESSAGES_STREAM, CLAUDE_SONNET_5_5_STREAM, CLAUDE_SONNET_5_5_THINKING, CLAUDE_SONNET_5_5_TOOL_CALL, @@ -41,26 +50,72 @@ from integration.cost_calculation.cost_tracking_case import ( from integration.cost_calculation.stream_parity.case import StreamParityTestCase, sse_frames -def _gemini_case_response(response: JsonResponse | SseResponse, model_version: str) -> JsonResponse | SseResponse: +def _gemini_case_response( + response: JsonResponse | SseResponse, + model_version: str, + usage_fields: dict[str, JsonValue] | None = None, +) -> JsonResponse | SseResponse: match response: case JsonResponse(): - return response.model_copy(update={"body": {**response.body, "modelVersion": model_version}}) + usage: Final = response.body.get("usageMetadata") + body: Final = {**response.body, "modelVersion": model_version} + return response.model_copy( + update={ + "body": { + **body, + **( + { + "usageMetadata": { + **cast(dict[str, JsonValue], usage), + **usage_fields, + } + } + if usage_fields is not None and isinstance(usage, dict) + else {} + ), + } + } + ) case SseResponse(): return response.model_copy( update={ - "frames": tuple(frame.replace("gemini-3.8-flash", model_version) for frame in response.frames) + "frames": tuple( + _gemini_case_frame(frame, model_version, usage_fields) for frame in response.frames + ) } ) +def _gemini_case_frame(frame: str, model_version: str, usage_fields: dict[str, JsonValue] | None) -> str: + replaced: Final = frame.replace("gemini-3.8-flash", model_version) + if usage_fields is None: + return replaced + prefix, separator, payload = replaced.partition("data: ") + if not separator or payload == "[DONE]": + return replaced + event: Final = cast(dict[str, JsonValue], json.loads(payload)) + usage: Final = event.get("usageMetadata") + if not isinstance(usage, dict): + return replaced + return f"{prefix}data: {json.dumps({**event, 'usageMetadata': {**usage, **usage_fields}})}" + + def _gemini_case( - sibling: CostTrackingTestCase, name: str, model: str, expected: ExactExpected + sibling: CostTrackingTestCase, + name: str, + model: str, + expected: ExactExpected, + *, + usage_fields: dict[str, JsonValue] | None = None, + model_version: str = "gemini-3.1-pro-preview", ) -> CostTrackingTestCase: return sibling.model_copy( update={ "name": name, "model": model, - "response": _gemini_case_response(cast(JsonResponse | SseResponse, sibling.response), "gemini-3.1-pro-preview"), + "response": _gemini_case_response( + cast(JsonResponse | SseResponse, sibling.response), model_version, usage_fields + ), "expected": expected, } ) @@ -97,6 +152,7 @@ def _vertex_claude_shape_case( expected: ExactExpected, response_content: list[dict[str, JsonValue]] | None = None, adaptive_thinking: bool = False, + thinking_effort: str = "medium", ) -> CostTrackingTestCase: response: Final = cast(JsonResponse, sibling.response) request: Final = ( @@ -107,7 +163,7 @@ def _vertex_claude_shape_case( if key not in {"reasoning_effort", "allowed_openai_params"} }, "thinking": {"type": "adaptive"}, - "output_config": {"effort": "medium"}, + "output_config": {"effort": thinking_effort}, "allowed_openai_params": ["thinking", "output_config"], } if adaptive_thinking @@ -1716,6 +1772,8 @@ GEMINI_3_8_FLASH_INPUT_TEXT: Final = CostTrackingTestCase( "candidatesTokenCount": 412, "totalTokenCount": 2252, "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 1840}], + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 412}], + "trafficType": "ON_DEMAND", }, "modelVersion": "gemini-3.8-flash", }, @@ -1770,6 +1828,8 @@ GEMINI_3_8_FLASH_CACHE_READ: Final = CostTrackingTestCase( "totalTokenCount": 13308, "cachedContentTokenCount": 12288, "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 12928}], + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 380}], + "trafficType": "ON_DEMAND", }, "modelVersion": "gemini-3.8-flash", }, @@ -2117,6 +2177,8 @@ GEMINI_3_8_FLASH_STREAM: Final = CostTrackingTestCase( "candidatesTokenCount": 412, "totalTokenCount": 2252, "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 1840}], + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 412}], + "trafficType": "ON_DEMAND", }, "modelVersion": "gemini-3.8-flash", }, @@ -3122,28 +3184,258 @@ VERTEX_AI_CLAUDE_FABLE_5_1_TOOL_CALL: Final = _vertex_claude_shape_case( ), ) +VERTEX_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT: Final = _vertex_claude_shape_case( + VERTEX_AI_CLAUDE_SONNET_5_5_INPUT_TEXT, + "vertex_ai/claude-haiku-5-5-input_text", + "vertex_ai/claude-haiku-5-5", + _vertex_claude_usage(input_tokens=1840, output_tokens=412), + ExactExpected(spend=0.00507, input_cost=0.002392, output_cost=0.002678, prompt_tokens=1840, completion_tokens=412), +) +VERTEX_AI_CLAUDE_HAIKU_5_5_STREAM: Final = VERTEX_AI_CLAUDE_SONNET_5_5_STREAM.model_copy( + update={ + "name": "vertex_ai/claude-haiku-5-5-stream", + "model": "vertex_ai/claude-haiku-5-5", + "response": _vertex_claude_stream_response("claude-haiku-5-5", "end_turn", ("scripted answer",)), + "expected": VERTEX_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT.expected, + } +) +VERTEX_AI_CLAUDE_HAIKU_5_5_CACHE_READ: Final = _vertex_claude_shape_case( + VERTEX_AI_CLAUDE_SONNET_5_5_CACHE_READ, + "vertex_ai/claude-haiku-5-5-cache_read", + "vertex_ai/claude-haiku-5-5", + _vertex_claude_usage(input_tokens=14, output_tokens=6, cache_read_input_tokens=27003), + ExactExpected( + spend=0.00356759, + input_cost=0.00352859, + output_cost=0.000039, + prompt_tokens=27017, + completion_tokens=6, + cache_read_cost=0.00351039, + ), +) +VERTEX_AI_CLAUDE_HAIKU_5_5_CACHE_WRITE_5M: Final = _vertex_claude_shape_case( + VERTEX_AI_CLAUDE_SONNET_5_5_CACHE_WRITE_5M, + "vertex_ai/claude-haiku-5-5-cache_write_5m", + "vertex_ai/claude-haiku-5-5", + _vertex_claude_usage( + input_tokens=14, output_tokens=6, cache_creation_input_tokens=27011, ephemeral_5m_input_tokens=27011 + ), + ExactExpected( + spend=0.043950075, + input_cost=0.043911075, + output_cost=0.000039, + prompt_tokens=27025, + completion_tokens=6, + cache_creation_cost=0.043892875, + ), +) +VERTEX_AI_CLAUDE_HAIKU_5_5_CACHE_WRITE_1H: Final = _vertex_claude_shape_case( + VERTEX_AI_CLAUDE_SONNET_5_5_CACHE_WRITE_1H, + "vertex_ai/claude-haiku-5-5-cache_write_1h", + "vertex_ai/claude-haiku-5-5", + _vertex_claude_usage( + input_tokens=14, output_tokens=6, cache_creation_input_tokens=27011, ephemeral_1h_input_tokens=27011 + ), + ExactExpected( + spend=0.0702858, + input_cost=0.0702468, + output_cost=0.000039, + prompt_tokens=27025, + completion_tokens=6, + cache_creation_cost=0.0702286, + ), +) +VERTEX_AI_CLAUDE_HAIKU_5_5_THINKING: Final = _vertex_claude_shape_case( + VERTEX_AI_CLAUDE_SONNET_5_5_THINKING, + "vertex_ai/claude-haiku-5-5-thinking", + "vertex_ai/claude-haiku-5-5", + _vertex_claude_usage(input_tokens=18, output_tokens=144, thinking_tokens=37), + ExactExpected( + spend=0.0009594, input_cost=0.0000234, output_cost=0.000936, prompt_tokens=18, completion_tokens=144 + ), + adaptive_thinking=True, + thinking_effort="low", +) +VERTEX_AI_CLAUDE_HAIKU_5_5_TOOL_CALL: Final = _vertex_claude_shape_case( + VERTEX_AI_CLAUDE_SONNET_5_5_TOOL_CALL, + "vertex_ai/claude-haiku-5-5-tool_call", + "vertex_ai/claude-haiku-5-5", + _vertex_claude_usage(input_tokens=385, output_tokens=50), + ExactExpected( + spend=0.0008255, input_cost=0.0005005, output_cost=0.000325, prompt_tokens=385, completion_tokens=50 + ), +) +VERTEX_AI_CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT: Final = CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT.model_copy( + update={ + "name": "vertex_ai/claude-haiku-5-5-messages_input_text", + "model": "vertex_ai/claude-haiku-5-5", + "endpoint": "/v1/messages", + "response": VERTEX_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT.response, + "expected": VERTEX_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT.expected, + } +) +VERTEX_AI_CLAUDE_HAIKU_5_5_MESSAGES_STREAM: Final = CLAUDE_HAIKU_5_5_MESSAGES_STREAM.model_copy( + update={ + "name": "vertex_ai/claude-haiku-5-5-messages_stream", + "model": "vertex_ai/claude-haiku-5-5", + "endpoint": "/v1/messages", + "response": VERTEX_AI_CLAUDE_HAIKU_5_5_STREAM.response, + "expected": VERTEX_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT.expected, + } +) + +VERTEX_AI_CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT: Final = CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT.model_copy( + update={ + "name": "vertex_ai/claude-fable-5-1-messages_input_text", + "model": "vertex_ai/claude-fable-5-1", + "endpoint": "/v1/messages", + "response": VERTEX_AI_CLAUDE_FABLE_5_1_INPUT_TEXT.response, + "expected": VERTEX_AI_CLAUDE_FABLE_5_1_INPUT_TEXT.expected, + } +) +VERTEX_AI_CLAUDE_FABLE_5_1_MESSAGES_STREAM: Final = CLAUDE_FABLE_5_1_MESSAGES_STREAM.model_copy( + update={ + "name": "vertex_ai/claude-fable-5-1-messages_stream", + "model": "vertex_ai/claude-fable-5-1", + "endpoint": "/v1/messages", + "response": VERTEX_AI_CLAUDE_FABLE_5_1_STREAM.response, + "expected": VERTEX_AI_CLAUDE_FABLE_5_1_INPUT_TEXT.expected, + } +) +VERTEX_AI_CLAUDE_FABLE_5_1_BRIDGE_PAIRS: Final = bridge_pairs( + (VERTEX_AI_CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT, VERTEX_AI_CLAUDE_FABLE_5_1_MESSAGES_STREAM), + ("/v1/responses",), +) +VERTEX_AI_CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT: Final = CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT.model_copy( + update={ + "name": "vertex_ai/claude-opus-5-5-messages_input_text", + "model": "vertex_ai/claude-opus-5-5", + "endpoint": "/v1/messages", + "response": VERTEX_AI_CLAUDE_OPUS_5_5_INPUT_TEXT.response, + "expected": VERTEX_AI_CLAUDE_OPUS_5_5_INPUT_TEXT.expected, + } +) +VERTEX_AI_CLAUDE_OPUS_5_5_MESSAGES_STREAM: Final = CLAUDE_OPUS_5_5_MESSAGES_STREAM.model_copy( + update={ + "name": "vertex_ai/claude-opus-5-5-messages_stream", + "model": "vertex_ai/claude-opus-5-5", + "endpoint": "/v1/messages", + "response": VERTEX_AI_CLAUDE_OPUS_5_5_STREAM.response, + "expected": VERTEX_AI_CLAUDE_OPUS_5_5_INPUT_TEXT.expected, + } +) +VERTEX_AI_CLAUDE_OPUS_5_5_BRIDGE_PAIRS: Final = bridge_pairs( + (VERTEX_AI_CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT, VERTEX_AI_CLAUDE_OPUS_5_5_MESSAGES_STREAM), + ("/v1/responses",), +) +VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_INPUT_TEXT: Final = CLAUDE_SONNET_5_5_MESSAGES_INPUT_TEXT.model_copy( + update={ + "name": "vertex_ai/claude-sonnet-5-5-messages_input_text", + "model": "vertex_ai/claude-sonnet-5-5", + "endpoint": "/v1/messages", + "response": VERTEX_AI_CLAUDE_SONNET_5_5_INPUT_TEXT.response, + "expected": VERTEX_AI_CLAUDE_SONNET_5_5_INPUT_TEXT.expected, + } +) +VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_STREAM: Final = CLAUDE_SONNET_5_5_MESSAGES_STREAM.model_copy( + update={ + "name": "vertex_ai/claude-sonnet-5-5-messages_stream", + "model": "vertex_ai/claude-sonnet-5-5", + "endpoint": "/v1/messages", + "response": VERTEX_AI_CLAUDE_SONNET_5_5_STREAM.response, + "expected": VERTEX_AI_CLAUDE_SONNET_5_5_INPUT_TEXT.expected, + } +) +VERTEX_AI_CLAUDE_SONNET_5_5_BRIDGE_PAIRS: Final = bridge_pairs( + (VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_INPUT_TEXT, VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_STREAM), + ("/v1/responses",), +) +VERTEX_AI_CLAUDE_HAIKU_5_5_BRIDGE_PAIRS: Final = bridge_pairs( + (VERTEX_AI_CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT, VERTEX_AI_CLAUDE_HAIKU_5_5_MESSAGES_STREAM), + ("/v1/responses",), +) + GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT: Final = _gemini_case( GEMINI_3_8_FLASH_INPUT_TEXT, "vertex_ai/gemini-3.1-pro-preview-input_text", "vertex_ai/gemini-3.1-pro-preview", - ExactExpected(spend=0.0094864, input_cost=0.004048, output_cost=0.0054384, prompt_tokens=1840, completion_tokens=412) + ExactExpected(spend=0.0094864, input_cost=0.004048, output_cost=0.0054384, prompt_tokens=1840, completion_tokens=412), + usage_fields={ + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 412}], + "trafficType": "ON_DEMAND", + }, ) GEMINI_3_1_PRO_PREVIEW_STREAM: Final = _gemini_case( GEMINI_3_8_FLASH_STREAM, "vertex_ai/gemini-3.1-pro-preview-stream", "vertex_ai/gemini-3.1-pro-preview", - ExactExpected(spend=0.0094864, input_cost=0.004048, output_cost=0.0054384, prompt_tokens=1840, completion_tokens=412) + ExactExpected(spend=0.0094864, input_cost=0.004048, output_cost=0.0054384, prompt_tokens=1840, completion_tokens=412), + usage_fields={ + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 412}], + "trafficType": "ON_DEMAND", + }, ) GEMINI_3_1_PRO_PREVIEW_CACHE_READ: Final = _gemini_case( GEMINI_3_8_FLASH_CACHE_READ, "vertex_ai/gemini-3.1-pro-preview-cache_read", "vertex_ai/gemini-3.1-pro-preview", - ExactExpected(spend=0.00912736, input_cost=0.00411136, output_cost=0.005016, prompt_tokens=12928, completion_tokens=380) + ExactExpected(spend=0.00912736, input_cost=0.00411136, output_cost=0.005016, prompt_tokens=12928, completion_tokens=380), + usage_fields={ + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 380}], + "trafficType": "ON_DEMAND", + }, ) GEMINI_3_1_PRO_PREVIEW_REASONING: Final = _gemini_case( GEMINI_3_1_PRO_REASONING, "vertex_ai/gemini-3.1-pro-preview-reasoning", "vertex_ai/gemini-3.1-pro-preview", - ExactExpected(spend=0.056056, input_cost=0.002728, output_cost=0.053328, prompt_tokens=1240, completion_tokens=4040) + ExactExpected(spend=0.056056, input_cost=0.002728, output_cost=0.053328, prompt_tokens=1240, completion_tokens=4040), + usage_fields={ + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 560}], + "trafficType": "ON_DEMAND", + }, +) +GEMINI_3_8_FLASH_REASONING: Final = _gemini_case( + GEMINI_3_1_PRO_REASONING, + "vertex_ai/gemini-3.8-flash-reasoning", + "vertex_ai/gemini-3.8-flash", + ExactExpected( + spend=0.01144, + input_cost=0.0006448, + output_cost=0.0107952, + prompt_tokens=1240, + completion_tokens=4040, + ), + usage_fields={ + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 560}], + "trafficType": "ON_DEMAND", + }, + model_version="gemini-3.8-flash", ) GEMINI_3_1_PRO_PREVIEW_TOOL_CALL: Final = _gemini_case( GEMINI_3_8_FLASH_TOOL_CALL, "vertex_ai/gemini-3.1-pro-preview-tool_call", "vertex_ai/gemini-3.1-pro-preview", ExactExpected(spend=0.0094864, input_cost=0.004048, output_cost=0.0054384, prompt_tokens=1840, completion_tokens=412) ) +GEMINI_3_1_PRO_PREVIEW_TIERED_INPUT_ABOVE_200K: Final = _gemini_case( + GEMINI_3_1_PRO_TIERED_INPUT_ABOVE_200K, + "vertex_ai/gemini-3.1-pro-preview-tiered_input_above_200k", + "vertex_ai/gemini-3.1-pro-preview", + ExactExpected( + spend=0.913396, input_cost=0.90112, output_cost=0.012276, prompt_tokens=204800, completion_tokens=620 + ), + usage_fields={ + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 620}], + "trafficType": "ON_DEMAND", + }, +) +GEMINI_3_1_PRO_PREVIEW_TIERED_CACHE_READ_ABOVE_200K: Final = _gemini_case( + GEMINI_3_1_PRO_TIERED_CACHE_READ_ABOVE_200K, + "vertex_ai/gemini-3.1-pro-preview-tiered_cache_read_above_200k", + "vertex_ai/gemini-3.1-pro-preview", + ExactExpected( + spend=0.11628672, input_cost=0.10678272, output_cost=0.009504, prompt_tokens=205824, completion_tokens=480 + ), + usage_fields={ + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 480}], + "trafficType": "ON_DEMAND", + "cacheTokensDetails": [{"modality": "TEXT", "tokenCount": 201728}], + }, +) + GEMINI_4_ARGON_INPUT_TEXT: Final = GEMINI_3_8_FLASH_INPUT_TEXT.model_copy( update={ "name": "vertex_ai/gemini-4-argon-input_text", @@ -3301,6 +3593,15 @@ GEMINI_4_ARGON_TOOL_CALL: Final = GEMINI_3_8_FLASH_TOOL_CALL.model_copy( } ) +GEMINI_4_ARGON_ROUTE_BRIDGES: Final = bridge_pairs( + (GEMINI_4_ARGON_INPUT_TEXT, GEMINI_4_ARGON_STREAM), + ("/v1/responses", "/v1/messages"), +) +GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES: Final = bridge_pairs( + (GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT, GEMINI_3_1_PRO_PREVIEW_STREAM), + ("/v1/responses", "/v1/messages"), +) + CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GEMINI_3_1_PRO_INPUT_TEXT, GEMINI_3_1_PRO_CACHE_READ, @@ -3326,6 +3627,7 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GEMINI_3_1_PRO_STREAM_TOOL_CALL, GEMINI_3_1_PRO_STREAM_FULL_USAGE, GEMINI_3_8_FLASH_INPUT_TEXT, + GEMINI_3_8_FLASH_REASONING, GEMINI_3_8_FLASH_CACHE_READ, GEMINI_3_8_FLASH_SERVICE_TIER_FLEX, GEMINI_3_8_FLASH_SERVICE_TIER_PRIORITY, @@ -3367,9 +3669,30 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( VERTEX_AI_CLAUDE_FABLE_5_1_CACHE_WRITE_1H, VERTEX_AI_CLAUDE_FABLE_5_1_THINKING, VERTEX_AI_CLAUDE_FABLE_5_1_TOOL_CALL, + VERTEX_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT, + VERTEX_AI_CLAUDE_HAIKU_5_5_STREAM, + VERTEX_AI_CLAUDE_HAIKU_5_5_CACHE_READ, + VERTEX_AI_CLAUDE_HAIKU_5_5_CACHE_WRITE_5M, + VERTEX_AI_CLAUDE_HAIKU_5_5_CACHE_WRITE_1H, + VERTEX_AI_CLAUDE_HAIKU_5_5_THINKING, + VERTEX_AI_CLAUDE_HAIKU_5_5_TOOL_CALL, + VERTEX_AI_CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT, + VERTEX_AI_CLAUDE_HAIKU_5_5_MESSAGES_STREAM, + *(pair.bridged for pair in VERTEX_AI_CLAUDE_HAIKU_5_5_BRIDGE_PAIRS), + VERTEX_AI_CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT, + VERTEX_AI_CLAUDE_FABLE_5_1_MESSAGES_STREAM, + *(pair.bridged for pair in VERTEX_AI_CLAUDE_FABLE_5_1_BRIDGE_PAIRS), + VERTEX_AI_CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT, + VERTEX_AI_CLAUDE_OPUS_5_5_MESSAGES_STREAM, + *(pair.bridged for pair in VERTEX_AI_CLAUDE_OPUS_5_5_BRIDGE_PAIRS), + VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_INPUT_TEXT, + VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_STREAM, + *(pair.bridged for pair in VERTEX_AI_CLAUDE_SONNET_5_5_BRIDGE_PAIRS), GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT, GEMINI_3_1_PRO_PREVIEW_STREAM, GEMINI_3_1_PRO_PREVIEW_CACHE_READ, + GEMINI_3_1_PRO_PREVIEW_TIERED_INPUT_ABOVE_200K, + GEMINI_3_1_PRO_PREVIEW_TIERED_CACHE_READ_ABOVE_200K, GEMINI_3_1_PRO_PREVIEW_REASONING, GEMINI_3_1_PRO_PREVIEW_TOOL_CALL, GEMINI_4_ARGON_INPUT_TEXT, @@ -3377,6 +3700,8 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GEMINI_4_ARGON_CACHE_READ, GEMINI_4_ARGON_REASONING, GEMINI_4_ARGON_TOOL_CALL, + *(pair.bridged for pair in GEMINI_4_ARGON_ROUTE_BRIDGES), + *(pair.bridged for pair in GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES), ) PARITY: Final[tuple[StreamParityTestCase, ...]] = ( @@ -3397,8 +3722,59 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = ( ), StreamParityTestCase(plain=VERTEX_AI_CLAUDE_OPUS_5_5_INPUT_TEXT, streamed=VERTEX_AI_CLAUDE_OPUS_5_5_STREAM), StreamParityTestCase(plain=VERTEX_AI_CLAUDE_FABLE_5_1_INPUT_TEXT, streamed=VERTEX_AI_CLAUDE_FABLE_5_1_STREAM), + StreamParityTestCase( + plain=VERTEX_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT, streamed=VERTEX_AI_CLAUDE_HAIKU_5_5_STREAM + ), + StreamParityTestCase( + plain=VERTEX_AI_CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT, + streamed=VERTEX_AI_CLAUDE_HAIKU_5_5_MESSAGES_STREAM, + ), + StreamParityTestCase( + plain=VERTEX_AI_CLAUDE_HAIKU_5_5_BRIDGE_PAIRS[0].bridged, + streamed=VERTEX_AI_CLAUDE_HAIKU_5_5_BRIDGE_PAIRS[1].bridged, + ), + StreamParityTestCase( + plain=VERTEX_AI_CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT, + streamed=VERTEX_AI_CLAUDE_FABLE_5_1_MESSAGES_STREAM, + ), + StreamParityTestCase( + plain=VERTEX_AI_CLAUDE_FABLE_5_1_BRIDGE_PAIRS[0].bridged, + streamed=VERTEX_AI_CLAUDE_FABLE_5_1_BRIDGE_PAIRS[1].bridged, + ), + StreamParityTestCase( + plain=VERTEX_AI_CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT, + streamed=VERTEX_AI_CLAUDE_OPUS_5_5_MESSAGES_STREAM, + ), + StreamParityTestCase( + plain=VERTEX_AI_CLAUDE_OPUS_5_5_BRIDGE_PAIRS[0].bridged, + streamed=VERTEX_AI_CLAUDE_OPUS_5_5_BRIDGE_PAIRS[1].bridged, + ), + StreamParityTestCase( + plain=VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_INPUT_TEXT, + streamed=VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_STREAM, + ), + StreamParityTestCase( + plain=VERTEX_AI_CLAUDE_SONNET_5_5_BRIDGE_PAIRS[0].bridged, + streamed=VERTEX_AI_CLAUDE_SONNET_5_5_BRIDGE_PAIRS[1].bridged, + ), StreamParityTestCase(plain=GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT, streamed=GEMINI_3_1_PRO_PREVIEW_STREAM), StreamParityTestCase(plain=GEMINI_4_ARGON_INPUT_TEXT, streamed=GEMINI_4_ARGON_STREAM), + StreamParityTestCase( + plain=GEMINI_4_ARGON_ROUTE_BRIDGES[0].bridged, + streamed=GEMINI_4_ARGON_ROUTE_BRIDGES[2].bridged, + ), + StreamParityTestCase( + plain=GEMINI_4_ARGON_ROUTE_BRIDGES[1].bridged, + streamed=GEMINI_4_ARGON_ROUTE_BRIDGES[3].bridged, + ), + StreamParityTestCase( + plain=GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES[0].bridged, + streamed=GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES[2].bridged, + ), + StreamParityTestCase( + plain=GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES[1].bridged, + streamed=GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES[3].bridged, + ), ) BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = ( @@ -3406,6 +3782,10 @@ BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = ( (GEMINI_3_8_FLASH_INPUT_TEXT, GEMINI_3_8_FLASH_CACHE_READ, GEMINI_3_8_FLASH_STREAM), ("/v1/responses", "/v1/messages"), ), - *bridge_pairs((GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT,), ("/v1/responses",)), - *bridge_pairs((GEMINI_4_ARGON_INPUT_TEXT,), ("/v1/responses",)), + *GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES, + *GEMINI_4_ARGON_ROUTE_BRIDGES, + *VERTEX_AI_CLAUDE_HAIKU_5_5_BRIDGE_PAIRS, + *VERTEX_AI_CLAUDE_FABLE_5_1_BRIDGE_PAIRS, + *VERTEX_AI_CLAUDE_OPUS_5_5_BRIDGE_PAIRS, + *VERTEX_AI_CLAUDE_SONNET_5_5_BRIDGE_PAIRS, ) diff --git a/tests/integration/cost_calculation/cost_map.py b/tests/integration/cost_calculation/cost_map.py index 2ef42249a7e..ba7be7647da 100644 --- a/tests/integration/cost_calculation/cost_map.py +++ b/tests/integration/cost_calculation/cost_map.py @@ -889,6 +889,26 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "max_output_tokens": 128000, "max_tokens": 128000, }, + "vertex_ai/claude-haiku-5-5": { + "litellm_provider": "vertex_ai-anthropic_models", + "mode": "chat", + "input_cost_per_token": 1.3e-06, + "input_cost_per_token_above_100k_tokens": 6.5e-06, + "output_cost_per_token": 6.5e-06, + "output_cost_per_token_above_100k_tokens": 3.25e-05, + "cache_creation_input_token_cost": 1.625e-06, + "cache_creation_input_token_cost_above_1hr": 2.6e-06, + "cache_creation_input_token_cost_above_100k_tokens": 8.125e-06, + "cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.3e-05, + "cache_read_input_token_cost": 1.3e-07, + "cache_read_input_token_cost_above_100k_tokens": 6.5e-07, + "supports_function_calling": True, + "supports_prompt_caching": True, + "supports_reasoning": True, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + }, "groq/qwen/qwen3.8-27b": { "litellm_provider": "groq", "mode": "chat",