From e74676961bcbc5caccbc65910c2cc8dc736c6573 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:42:25 +0000 Subject: [PATCH] test(cost): add gpt-6-luna 12h cache write, gemini above-200k flex and vertex tier cost cases --- .../cost_calculation/cases/gemini.py | 16 +++++ .../cost_calculation/cases/openai.py | 12 ++-- .../cost_calculation/cases/vertex_ai.py | 61 +++++++++++++++++++ .../integration/cost_calculation/cost_map.py | 11 ++++ .../cost_calculation/cost_tracking_case.py | 3 + 5 files changed, 97 insertions(+), 6 deletions(-) diff --git a/tests/integration/cost_calculation/cases/gemini.py b/tests/integration/cost_calculation/cases/gemini.py index 9fe4b1c50d5..2af00dbc308 100644 --- a/tests/integration/cost_calculation/cases/gemini.py +++ b/tests/integration/cost_calculation/cases/gemini.py @@ -3480,6 +3480,21 @@ GEMINI_GEMINI_3_1_PRO_PREVIEW_TIERED_INPUT_ABOVE_200K: Final = _gemini_case( ), usage_fields={"serviceTier": "standard"}, ) +GEMINI_GEMINI_3_1_PRO_PREVIEW_TIERED_FLEX_ABOVE_200K: Final = _gemini_case( + GEMINI_GEMINI_3_1_PRO_TIERED_INPUT_ABOVE_200K, + "gemini-gemini-3.1-pro-preview-tiered_flex_above_200k", + "gemini/gemini-3.1-pro-preview", + "gemini-3.1-pro-preview", + ExactExpected( + spend=0.539734, input_cost=0.53248, output_cost=0.007254, prompt_tokens=204800, completion_tokens=620 + ), + usage_fields={"serviceTier": "flex"}, +).model_copy( + update={ + "known_bug": "gemini route bills the standard above-200k rate (1.079468) when the Gemini API reply marks flex only " + "through usageMetadata.serviceTier, so the above_200k_tokens_flex prices are never applied" + } +) GEMINI_GEMINI_3_1_PRO_PREVIEW_TIERED_CACHE_READ_ABOVE_200K: Final = _gemini_case( GEMINI_GEMINI_3_1_PRO_TIERED_CACHE_READ_ABOVE_200K, "gemini-gemini-3.1-pro-preview-tiered_cache_read_above_200k", @@ -3572,6 +3587,7 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GEMINI_GEMINI_3_1_PRO_PREVIEW_CACHE_READ, GEMINI_GEMINI_3_1_PRO_PREVIEW_TIERED_INPUT_ABOVE_200K, GEMINI_GEMINI_3_1_PRO_PREVIEW_TIERED_CACHE_READ_ABOVE_200K, + GEMINI_GEMINI_3_1_PRO_PREVIEW_TIERED_FLEX_ABOVE_200K, GEMINI_GEMINI_3_1_PRO_PREVIEW_REASONING, GEMINI_GEMINI_3_1_PRO_PREVIEW_TOOL_CALL, GEMINI_GEMINI_3_1_PRO_PREVIEW_SERVICE_TIER_PRIORITY, diff --git a/tests/integration/cost_calculation/cases/openai.py b/tests/integration/cost_calculation/cases/openai.py index 270bb6e105d..f5395a9dd56 100644 --- a/tests/integration/cost_calculation/cases/openai.py +++ b/tests/integration/cost_calculation/cases/openai.py @@ -7036,8 +7036,8 @@ GPT_5_6_RESPONSES_CACHE_WRITE_12H: Final = CostTrackingTestCase( ), ) -GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE: Final = CostTrackingTestCase( - name="gpt-6-luna-responses_cache_write_12h_no_12h_rate", +GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H: Final = CostTrackingTestCase( + name="gpt-6-luna-responses_cache_write_12h", covers="quota_management.spend_tracking.cost_matrix.logs_cost", model="gpt-6-luna", endpoint="/v1/responses", @@ -7077,12 +7077,12 @@ GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE: Final = CostTrackingTestCase( }, ), expected=ExactExpected( - spend=0.0016078, - input_cost=0.0014178, + spend=0.0018578, + input_cost=0.0016678, output_cost=0.00019, prompt_tokens=12928, completion_tokens=380, - cache_creation_cost=0.000625, + cache_creation_cost=0.000875, ), ) @@ -12224,7 +12224,7 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( OMNI_MODERATIONS_NEXT_LIST, GPT_5_6_RESPONSES_CACHE_READ, GPT_5_6_RESPONSES_CACHE_WRITE_12H, - GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE, + GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H, GPT_5_6_RESPONSES_REASONING, GPT_5_6_RESPONSES_STREAM, GPT_5_6_RESPONSES_STREAM_CACHE_READ, diff --git a/tests/integration/cost_calculation/cases/vertex_ai.py b/tests/integration/cost_calculation/cases/vertex_ai.py index 8590a19646b..bc5f88e2ebf 100644 --- a/tests/integration/cost_calculation/cases/vertex_ai.py +++ b/tests/integration/cost_calculation/cases/vertex_ai.py @@ -3694,6 +3694,40 @@ GEMINI_3_1_PRO_PREVIEW_TIERED_CACHE_READ_ABOVE_200K: Final = _gemini_case( }, ) +GEMINI_3_1_PRO_PREVIEW_TIERED_FLEX_ABOVE_200K: Final = _gemini_case( + GEMINI_3_1_PRO_TIERED_INPUT_ABOVE_200K, + "vertex_ai/gemini-3.1-pro-preview-tiered_flex_above_200k", + "vertex_ai/gemini-3.1-pro-preview", + ExactExpected( + spend=0.456698, input_cost=0.45056, output_cost=0.006138, prompt_tokens=204800, completion_tokens=620 + ), + usage_fields={ + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 620}], + "trafficType": "ON_DEMAND_FLEX", + }, +) +GEMINI_3_1_PRO_PREVIEW_SERVICE_TIER_FLEX: Final = _gemini_case( + GEMINI_3_8_FLASH_SERVICE_TIER_FLEX, + "vertex_ai/gemini-3.1-pro-preview-service_tier_flex", + "vertex_ai/gemini-3.1-pro-preview", + ExactExpected(spend=0.0047432, input_cost=0.002024, output_cost=0.0027192, prompt_tokens=1840, completion_tokens=412), + usage_fields={ + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 412}], + "trafficType": "ON_DEMAND_FLEX", + }, +) +GEMINI_3_1_PRO_PREVIEW_SERVICE_TIER_PRIORITY: Final = _gemini_case( + GEMINI_3_8_FLASH_SERVICE_TIER_PRIORITY, + "vertex_ai/gemini-3.1-pro-preview-service_tier_priority", + "vertex_ai/gemini-3.1-pro-preview", + ExactExpected( + spend=0.01707552, input_cost=0.0072864, output_cost=0.00978912, prompt_tokens=1840, completion_tokens=412 + ), + usage_fields={ + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 412}], + "trafficType": "ON_DEMAND_PRIORITY", + }, +) GEMINI_4_ARGON_INPUT_TEXT: Final = GEMINI_3_8_FLASH_INPUT_TEXT.model_copy( update={ "name": "vertex_ai/gemini-4-argon-input_text", @@ -3851,6 +3885,28 @@ GEMINI_4_ARGON_TOOL_CALL: Final = GEMINI_3_8_FLASH_TOOL_CALL.model_copy( } ) +GEMINI_4_ARGON_SERVICE_TIER_FLEX: Final = _gemini_case( + GEMINI_3_8_FLASH_SERVICE_TIER_FLEX, + "vertex_ai/gemini-4-argon-service_tier_flex", + "vertex_ai/gemini-4-argon", + ExactExpected(spend=0.00468, input_cost=0.002208, output_cost=0.002472, prompt_tokens=1840, completion_tokens=412), + usage_fields={ + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 412}], + "trafficType": "ON_DEMAND_FLEX", + }, + model_version="gemini-4-argon", +) +GEMINI_4_ARGON_SERVICE_TIER_PRIORITY: Final = _gemini_case( + GEMINI_3_8_FLASH_SERVICE_TIER_PRIORITY, + "vertex_ai/gemini-4-argon-service_tier_priority", + "vertex_ai/gemini-4-argon", + ExactExpected(spend=0.016848, input_cost=0.0079488, output_cost=0.0088992, prompt_tokens=1840, completion_tokens=412), + usage_fields={ + "candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 412}], + "trafficType": "ON_DEMAND_PRIORITY", + }, + model_version="gemini-4-argon", +) GEMINI_4_ARGON_ROUTE_BRIDGES: Final = bridge_pairs( (GEMINI_4_ARGON_INPUT_TEXT, GEMINI_4_ARGON_STREAM), ("/v1/responses", "/v1/messages"), @@ -3965,13 +4021,18 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GEMINI_3_1_PRO_PREVIEW_CACHE_READ, GEMINI_3_1_PRO_PREVIEW_TIERED_INPUT_ABOVE_200K, GEMINI_3_1_PRO_PREVIEW_TIERED_CACHE_READ_ABOVE_200K, + GEMINI_3_1_PRO_PREVIEW_TIERED_FLEX_ABOVE_200K, GEMINI_3_1_PRO_PREVIEW_REASONING, GEMINI_3_1_PRO_PREVIEW_TOOL_CALL, + GEMINI_3_1_PRO_PREVIEW_SERVICE_TIER_FLEX, + GEMINI_3_1_PRO_PREVIEW_SERVICE_TIER_PRIORITY, GEMINI_4_ARGON_INPUT_TEXT, GEMINI_4_ARGON_STREAM, GEMINI_4_ARGON_CACHE_READ, GEMINI_4_ARGON_REASONING, GEMINI_4_ARGON_TOOL_CALL, + GEMINI_4_ARGON_SERVICE_TIER_FLEX, + GEMINI_4_ARGON_SERVICE_TIER_PRIORITY, *(pair.bridged for pair in GEMINI_4_ARGON_ROUTE_BRIDGES), *(pair.bridged for pair in GEMINI_3_8_FLASH_ROUTE_BRIDGES), *(pair.bridged for pair in GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES), diff --git a/tests/integration/cost_calculation/cost_map.py b/tests/integration/cost_calculation/cost_map.py index ffdb6752bf2..598d56d31b6 100644 --- a/tests/integration/cost_calculation/cost_map.py +++ b/tests/integration/cost_calculation/cost_map.py @@ -94,6 +94,12 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "cache_read_input_token_cost_above_272k_tokens_priority": 4e-08, "cache_read_input_token_cost_above_272k_tokens": 2e-08, "cache_read_input_token_cost_above_272k_tokens_decisions": 2e-07, + "cache_creation_input_token_cost_above_12hr": 2.5e-07, + "cache_creation_input_token_cost_above_12hr_flex": 1.25e-07, + "cache_creation_input_token_cost_above_12hr_priority": 5e-07, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-07, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-07, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 1e-06, "input_cost_per_token": 1e-07, "input_cost_per_token_decisions": 1e-07, "input_cost_per_token_above_272k_tokens": 2e-07, @@ -501,6 +507,9 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "cache_read_input_token_cost_above_200k_tokens_priority": 9.36e-07, "cache_read_input_token_cost_flex": 2.6e-07, "input_cost_per_token_flex": 1.3e-06, + "input_cost_per_token_above_200k_tokens_flex": 2.6e-06, + "output_cost_per_token_above_200k_tokens_flex": 1.17e-05, + "cache_read_input_token_cost_above_200k_tokens_flex": 5.2e-07, "output_cost_per_token_flex": 7.8e-06, }, "gemini/gemini-3.8-flash": { @@ -558,6 +567,8 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "input_cost_per_token_above_200k_tokens": 4.4e-06, "input_cost_per_token_above_200k_tokens_priority": 7.92e-06, "input_cost_per_token_flex": 1.1e-06, + "input_cost_per_token_above_200k_tokens_flex": 2.2e-06, + "output_cost_per_token_above_200k_tokens_flex": 9.9e-06, "input_cost_per_token_priority": 3.96e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 1048576, diff --git a/tests/integration/cost_calculation/cost_tracking_case.py b/tests/integration/cost_calculation/cost_tracking_case.py index 95363e92366..4271dcd054d 100644 --- a/tests/integration/cost_calculation/cost_tracking_case.py +++ b/tests/integration/cost_calculation/cost_tracking_case.py @@ -125,6 +125,8 @@ class CostMapEntry(BaseModel): output_cost_per_token_above_272k_tokens_ultrafast: float | None = None input_cost_per_token_above_200k_tokens_priority: float | None = None output_cost_per_token_above_200k_tokens_priority: float | None = None + input_cost_per_token_above_200k_tokens_flex: float | None = None + output_cost_per_token_above_200k_tokens_flex: float | None = None cache_read_input_token_cost_flex: float | None = None cache_read_input_token_cost_priority: float | None = None cache_read_input_token_cost_ultrafast: float | None = None @@ -132,6 +134,7 @@ class CostMapEntry(BaseModel): cache_read_input_token_cost_above_272k_tokens_priority: float | None = None cache_read_input_token_cost_above_272k_tokens_ultrafast: float | None = None cache_read_input_token_cost_above_200k_tokens_priority: float | None = None + cache_read_input_token_cost_above_200k_tokens_flex: float | None = None cache_creation_input_token_cost_flex: float | None = None cache_creation_input_token_cost_priority: float | None = None cache_creation_input_token_cost_ultrafast: float | None = None