mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-17 04:34:25 +00:00
test(cost): add tier 1 cost tracking cases for gpt-5.6-terra, gemini-4-argon and gemini-3.1-pro-preview (#46931)
* test(cost): add tier 1 cost tracking cases for gpt-5.6-terra, gemini-4-argon and gemini-3.1-pro-preview Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(cost): add audit 3 shape cases for vertex_ai and azure_ai claude Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(cost): mark azure gpt-5.6-terra priority case as known bug Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
7ddc9f1c96
commit
ee4860dd84
7 changed files with 1180 additions and 7 deletions
|
|
@ -4111,6 +4111,7 @@ CLAUDE_OPUS_5_5_INPUT_TEXT: Final = CostTrackingTestCase(
|
|||
"output_tokens_details": {"thinking_tokens": 0},
|
||||
"service_tier": "standard",
|
||||
"inference_geo": "global",
|
||||
"speed": "standard",
|
||||
},
|
||||
},
|
||||
),
|
||||
|
|
@ -4170,6 +4171,7 @@ CLAUDE_SONNET_5_5_INPUT_TEXT: Final = CostTrackingTestCase(
|
|||
"output_tokens_details": {"thinking_tokens": 0},
|
||||
"service_tier": "standard",
|
||||
"inference_geo": "global",
|
||||
"speed": "standard",
|
||||
},
|
||||
"diagnostics": None,
|
||||
},
|
||||
|
|
@ -4232,6 +4234,7 @@ CLAUDE_SONNET_5_5_STREAM: Final = CostTrackingTestCase(
|
|||
"output_tokens": 1,
|
||||
"service_tier": "standard",
|
||||
"inference_geo": "global",
|
||||
"speed": "standard",
|
||||
},
|
||||
"diagnostics": None,
|
||||
},
|
||||
|
|
@ -4314,6 +4317,7 @@ CLAUDE_SONNET_5_5_CACHE_READ: Final = CostTrackingTestCase(
|
|||
"output_tokens_details": {"thinking_tokens": 0},
|
||||
"service_tier": "standard",
|
||||
"inference_geo": "global",
|
||||
"speed": "standard",
|
||||
},
|
||||
"diagnostics": None,
|
||||
},
|
||||
|
|
@ -4566,6 +4570,7 @@ CLAUDE_HAIKU_5_5_INPUT_TEXT: Final = CostTrackingTestCase(
|
|||
"output_tokens_details": {"thinking_tokens": 0},
|
||||
"service_tier": "standard",
|
||||
"inference_geo": "global",
|
||||
"speed": "standard",
|
||||
},
|
||||
"diagnostics": None,
|
||||
},
|
||||
|
|
@ -4628,6 +4633,7 @@ CLAUDE_HAIKU_5_5_STREAM: Final = CostTrackingTestCase(
|
|||
"output_tokens": 1,
|
||||
"service_tier": "standard",
|
||||
"inference_geo": "global",
|
||||
"speed": "standard",
|
||||
},
|
||||
"diagnostics": None,
|
||||
},
|
||||
|
|
@ -4710,6 +4716,7 @@ CLAUDE_HAIKU_5_5_CACHE_READ: Final = CostTrackingTestCase(
|
|||
"output_tokens_details": {"thinking_tokens": 0},
|
||||
"service_tier": "standard",
|
||||
"inference_geo": "global",
|
||||
"speed": "standard",
|
||||
},
|
||||
"diagnostics": None,
|
||||
},
|
||||
|
|
@ -5325,6 +5332,7 @@ CLAUDE_FABLE_5_1_INPUT_TEXT: Final = CostTrackingTestCase(
|
|||
"output_tokens_details": {"thinking_tokens": 0},
|
||||
"service_tier": "standard",
|
||||
"inference_geo": "global",
|
||||
"speed": "standard",
|
||||
},
|
||||
"diagnostics": None,
|
||||
},
|
||||
|
|
@ -5387,6 +5395,7 @@ CLAUDE_FABLE_5_1_STREAM: Final = CostTrackingTestCase(
|
|||
"output_tokens": 1,
|
||||
"service_tier": "standard",
|
||||
"inference_geo": "global",
|
||||
"speed": "standard",
|
||||
},
|
||||
"diagnostics": None,
|
||||
},
|
||||
|
|
@ -5469,6 +5478,7 @@ CLAUDE_FABLE_5_1_CACHE_READ: Final = CostTrackingTestCase(
|
|||
"output_tokens_details": {"thinking_tokens": 0},
|
||||
"service_tier": "standard",
|
||||
"inference_geo": "global",
|
||||
"speed": "standard",
|
||||
},
|
||||
"diagnostics": None,
|
||||
},
|
||||
|
|
@ -5720,6 +5730,7 @@ CLAUDE_OPUS_5_5_STREAM: Final = CostTrackingTestCase(
|
|||
"output_tokens": 1,
|
||||
"service_tier": "standard",
|
||||
"inference_geo": "global",
|
||||
"speed": "standard",
|
||||
},
|
||||
"diagnostics": None,
|
||||
},
|
||||
|
|
@ -5802,6 +5813,7 @@ CLAUDE_OPUS_5_5_CACHE_READ: Final = CostTrackingTestCase(
|
|||
"output_tokens_details": {"thinking_tokens": 0},
|
||||
"service_tier": "standard",
|
||||
"inference_geo": "global",
|
||||
"speed": "standard",
|
||||
},
|
||||
"diagnostics": None,
|
||||
},
|
||||
|
|
@ -6141,6 +6153,7 @@ CLAUDE_SONNET_5_5_THINKING: Final = CostTrackingTestCase(
|
|||
"output_tokens_details": {"thinking_tokens": 0},
|
||||
"service_tier": "standard",
|
||||
"inference_geo": "global",
|
||||
"speed": "standard",
|
||||
},
|
||||
"diagnostics": None,
|
||||
},
|
||||
|
|
@ -6294,6 +6307,7 @@ CLAUDE_HAIKU_5_5_THINKING: Final = CostTrackingTestCase(
|
|||
"output_tokens_details": {"thinking_tokens": 3480},
|
||||
"service_tier": "standard",
|
||||
"inference_geo": "global",
|
||||
"speed": "standard",
|
||||
},
|
||||
"diagnostics": None,
|
||||
},
|
||||
|
|
@ -6593,6 +6607,7 @@ CLAUDE_FABLE_5_1_THINKING: Final = CostTrackingTestCase(
|
|||
"output_tokens_details": {"thinking_tokens": 0},
|
||||
"service_tier": "standard",
|
||||
"inference_geo": "global",
|
||||
"speed": "standard",
|
||||
},
|
||||
"diagnostics": None,
|
||||
},
|
||||
|
|
@ -6746,6 +6761,7 @@ CLAUDE_OPUS_5_5_THINKING: Final = CostTrackingTestCase(
|
|||
"output_tokens_details": {"thinking_tokens": 3480},
|
||||
"service_tier": "standard",
|
||||
"inference_geo": "global",
|
||||
"speed": "standard",
|
||||
},
|
||||
"diagnostics": None,
|
||||
},
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
PARITY pairs the streamed case with its plain twin where both bill the same row."""
|
||||
|
||||
from typing import Final
|
||||
from typing import Final, cast
|
||||
|
||||
from integration.cost_calculation.cost_tracking_case import (
|
||||
with_response_usage,
|
||||
|
|
@ -23,6 +23,20 @@ from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParit
|
|||
from integration.cost_calculation.stream_parity.case import StreamParityTestCase, sse_frames
|
||||
|
||||
|
||||
def _response_for_model(response: JsonResponse, model: str) -> JsonResponse:
|
||||
return response.model_copy(update={"body": {**response.body, "model": model}})
|
||||
|
||||
|
||||
def _stream_for_model(response: SseResponse, model: str) -> SseResponse:
|
||||
return response.model_copy(
|
||||
update={
|
||||
"frames": tuple(
|
||||
frame.replace("gpt-6-luna-2026-09-22", model).replace("gpt-6-luna", model) for frame in response.frames
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# azure/gpt-5.4-mini
|
||||
AZURE_GPT_5_4_MINI_INPUT_TEXT: Final = CostTrackingTestCase(
|
||||
name="azure-gpt-5.4-mini-input_text",
|
||||
|
|
@ -6174,6 +6188,152 @@ AZURE_GPT_6_LUNA_CACHE_WRITE_5M: Final = with_response_usage(
|
|||
"content_filter_result": {"error": {"code": "content_filter_error", "message": "The contents are not filtered"}}
|
||||
},
|
||||
)
|
||||
|
||||
AZURE_GPT_5_6_TERRA_INPUT_TEXT: Final = AZURE_GPT_6_LUNA_INPUT_TEXT.model_copy(
|
||||
update={
|
||||
"name": "azure-gpt-5.6-terra-input_text",
|
||||
"model": "azure/gpt-5.6-terra",
|
||||
"response": _response_for_model(
|
||||
cast(JsonResponse, AZURE_GPT_6_LUNA_INPUT_TEXT.response), "gpt-5.6-terra-2026-07-09"
|
||||
),
|
||||
"expected": ExactExpected(
|
||||
spend=0.0094864,
|
||||
input_cost=0.004048,
|
||||
output_cost=0.0054384,
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
),
|
||||
}
|
||||
)
|
||||
AZURE_GPT_5_6_TERRA_STREAM: Final = AZURE_GPT_6_LUNA_STREAM.model_copy(
|
||||
update={
|
||||
"name": "azure-gpt-5.6-terra-stream",
|
||||
"model": "azure/gpt-5.6-terra",
|
||||
"response": _stream_for_model(
|
||||
cast(SseResponse, AZURE_GPT_6_LUNA_STREAM.response), "gpt-5.6-terra-2026-07-09"
|
||||
),
|
||||
"expected": ExactExpected(
|
||||
spend=0.0094864,
|
||||
input_cost=0.004048,
|
||||
output_cost=0.0054384,
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
),
|
||||
}
|
||||
)
|
||||
AZURE_GPT_5_6_TERRA_SERVICE_TIER_FLEX: Final = AZURE_GPT_6_LUNA_SERVICE_TIER_FLEX.model_copy(
|
||||
update={
|
||||
"name": "azure-gpt-5.6-terra-service_tier_flex",
|
||||
"model": "azure/gpt-5.6-terra",
|
||||
"response": _response_for_model(
|
||||
cast(JsonResponse, AZURE_GPT_6_LUNA_SERVICE_TIER_FLEX.response), "gpt-5.6-terra-2026-07-09"
|
||||
),
|
||||
"expected": ExactExpected(
|
||||
spend=0.0047432,
|
||||
input_cost=0.002024,
|
||||
output_cost=0.0027192,
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
),
|
||||
}
|
||||
)
|
||||
AZURE_GPT_5_6_TERRA_SERVICE_TIER_PRIORITY: Final = AZURE_GPT_6_LUNA_SERVICE_TIER_PRIORITY.model_copy(
|
||||
update={
|
||||
"name": "azure-gpt-5.6-terra-service_tier_priority",
|
||||
"model": "azure/gpt-5.6-terra",
|
||||
"response": _response_for_model(
|
||||
cast(JsonResponse, AZURE_GPT_6_LUNA_SERVICE_TIER_PRIORITY.response), "gpt-5.6-terra-2026-07-09"
|
||||
),
|
||||
"expected": ExactExpected(
|
||||
spend=0.0189728,
|
||||
input_cost=0.008096,
|
||||
output_cost=0.0108768,
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
),
|
||||
"known_bug": "Azure GPT Terra priority service tier is billed at the base rate",
|
||||
}
|
||||
)
|
||||
AZURE_GPT_5_6_TERRA_MESSAGES_BRIDGED_JSON: Final = AZURE_GPT_6_LUNA_MESSAGES_BRIDGED_JSON.model_copy(
|
||||
update={
|
||||
"name": "azure-gpt-5.6-terra-messages_bridged_json",
|
||||
"model": "azure/gpt-5.6-terra",
|
||||
"response": _response_for_model(
|
||||
cast(JsonResponse, AZURE_GPT_6_LUNA_MESSAGES_BRIDGED_JSON.response), "gpt-5.6-terra-2026-07-09"
|
||||
),
|
||||
"expected": ExactExpected(
|
||||
spend=0.0094864,
|
||||
input_cost=0.004048,
|
||||
output_cost=0.0054384,
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
),
|
||||
}
|
||||
)
|
||||
AZURE_GPT_5_6_TERRA_CACHE_READ: Final = AZURE_GPT_6_LUNA_CACHE_READ.model_copy(
|
||||
update={
|
||||
"name": "azure/gpt-5.6-terra-cache_read",
|
||||
"model": "azure/gpt-5.6-terra",
|
||||
"response": _response_for_model(
|
||||
cast(JsonResponse, AZURE_GPT_6_LUNA_CACHE_READ.response), "gpt-5.6-terra-2026-07-09"
|
||||
),
|
||||
"expected": ExactExpected(
|
||||
spend=0.00912736,
|
||||
input_cost=0.00411136,
|
||||
output_cost=0.005016,
|
||||
prompt_tokens=12928,
|
||||
completion_tokens=380,
|
||||
),
|
||||
}
|
||||
)
|
||||
AZURE_GPT_5_6_TERRA_CACHE_WRITE_5M: Final = AZURE_GPT_6_LUNA_CACHE_WRITE_5M.model_copy(
|
||||
update={
|
||||
"name": "azure/gpt-5.6-terra-cache_write_5m",
|
||||
"model": "azure/gpt-5.6-terra",
|
||||
"response": _response_for_model(
|
||||
cast(JsonResponse, AZURE_GPT_6_LUNA_CACHE_WRITE_5M.response), "gpt-5.6-terra-2026-07-09"
|
||||
),
|
||||
"expected": ExactExpected(
|
||||
spend=0.040216,
|
||||
input_cost=0.0352,
|
||||
output_cost=0.005016,
|
||||
prompt_tokens=12928,
|
||||
completion_tokens=380,
|
||||
),
|
||||
}
|
||||
)
|
||||
AZURE_GPT_5_6_TERRA_REASONING: Final = AZURE_GPT_6_LUNA_REASONING.model_copy(
|
||||
update={
|
||||
"name": "azure/gpt-5.6-terra-reasoning",
|
||||
"model": "azure/gpt-5.6-terra",
|
||||
"response": _response_for_model(
|
||||
cast(JsonResponse, AZURE_GPT_6_LUNA_REASONING.response), "gpt-5.6-terra-2026-07-09"
|
||||
),
|
||||
"expected": ExactExpected(
|
||||
spend=0.056056,
|
||||
input_cost=0.002728,
|
||||
output_cost=0.053328,
|
||||
prompt_tokens=1240,
|
||||
completion_tokens=4040,
|
||||
),
|
||||
}
|
||||
)
|
||||
AZURE_GPT_5_6_TERRA_TOOL_CALL: Final = AZURE_GPT_6_LUNA_TOOL_CALL.model_copy(
|
||||
update={
|
||||
"name": "azure/gpt-5.6-terra-tool_call",
|
||||
"model": "azure/gpt-5.6-terra",
|
||||
"response": _response_for_model(
|
||||
cast(JsonResponse, AZURE_GPT_6_LUNA_TOOL_CALL.response), "gpt-5.6-terra-2026-07-09"
|
||||
),
|
||||
"expected": ExactExpected(
|
||||
spend=0.0094864,
|
||||
input_cost=0.004048,
|
||||
output_cost=0.0054384,
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
),
|
||||
}
|
||||
)
|
||||
AZURE_GPT_6_ASTRA_CACHE_WRITE_5M: Final = with_response_usage(
|
||||
AZURE_GPT_6_ASTRA_CACHE_READ,
|
||||
"azure/gpt-6-astra-cache_write_5m",
|
||||
|
|
@ -6304,6 +6464,15 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
AZURE_GPT_6_1_SOL_CACHE_WRITE_5M,
|
||||
AZURE_GPT_6_LUNA_CACHE_WRITE_5M,
|
||||
AZURE_GPT_6_ASTRA_CACHE_WRITE_5M,
|
||||
AZURE_GPT_5_6_TERRA_INPUT_TEXT,
|
||||
AZURE_GPT_5_6_TERRA_STREAM,
|
||||
AZURE_GPT_5_6_TERRA_SERVICE_TIER_FLEX,
|
||||
AZURE_GPT_5_6_TERRA_SERVICE_TIER_PRIORITY,
|
||||
AZURE_GPT_5_6_TERRA_MESSAGES_BRIDGED_JSON,
|
||||
AZURE_GPT_5_6_TERRA_CACHE_READ,
|
||||
AZURE_GPT_5_6_TERRA_CACHE_WRITE_5M,
|
||||
AZURE_GPT_5_6_TERRA_REASONING,
|
||||
AZURE_GPT_5_6_TERRA_TOOL_CALL,
|
||||
)
|
||||
|
||||
PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
||||
|
|
@ -6316,6 +6485,7 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
|||
StreamParityTestCase(plain=AZURE_GPT_6_LUNA_INPUT_TEXT, streamed=AZURE_GPT_6_LUNA_STREAM),
|
||||
StreamParityTestCase(plain=AZURE_GPT_6_ASTRA_INPUT_TEXT, streamed=AZURE_GPT_6_ASTRA_STREAM),
|
||||
StreamParityTestCase(plain=AZURE_GPT_6_1_SOL_INPUT_TEXT, streamed=AZURE_GPT_6_1_SOL_STREAM),
|
||||
StreamParityTestCase(plain=AZURE_GPT_5_6_TERRA_INPUT_TEXT, streamed=AZURE_GPT_5_6_TERRA_STREAM),
|
||||
)
|
||||
|
||||
BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = (
|
||||
|
|
@ -6327,4 +6497,7 @@ BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = (
|
|||
BridgeCostParityTestCase(native=AZURE_GPT_6_LUNA_INPUT_TEXT, bridged=AZURE_GPT_6_LUNA_MESSAGES_BRIDGED_JSON),
|
||||
BridgeCostParityTestCase(native=AZURE_GPT_6_ASTRA_INPUT_TEXT, bridged=AZURE_GPT_6_ASTRA_MESSAGES_BRIDGED_JSON),
|
||||
BridgeCostParityTestCase(native=AZURE_GPT_6_1_SOL_INPUT_TEXT, bridged=AZURE_GPT_6_1_SOL_MESSAGES_BRIDGED_JSON),
|
||||
BridgeCostParityTestCase(
|
||||
native=AZURE_GPT_5_6_TERRA_INPUT_TEXT, bridged=AZURE_GPT_5_6_TERRA_MESSAGES_BRIDGED_JSON
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
PARITY pairs the streamed case with its plain twin where both bill the same row."""
|
||||
|
||||
from typing import Final
|
||||
from typing import Final, cast
|
||||
|
||||
from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs
|
||||
from integration.cost_calculation.cost_tracking_case import (
|
||||
|
|
@ -16,6 +16,32 @@ from integration.cost_calculation.cost_tracking_case import (
|
|||
)
|
||||
from integration.cost_calculation.stream_parity.case import StreamParityTestCase, sse_frames
|
||||
|
||||
|
||||
def _gemini_case_response(response: JsonResponse | SseResponse, model_version: str) -> JsonResponse | SseResponse:
|
||||
match response:
|
||||
case JsonResponse():
|
||||
return response.model_copy(update={"body": {**response.body, "modelVersion": model_version}})
|
||||
case SseResponse():
|
||||
return response.model_copy(
|
||||
update={
|
||||
"frames": tuple(frame.replace("gemini-3.8-flash", model_version) for frame in response.frames)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _gemini_case(
|
||||
sibling: CostTrackingTestCase, name: str, model: str, model_version: str, expected: ExactExpected
|
||||
) -> CostTrackingTestCase:
|
||||
return sibling.model_copy(
|
||||
update={
|
||||
"name": name,
|
||||
"model": model,
|
||||
"response": _gemini_case_response(cast(JsonResponse | SseResponse, sibling.response), model_version),
|
||||
"expected": expected,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# gemini/gemini-3.1-pro
|
||||
GEMINI_GEMINI_3_1_PRO_INPUT_TEXT: Final = CostTrackingTestCase(
|
||||
name="gemini-gemini-3.1-pro-input_text",
|
||||
|
|
@ -3543,6 +3569,51 @@ GEMINI_GEMINI_3_1_PRO_CACHE_CREATION: Final = CostTrackingTestCase(
|
|||
),
|
||||
)
|
||||
|
||||
GEMINI_GEMINI_4_ARGON_INPUT_TEXT: Final = _gemini_case(
|
||||
GEMINI_GEMINI_3_8_FLASH_INPUT_TEXT, "gemini-gemini-4-argon-input_text", "gemini/gemini-4-argon",
|
||||
"gemini-4-argon", ExactExpected(spend=0.00858, input_cost=0.004048, output_cost=0.004532, prompt_tokens=1840, completion_tokens=412)
|
||||
)
|
||||
GEMINI_GEMINI_4_ARGON_STREAM: Final = _gemini_case(
|
||||
GEMINI_GEMINI_3_8_FLASH_STREAM, "gemini-gemini-4-argon-stream", "gemini/gemini-4-argon",
|
||||
"gemini-4-argon", ExactExpected(spend=0.00858, input_cost=0.004048, output_cost=0.004532, prompt_tokens=1840, completion_tokens=412)
|
||||
)
|
||||
GEMINI_GEMINI_4_ARGON_CACHE_READ: Final = _gemini_case(
|
||||
GEMINI_GEMINI_3_8_FLASH_CACHE_READ, "gemini-gemini-4-argon-cache_read", "gemini/gemini-4-argon",
|
||||
"gemini-4-argon", ExactExpected(spend=0.00693968, input_cost=0.00275968, output_cost=0.00418, prompt_tokens=12928, completion_tokens=380)
|
||||
)
|
||||
GEMINI_GEMINI_4_ARGON_REASONING: Final = _gemini_case(
|
||||
GEMINI_GEMINI_3_8_FLASH_REASONING, "gemini-gemini-4-argon-reasoning", "gemini/gemini-4-argon",
|
||||
"gemini-4-argon", ExactExpected(spend=0.047168, input_cost=0.002728, output_cost=0.04444, prompt_tokens=1240, completion_tokens=4040)
|
||||
)
|
||||
GEMINI_GEMINI_4_ARGON_TOOL_CALL: Final = _gemini_case(
|
||||
GEMINI_GEMINI_3_8_FLASH_TOOL_CALL, "gemini-gemini-4-argon-tool_call", "gemini/gemini-4-argon",
|
||||
"gemini-4-argon", ExactExpected(spend=0.00858, input_cost=0.004048, output_cost=0.004532, prompt_tokens=1840, completion_tokens=412)
|
||||
)
|
||||
GEMINI_GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT: Final = _gemini_case(
|
||||
GEMINI_GEMINI_3_8_FLASH_INPUT_TEXT, "gemini-gemini-3.1-pro-preview-input_text",
|
||||
"gemini/gemini-3.1-pro-preview", "gemini-3.1-pro-preview",
|
||||
ExactExpected(spend=0.0112112, input_cost=0.004784, output_cost=0.0064272, prompt_tokens=1840, completion_tokens=412)
|
||||
)
|
||||
GEMINI_GEMINI_3_1_PRO_PREVIEW_STREAM: Final = _gemini_case(
|
||||
GEMINI_GEMINI_3_8_FLASH_STREAM, "gemini-gemini-3.1-pro-preview-stream",
|
||||
"gemini/gemini-3.1-pro-preview", "gemini-3.1-pro-preview",
|
||||
ExactExpected(spend=0.0112112, input_cost=0.004784, output_cost=0.0064272, prompt_tokens=1840, completion_tokens=412)
|
||||
)
|
||||
GEMINI_GEMINI_3_1_PRO_PREVIEW_CACHE_READ: Final = _gemini_case(
|
||||
GEMINI_GEMINI_3_8_FLASH_CACHE_READ, "gemini-gemini-3.1-pro-preview-cache_read",
|
||||
"gemini/gemini-3.1-pro-preview", "gemini-3.1-pro-preview",
|
||||
ExactExpected(spend=0.01078688, input_cost=0.00485888, output_cost=0.005928, prompt_tokens=12928, completion_tokens=380)
|
||||
)
|
||||
GEMINI_GEMINI_3_1_PRO_PREVIEW_REASONING: Final = _gemini_case(
|
||||
GEMINI_GEMINI_3_8_FLASH_REASONING, "gemini-gemini-3.1-pro-preview-reasoning",
|
||||
"gemini/gemini-3.1-pro-preview", "gemini-3.1-pro-preview",
|
||||
ExactExpected(spend=0.066248, input_cost=0.003224, output_cost=0.063024, prompt_tokens=1240, completion_tokens=4040)
|
||||
)
|
||||
GEMINI_GEMINI_3_1_PRO_PREVIEW_TOOL_CALL: Final = _gemini_case(
|
||||
GEMINI_GEMINI_3_8_FLASH_TOOL_CALL, "gemini-gemini-3.1-pro-preview-tool_call",
|
||||
"gemini/gemini-3.1-pro-preview", "gemini-3.1-pro-preview",
|
||||
ExactExpected(spend=0.0112112, input_cost=0.004784, output_cost=0.0064272, prompt_tokens=1840, completion_tokens=412)
|
||||
)
|
||||
|
||||
CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
||||
GEMINI_GEMINI_3_1_PRO_INPUT_TEXT,
|
||||
|
|
@ -3605,6 +3676,16 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
GEMINI_NANO_BANANA_2_1_WEB_SEARCH_SINGLE_QUERY,
|
||||
GEMINI_NANO_BANANA_2_1_WEB_SEARCH_MULTIPLE_QUERIES,
|
||||
GEMINI_GEMINI_3_1_PRO_CACHE_CREATION,
|
||||
GEMINI_GEMINI_4_ARGON_INPUT_TEXT,
|
||||
GEMINI_GEMINI_4_ARGON_STREAM,
|
||||
GEMINI_GEMINI_4_ARGON_CACHE_READ,
|
||||
GEMINI_GEMINI_4_ARGON_REASONING,
|
||||
GEMINI_GEMINI_4_ARGON_TOOL_CALL,
|
||||
GEMINI_GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT,
|
||||
GEMINI_GEMINI_3_1_PRO_PREVIEW_STREAM,
|
||||
GEMINI_GEMINI_3_1_PRO_PREVIEW_CACHE_READ,
|
||||
GEMINI_GEMINI_3_1_PRO_PREVIEW_REASONING,
|
||||
GEMINI_GEMINI_3_1_PRO_PREVIEW_TOOL_CALL,
|
||||
)
|
||||
|
||||
PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
||||
|
|
@ -3626,6 +3707,10 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
|||
streamed=GEMINI_GEMINI_3_8_FLASH_STREAM_RESPONSE_MODEL_OVERRIDE,
|
||||
),
|
||||
StreamParityTestCase(plain=GEMINI_GEMINI_3_8_FLASH_TOOL_CALL, streamed=GEMINI_GEMINI_3_8_FLASH_STREAM_TOOL_CALL),
|
||||
StreamParityTestCase(plain=GEMINI_GEMINI_4_ARGON_INPUT_TEXT, streamed=GEMINI_GEMINI_4_ARGON_STREAM),
|
||||
StreamParityTestCase(
|
||||
plain=GEMINI_GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT, streamed=GEMINI_GEMINI_3_1_PRO_PREVIEW_STREAM
|
||||
),
|
||||
)
|
||||
|
||||
BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = (
|
||||
|
|
@ -3638,4 +3723,6 @@ BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = (
|
|||
),
|
||||
("/v1/responses", "/v1/messages"),
|
||||
),
|
||||
*bridge_pairs((GEMINI_GEMINI_4_ARGON_INPUT_TEXT,), ("/v1/responses",)),
|
||||
*bridge_pairs((GEMINI_GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT,), ("/v1/responses",)),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,9 @@
|
|||
|
||||
PARITY pairs the streamed case with its plain twin where both bill the same row."""
|
||||
|
||||
from typing import Final
|
||||
from typing import Final, cast
|
||||
|
||||
from pydantic import JsonValue
|
||||
|
||||
from integration.cost_calculation.cost_tracking_case import (
|
||||
ALWAYS_OPEN_OFF_PEAK_WINDOW,
|
||||
|
|
@ -20,13 +22,91 @@ from integration.cost_calculation.cost_tracking_case import (
|
|||
)
|
||||
from integration.cost_calculation.stream_parity.case import StreamParityTestCase, sse_frames
|
||||
from integration.cost_calculation.cases.anthropic import (
|
||||
CLAUDE_FABLE_5_1_CACHE_READ,
|
||||
CLAUDE_FABLE_5_1_CACHE_WRITE_1H,
|
||||
CLAUDE_FABLE_5_1_CACHE_WRITE_5M,
|
||||
CLAUDE_FABLE_5_1_INPUT_TEXT,
|
||||
CLAUDE_FABLE_5_1_STREAM,
|
||||
CLAUDE_FABLE_5_1_THINKING,
|
||||
CLAUDE_FABLE_5_1_TOOL_CALL,
|
||||
CLAUDE_HAIKU_5_5_CACHE_READ,
|
||||
CLAUDE_HAIKU_5_5_CACHE_WRITE_1H,
|
||||
CLAUDE_HAIKU_5_5_CACHE_WRITE_5M,
|
||||
CLAUDE_HAIKU_5_5_INPUT_TEXT,
|
||||
CLAUDE_HAIKU_5_5_STREAM,
|
||||
CLAUDE_HAIKU_5_5_THINKING,
|
||||
CLAUDE_HAIKU_5_5_TOOL_CALL,
|
||||
)
|
||||
|
||||
|
||||
def _azure_ai_claude_usage(
|
||||
input_tokens: int,
|
||||
output_tokens: int,
|
||||
inference_geo: str,
|
||||
*,
|
||||
cache_creation_input_tokens: int = 0,
|
||||
cache_read_input_tokens: int = 0,
|
||||
ephemeral_5m_input_tokens: int = 0,
|
||||
ephemeral_1h_input_tokens: int = 0,
|
||||
thinking_tokens: int = 0,
|
||||
) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"input_tokens": input_tokens,
|
||||
"cache_creation_input_tokens": cache_creation_input_tokens,
|
||||
"cache_read_input_tokens": cache_read_input_tokens,
|
||||
"cache_creation": {
|
||||
"ephemeral_5m_input_tokens": ephemeral_5m_input_tokens,
|
||||
"ephemeral_1h_input_tokens": ephemeral_1h_input_tokens,
|
||||
},
|
||||
"output_tokens": output_tokens,
|
||||
"output_tokens_details": {"thinking_tokens": thinking_tokens},
|
||||
"service_tier": "standard",
|
||||
"inference_geo": inference_geo,
|
||||
"speed": "standard",
|
||||
}
|
||||
|
||||
|
||||
def _azure_ai_claude_shape_case(
|
||||
sibling: CostTrackingTestCase,
|
||||
name: str,
|
||||
model: str,
|
||||
usage: dict[str, JsonValue],
|
||||
expected: ExactExpected,
|
||||
response_content: list[dict[str, JsonValue]] | None = None,
|
||||
adaptive_thinking: bool = False,
|
||||
) -> CostTrackingTestCase:
|
||||
response: Final = cast(JsonResponse, sibling.response)
|
||||
request: Final = (
|
||||
{
|
||||
**{
|
||||
key: value
|
||||
for key, value in sibling.request.items()
|
||||
if key not in {"reasoning_effort", "allowed_openai_params"}
|
||||
},
|
||||
"thinking": {"type": "adaptive"},
|
||||
"output_config": {"effort": "medium"},
|
||||
"allowed_openai_params": ["thinking", "output_config"],
|
||||
}
|
||||
if adaptive_thinking
|
||||
else sibling.request
|
||||
)
|
||||
body: Final = {
|
||||
**response.body,
|
||||
"model": model.rsplit("/", 1)[-1],
|
||||
"usage": usage,
|
||||
**({"content": response_content} if response_content is not None else {}),
|
||||
}
|
||||
return sibling.model_copy(
|
||||
update={
|
||||
"name": name,
|
||||
"model": model,
|
||||
"request": request,
|
||||
"response": response.model_copy(update={"body": body}),
|
||||
"expected": expected,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# nova-next
|
||||
NOVA_NEXT_TRANSCRIPTIONS_PER_SECOND: Final = CostTrackingTestCase(
|
||||
name="nova-next-transcriptions-per-second",
|
||||
|
|
@ -1457,6 +1537,7 @@ AZURE_AI_CLAUDE_FABLE_5_1_INPUT_TEXT: Final = CLAUDE_FABLE_5_1_INPUT_TEXT.model_
|
|||
"output_tokens_details": {"thinking_tokens": 0},
|
||||
"service_tier": "standard",
|
||||
"inference_geo": "not_available",
|
||||
"speed": "standard",
|
||||
},
|
||||
},
|
||||
),
|
||||
|
|
@ -1495,6 +1576,7 @@ AZURE_AI_CLAUDE_FABLE_5_1_STREAM: Final = CLAUDE_FABLE_5_1_STREAM.model_copy(
|
|||
"cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0},
|
||||
"service_tier": "standard",
|
||||
"inference_geo": "not_available",
|
||||
"speed": "standard",
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
@ -1582,6 +1664,7 @@ AZURE_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT: Final = CLAUDE_HAIKU_5_5_INPUT_TEXT.model_
|
|||
"output_tokens_details": {"thinking_tokens": 0},
|
||||
"service_tier": "standard",
|
||||
"inference_geo": "global",
|
||||
"speed": "standard",
|
||||
},
|
||||
"diagnostics": None,
|
||||
},
|
||||
|
|
@ -1624,6 +1707,7 @@ AZURE_AI_CLAUDE_HAIKU_5_5_STREAM: Final = CLAUDE_HAIKU_5_5_STREAM.model_copy(
|
|||
},
|
||||
"service_tier": "standard",
|
||||
"inference_geo": "global",
|
||||
"speed": "standard",
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
@ -1713,6 +1797,157 @@ PERPLEXITY_SONAR_NEXT_OFF_PEAK: Final = PERPLEXITY_SONAR_NEXT_NO_SEARCH.model_co
|
|||
}
|
||||
)
|
||||
|
||||
|
||||
AZURE_AI_CLAUDE_FABLE_5_1_CACHE_READ: Final = _azure_ai_claude_shape_case(
|
||||
CLAUDE_FABLE_5_1_CACHE_READ,
|
||||
"azure_ai/claude-fable-5-1-cache_read",
|
||||
"azure_ai/claude-fable-5-1",
|
||||
_azure_ai_claude_usage(input_tokens=14, output_tokens=6, inference_geo="not_available", cache_read_input_tokens=27003),
|
||||
ExactExpected(
|
||||
spend=0.012224275,
|
||||
input_cost=0.011714275,
|
||||
output_cost=0.00051,
|
||||
prompt_tokens=27017,
|
||||
completion_tokens=6,
|
||||
),
|
||||
)
|
||||
AZURE_AI_CLAUDE_FABLE_5_1_CACHE_WRITE_5M: Final = _azure_ai_claude_shape_case(
|
||||
CLAUDE_FABLE_5_1_CACHE_WRITE_5M,
|
||||
"azure_ai/claude-fable-5-1-cache_write_5m",
|
||||
"azure_ai/claude-fable-5-1",
|
||||
_azure_ai_claude_usage(
|
||||
input_tokens=14,
|
||||
output_tokens=5,
|
||||
inference_geo="not_available",
|
||||
cache_creation_input_tokens=27011,
|
||||
ephemeral_5m_input_tokens=27011,
|
||||
),
|
||||
ExactExpected(
|
||||
spend=0.57464675,
|
||||
input_cost=0.57422175,
|
||||
output_cost=0.000425,
|
||||
prompt_tokens=27025,
|
||||
completion_tokens=5,
|
||||
),
|
||||
)
|
||||
AZURE_AI_CLAUDE_FABLE_5_1_CACHE_WRITE_1H: Final = _azure_ai_claude_shape_case(
|
||||
CLAUDE_FABLE_5_1_CACHE_WRITE_1H,
|
||||
"azure_ai/claude-fable-5-1-cache_write_1h",
|
||||
"azure_ai/claude-fable-5-1",
|
||||
_azure_ai_claude_usage(
|
||||
input_tokens=14,
|
||||
output_tokens=5,
|
||||
inference_geo="not_available",
|
||||
cache_creation_input_tokens=27011,
|
||||
ephemeral_1h_input_tokens=27011,
|
||||
),
|
||||
ExactExpected(
|
||||
spend=0.919037,
|
||||
input_cost=0.918612,
|
||||
output_cost=0.000425,
|
||||
prompt_tokens=27025,
|
||||
completion_tokens=5,
|
||||
),
|
||||
)
|
||||
AZURE_AI_CLAUDE_FABLE_5_1_THINKING: Final = _azure_ai_claude_shape_case(
|
||||
CLAUDE_FABLE_5_1_THINKING,
|
||||
"azure_ai/claude-fable-5-1-thinking",
|
||||
"azure_ai/claude-fable-5-1",
|
||||
_azure_ai_claude_usage(input_tokens=18, output_tokens=99, inference_geo="not_available", thinking_tokens=33),
|
||||
ExactExpected(
|
||||
spend=0.008721, input_cost=0.000306, output_cost=0.008415, prompt_tokens=18, completion_tokens=99
|
||||
),
|
||||
[
|
||||
{"type": "thinking", "thinking": "scripted thought", "signature": "scripted signature"},
|
||||
{"type": "text", "text": "scripted answer for cost fixtures"},
|
||||
],
|
||||
adaptive_thinking=True,
|
||||
)
|
||||
AZURE_AI_CLAUDE_FABLE_5_1_TOOL_CALL: Final = _azure_ai_claude_shape_case(
|
||||
CLAUDE_FABLE_5_1_TOOL_CALL,
|
||||
"azure_ai/claude-fable-5-1-tool_call",
|
||||
"azure_ai/claude-fable-5-1",
|
||||
_azure_ai_claude_usage(input_tokens=385, output_tokens=50, inference_geo="not_available"),
|
||||
ExactExpected(
|
||||
spend=0.010795, input_cost=0.006545, output_cost=0.00425, prompt_tokens=385, completion_tokens=50
|
||||
),
|
||||
)
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_CACHE_READ: Final = _azure_ai_claude_shape_case(
|
||||
CLAUDE_HAIKU_5_5_CACHE_READ,
|
||||
"azure_ai/claude-haiku-5-5-cache_read",
|
||||
"azure_ai/claude-haiku-5-5",
|
||||
_azure_ai_claude_usage(input_tokens=14, output_tokens=5, inference_geo="global", cache_read_input_tokens=27003),
|
||||
ExactExpected(
|
||||
spend=0.000520467,
|
||||
input_cost=0.000515717,
|
||||
output_cost=0.00000475,
|
||||
prompt_tokens=27017,
|
||||
completion_tokens=5,
|
||||
),
|
||||
)
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_CACHE_WRITE_5M: Final = _azure_ai_claude_shape_case(
|
||||
CLAUDE_HAIKU_5_5_CACHE_WRITE_5M,
|
||||
"azure_ai/claude-haiku-5-5-cache_write_5m",
|
||||
"azure_ai/claude-haiku-5-5",
|
||||
_azure_ai_claude_usage(
|
||||
input_tokens=14,
|
||||
output_tokens=6,
|
||||
inference_geo="global",
|
||||
cache_creation_input_tokens=27011,
|
||||
ephemeral_5m_input_tokens=27011,
|
||||
),
|
||||
ExactExpected(
|
||||
spend=0.0064234725,
|
||||
input_cost=0.0064177725,
|
||||
output_cost=0.0000057,
|
||||
prompt_tokens=27025,
|
||||
completion_tokens=6,
|
||||
),
|
||||
)
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_CACHE_WRITE_1H: Final = _azure_ai_claude_shape_case(
|
||||
CLAUDE_HAIKU_5_5_CACHE_WRITE_1H,
|
||||
"azure_ai/claude-haiku-5-5-cache_write_1h",
|
||||
"azure_ai/claude-haiku-5-5",
|
||||
_azure_ai_claude_usage(
|
||||
input_tokens=14,
|
||||
output_tokens=6,
|
||||
inference_geo="global",
|
||||
cache_creation_input_tokens=27011,
|
||||
ephemeral_1h_input_tokens=27011,
|
||||
),
|
||||
ExactExpected(
|
||||
spend=0.01027254,
|
||||
input_cost=0.01026684,
|
||||
output_cost=0.0000057,
|
||||
prompt_tokens=27025,
|
||||
completion_tokens=6,
|
||||
),
|
||||
)
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_THINKING: Final = _azure_ai_claude_shape_case(
|
||||
CLAUDE_HAIKU_5_5_THINKING,
|
||||
"azure_ai/claude-haiku-5-5-thinking",
|
||||
"azure_ai/claude-haiku-5-5",
|
||||
_azure_ai_claude_usage(input_tokens=18, output_tokens=95, inference_geo="global", thinking_tokens=54),
|
||||
ExactExpected(
|
||||
spend=0.00009367, input_cost=0.00000342, output_cost=0.00009025, prompt_tokens=18, completion_tokens=95
|
||||
),
|
||||
[
|
||||
{"type": "thinking", "thinking": "scripted thought", "signature": "scripted signature"},
|
||||
{"type": "text", "text": "scripted answer for cost fixtures"},
|
||||
],
|
||||
adaptive_thinking=True,
|
||||
)
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_TOOL_CALL: Final = _azure_ai_claude_shape_case(
|
||||
CLAUDE_HAIKU_5_5_TOOL_CALL,
|
||||
"azure_ai/claude-haiku-5-5-tool_call",
|
||||
"azure_ai/claude-haiku-5-5",
|
||||
_azure_ai_claude_usage(input_tokens=511, output_tokens=34, inference_geo="global"),
|
||||
ExactExpected(
|
||||
spend=0.00012939, input_cost=0.00009709, output_cost=0.0000323, prompt_tokens=511, completion_tokens=34
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
||||
PERPLEXITY_SONAR_NEXT_OFF_PEAK,
|
||||
NOVA_NEXT_TRANSCRIPTIONS_PER_SECOND,
|
||||
|
|
@ -1764,8 +1999,18 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
XAI_GROK_4_7_INPUT_TEXT,
|
||||
AZURE_AI_CLAUDE_FABLE_5_1_INPUT_TEXT,
|
||||
AZURE_AI_CLAUDE_FABLE_5_1_STREAM,
|
||||
AZURE_AI_CLAUDE_FABLE_5_1_CACHE_READ,
|
||||
AZURE_AI_CLAUDE_FABLE_5_1_CACHE_WRITE_5M,
|
||||
AZURE_AI_CLAUDE_FABLE_5_1_CACHE_WRITE_1H,
|
||||
AZURE_AI_CLAUDE_FABLE_5_1_THINKING,
|
||||
AZURE_AI_CLAUDE_FABLE_5_1_TOOL_CALL,
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT,
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_STREAM,
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_CACHE_READ,
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_CACHE_WRITE_5M,
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_CACHE_WRITE_1H,
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_THINKING,
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_TOOL_CALL,
|
||||
)
|
||||
|
||||
PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
||||
|
|
|
|||
|
|
@ -26,6 +26,14 @@ from integration.cost_calculation.stream_parity.case import StreamParityTestCase
|
|||
from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs
|
||||
|
||||
|
||||
def _response_for_model(response: JsonResponse, model: str) -> JsonResponse:
|
||||
return response.model_copy(update={"body": {**response.body, "model": model}})
|
||||
|
||||
|
||||
def _stream_for_model(response: SseResponse, model: str) -> SseResponse:
|
||||
return response.model_copy(update={"frames": tuple(frame.replace("gpt-6-luna", model) for frame in response.frames)})
|
||||
|
||||
|
||||
# gpt-5.3-codex
|
||||
GPT_5_3_CODEX_INPUT_TEXT: Final = CostTrackingTestCase(
|
||||
name="gpt-5.3-codex-input_text",
|
||||
|
|
@ -11183,6 +11191,171 @@ GPT_6_ASTRA_CACHE_WRITE_5M: Final = with_response_usage(
|
|||
ExactExpected(spend=0.2148, input_cost=0.192, output_cost=0.0228, prompt_tokens=12928, completion_tokens=380),
|
||||
)
|
||||
|
||||
GPT_5_6_TERRA_INPUT_TEXT: Final = GPT_6_LUNA_INPUT_TEXT.model_copy(
|
||||
update={
|
||||
"name": "gpt-5.6-terra-input_text",
|
||||
"model": "gpt-5.6-terra",
|
||||
"response": _response_for_model(cast(JsonResponse, GPT_6_LUNA_INPUT_TEXT.response), "gpt-5.6-terra"),
|
||||
"expected": ExactExpected(
|
||||
spend=0.0112112,
|
||||
input_cost=0.004784,
|
||||
output_cost=0.0064272,
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
),
|
||||
}
|
||||
)
|
||||
GPT_5_6_TERRA_STREAM: Final = GPT_6_LUNA_STREAM.model_copy(
|
||||
update={
|
||||
"name": "gpt-5.6-terra-stream",
|
||||
"model": "gpt-5.6-terra",
|
||||
"response": _stream_for_model(cast(SseResponse, GPT_6_LUNA_STREAM.response), "gpt-5.6-terra"),
|
||||
"expected": ExactExpected(
|
||||
spend=0.0112112,
|
||||
input_cost=0.004784,
|
||||
output_cost=0.0064272,
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
),
|
||||
}
|
||||
)
|
||||
GPT_5_6_TERRA_SERVICE_TIER_FLEX: Final = GPT_6_LUNA_SERVICE_TIER_FLEX.model_copy(
|
||||
update={
|
||||
"name": "gpt-5.6-terra-service_tier_flex",
|
||||
"model": "gpt-5.6-terra",
|
||||
"response": _response_for_model(cast(JsonResponse, GPT_6_LUNA_SERVICE_TIER_FLEX.response), "gpt-5.6-terra"),
|
||||
"expected": ExactExpected(
|
||||
spend=0.0056056,
|
||||
input_cost=0.002392,
|
||||
output_cost=0.0032136,
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
),
|
||||
}
|
||||
)
|
||||
GPT_5_6_TERRA_SERVICE_TIER_PRIORITY: Final = GPT_6_LUNA_SERVICE_TIER_PRIORITY.model_copy(
|
||||
update={
|
||||
"name": "gpt-5.6-terra-service_tier_priority",
|
||||
"model": "gpt-5.6-terra",
|
||||
"response": _response_for_model(
|
||||
cast(JsonResponse, GPT_6_LUNA_SERVICE_TIER_PRIORITY.response), "gpt-5.6-terra"
|
||||
),
|
||||
"expected": ExactExpected(
|
||||
spend=0.0224224,
|
||||
input_cost=0.009568,
|
||||
output_cost=0.0128544,
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
),
|
||||
}
|
||||
)
|
||||
GPT_5_6_TERRA_RESPONSES_NATIVE_JSON: Final = GPT_6_LUNA_RESPONSES_NATIVE_JSON.model_copy(
|
||||
update={
|
||||
"name": "gpt-5.6-terra-responses_native_json",
|
||||
"model": "gpt-5.6-terra",
|
||||
"response": _response_for_model(
|
||||
cast(JsonResponse, GPT_6_LUNA_RESPONSES_NATIVE_JSON.response), "gpt-5.6-terra"
|
||||
),
|
||||
"expected": ExactExpected(
|
||||
spend=0.0112112,
|
||||
input_cost=0.004784,
|
||||
output_cost=0.0064272,
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
),
|
||||
}
|
||||
)
|
||||
GPT_5_6_TERRA_RESPONSES_DEPLOYMENT_JSON: Final = GPT_6_LUNA_RESPONSES_DEPLOYMENT_JSON.model_copy(
|
||||
update={
|
||||
"name": "gpt-5.6-terra-responses_deployment_json",
|
||||
"model": "gpt-5.6-terra",
|
||||
"deployment": Deployment(model="openai/responses/gpt-5.6-terra"),
|
||||
"response": _response_for_model(
|
||||
cast(JsonResponse, GPT_6_LUNA_RESPONSES_DEPLOYMENT_JSON.response), "gpt-5.6-terra"
|
||||
),
|
||||
"expected": ExactExpected(
|
||||
spend=0.0112112,
|
||||
input_cost=0.004784,
|
||||
output_cost=0.0064272,
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
),
|
||||
}
|
||||
)
|
||||
GPT_5_6_TERRA_CHAT_BRIDGED_JSON: Final = GPT_6_LUNA_CHAT_BRIDGED_JSON.model_copy(
|
||||
update={
|
||||
"name": "gpt-5.6-terra-chat_bridged_json",
|
||||
"model": "gpt-5.6-terra",
|
||||
"deployment": Deployment(model="openai/responses/gpt-5.6-terra"),
|
||||
"response": _response_for_model(cast(JsonResponse, GPT_6_LUNA_CHAT_BRIDGED_JSON.response), "gpt-5.6-terra"),
|
||||
"expected": ExactExpected(
|
||||
spend=0.0112112,
|
||||
input_cost=0.004784,
|
||||
output_cost=0.0064272,
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
),
|
||||
}
|
||||
)
|
||||
GPT_5_6_TERRA_CACHE_READ: Final = GPT_6_LUNA_CACHE_READ.model_copy(
|
||||
update={
|
||||
"name": "gpt-5.6-terra-cache_read",
|
||||
"model": "gpt-5.6-terra",
|
||||
"response": _response_for_model(cast(JsonResponse, GPT_6_LUNA_CACHE_READ.response), "gpt-5.6-terra"),
|
||||
"expected": ExactExpected(
|
||||
spend=0.01078688,
|
||||
input_cost=0.00485888,
|
||||
output_cost=0.005928,
|
||||
prompt_tokens=12928,
|
||||
completion_tokens=380,
|
||||
),
|
||||
}
|
||||
)
|
||||
GPT_5_6_TERRA_CACHE_WRITE_5M: Final = GPT_6_LUNA_CACHE_WRITE_5M.model_copy(
|
||||
update={
|
||||
"name": "gpt-5.6-terra-cache_write_5m",
|
||||
"model": "gpt-5.6-terra",
|
||||
"response": _response_for_model(
|
||||
cast(JsonResponse, GPT_6_LUNA_CACHE_WRITE_5M.response), "gpt-5.6-terra"
|
||||
),
|
||||
"expected": ExactExpected(
|
||||
spend=0.047528,
|
||||
input_cost=0.0416,
|
||||
output_cost=0.005928,
|
||||
prompt_tokens=12928,
|
||||
completion_tokens=380,
|
||||
),
|
||||
}
|
||||
)
|
||||
GPT_5_6_TERRA_REASONING: Final = GPT_6_LUNA_REASONING.model_copy(
|
||||
update={
|
||||
"name": "gpt-5.6-terra-reasoning",
|
||||
"model": "gpt-5.6-terra",
|
||||
"response": _response_for_model(cast(JsonResponse, GPT_6_LUNA_REASONING.response), "gpt-5.6-terra"),
|
||||
"expected": ExactExpected(
|
||||
spend=0.066248,
|
||||
input_cost=0.003224,
|
||||
output_cost=0.063024,
|
||||
prompt_tokens=1240,
|
||||
completion_tokens=4040,
|
||||
),
|
||||
}
|
||||
)
|
||||
GPT_5_6_TERRA_TOOL_CALL: Final = GPT_6_LUNA_TOOL_CALL.model_copy(
|
||||
update={
|
||||
"name": "gpt-5.6-terra-tool_call",
|
||||
"model": "gpt-5.6-terra",
|
||||
"response": _response_for_model(cast(JsonResponse, GPT_6_LUNA_TOOL_CALL.response), "gpt-5.6-terra"),
|
||||
"expected": ExactExpected(
|
||||
spend=0.0112112,
|
||||
input_cost=0.004784,
|
||||
output_cost=0.0064272,
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
GPT_5_6_SLASH_ALIAS: Final = GPT_5_6_INPUT_TEXT.model_copy(
|
||||
update={"name": "gpt-5.6-slash_alias", "alias": "openai/custom/gpt-5.6-$MARKER"}
|
||||
|
|
@ -11485,6 +11658,17 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
GPT_6_1_SOL_CACHE_WRITE_5M,
|
||||
GPT_6_LUNA_CACHE_WRITE_5M,
|
||||
GPT_6_ASTRA_CACHE_WRITE_5M,
|
||||
GPT_5_6_TERRA_INPUT_TEXT,
|
||||
GPT_5_6_TERRA_STREAM,
|
||||
GPT_5_6_TERRA_RESPONSES_NATIVE_JSON,
|
||||
GPT_5_6_TERRA_RESPONSES_DEPLOYMENT_JSON,
|
||||
GPT_5_6_TERRA_CHAT_BRIDGED_JSON,
|
||||
GPT_5_6_TERRA_CACHE_READ,
|
||||
GPT_5_6_TERRA_CACHE_WRITE_5M,
|
||||
GPT_5_6_TERRA_REASONING,
|
||||
GPT_5_6_TERRA_SERVICE_TIER_FLEX,
|
||||
GPT_5_6_TERRA_SERVICE_TIER_PRIORITY,
|
||||
GPT_5_6_TERRA_TOOL_CALL,
|
||||
)
|
||||
|
||||
PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
||||
|
|
@ -11513,6 +11697,7 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
|||
StreamParityTestCase(plain=GPT_6_LUNA_INPUT_TEXT, streamed=GPT_6_LUNA_STREAM),
|
||||
StreamParityTestCase(plain=GPT_6_ASTRA_INPUT_TEXT, streamed=GPT_6_ASTRA_STREAM),
|
||||
StreamParityTestCase(plain=GPT_6_1_SOL_INPUT_TEXT, streamed=GPT_6_1_SOL_STREAM),
|
||||
StreamParityTestCase(plain=GPT_5_6_TERRA_INPUT_TEXT, streamed=GPT_5_6_TERRA_STREAM),
|
||||
)
|
||||
|
||||
BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = (
|
||||
|
|
@ -11545,4 +11730,7 @@ BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = (
|
|||
BridgeCostParityTestCase(native=GPT_6_LUNA_RESPONSES_DEPLOYMENT_JSON, bridged=GPT_6_LUNA_CHAT_BRIDGED_JSON),
|
||||
BridgeCostParityTestCase(native=GPT_6_ASTRA_RESPONSES_DEPLOYMENT_JSON, bridged=GPT_6_ASTRA_CHAT_BRIDGED_JSON),
|
||||
BridgeCostParityTestCase(native=GPT_6_1_SOL_RESPONSES_DEPLOYMENT_JSON, bridged=GPT_6_1_SOL_CHAT_BRIDGED_JSON),
|
||||
BridgeCostParityTestCase(
|
||||
native=GPT_5_6_TERRA_RESPONSES_DEPLOYMENT_JSON, bridged=GPT_5_6_TERRA_CHAT_BRIDGED_JSON
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,15 +2,32 @@
|
|||
|
||||
PARITY pairs the streamed case with its plain twin where both bill the same row."""
|
||||
|
||||
from typing import Final
|
||||
from typing import Final, cast
|
||||
|
||||
from pydantic import JsonValue
|
||||
|
||||
from integration.cost_calculation.cases.anthropic import (
|
||||
CLAUDE_FABLE_5_1_CACHE_READ,
|
||||
CLAUDE_FABLE_5_1_CACHE_WRITE_1H,
|
||||
CLAUDE_FABLE_5_1_CACHE_WRITE_5M,
|
||||
CLAUDE_FABLE_5_1_INPUT_TEXT,
|
||||
CLAUDE_FABLE_5_1_STREAM,
|
||||
CLAUDE_FABLE_5_1_THINKING,
|
||||
CLAUDE_FABLE_5_1_TOOL_CALL,
|
||||
CLAUDE_OPUS_5_5_CACHE_READ,
|
||||
CLAUDE_OPUS_5_5_CACHE_WRITE_1H,
|
||||
CLAUDE_OPUS_5_5_CACHE_WRITE_5M,
|
||||
CLAUDE_OPUS_5_5_INPUT_TEXT,
|
||||
CLAUDE_OPUS_5_5_STREAM,
|
||||
CLAUDE_OPUS_5_5_THINKING,
|
||||
CLAUDE_OPUS_5_5_TOOL_CALL,
|
||||
CLAUDE_SONNET_5_5_CACHE_READ,
|
||||
CLAUDE_SONNET_5_5_CACHE_WRITE_1H,
|
||||
CLAUDE_SONNET_5_5_CACHE_WRITE_5M,
|
||||
CLAUDE_SONNET_5_5_INPUT_TEXT,
|
||||
CLAUDE_SONNET_5_5_STREAM,
|
||||
CLAUDE_SONNET_5_5_THINKING,
|
||||
CLAUDE_SONNET_5_5_TOOL_CALL,
|
||||
)
|
||||
from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs
|
||||
from integration.cost_calculation.cost_tracking_case import (
|
||||
|
|
@ -24,6 +41,99 @@ from integration.cost_calculation.cost_tracking_case import (
|
|||
from integration.cost_calculation.stream_parity.case import StreamParityTestCase, sse_frames
|
||||
|
||||
|
||||
def _gemini_case_response(response: JsonResponse | SseResponse, model_version: str) -> JsonResponse | SseResponse:
|
||||
match response:
|
||||
case JsonResponse():
|
||||
return response.model_copy(update={"body": {**response.body, "modelVersion": model_version}})
|
||||
case SseResponse():
|
||||
return response.model_copy(
|
||||
update={
|
||||
"frames": tuple(frame.replace("gemini-3.8-flash", model_version) for frame in response.frames)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _gemini_case(
|
||||
sibling: CostTrackingTestCase, name: str, model: str, expected: ExactExpected
|
||||
) -> CostTrackingTestCase:
|
||||
return sibling.model_copy(
|
||||
update={
|
||||
"name": name,
|
||||
"model": model,
|
||||
"response": _gemini_case_response(cast(JsonResponse | SseResponse, sibling.response), "gemini-3.1-pro-preview"),
|
||||
"expected": expected,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _vertex_claude_usage(
|
||||
input_tokens: int,
|
||||
output_tokens: int,
|
||||
*,
|
||||
cache_creation_input_tokens: int = 0,
|
||||
cache_read_input_tokens: int = 0,
|
||||
ephemeral_5m_input_tokens: int = 0,
|
||||
ephemeral_1h_input_tokens: int = 0,
|
||||
thinking_tokens: int = 0,
|
||||
) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"input_tokens": input_tokens,
|
||||
"cache_creation_input_tokens": cache_creation_input_tokens,
|
||||
"cache_read_input_tokens": cache_read_input_tokens,
|
||||
"cache_creation": {
|
||||
"ephemeral_5m_input_tokens": ephemeral_5m_input_tokens,
|
||||
"ephemeral_1h_input_tokens": ephemeral_1h_input_tokens,
|
||||
},
|
||||
"output_tokens": output_tokens,
|
||||
"output_tokens_details": {"thinking_tokens": thinking_tokens},
|
||||
}
|
||||
|
||||
|
||||
def _vertex_claude_shape_case(
|
||||
sibling: CostTrackingTestCase,
|
||||
name: str,
|
||||
model: str,
|
||||
usage: dict[str, JsonValue],
|
||||
expected: ExactExpected,
|
||||
response_content: list[dict[str, JsonValue]] | None = None,
|
||||
adaptive_thinking: bool = False,
|
||||
) -> CostTrackingTestCase:
|
||||
response: Final = cast(JsonResponse, sibling.response)
|
||||
request: Final = (
|
||||
{
|
||||
**{
|
||||
key: value
|
||||
for key, value in sibling.request.items()
|
||||
if key not in {"reasoning_effort", "allowed_openai_params"}
|
||||
},
|
||||
"thinking": {"type": "adaptive"},
|
||||
"output_config": {"effort": "medium"},
|
||||
"allowed_openai_params": ["thinking", "output_config"],
|
||||
}
|
||||
if adaptive_thinking
|
||||
else sibling.request
|
||||
)
|
||||
body: Final = {
|
||||
key: value
|
||||
for key, value in {
|
||||
**response.body,
|
||||
"model": model.rsplit("/", 1)[-1],
|
||||
"usage": usage,
|
||||
**({"content": response_content} if response_content is not None else {}),
|
||||
}.items()
|
||||
if key != "diagnostics"
|
||||
}
|
||||
return sibling.model_copy(
|
||||
update={
|
||||
"name": name,
|
||||
"model": model,
|
||||
"request": request,
|
||||
"response": response.model_copy(update={"body": body}),
|
||||
"expected": expected,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _vertex_claude_stream_response(model: str, stop_reason: str, text_deltas: tuple[str, ...]) -> SseResponse:
|
||||
text_frames: Final = (
|
||||
(
|
||||
|
|
@ -3081,6 +3191,191 @@ VERTEX_AI_CLAUDE_FABLE_5_1_STREAM: Final = CLAUDE_FABLE_5_1_STREAM.model_copy(
|
|||
}
|
||||
)
|
||||
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_CACHE_READ: Final = _vertex_claude_shape_case(
|
||||
CLAUDE_SONNET_5_5_CACHE_READ,
|
||||
"vertex_ai/claude-sonnet-5-5-cache_read",
|
||||
"vertex_ai/claude-sonnet-5-5",
|
||||
_vertex_claude_usage(input_tokens=14, output_tokens=6, cache_read_input_tokens=27003),
|
||||
ExactExpected(
|
||||
spend=0.00474011, input_cost=0.00463811, output_cost=0.000102, prompt_tokens=27017, completion_tokens=6
|
||||
),
|
||||
)
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_CACHE_WRITE_5M: Final = _vertex_claude_shape_case(
|
||||
CLAUDE_SONNET_5_5_CACHE_WRITE_5M,
|
||||
"vertex_ai/claude-sonnet-5-5-cache_write_5m",
|
||||
"vertex_ai/claude-sonnet-5-5",
|
||||
_vertex_claude_usage(
|
||||
input_tokens=14, output_tokens=6, cache_creation_input_tokens=27011, ephemeral_5m_input_tokens=27011
|
||||
),
|
||||
ExactExpected(
|
||||
spend=0.11494635, input_cost=0.11484435, output_cost=0.000102, prompt_tokens=27025, completion_tokens=6
|
||||
),
|
||||
)
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_CACHE_WRITE_1H: Final = _vertex_claude_shape_case(
|
||||
CLAUDE_SONNET_5_5_CACHE_WRITE_1H,
|
||||
"vertex_ai/claude-sonnet-5-5-cache_write_1h",
|
||||
"vertex_ai/claude-sonnet-5-5",
|
||||
_vertex_claude_usage(
|
||||
input_tokens=14, output_tokens=6, cache_creation_input_tokens=27011, ephemeral_1h_input_tokens=27011
|
||||
),
|
||||
ExactExpected(
|
||||
spend=0.1838244, input_cost=0.1837224, output_cost=0.000102, prompt_tokens=27025, completion_tokens=6
|
||||
),
|
||||
)
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_THINKING: Final = _vertex_claude_shape_case(
|
||||
CLAUDE_SONNET_5_5_THINKING,
|
||||
"vertex_ai/claude-sonnet-5-5-thinking",
|
||||
"vertex_ai/claude-sonnet-5-5",
|
||||
_vertex_claude_usage(input_tokens=18, output_tokens=144, thinking_tokens=37),
|
||||
ExactExpected(
|
||||
spend=0.0025092, input_cost=0.0000612, output_cost=0.002448, prompt_tokens=18, completion_tokens=144
|
||||
),
|
||||
[
|
||||
{"type": "thinking", "thinking": "scripted thought", "signature": "scripted signature"},
|
||||
{"type": "text", "text": "scripted answer for cost fixtures"},
|
||||
],
|
||||
adaptive_thinking=True,
|
||||
)
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_TOOL_CALL: Final = _vertex_claude_shape_case(
|
||||
CLAUDE_SONNET_5_5_TOOL_CALL,
|
||||
"vertex_ai/claude-sonnet-5-5-tool_call",
|
||||
"vertex_ai/claude-sonnet-5-5",
|
||||
_vertex_claude_usage(input_tokens=385, output_tokens=50),
|
||||
ExactExpected(
|
||||
spend=0.002159, input_cost=0.001309, output_cost=0.00085, prompt_tokens=385, completion_tokens=50
|
||||
),
|
||||
)
|
||||
VERTEX_AI_CLAUDE_OPUS_5_5_CACHE_READ: Final = _vertex_claude_shape_case(
|
||||
CLAUDE_OPUS_5_5_CACHE_READ,
|
||||
"vertex_ai/claude-opus-5-5-cache_read",
|
||||
"vertex_ai/claude-opus-5-5",
|
||||
_vertex_claude_usage(input_tokens=14, output_tokens=6, cache_read_input_tokens=27003),
|
||||
ExactExpected(
|
||||
spend=0.01003788, input_cost=0.00982188, output_cost=0.000216, prompt_tokens=27017, completion_tokens=6
|
||||
),
|
||||
)
|
||||
VERTEX_AI_CLAUDE_OPUS_5_5_CACHE_WRITE_5M: Final = _vertex_claude_shape_case(
|
||||
CLAUDE_OPUS_5_5_CACHE_WRITE_5M,
|
||||
"vertex_ai/claude-opus-5-5-cache_write_5m",
|
||||
"vertex_ai/claude-opus-5-5",
|
||||
_vertex_claude_usage(
|
||||
input_tokens=14, output_tokens=6, cache_creation_input_tokens=27011, ephemeral_5m_input_tokens=27011
|
||||
),
|
||||
ExactExpected(
|
||||
spend=0.2434158, input_cost=0.2431998, output_cost=0.000216, prompt_tokens=27025, completion_tokens=6
|
||||
),
|
||||
)
|
||||
VERTEX_AI_CLAUDE_OPUS_5_5_CACHE_WRITE_1H: Final = _vertex_claude_shape_case(
|
||||
CLAUDE_OPUS_5_5_CACHE_WRITE_1H,
|
||||
"vertex_ai/claude-opus-5-5-cache_write_1h",
|
||||
"vertex_ai/claude-opus-5-5",
|
||||
_vertex_claude_usage(
|
||||
input_tokens=14, output_tokens=6, cache_creation_input_tokens=27011, ephemeral_1h_input_tokens=27011
|
||||
),
|
||||
ExactExpected(
|
||||
spend=0.3892752, input_cost=0.3890592, output_cost=0.000216, prompt_tokens=27025, completion_tokens=6
|
||||
),
|
||||
)
|
||||
VERTEX_AI_CLAUDE_OPUS_5_5_THINKING: Final = _vertex_claude_shape_case(
|
||||
CLAUDE_OPUS_5_5_THINKING,
|
||||
"vertex_ai/claude-opus-5-5-thinking",
|
||||
"vertex_ai/claude-opus-5-5",
|
||||
_vertex_claude_usage(input_tokens=18, output_tokens=162, thinking_tokens=66),
|
||||
ExactExpected(
|
||||
spend=0.0059616, input_cost=0.0001296, output_cost=0.005832, prompt_tokens=18, completion_tokens=162
|
||||
),
|
||||
[
|
||||
{"type": "thinking", "thinking": "scripted thought", "signature": "scripted signature"},
|
||||
{"type": "text", "text": "scripted answer for cost fixtures"},
|
||||
],
|
||||
adaptive_thinking=True,
|
||||
)
|
||||
VERTEX_AI_CLAUDE_OPUS_5_5_TOOL_CALL: Final = _vertex_claude_shape_case(
|
||||
CLAUDE_OPUS_5_5_TOOL_CALL,
|
||||
"vertex_ai/claude-opus-5-5-tool_call",
|
||||
"vertex_ai/claude-opus-5-5",
|
||||
_vertex_claude_usage(input_tokens=385, output_tokens=50),
|
||||
ExactExpected(
|
||||
spend=0.004572, input_cost=0.002772, output_cost=0.0018, prompt_tokens=385, completion_tokens=50
|
||||
),
|
||||
)
|
||||
VERTEX_AI_CLAUDE_FABLE_5_1_CACHE_READ: Final = _vertex_claude_shape_case(
|
||||
CLAUDE_FABLE_5_1_CACHE_READ,
|
||||
"vertex_ai/claude-fable-5-1-cache_read",
|
||||
"vertex_ai/claude-fable-5-1",
|
||||
_vertex_claude_usage(input_tokens=14, output_tokens=5, cache_read_input_tokens=27003),
|
||||
ExactExpected(
|
||||
spend=0.013567425, input_cost=0.013092425, output_cost=0.000475, prompt_tokens=27017, completion_tokens=5
|
||||
),
|
||||
)
|
||||
VERTEX_AI_CLAUDE_FABLE_5_1_CACHE_WRITE_5M: Final = _vertex_claude_shape_case(
|
||||
CLAUDE_FABLE_5_1_CACHE_WRITE_5M,
|
||||
"vertex_ai/claude-fable-5-1-cache_write_5m",
|
||||
"vertex_ai/claude-fable-5-1",
|
||||
_vertex_claude_usage(
|
||||
input_tokens=14, output_tokens=5, cache_creation_input_tokens=27011, ephemeral_5m_input_tokens=27011
|
||||
),
|
||||
ExactExpected(
|
||||
spend=0.64225225, input_cost=0.64177725, output_cost=0.000475, prompt_tokens=27025, completion_tokens=5
|
||||
),
|
||||
)
|
||||
VERTEX_AI_CLAUDE_FABLE_5_1_CACHE_WRITE_1H: Final = _vertex_claude_shape_case(
|
||||
CLAUDE_FABLE_5_1_CACHE_WRITE_1H,
|
||||
"vertex_ai/claude-fable-5-1-cache_write_1h",
|
||||
"vertex_ai/claude-fable-5-1",
|
||||
_vertex_claude_usage(
|
||||
input_tokens=14, output_tokens=5, cache_creation_input_tokens=27011, ephemeral_1h_input_tokens=27011
|
||||
),
|
||||
ExactExpected(
|
||||
spend=1.027159, input_cost=1.026684, output_cost=0.000475, prompt_tokens=27025, completion_tokens=5
|
||||
),
|
||||
)
|
||||
VERTEX_AI_CLAUDE_FABLE_5_1_THINKING: Final = _vertex_claude_shape_case(
|
||||
CLAUDE_FABLE_5_1_THINKING,
|
||||
"vertex_ai/claude-fable-5-1-thinking",
|
||||
"vertex_ai/claude-fable-5-1",
|
||||
_vertex_claude_usage(input_tokens=18, output_tokens=98, thinking_tokens=33),
|
||||
ExactExpected(
|
||||
spend=0.009652, input_cost=0.000342, output_cost=0.00931, prompt_tokens=18, completion_tokens=98
|
||||
),
|
||||
[
|
||||
{"type": "thinking", "thinking": "scripted thought", "signature": "scripted signature"},
|
||||
{"type": "text", "text": "scripted answer for cost fixtures"},
|
||||
],
|
||||
adaptive_thinking=True,
|
||||
)
|
||||
VERTEX_AI_CLAUDE_FABLE_5_1_TOOL_CALL: Final = _vertex_claude_shape_case(
|
||||
CLAUDE_FABLE_5_1_TOOL_CALL,
|
||||
"vertex_ai/claude-fable-5-1-tool_call",
|
||||
"vertex_ai/claude-fable-5-1",
|
||||
_vertex_claude_usage(input_tokens=385, output_tokens=50),
|
||||
ExactExpected(
|
||||
spend=0.012065, input_cost=0.007315, output_cost=0.00475, prompt_tokens=385, completion_tokens=50
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT: Final = _gemini_case(
|
||||
GEMINI_3_8_FLASH_INPUT_TEXT, "gemini-3.1-pro-preview-input_text", "gemini-3.1-pro-preview",
|
||||
ExactExpected(spend=0.0094864, input_cost=0.004048, output_cost=0.0054384, prompt_tokens=1840, completion_tokens=412)
|
||||
)
|
||||
GEMINI_3_1_PRO_PREVIEW_STREAM: Final = _gemini_case(
|
||||
GEMINI_3_8_FLASH_STREAM, "gemini-3.1-pro-preview-stream", "gemini-3.1-pro-preview",
|
||||
ExactExpected(spend=0.0094864, input_cost=0.004048, output_cost=0.0054384, prompt_tokens=1840, completion_tokens=412)
|
||||
)
|
||||
GEMINI_3_1_PRO_PREVIEW_CACHE_READ: Final = _gemini_case(
|
||||
GEMINI_3_8_FLASH_CACHE_READ, "gemini-3.1-pro-preview-cache_read", "gemini-3.1-pro-preview",
|
||||
ExactExpected(spend=0.00912736, input_cost=0.00411136, output_cost=0.005016, prompt_tokens=12928, completion_tokens=380)
|
||||
)
|
||||
GEMINI_3_1_PRO_PREVIEW_REASONING: Final = _gemini_case(
|
||||
GEMINI_3_8_FLASH_FALLBACK_REASONING_AT_OUTPUT_RATE, "gemini-3.1-pro-preview-reasoning", "gemini-3.1-pro-preview",
|
||||
ExactExpected(spend=0.056056, input_cost=0.002728, output_cost=0.053328, prompt_tokens=1240, completion_tokens=4040)
|
||||
)
|
||||
GEMINI_3_1_PRO_PREVIEW_TOOL_CALL: Final = _gemini_case(
|
||||
GEMINI_3_8_FLASH_TOOL_CALL, "gemini-3.1-pro-preview-tool_call", "gemini-3.1-pro-preview",
|
||||
ExactExpected(spend=0.0094864, input_cost=0.004048, output_cost=0.0054384, prompt_tokens=1840, completion_tokens=412)
|
||||
)
|
||||
|
||||
|
||||
CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
||||
GEMINI_3_1_PRO_INPUT_TEXT,
|
||||
|
|
@ -3137,6 +3432,26 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
VERTEX_AI_CLAUDE_OPUS_5_5_STREAM,
|
||||
VERTEX_AI_CLAUDE_FABLE_5_1_INPUT_TEXT,
|
||||
VERTEX_AI_CLAUDE_FABLE_5_1_STREAM,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_CACHE_READ,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_CACHE_WRITE_5M,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_CACHE_WRITE_1H,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_THINKING,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_TOOL_CALL,
|
||||
VERTEX_AI_CLAUDE_OPUS_5_5_CACHE_READ,
|
||||
VERTEX_AI_CLAUDE_OPUS_5_5_CACHE_WRITE_5M,
|
||||
VERTEX_AI_CLAUDE_OPUS_5_5_CACHE_WRITE_1H,
|
||||
VERTEX_AI_CLAUDE_OPUS_5_5_THINKING,
|
||||
VERTEX_AI_CLAUDE_OPUS_5_5_TOOL_CALL,
|
||||
VERTEX_AI_CLAUDE_FABLE_5_1_CACHE_READ,
|
||||
VERTEX_AI_CLAUDE_FABLE_5_1_CACHE_WRITE_5M,
|
||||
VERTEX_AI_CLAUDE_FABLE_5_1_CACHE_WRITE_1H,
|
||||
VERTEX_AI_CLAUDE_FABLE_5_1_THINKING,
|
||||
VERTEX_AI_CLAUDE_FABLE_5_1_TOOL_CALL,
|
||||
GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT,
|
||||
GEMINI_3_1_PRO_PREVIEW_STREAM,
|
||||
GEMINI_3_1_PRO_PREVIEW_CACHE_READ,
|
||||
GEMINI_3_1_PRO_PREVIEW_REASONING,
|
||||
GEMINI_3_1_PRO_PREVIEW_TOOL_CALL,
|
||||
)
|
||||
|
||||
PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
||||
|
|
@ -3157,9 +3472,13 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
|||
),
|
||||
StreamParityTestCase(plain=VERTEX_AI_CLAUDE_OPUS_5_5_INPUT_TEXT, streamed=VERTEX_AI_CLAUDE_OPUS_5_5_STREAM),
|
||||
StreamParityTestCase(plain=VERTEX_AI_CLAUDE_FABLE_5_1_INPUT_TEXT, streamed=VERTEX_AI_CLAUDE_FABLE_5_1_STREAM),
|
||||
StreamParityTestCase(plain=GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT, streamed=GEMINI_3_1_PRO_PREVIEW_STREAM),
|
||||
)
|
||||
|
||||
BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = bridge_pairs(
|
||||
(GEMINI_3_8_FLASH_INPUT_TEXT, GEMINI_3_8_FLASH_CACHE_READ, GEMINI_3_8_FLASH_STREAM),
|
||||
("/v1/responses", "/v1/messages"),
|
||||
BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = (
|
||||
*bridge_pairs(
|
||||
(GEMINI_3_8_FLASH_INPUT_TEXT, GEMINI_3_8_FLASH_CACHE_READ, GEMINI_3_8_FLASH_STREAM),
|
||||
("/v1/responses", "/v1/messages"),
|
||||
),
|
||||
*bridge_pairs((GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT,), ("/v1/responses",)),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -32,6 +32,41 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = {
|
|||
"input_cost_per_token": 3.2e-06,
|
||||
"output_cost_per_token": 1.6e-05,
|
||||
},
|
||||
"gpt-5.6-terra": {
|
||||
"cache_creation_input_token_cost": 3.25e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 6.5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 3.25e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 1.3e-05,
|
||||
"cache_creation_input_token_cost_flex": 1.625e-06,
|
||||
"cache_creation_input_token_cost_priority": 6.5e-06,
|
||||
"cache_read_input_token_cost": 2.6e-07,
|
||||
"cache_read_input_token_cost_above_272k_tokens": 5.2e-07,
|
||||
"cache_read_input_token_cost_above_272k_tokens_flex": 2.6e-07,
|
||||
"cache_read_input_token_cost_above_272k_tokens_priority": 1.04e-06,
|
||||
"cache_read_input_token_cost_flex": 1.3e-07,
|
||||
"cache_read_input_token_cost_priority": 5.2e-07,
|
||||
"input_cost_per_token": 2.6e-06,
|
||||
"input_cost_per_token_above_272k_tokens": 5.2e-06,
|
||||
"input_cost_per_token_above_272k_tokens_flex": 2.6e-06,
|
||||
"input_cost_per_token_above_272k_tokens_priority": 1.04e-05,
|
||||
"input_cost_per_token_flex": 1.3e-06,
|
||||
"input_cost_per_token_priority": 5.2e-06,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 922000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.56e-05,
|
||||
"output_cost_per_token_above_272k_tokens": 2.34e-05,
|
||||
"output_cost_per_token_above_272k_tokens_flex": 1.17e-05,
|
||||
"output_cost_per_token_above_272k_tokens_priority": 4.68e-05,
|
||||
"output_cost_per_token_flex": 7.8e-06,
|
||||
"output_cost_per_token_priority": 3.12e-05,
|
||||
"supports_function_calling": True,
|
||||
"supports_none_reasoning_effort": True,
|
||||
"supports_prompt_caching": True,
|
||||
"supports_reasoning": True,
|
||||
},
|
||||
"gpt-6-luna": {
|
||||
"output_cost_per_token_priority": 1e-06,
|
||||
"input_cost_per_token_priority": 2e-07,
|
||||
|
|
@ -407,8 +442,55 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = {
|
|||
"supports_function_calling": True,
|
||||
"web_search_billing_unit": "per_query",
|
||||
},
|
||||
"gemini/gemini-4-argon": {
|
||||
"cache_read_input_token_cost": 1.1e-07,
|
||||
"cache_read_input_token_cost_flex": 5.5e-08,
|
||||
"cache_read_input_token_cost_priority": 1.98e-07,
|
||||
"input_cost_per_token": 2.2e-06,
|
||||
"input_cost_per_token_flex": 1.1e-06,
|
||||
"input_cost_per_token_priority": 3.96e-06,
|
||||
"litellm_provider": "gemini",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 1048576,
|
||||
"max_tokens": 1048576,
|
||||
"mode": "chat",
|
||||
"output_cost_per_reasoning_token": 1.1e-05,
|
||||
"output_cost_per_token": 1.1e-05,
|
||||
"output_cost_per_token_flex": 5.5e-06,
|
||||
"output_cost_per_token_priority": 1.98e-05,
|
||||
"supports_function_calling": True,
|
||||
"supports_prompt_caching": True,
|
||||
"supports_reasoning": True,
|
||||
},
|
||||
"gemini/gemini-3.1-pro-preview": {
|
||||
"cache_read_input_token_cost": 2.6e-07,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 5.2e-07,
|
||||
"input_cost_per_token": 2.6e-06,
|
||||
"input_cost_per_token_above_200k_tokens": 5.2e-06,
|
||||
"litellm_provider": "gemini",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 65536,
|
||||
"max_tokens": 65536,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.56e-05,
|
||||
"output_cost_per_token_above_200k_tokens": 2.34e-05,
|
||||
"supports_function_calling": True,
|
||||
"supports_prompt_caching": True,
|
||||
"supports_reasoning": True,
|
||||
"input_cost_per_token_priority": 4.68e-06,
|
||||
"input_cost_per_token_above_200k_tokens_priority": 9.36e-06,
|
||||
"output_cost_per_token_priority": 2.808e-05,
|
||||
"output_cost_per_token_above_200k_tokens_priority": 4.212e-05,
|
||||
"cache_read_input_token_cost_priority": 4.68e-07,
|
||||
"cache_read_input_token_cost_above_200k_tokens_priority": 9.36e-07,
|
||||
"cache_read_input_token_cost_flex": 2.6e-07,
|
||||
"input_cost_per_token_flex": 1.3e-06,
|
||||
"output_cost_per_token_flex": 7.8e-06,
|
||||
},
|
||||
"gemini/gemini-3.8-flash": {
|
||||
"cache_read_input_token_cost": 5e-08,
|
||||
"cache_read_input_token_cost_flex": 2.5e-08,
|
||||
"cache_read_input_token_cost_priority": 9e-08,
|
||||
"google_maps_grounding_cost_per_query": 0.025,
|
||||
"input_cost_per_audio_token": 1e-06,
|
||||
"input_cost_per_image_token": 5.5e-07,
|
||||
|
|
@ -454,8 +536,36 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = {
|
|||
"supports_function_calling": True,
|
||||
"web_search_billing_unit": "per_query",
|
||||
},
|
||||
"gemini-3.1-pro-preview": {
|
||||
"cache_creation_input_token_cost_above_200k_tokens": 2.75e-07,
|
||||
"cache_read_input_token_cost": 2.2e-07,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 4.4e-07,
|
||||
"cache_read_input_token_cost_above_200k_tokens_priority": 7.92e-07,
|
||||
"cache_read_input_token_cost_priority": 3.96e-07,
|
||||
"input_cost_per_token": 2.2e-06,
|
||||
"input_cost_per_token_above_200k_tokens": 4.4e-06,
|
||||
"input_cost_per_token_above_200k_tokens_priority": 7.92e-06,
|
||||
"input_cost_per_token_flex": 1.1e-06,
|
||||
"input_cost_per_token_priority": 3.96e-06,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 65536,
|
||||
"max_tokens": 65536,
|
||||
"mode": "chat",
|
||||
"output_cost_per_image": 0.000132,
|
||||
"output_cost_per_token": 1.32e-05,
|
||||
"output_cost_per_token_above_200k_tokens": 1.98e-05,
|
||||
"output_cost_per_token_above_200k_tokens_priority": 3.564e-05,
|
||||
"output_cost_per_token_flex": 6.6e-06,
|
||||
"output_cost_per_token_priority": 2.376e-05,
|
||||
"supports_function_calling": True,
|
||||
"supports_prompt_caching": True,
|
||||
"supports_reasoning": True,
|
||||
},
|
||||
"gemini-3.8-flash": {
|
||||
"cache_read_input_token_cost": 5.2e-08,
|
||||
"cache_read_input_token_cost_flex": 2.6e-08,
|
||||
"cache_read_input_token_cost_priority": 9.36e-08,
|
||||
"google_maps_grounding_cost_per_query": 0.025,
|
||||
"input_cost_per_audio_token": 1.04e-06,
|
||||
"input_cost_per_token": 5.2e-07,
|
||||
|
|
@ -1259,6 +1369,41 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = {
|
|||
"output_cost_per_token_above_272k_tokens_priority": 4.6e-05,
|
||||
"cache_read_input_token_cost_above_272k_tokens": 4.6e-07,
|
||||
},
|
||||
"azure/gpt-5.6-terra": {
|
||||
"cache_creation_input_token_cost": 2.75e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 5.5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.75e-06,
|
||||
"cache_creation_input_token_cost_priority": 5.5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 1.1e-05,
|
||||
"cache_creation_input_token_cost_flex": 1.375e-06,
|
||||
"cache_read_input_token_cost": 2.2e-07,
|
||||
"cache_read_input_token_cost_above_272k_tokens": 4.4e-07,
|
||||
"cache_read_input_token_cost_above_272k_tokens_flex": 2.2e-07,
|
||||
"cache_read_input_token_cost_priority": 4.4e-07,
|
||||
"cache_read_input_token_cost_above_272k_tokens_priority": 8.8e-07,
|
||||
"cache_read_input_token_cost_flex": 1.1e-07,
|
||||
"input_cost_per_token": 2.2e-06,
|
||||
"input_cost_per_token_above_272k_tokens": 4.4e-06,
|
||||
"input_cost_per_token_above_272k_tokens_flex": 2.2e-06,
|
||||
"input_cost_per_token_priority": 4.4e-06,
|
||||
"input_cost_per_token_above_272k_tokens_priority": 8.8e-06,
|
||||
"input_cost_per_token_flex": 1.1e-06,
|
||||
"litellm_provider": "azure",
|
||||
"max_input_tokens": 922000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.32e-05,
|
||||
"output_cost_per_token_above_272k_tokens": 1.98e-05,
|
||||
"output_cost_per_token_above_272k_tokens_flex": 9.9e-06,
|
||||
"output_cost_per_token_priority": 2.64e-05,
|
||||
"output_cost_per_token_above_272k_tokens_priority": 3.96e-05,
|
||||
"output_cost_per_token_flex": 6.6e-06,
|
||||
"supports_function_calling": True,
|
||||
"supports_prompt_caching": True,
|
||||
"supports_reasoning": True,
|
||||
"supports_none_reasoning_effort": True,
|
||||
},
|
||||
"azure/gpt-6-luna": {
|
||||
"cache_read_input_token_cost": 1.3e-08,
|
||||
"cache_creation_input_token_cost": 1.625e-07,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue