mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-18 04:44:37 +00:00
test(cost): add tier 1 cost tracking cells and sync usage shapes for 2026-10-17 (#47141)
* test(cost): add tier 1 cost tracking cells and sync usage shapes for 2026-10-17 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(cost): restore tiered cost cases and revert out of scope usage edits Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(cost): add remaining tier 1 endpoint cells for 2026-10-17 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
836f326611
commit
cd20ecace1
9 changed files with 1601 additions and 38 deletions
|
|
@ -6899,6 +6899,35 @@ CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_JSON: Final = CLAUDE_OPUS_5_5_MESSAGES_INPUT_T
|
|||
}
|
||||
)
|
||||
|
||||
def _responses_bridged_stream_case(
|
||||
sibling: CostTrackingTestCase, stream_sibling: CostTrackingTestCase, name: str
|
||||
) -> CostTrackingTestCase:
|
||||
return sibling.model_copy(
|
||||
update={
|
||||
"name": name,
|
||||
"request": {**sibling.request, "stream": True},
|
||||
"response": stream_sibling.response,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_STREAM: Final = _responses_bridged_stream_case(
|
||||
CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_JSON, CLAUDE_FABLE_5_1_MESSAGES_STREAM,
|
||||
"claude-fable-5-1-responses_bridged_stream",
|
||||
)
|
||||
CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_STREAM: Final = _responses_bridged_stream_case(
|
||||
CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_JSON, CLAUDE_HAIKU_5_5_MESSAGES_STREAM,
|
||||
"claude-haiku-5-5-responses_bridged_stream",
|
||||
)
|
||||
CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_STREAM: Final = _responses_bridged_stream_case(
|
||||
CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_JSON, CLAUDE_OPUS_5_5_MESSAGES_STREAM,
|
||||
"claude-opus-5-5-responses_bridged_stream",
|
||||
)
|
||||
CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_STREAM: Final = _responses_bridged_stream_case(
|
||||
CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_JSON, CLAUDE_SONNET_5_5_MESSAGES_STREAM,
|
||||
"claude-sonnet-5-5-responses_bridged_stream",
|
||||
)
|
||||
|
||||
CLAUDE_HAIKU_5_5_TIERED_INPUT_ABOVE_100K: Final = with_usage(
|
||||
CLAUDE_HAIKU_5_5_INPUT_TEXT,
|
||||
"claude-haiku-5-5-tiered_input_above_100k",
|
||||
|
|
@ -6994,6 +7023,10 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
CLAUDE_SONNET_5_RESPONSES_BRIDGED_JSON,
|
||||
CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_JSON,
|
||||
CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_JSON,
|
||||
CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_STREAM,
|
||||
CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_STREAM,
|
||||
CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_STREAM,
|
||||
CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_STREAM,
|
||||
CLAUDE_FABLE_5_RESPONSES_BRIDGED_JSON,
|
||||
CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_JSON,
|
||||
CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_JSON,
|
||||
|
|
@ -7119,6 +7152,18 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
)
|
||||
|
||||
PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
||||
StreamParityTestCase(
|
||||
plain=CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_JSON, streamed=CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_STREAM
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_JSON, streamed=CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_STREAM
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_JSON, streamed=CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_STREAM
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_JSON, streamed=CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_STREAM
|
||||
),
|
||||
StreamParityTestCase(plain=CLAUDE_HAIKU_4_5_INPUT_TEXT, streamed=CLAUDE_HAIKU_4_5_STREAM),
|
||||
StreamParityTestCase(
|
||||
plain=CLAUDE_HAIKU_4_5_RESPONSE_MODEL_OVERRIDE, streamed=CLAUDE_HAIKU_4_5_STREAM_RESPONSE_MODEL_OVERRIDE
|
||||
|
|
|
|||
|
|
@ -21,6 +21,8 @@ from integration.cost_calculation.cost_tracking_case import (
|
|||
)
|
||||
from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs
|
||||
from integration.cost_calculation.stream_parity.case import StreamParityTestCase, sse_frames
|
||||
from integration.cost_calculation.cases.openai import _responses_json_body, _responses_stream_case
|
||||
from pydantic import JsonValue
|
||||
|
||||
|
||||
def _response_for_model(
|
||||
|
|
@ -47,6 +49,37 @@ def _stream_for_model(response: SseResponse, model: str) -> SseResponse:
|
|||
)
|
||||
|
||||
|
||||
def _azure_responses_json_case(
|
||||
base: CostTrackingTestCase, name: str, response_model: str, content_filters: JsonValue
|
||||
) -> CostTrackingTestCase:
|
||||
response_case: Final = bridge_pairs((base,), ("/v1/responses",))[0].bridged
|
||||
return response_case.model_copy(
|
||||
update={
|
||||
"name": name,
|
||||
"response": JsonResponse(
|
||||
content_type="application/json",
|
||||
body=_responses_json_body(
|
||||
response_model,
|
||||
include_content_filters=True,
|
||||
content_filters=content_filters,
|
||||
),
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _azure_messages_stream_case(
|
||||
messages_json: CostTrackingTestCase, stream_case: CostTrackingTestCase, name: str
|
||||
) -> CostTrackingTestCase:
|
||||
return messages_json.model_copy(
|
||||
update={
|
||||
"name": name,
|
||||
"request": {**messages_json.request, "stream": True},
|
||||
"response": stream_case.response,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# azure/gpt-5.4-mini
|
||||
AZURE_GPT_5_4_MINI_INPUT_TEXT: Final = CostTrackingTestCase(
|
||||
name="azure-gpt-5.4-mini-input_text",
|
||||
|
|
@ -5812,6 +5845,85 @@ AZURE_GPT_6_ASTRA_CACHE_WRITE_5M: Final = with_response_usage(
|
|||
ExactExpected(spend=0.22375, input_cost=0.2, output_cost=0.02375, prompt_tokens=12928, completion_tokens=380),
|
||||
)
|
||||
|
||||
AZURE_GPT_5_6_TERRA_TIERED_INPUT_ABOVE_272K: Final = with_usage(
|
||||
AZURE_GPT_5_6_TERRA_INPUT_TEXT,
|
||||
"azure/gpt-5.6-terra-tiered_input_above_272k",
|
||||
{
|
||||
"prompt_tokens": 300000,
|
||||
"total_tokens": 300412,
|
||||
"prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 0},
|
||||
},
|
||||
ExactExpected(
|
||||
spend=1.3281576, input_cost=1.32, output_cost=0.0081576, prompt_tokens=300000, completion_tokens=412
|
||||
),
|
||||
)
|
||||
AZURE_GPT_5_6_TERRA_TIERED_CACHE_READ_ABOVE_272K: Final = with_usage(
|
||||
AZURE_GPT_5_6_TERRA_INPUT_TEXT,
|
||||
"azure/gpt-5.6-terra-tiered_cache_read_above_272k",
|
||||
{
|
||||
"prompt_tokens": 300000,
|
||||
"total_tokens": 300412,
|
||||
"prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 200000},
|
||||
},
|
||||
ExactExpected(
|
||||
spend=0.5361576, input_cost=0.528, output_cost=0.0081576, prompt_tokens=300000, completion_tokens=412
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
AZURE_GPT_6_1_SOL_RESPONSES_JSON: Final = _azure_responses_json_case(
|
||||
AZURE_GPT_6_1_SOL_INPUT_TEXT, "azure-gpt-6.1-sol-responses_json", "gpt-6.1-sol", None
|
||||
)
|
||||
AZURE_GPT_6_LUNA_RESPONSES_JSON: Final = _azure_responses_json_case(
|
||||
AZURE_GPT_6_LUNA_INPUT_TEXT, "azure-gpt-6-luna-responses_json", "gpt-6-luna", None
|
||||
)
|
||||
AZURE_GPT_6_ASTRA_RESPONSES_JSON: Final = _azure_responses_json_case(
|
||||
AZURE_GPT_6_ASTRA_INPUT_TEXT, "azure-gpt-6-astra-responses_json", "gpt-6-astra", []
|
||||
)
|
||||
AZURE_GPT_5_6_TERRA_RESPONSES_JSON: Final = _azure_responses_json_case(
|
||||
AZURE_GPT_5_6_TERRA_INPUT_TEXT, "azure-gpt-5.6-terra-responses_json", "gpt-5.6-terra", None
|
||||
)
|
||||
AZURE_GPT_6_1_SOL_RESPONSES_STREAM: Final = _responses_stream_case(
|
||||
AZURE_GPT_6_1_SOL_RESPONSES_JSON,
|
||||
"azure-gpt-6.1-sol-responses_stream",
|
||||
"gpt-6.1-sol",
|
||||
14,
|
||||
include_content_filters=True,
|
||||
)
|
||||
AZURE_GPT_6_LUNA_RESPONSES_STREAM: Final = _responses_stream_case(
|
||||
AZURE_GPT_6_LUNA_RESPONSES_JSON,
|
||||
"azure-gpt-6-luna-responses_stream",
|
||||
"gpt-6-luna",
|
||||
25,
|
||||
include_content_filters=True,
|
||||
)
|
||||
AZURE_GPT_6_ASTRA_RESPONSES_STREAM: Final = _responses_stream_case(
|
||||
AZURE_GPT_6_ASTRA_RESPONSES_JSON,
|
||||
"azure-gpt-6-astra-responses_stream",
|
||||
"gpt-6-astra",
|
||||
16,
|
||||
include_content_filters=True,
|
||||
content_filters=[],
|
||||
)
|
||||
AZURE_GPT_5_6_TERRA_RESPONSES_STREAM: Final = _responses_stream_case(
|
||||
AZURE_GPT_5_6_TERRA_RESPONSES_JSON,
|
||||
"azure-gpt-5.6-terra-responses_stream",
|
||||
"gpt-5.6-terra",
|
||||
15,
|
||||
include_content_filters=True,
|
||||
)
|
||||
AZURE_GPT_6_1_SOL_MESSAGES_BRIDGED_STREAM: Final = _azure_messages_stream_case(
|
||||
AZURE_GPT_6_1_SOL_MESSAGES_BRIDGED_JSON, AZURE_GPT_6_1_SOL_STREAM, "azure-gpt-6.1-sol-messages_bridged_stream"
|
||||
)
|
||||
AZURE_GPT_6_LUNA_MESSAGES_BRIDGED_STREAM: Final = _azure_messages_stream_case(
|
||||
AZURE_GPT_6_LUNA_MESSAGES_BRIDGED_JSON, AZURE_GPT_6_LUNA_STREAM, "azure-gpt-6-luna-messages_bridged_stream"
|
||||
)
|
||||
AZURE_GPT_6_ASTRA_MESSAGES_BRIDGED_STREAM: Final = _azure_messages_stream_case(
|
||||
AZURE_GPT_6_ASTRA_MESSAGES_BRIDGED_JSON, AZURE_GPT_6_ASTRA_STREAM, "azure-gpt-6-astra-messages_bridged_stream"
|
||||
)
|
||||
AZURE_GPT_5_6_TERRA_MESSAGES_BRIDGED_STREAM: Final = _azure_messages_stream_case(
|
||||
AZURE_GPT_5_6_TERRA_MESSAGES_BRIDGED_JSON, AZURE_GPT_5_6_TERRA_STREAM, "azure-gpt-5.6-terra-messages_bridged_stream"
|
||||
)
|
||||
|
||||
CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
||||
AZURE_GPT_6_SOL_MESSAGES_BRIDGED_JSON,
|
||||
|
|
@ -5903,6 +6015,8 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
AZURE_GPT_6_LUNA_CACHE_WRITE_5M,
|
||||
AZURE_GPT_6_ASTRA_CACHE_WRITE_5M,
|
||||
AZURE_GPT_5_6_TERRA_INPUT_TEXT,
|
||||
AZURE_GPT_5_6_TERRA_TIERED_INPUT_ABOVE_272K,
|
||||
AZURE_GPT_5_6_TERRA_TIERED_CACHE_READ_ABOVE_272K,
|
||||
AZURE_GPT_5_6_TERRA_STREAM,
|
||||
AZURE_GPT_5_6_TERRA_SERVICE_TIER_FLEX,
|
||||
AZURE_GPT_5_6_TERRA_SERVICE_TIER_PRIORITY,
|
||||
|
|
@ -5911,6 +6025,18 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
AZURE_GPT_5_6_TERRA_CACHE_WRITE_5M,
|
||||
AZURE_GPT_5_6_TERRA_REASONING,
|
||||
AZURE_GPT_5_6_TERRA_TOOL_CALL,
|
||||
AZURE_GPT_6_1_SOL_RESPONSES_JSON,
|
||||
AZURE_GPT_6_1_SOL_RESPONSES_STREAM,
|
||||
AZURE_GPT_6_LUNA_RESPONSES_JSON,
|
||||
AZURE_GPT_6_LUNA_RESPONSES_STREAM,
|
||||
AZURE_GPT_6_ASTRA_RESPONSES_JSON,
|
||||
AZURE_GPT_6_ASTRA_RESPONSES_STREAM,
|
||||
AZURE_GPT_5_6_TERRA_RESPONSES_JSON,
|
||||
AZURE_GPT_5_6_TERRA_RESPONSES_STREAM,
|
||||
AZURE_GPT_6_1_SOL_MESSAGES_BRIDGED_STREAM,
|
||||
AZURE_GPT_6_LUNA_MESSAGES_BRIDGED_STREAM,
|
||||
AZURE_GPT_6_ASTRA_MESSAGES_BRIDGED_STREAM,
|
||||
AZURE_GPT_5_6_TERRA_MESSAGES_BRIDGED_STREAM,
|
||||
)
|
||||
|
||||
PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
||||
|
|
@ -5924,6 +6050,22 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
|||
StreamParityTestCase(plain=AZURE_GPT_6_ASTRA_INPUT_TEXT, streamed=AZURE_GPT_6_ASTRA_STREAM),
|
||||
StreamParityTestCase(plain=AZURE_GPT_6_1_SOL_INPUT_TEXT, streamed=AZURE_GPT_6_1_SOL_STREAM),
|
||||
StreamParityTestCase(plain=AZURE_GPT_5_6_TERRA_INPUT_TEXT, streamed=AZURE_GPT_5_6_TERRA_STREAM),
|
||||
StreamParityTestCase(plain=AZURE_GPT_6_1_SOL_RESPONSES_JSON, streamed=AZURE_GPT_6_1_SOL_RESPONSES_STREAM),
|
||||
StreamParityTestCase(plain=AZURE_GPT_6_LUNA_RESPONSES_JSON, streamed=AZURE_GPT_6_LUNA_RESPONSES_STREAM),
|
||||
StreamParityTestCase(plain=AZURE_GPT_6_ASTRA_RESPONSES_JSON, streamed=AZURE_GPT_6_ASTRA_RESPONSES_STREAM),
|
||||
StreamParityTestCase(plain=AZURE_GPT_5_6_TERRA_RESPONSES_JSON, streamed=AZURE_GPT_5_6_TERRA_RESPONSES_STREAM),
|
||||
StreamParityTestCase(
|
||||
plain=AZURE_GPT_6_1_SOL_MESSAGES_BRIDGED_JSON, streamed=AZURE_GPT_6_1_SOL_MESSAGES_BRIDGED_STREAM
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=AZURE_GPT_6_LUNA_MESSAGES_BRIDGED_JSON, streamed=AZURE_GPT_6_LUNA_MESSAGES_BRIDGED_STREAM
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=AZURE_GPT_6_ASTRA_MESSAGES_BRIDGED_JSON, streamed=AZURE_GPT_6_ASTRA_MESSAGES_BRIDGED_STREAM
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=AZURE_GPT_5_6_TERRA_MESSAGES_BRIDGED_JSON, streamed=AZURE_GPT_5_6_TERRA_MESSAGES_BRIDGED_STREAM
|
||||
),
|
||||
)
|
||||
|
||||
BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = (
|
||||
|
|
@ -5938,4 +6080,10 @@ BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = (
|
|||
BridgeCostParityTestCase(
|
||||
native=AZURE_GPT_5_6_TERRA_INPUT_TEXT, bridged=AZURE_GPT_5_6_TERRA_MESSAGES_BRIDGED_JSON
|
||||
),
|
||||
BridgeCostParityTestCase(native=AZURE_GPT_6_1_SOL_STREAM, bridged=AZURE_GPT_6_1_SOL_MESSAGES_BRIDGED_STREAM),
|
||||
BridgeCostParityTestCase(native=AZURE_GPT_6_LUNA_STREAM, bridged=AZURE_GPT_6_LUNA_MESSAGES_BRIDGED_STREAM),
|
||||
BridgeCostParityTestCase(native=AZURE_GPT_6_ASTRA_STREAM, bridged=AZURE_GPT_6_ASTRA_MESSAGES_BRIDGED_STREAM),
|
||||
BridgeCostParityTestCase(
|
||||
native=AZURE_GPT_5_6_TERRA_STREAM, bridged=AZURE_GPT_5_6_TERRA_MESSAGES_BRIDGED_STREAM
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -6088,6 +6088,71 @@ US_ANTHROPIC_CLAUDE_HAIKU_5_5_TIERED_CACHE_WRITE_1H_ABOVE_100K: Final = with_usa
|
|||
),
|
||||
)
|
||||
|
||||
def _us_anthropic_messages_bridge_pairs(
|
||||
input_case: CostTrackingTestCase, stream_case: CostTrackingTestCase
|
||||
) -> tuple[BridgeCostParityTestCase, ...]:
|
||||
return tuple(
|
||||
BridgeCostParityTestCase(
|
||||
native=pair.native,
|
||||
bridged=pair.bridged.model_copy(
|
||||
update={
|
||||
"name": (
|
||||
f"us.anthropic."
|
||||
f"{pair.native.name.removeprefix('us.anthropic.').removesuffix('-input_text').removesuffix('-stream')}"
|
||||
f"-messages_bridged_{'stream' if pair.native.request.get('stream') is True else 'json'}"
|
||||
)
|
||||
}
|
||||
),
|
||||
)
|
||||
for pair in bridge_pairs((input_case, stream_case), ("/v1/messages",))
|
||||
)
|
||||
|
||||
|
||||
def _us_anthropic_responses_stream_case(
|
||||
json_case: CostTrackingTestCase, stream_case: CostTrackingTestCase, name: str
|
||||
) -> CostTrackingTestCase:
|
||||
return json_case.model_copy(
|
||||
update={
|
||||
"name": name,
|
||||
"request": {**json_case.request, "stream": True},
|
||||
"response": stream_case.response,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
US_ANTHROPIC_CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_STREAM: Final = _us_anthropic_responses_stream_case(
|
||||
US_ANTHROPIC_CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_JSON,
|
||||
US_ANTHROPIC_CLAUDE_FABLE_5_1_STREAM,
|
||||
"us.anthropic.claude-fable-5-1-responses_bridged_stream",
|
||||
)
|
||||
US_ANTHROPIC_CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_STREAM: Final = _us_anthropic_responses_stream_case(
|
||||
US_ANTHROPIC_CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_JSON,
|
||||
US_ANTHROPIC_CLAUDE_HAIKU_5_5_STREAM,
|
||||
"us.anthropic.claude-haiku-5-5-responses_bridged_stream",
|
||||
)
|
||||
US_ANTHROPIC_CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_STREAM: Final = _us_anthropic_responses_stream_case(
|
||||
US_ANTHROPIC_CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_JSON,
|
||||
US_ANTHROPIC_CLAUDE_OPUS_5_5_STREAM,
|
||||
"us.anthropic.claude-opus-5-5-responses_bridged_stream",
|
||||
)
|
||||
US_ANTHROPIC_CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_STREAM: Final = _us_anthropic_responses_stream_case(
|
||||
US_ANTHROPIC_CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_JSON,
|
||||
US_ANTHROPIC_CLAUDE_SONNET_5_5_STREAM,
|
||||
"us.anthropic.claude-sonnet-5-5-responses_bridged_stream",
|
||||
)
|
||||
_US_ANTHROPIC_FABLE_5_1_MESSAGES_PAIRS: Final = _us_anthropic_messages_bridge_pairs(
|
||||
US_ANTHROPIC_CLAUDE_FABLE_5_1_INPUT_TEXT, US_ANTHROPIC_CLAUDE_FABLE_5_1_STREAM
|
||||
)
|
||||
_US_ANTHROPIC_HAIKU_5_5_MESSAGES_PAIRS: Final = _us_anthropic_messages_bridge_pairs(
|
||||
US_ANTHROPIC_CLAUDE_HAIKU_5_5_INPUT_TEXT, US_ANTHROPIC_CLAUDE_HAIKU_5_5_STREAM
|
||||
)
|
||||
_US_ANTHROPIC_OPUS_5_5_MESSAGES_PAIRS: Final = _us_anthropic_messages_bridge_pairs(
|
||||
US_ANTHROPIC_CLAUDE_OPUS_5_5_INPUT_TEXT, US_ANTHROPIC_CLAUDE_OPUS_5_5_STREAM
|
||||
)
|
||||
_US_ANTHROPIC_SONNET_5_5_MESSAGES_PAIRS: Final = _us_anthropic_messages_bridge_pairs(
|
||||
US_ANTHROPIC_CLAUDE_SONNET_5_5_INPUT_TEXT, US_ANTHROPIC_CLAUDE_SONNET_5_5_STREAM
|
||||
)
|
||||
|
||||
CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
||||
US_ANTHROPIC_CLAUDE_SONNET_5_RESPONSES_BRIDGED_JSON,
|
||||
US_ANTHROPIC_CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_JSON,
|
||||
|
|
@ -6210,6 +6275,14 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
US_ANTHROPIC_CLAUDE_FABLE_5_1_THINKING,
|
||||
US_ANTHROPIC_CLAUDE_OPUS_5_5_UPSTREAM_400_ZERO_SPEND,
|
||||
US_ANTHROPIC_CLAUDE_OPUS_5_5_INPUT_TEXT_ROLLUPS,
|
||||
US_ANTHROPIC_CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_STREAM,
|
||||
US_ANTHROPIC_CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_STREAM,
|
||||
US_ANTHROPIC_CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_STREAM,
|
||||
US_ANTHROPIC_CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_STREAM,
|
||||
*tuple(pair.bridged for pair in _US_ANTHROPIC_FABLE_5_1_MESSAGES_PAIRS),
|
||||
*tuple(pair.bridged for pair in _US_ANTHROPIC_HAIKU_5_5_MESSAGES_PAIRS),
|
||||
*tuple(pair.bridged for pair in _US_ANTHROPIC_OPUS_5_5_MESSAGES_PAIRS),
|
||||
*tuple(pair.bridged for pair in _US_ANTHROPIC_SONNET_5_5_MESSAGES_PAIRS),
|
||||
)
|
||||
|
||||
PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
||||
|
|
@ -6235,6 +6308,38 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
|||
StreamParityTestCase(plain=US_ANTHROPIC_CLAUDE_HAIKU_5_5_INPUT_TEXT, streamed=US_ANTHROPIC_CLAUDE_HAIKU_5_5_STREAM),
|
||||
StreamParityTestCase(plain=US_ANTHROPIC_CLAUDE_FABLE_5_INPUT_TEXT, streamed=US_ANTHROPIC_CLAUDE_FABLE_5_STREAM),
|
||||
StreamParityTestCase(plain=US_ANTHROPIC_CLAUDE_FABLE_5_1_INPUT_TEXT, streamed=US_ANTHROPIC_CLAUDE_FABLE_5_1_STREAM),
|
||||
StreamParityTestCase(
|
||||
plain=US_ANTHROPIC_CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_JSON,
|
||||
streamed=US_ANTHROPIC_CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_STREAM,
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=US_ANTHROPIC_CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_JSON,
|
||||
streamed=US_ANTHROPIC_CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_STREAM,
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=US_ANTHROPIC_CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_JSON,
|
||||
streamed=US_ANTHROPIC_CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_STREAM,
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=US_ANTHROPIC_CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_JSON,
|
||||
streamed=US_ANTHROPIC_CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_STREAM,
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=_US_ANTHROPIC_FABLE_5_1_MESSAGES_PAIRS[0].bridged,
|
||||
streamed=_US_ANTHROPIC_FABLE_5_1_MESSAGES_PAIRS[1].bridged,
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=_US_ANTHROPIC_HAIKU_5_5_MESSAGES_PAIRS[0].bridged,
|
||||
streamed=_US_ANTHROPIC_HAIKU_5_5_MESSAGES_PAIRS[1].bridged,
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=_US_ANTHROPIC_OPUS_5_5_MESSAGES_PAIRS[0].bridged,
|
||||
streamed=_US_ANTHROPIC_OPUS_5_5_MESSAGES_PAIRS[1].bridged,
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=_US_ANTHROPIC_SONNET_5_5_MESSAGES_PAIRS[0].bridged,
|
||||
streamed=_US_ANTHROPIC_SONNET_5_5_MESSAGES_PAIRS[1].bridged,
|
||||
),
|
||||
)
|
||||
|
||||
BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = (
|
||||
|
|
@ -6267,4 +6372,24 @@ BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = (
|
|||
BridgeCostParityTestCase(
|
||||
native=US_ANTHROPIC_CLAUDE_FABLE_5_1_INPUT_TEXT, bridged=US_ANTHROPIC_CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_JSON
|
||||
),
|
||||
BridgeCostParityTestCase(
|
||||
native=US_ANTHROPIC_CLAUDE_FABLE_5_1_STREAM,
|
||||
bridged=US_ANTHROPIC_CLAUDE_FABLE_5_1_RESPONSES_BRIDGED_STREAM,
|
||||
),
|
||||
BridgeCostParityTestCase(
|
||||
native=US_ANTHROPIC_CLAUDE_HAIKU_5_5_STREAM,
|
||||
bridged=US_ANTHROPIC_CLAUDE_HAIKU_5_5_RESPONSES_BRIDGED_STREAM,
|
||||
),
|
||||
BridgeCostParityTestCase(
|
||||
native=US_ANTHROPIC_CLAUDE_OPUS_5_5_STREAM,
|
||||
bridged=US_ANTHROPIC_CLAUDE_OPUS_5_5_RESPONSES_BRIDGED_STREAM,
|
||||
),
|
||||
BridgeCostParityTestCase(
|
||||
native=US_ANTHROPIC_CLAUDE_SONNET_5_5_STREAM,
|
||||
bridged=US_ANTHROPIC_CLAUDE_SONNET_5_5_RESPONSES_BRIDGED_STREAM,
|
||||
),
|
||||
*_US_ANTHROPIC_FABLE_5_1_MESSAGES_PAIRS,
|
||||
*_US_ANTHROPIC_HAIKU_5_5_MESSAGES_PAIRS,
|
||||
*_US_ANTHROPIC_OPUS_5_5_MESSAGES_PAIRS,
|
||||
*_US_ANTHROPIC_SONNET_5_5_MESSAGES_PAIRS,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ from integration.cost_calculation.cases.anthropic import (
|
|||
CLAUDE_OPUS_5_5_THINKING,
|
||||
CLAUDE_OPUS_5_5_TOOL_CALL,
|
||||
)
|
||||
from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase
|
||||
from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs
|
||||
from integration.cost_calculation.cost_tracking_case import (
|
||||
CostTrackingTestCase,
|
||||
Deployment,
|
||||
|
|
@ -535,14 +535,40 @@ _MANTLE_GPT_CHAT_REQUEST: Final = {
|
|||
"messages": [{"role": "user", "content": "cost tracking fixture"}],
|
||||
"max_completion_tokens": 64,
|
||||
}
|
||||
_MANTLE_GPT_TOOL_REQUEST: Final = {
|
||||
**_MANTLE_GPT_RESPONSES_REQUEST,
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"name": "get_weather",
|
||||
"description": "Get the weather for a city",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
}
|
||||
],
|
||||
"tool_choice": "required",
|
||||
}
|
||||
_MANTLE_GPT_CACHE_WRITE_REQUEST: Final = {
|
||||
**_MANTLE_GPT_RESPONSES_REQUEST,
|
||||
"input": "cacheable pricing-harness prefix " * 1000,
|
||||
"prompt_cache_key": "cost-cases-20261017",
|
||||
"max_output_tokens": 64,
|
||||
}
|
||||
|
||||
|
||||
def _mantle_gpt_usage(
|
||||
input_tokens: int, output_tokens: int, reasoning_tokens: int, cached_tokens: int = 0
|
||||
input_tokens: int,
|
||||
output_tokens: int,
|
||||
reasoning_tokens: int,
|
||||
cached_tokens: int = 0,
|
||||
cache_write_tokens: int = 0,
|
||||
) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"input_tokens": input_tokens,
|
||||
"input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": cached_tokens},
|
||||
"input_tokens_details": {"cache_write_tokens": cache_write_tokens, "cached_tokens": cached_tokens},
|
||||
"output_tokens": output_tokens,
|
||||
"output_tokens_details": {"reasoning_tokens": reasoning_tokens},
|
||||
"total_tokens": input_tokens + output_tokens,
|
||||
|
|
@ -628,6 +654,20 @@ def _mantle_gpt_cases(
|
|||
response_expected,
|
||||
_MANTLE_GPT_CHAT_REQUEST,
|
||||
)
|
||||
chat_stream_case: Final = chat_case.model_copy(
|
||||
update={
|
||||
"name": f"bedrock_mantle-{provider_model}-chat_stream",
|
||||
"request": {**chat_case.request, "stream": True},
|
||||
"response": stream_case.response,
|
||||
}
|
||||
)
|
||||
messages_pairs: Final = bridge_pairs((response_case, stream_case), ("/v1/messages",))
|
||||
messages_json: Final = messages_pairs[0].bridged.model_copy(
|
||||
update={"name": f"bedrock_mantle-{provider_model}-messages_bridged_json"}
|
||||
)
|
||||
messages_stream: Final = messages_pairs[1].bridged.model_copy(
|
||||
update={"name": f"bedrock_mantle-{provider_model}-messages_bridged_stream"}
|
||||
)
|
||||
cache_case: Final = _mantle_gpt_json_case(
|
||||
BEDROCK_MANTLE_OPENAI_GPT_6_SOL_RESPONSES_JSON,
|
||||
provider_model,
|
||||
|
|
@ -660,7 +700,73 @@ def _mantle_gpt_cases(
|
|||
tiered_boundary_expected,
|
||||
{**_MANTLE_GPT_RESPONSES_REQUEST, "max_output_tokens": 412},
|
||||
)
|
||||
return (response_case, stream_case, reasoning_case, chat_case, cache_case, tiered_input, tiered_cache, tiered_boundary)
|
||||
return (
|
||||
response_case,
|
||||
stream_case,
|
||||
reasoning_case,
|
||||
chat_case,
|
||||
cache_case,
|
||||
tiered_input,
|
||||
tiered_cache,
|
||||
tiered_boundary,
|
||||
chat_stream_case,
|
||||
messages_json,
|
||||
messages_stream,
|
||||
)
|
||||
|
||||
|
||||
def _mantle_gpt_audit3_cases(
|
||||
provider_model: str,
|
||||
tool_input_tokens: int,
|
||||
tool_output_tokens: int,
|
||||
tool_reasoning_tokens: int,
|
||||
tool_expected: ExactExpected,
|
||||
cache_output_tokens: int,
|
||||
cache_reasoning_tokens: int,
|
||||
cache_expected: ExactExpected,
|
||||
) -> tuple[CostTrackingTestCase, ...]:
|
||||
tool_case: Final = _mantle_gpt_json_case(
|
||||
BEDROCK_MANTLE_OPENAI_GPT_6_SOL_RESPONSES_JSON,
|
||||
provider_model,
|
||||
"tool_call",
|
||||
_mantle_gpt_usage(tool_input_tokens, tool_output_tokens, tool_reasoning_tokens),
|
||||
tool_expected,
|
||||
_MANTLE_GPT_TOOL_REQUEST,
|
||||
)
|
||||
if not isinstance(tool_case.response, JsonResponse):
|
||||
raise TypeError(f"{tool_case.name}: expected a JSON response fixture")
|
||||
tool_response: Final = tool_case.response.model_copy(
|
||||
update={
|
||||
"body": {
|
||||
**tool_case.response.body,
|
||||
"output": [
|
||||
{
|
||||
"arguments": '{"city":"Paris"}',
|
||||
"call_id": "call_$REQUEST_ID",
|
||||
"id": "fc_$REQUEST_ID",
|
||||
"name": "get_weather",
|
||||
"status": "completed",
|
||||
"type": "function_call",
|
||||
}
|
||||
],
|
||||
}
|
||||
}
|
||||
)
|
||||
tool_case_with_response: Final = tool_case.model_copy(update={"response": tool_response})
|
||||
cache_case: Final = _mantle_gpt_json_case(
|
||||
BEDROCK_MANTLE_OPENAI_GPT_6_SOL_RESPONSES_JSON,
|
||||
provider_model,
|
||||
"cache_write_5m",
|
||||
_mantle_gpt_usage(
|
||||
1667,
|
||||
cache_output_tokens,
|
||||
cache_reasoning_tokens,
|
||||
cache_write_tokens=1665,
|
||||
),
|
||||
cache_expected,
|
||||
_MANTLE_GPT_CACHE_WRITE_REQUEST,
|
||||
)
|
||||
return (tool_case_with_response, cache_case)
|
||||
|
||||
|
||||
def _mantle_gpt_bridge(case: CostTrackingTestCase) -> BridgeCostParityTestCase:
|
||||
|
|
@ -723,6 +829,83 @@ _MANTLE_GPT_56_TERRA_CASES: Final = _mantle_gpt_cases(
|
|||
ExactExpected(spend=0.84537376, input_cost=0.83776, output_cost=0.00761376, prompt_tokens=272000, completion_tokens=412),
|
||||
)
|
||||
|
||||
_MANTLE_GPT_AUDIT3_CASES: Final = (
|
||||
*_mantle_gpt_audit3_cases(
|
||||
"openai.gpt-6.1-sol",
|
||||
50,
|
||||
20,
|
||||
0,
|
||||
ExactExpected(spend=0.000363, input_cost=0.000121, output_cost=0.000242, prompt_tokens=50, completion_tokens=20),
|
||||
28,
|
||||
0,
|
||||
ExactExpected(
|
||||
spend=0.005381765,
|
||||
input_cost=0.005042965,
|
||||
output_cost=0.0003388,
|
||||
prompt_tokens=1667,
|
||||
completion_tokens=28,
|
||||
cache_creation_cost=0.005038125,
|
||||
),
|
||||
),
|
||||
*_mantle_gpt_audit3_cases(
|
||||
"openai.gpt-6-luna",
|
||||
51,
|
||||
18,
|
||||
0,
|
||||
ExactExpected(
|
||||
spend=0.000018612, input_cost=0.000006732, output_cost=0.00001188, prompt_tokens=51, completion_tokens=18
|
||||
),
|
||||
64,
|
||||
40,
|
||||
ExactExpected(
|
||||
spend=0.000317229,
|
||||
input_cost=0.000274989,
|
||||
output_cost=0.00004224,
|
||||
prompt_tokens=1667,
|
||||
completion_tokens=64,
|
||||
cache_creation_cost=0.000274725,
|
||||
),
|
||||
),
|
||||
*_mantle_gpt_audit3_cases(
|
||||
"openai.gpt-6-astra",
|
||||
51,
|
||||
18,
|
||||
0,
|
||||
ExactExpected(
|
||||
spend=0.0020163, input_cost=0.0007293, output_cost=0.001287, prompt_tokens=51, completion_tokens=18
|
||||
),
|
||||
27,
|
||||
0,
|
||||
ExactExpected(
|
||||
spend=0.031720975,
|
||||
input_cost=0.029790475,
|
||||
output_cost=0.0019305,
|
||||
prompt_tokens=1667,
|
||||
completion_tokens=27,
|
||||
cache_creation_cost=0.029761875,
|
||||
),
|
||||
),
|
||||
*_mantle_gpt_audit3_cases(
|
||||
"openai.gpt-5.6-terra",
|
||||
51,
|
||||
28,
|
||||
8,
|
||||
ExactExpected(
|
||||
spend=0.00067452, input_cost=0.00015708, output_cost=0.00051744, prompt_tokens=51, completion_tokens=28
|
||||
),
|
||||
28,
|
||||
0,
|
||||
ExactExpected(
|
||||
spend=0.00693385,
|
||||
input_cost=0.00641641,
|
||||
output_cost=0.00051744,
|
||||
prompt_tokens=1667,
|
||||
completion_tokens=28,
|
||||
cache_creation_cost=0.00641025,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
_MANTLE_CLAUDE_OPUS_MODEL: Final = "bedrock_mantle/anthropic.claude-opus-5-5"
|
||||
_ADAPTIVE_USAGE: Final = {
|
||||
"input_tokens": 24,
|
||||
|
|
@ -737,7 +920,25 @@ _ADAPTIVE_EXPECTED: Final = ExactExpected(
|
|||
spend=0.0007524, input_cost=0.0001584, output_cost=0.000594, prompt_tokens=24, completion_tokens=18
|
||||
)
|
||||
|
||||
BEDROCK_MANTLE_CLAUDE_OPUS_5_5_INPUT_TEXT: Final = with_usage(
|
||||
|
||||
def _mantle_without_usage_fields(case: CostTrackingTestCase) -> CostTrackingTestCase:
|
||||
if not isinstance(case.response, JsonResponse):
|
||||
raise TypeError(f"{case.name}: expected a JSON response fixture")
|
||||
usage: Final = case.response.body.get("usage")
|
||||
if not isinstance(usage, dict):
|
||||
raise TypeError(f"{case.name}: expected a usage object")
|
||||
response: Final = case.response.model_copy(
|
||||
update={
|
||||
"body": {
|
||||
**case.response.body,
|
||||
"usage": {key: value for key, value in usage.items() if key not in {"inference_geo", "speed"}},
|
||||
}
|
||||
}
|
||||
)
|
||||
return case.model_copy(update={"response": response})
|
||||
|
||||
|
||||
BEDROCK_MANTLE_CLAUDE_OPUS_5_5_INPUT_TEXT: Final = _mantle_without_usage_fields(with_usage(
|
||||
CLAUDE_OPUS_5_5_INPUT_TEXT,
|
||||
"bedrock_mantle-anthropic.claude-opus-5-5-input_text",
|
||||
_ADAPTIVE_USAGE,
|
||||
|
|
@ -751,7 +952,7 @@ BEDROCK_MANTLE_CLAUDE_OPUS_5_5_INPUT_TEXT: Final = with_usage(
|
|||
"allowed_openai_params": ["reasoning_effort"],
|
||||
},
|
||||
}
|
||||
)
|
||||
))
|
||||
|
||||
_MANTLE_CLAUDE_STREAM_RESPONSE: Final = SseResponse(
|
||||
content_type="text/event-stream",
|
||||
|
|
@ -841,7 +1042,7 @@ BEDROCK_MANTLE_CLAUDE_OPUS_5_5_MESSAGES_STREAM: Final = CLAUDE_OPUS_5_5_MESSAGES
|
|||
}
|
||||
)
|
||||
|
||||
BEDROCK_MANTLE_CLAUDE_OPUS_5_5_CACHE_READ: Final = with_usage(
|
||||
BEDROCK_MANTLE_CLAUDE_OPUS_5_5_CACHE_READ: Final = _mantle_without_usage_fields(with_usage(
|
||||
CLAUDE_OPUS_5_5_CACHE_READ,
|
||||
"bedrock_mantle-anthropic.claude-opus-5-5-cache_read",
|
||||
{
|
||||
|
|
@ -856,7 +1057,7 @@ BEDROCK_MANTLE_CLAUDE_OPUS_5_5_CACHE_READ: Final = with_usage(
|
|||
ExactExpected(
|
||||
spend=0.00308319, input_cost=0.00153219, output_cost=0.001551, prompt_tokens=4225, completion_tokens=47
|
||||
),
|
||||
).model_copy(update={"model": _MANTLE_CLAUDE_OPUS_MODEL})
|
||||
).model_copy(update={"model": _MANTLE_CLAUDE_OPUS_MODEL}))
|
||||
|
||||
BEDROCK_MANTLE_CLAUDE_OPUS_5_5_CACHE_WRITE_5M: Final = with_usage(
|
||||
CLAUDE_OPUS_5_5_CACHE_WRITE_5M,
|
||||
|
|
@ -892,12 +1093,12 @@ BEDROCK_MANTLE_CLAUDE_OPUS_5_5_CACHE_WRITE_1H: Final = with_usage(
|
|||
),
|
||||
).model_copy(update={"model": _MANTLE_CLAUDE_OPUS_MODEL})
|
||||
|
||||
BEDROCK_MANTLE_CLAUDE_OPUS_5_5_THINKING: Final = with_usage(
|
||||
BEDROCK_MANTLE_CLAUDE_OPUS_5_5_THINKING: Final = _mantle_without_usage_fields(with_usage(
|
||||
CLAUDE_OPUS_5_5_THINKING,
|
||||
"bedrock_mantle-anthropic.claude-opus-5-5-thinking",
|
||||
_ADAPTIVE_USAGE,
|
||||
_ADAPTIVE_EXPECTED,
|
||||
).model_copy(update={"model": _MANTLE_CLAUDE_OPUS_MODEL})
|
||||
).model_copy(update={"model": _MANTLE_CLAUDE_OPUS_MODEL}))
|
||||
|
||||
BEDROCK_MANTLE_CLAUDE_OPUS_5_5_TOOL_CALL: Final = with_usage(
|
||||
CLAUDE_OPUS_5_5_TOOL_CALL,
|
||||
|
|
@ -921,6 +1122,21 @@ BEDROCK_MANTLE_CLAUDE_OPUS_5_5_CHAT_BRIDGED_JSON: Final = BEDROCK_MANTLE_CLAUDE_
|
|||
}
|
||||
)
|
||||
|
||||
BEDROCK_MANTLE_CLAUDE_OPUS_5_5_RESPONSES_JSON: Final = BEDROCK_MANTLE_CLAUDE_OPUS_5_5_INPUT_TEXT.model_copy(
|
||||
update={
|
||||
"name": "bedrock_mantle-anthropic.claude-opus-5-5-responses_json",
|
||||
"endpoint": "/v1/responses",
|
||||
"request": {"model": "$MODEL", "input": "summarize the attached material in one line"},
|
||||
}
|
||||
)
|
||||
BEDROCK_MANTLE_CLAUDE_OPUS_5_5_RESPONSES_STREAM: Final = BEDROCK_MANTLE_CLAUDE_OPUS_5_5_STREAM.model_copy(
|
||||
update={
|
||||
"name": "bedrock_mantle-anthropic.claude-opus-5-5-responses_stream",
|
||||
"endpoint": "/v1/responses",
|
||||
"request": {"model": "$MODEL", "input": "summarize the attached material in one line", "stream": True},
|
||||
}
|
||||
)
|
||||
|
||||
BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = (
|
||||
_mantle_gpt_bridge(_MANTLE_GPT_61_SOL_CASES[0]),
|
||||
_mantle_gpt_bridge(_MANTLE_GPT_6_LUNA_CASES[0]),
|
||||
|
|
@ -930,6 +1146,14 @@ BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = (
|
|||
native=BEDROCK_MANTLE_CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT,
|
||||
bridged=BEDROCK_MANTLE_CLAUDE_OPUS_5_5_CHAT_BRIDGED_JSON,
|
||||
),
|
||||
*tuple(
|
||||
BridgeCostParityTestCase(native=cases[0], bridged=cases[9])
|
||||
for cases in (_MANTLE_GPT_61_SOL_CASES, _MANTLE_GPT_6_LUNA_CASES, _MANTLE_GPT_6_ASTRA_CASES, _MANTLE_GPT_56_TERRA_CASES)
|
||||
),
|
||||
*tuple(
|
||||
BridgeCostParityTestCase(native=cases[1], bridged=cases[10])
|
||||
for cases in (_MANTLE_GPT_61_SOL_CASES, _MANTLE_GPT_6_LUNA_CASES, _MANTLE_GPT_6_ASTRA_CASES, _MANTLE_GPT_56_TERRA_CASES)
|
||||
),
|
||||
)
|
||||
|
||||
CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
||||
|
|
@ -945,11 +1169,14 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
*_MANTLE_GPT_6_LUNA_CASES,
|
||||
*_MANTLE_GPT_6_ASTRA_CASES,
|
||||
*_MANTLE_GPT_56_TERRA_CASES,
|
||||
*_MANTLE_GPT_AUDIT3_CASES,
|
||||
BEDROCK_MANTLE_CLAUDE_OPUS_5_5_INPUT_TEXT,
|
||||
BEDROCK_MANTLE_CLAUDE_OPUS_5_5_STREAM,
|
||||
BEDROCK_MANTLE_CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT,
|
||||
BEDROCK_MANTLE_CLAUDE_OPUS_5_5_MESSAGES_STREAM,
|
||||
BEDROCK_MANTLE_CLAUDE_OPUS_5_5_CHAT_BRIDGED_JSON,
|
||||
BEDROCK_MANTLE_CLAUDE_OPUS_5_5_RESPONSES_JSON,
|
||||
BEDROCK_MANTLE_CLAUDE_OPUS_5_5_RESPONSES_STREAM,
|
||||
BEDROCK_MANTLE_CLAUDE_OPUS_5_5_CACHE_READ,
|
||||
BEDROCK_MANTLE_CLAUDE_OPUS_5_5_CACHE_WRITE_5M,
|
||||
BEDROCK_MANTLE_CLAUDE_OPUS_5_5_CACHE_WRITE_1H,
|
||||
|
|
@ -972,4 +1199,16 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
|||
plain=BEDROCK_MANTLE_CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT,
|
||||
streamed=BEDROCK_MANTLE_CLAUDE_OPUS_5_5_MESSAGES_STREAM,
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=BEDROCK_MANTLE_CLAUDE_OPUS_5_5_RESPONSES_JSON,
|
||||
streamed=BEDROCK_MANTLE_CLAUDE_OPUS_5_5_RESPONSES_STREAM,
|
||||
),
|
||||
*tuple(
|
||||
StreamParityTestCase(plain=cases[3], streamed=cases[8])
|
||||
for cases in (_MANTLE_GPT_61_SOL_CASES, _MANTLE_GPT_6_LUNA_CASES, _MANTLE_GPT_6_ASTRA_CASES, _MANTLE_GPT_56_TERRA_CASES)
|
||||
),
|
||||
*tuple(
|
||||
StreamParityTestCase(plain=cases[9], streamed=cases[10])
|
||||
for cases in (_MANTLE_GPT_61_SOL_CASES, _MANTLE_GPT_6_LUNA_CASES, _MANTLE_GPT_6_ASTRA_CASES, _MANTLE_GPT_56_TERRA_CASES)
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,8 +2,10 @@
|
|||
|
||||
PARITY pairs the streamed case with its plain twin where both bill the same row."""
|
||||
|
||||
import json
|
||||
from typing import Final, cast
|
||||
|
||||
from pydantic import JsonValue
|
||||
from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs
|
||||
from integration.cost_calculation.cost_tracking_case import (
|
||||
CostTrackingTestCase,
|
||||
|
|
@ -17,26 +19,74 @@ from integration.cost_calculation.cost_tracking_case import (
|
|||
from integration.cost_calculation.stream_parity.case import StreamParityTestCase, sse_frames
|
||||
|
||||
|
||||
def _gemini_case_response(response: JsonResponse | SseResponse, model_version: str) -> JsonResponse | SseResponse:
|
||||
def _gemini_case_response(
|
||||
response: JsonResponse | SseResponse,
|
||||
model_version: str,
|
||||
usage_fields: dict[str, JsonValue] | None = None,
|
||||
) -> JsonResponse | SseResponse:
|
||||
match response:
|
||||
case JsonResponse():
|
||||
return response.model_copy(update={"body": {**response.body, "modelVersion": model_version}})
|
||||
usage: Final = response.body.get("usageMetadata")
|
||||
body: Final = {**response.body, "modelVersion": model_version}
|
||||
return response.model_copy(
|
||||
update={
|
||||
"body": {
|
||||
**body,
|
||||
**(
|
||||
{
|
||||
"usageMetadata": {
|
||||
**cast(dict[str, JsonValue], usage),
|
||||
**usage_fields,
|
||||
}
|
||||
}
|
||||
if usage_fields is not None and isinstance(usage, dict)
|
||||
else {}
|
||||
),
|
||||
}
|
||||
}
|
||||
)
|
||||
case SseResponse():
|
||||
return response.model_copy(
|
||||
update={
|
||||
"frames": tuple(frame.replace("gemini-3.8-flash", model_version) for frame in response.frames)
|
||||
"frames": tuple(
|
||||
_gemini_case_frame(frame, model_version, usage_fields) for frame in response.frames
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _gemini_case_frame(
|
||||
frame: str, model_version: str, usage_fields: dict[str, JsonValue] | None
|
||||
) -> str:
|
||||
replaced: Final = frame.replace("gemini-3.8-flash", model_version)
|
||||
if usage_fields is None:
|
||||
return replaced
|
||||
prefix, separator, payload = replaced.partition("data: ")
|
||||
if not separator or payload == "[DONE]":
|
||||
return replaced
|
||||
event: Final = cast(dict[str, JsonValue], json.loads(payload))
|
||||
usage: Final = event.get("usageMetadata")
|
||||
if not isinstance(usage, dict):
|
||||
return replaced
|
||||
return f"{prefix}data: {json.dumps({**event, 'usageMetadata': {**usage, **usage_fields}})}"
|
||||
|
||||
|
||||
def _gemini_case(
|
||||
sibling: CostTrackingTestCase, name: str, model: str, model_version: str, expected: ExactExpected
|
||||
sibling: CostTrackingTestCase,
|
||||
name: str,
|
||||
model: str,
|
||||
model_version: str,
|
||||
expected: ExactExpected,
|
||||
*,
|
||||
usage_fields: dict[str, JsonValue] | None = None,
|
||||
) -> CostTrackingTestCase:
|
||||
return sibling.model_copy(
|
||||
update={
|
||||
"name": name,
|
||||
"model": model,
|
||||
"response": _gemini_case_response(cast(JsonResponse | SseResponse, sibling.response), model_version),
|
||||
"response": _gemini_case_response(
|
||||
cast(JsonResponse | SseResponse, sibling.response), model_version, usage_fields
|
||||
),
|
||||
"expected": expected,
|
||||
}
|
||||
)
|
||||
|
|
@ -1540,6 +1590,7 @@ GEMINI_GEMINI_3_8_FLASH_INPUT_TEXT: Final = CostTrackingTestCase(
|
|||
"candidatesTokenCount": 412,
|
||||
"totalTokenCount": 2252,
|
||||
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 1840}],
|
||||
"serviceTier": "standard",
|
||||
},
|
||||
"modelVersion": "gemini-3.8-flash",
|
||||
},
|
||||
|
|
@ -1594,6 +1645,7 @@ GEMINI_GEMINI_3_8_FLASH_CACHE_READ: Final = CostTrackingTestCase(
|
|||
"totalTokenCount": 13308,
|
||||
"cachedContentTokenCount": 12288,
|
||||
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 12928}],
|
||||
"serviceTier": "standard",
|
||||
},
|
||||
"modelVersion": "gemini-3.8-flash",
|
||||
},
|
||||
|
|
@ -1653,6 +1705,7 @@ GEMINI_GEMINI_3_8_FLASH_REASONING: Final = CostTrackingTestCase(
|
|||
"thoughtsTokenCount": 3480,
|
||||
"totalTokenCount": 5280,
|
||||
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 1240}],
|
||||
"serviceTier": "standard",
|
||||
},
|
||||
"modelVersion": "gemini-3.8-flash",
|
||||
},
|
||||
|
|
@ -1934,6 +1987,7 @@ GEMINI_GEMINI_3_8_FLASH_STREAM: Final = CostTrackingTestCase(
|
|||
"candidatesTokenCount": 412,
|
||||
"totalTokenCount": 2252,
|
||||
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 1840}],
|
||||
"serviceTier": "standard",
|
||||
},
|
||||
"modelVersion": "gemini-3.8-flash",
|
||||
},
|
||||
|
|
@ -3275,19 +3329,46 @@ GEMINI_GEMINI_3_1_PRO_CACHE_CREATION: Final = CostTrackingTestCase(
|
|||
|
||||
GEMINI_GEMINI_4_ARGON_INPUT_TEXT: Final = _gemini_case(
|
||||
GEMINI_GEMINI_3_8_FLASH_INPUT_TEXT, "gemini-gemini-4-argon-input_text", "gemini/gemini-4-argon",
|
||||
"gemini-4-argon", ExactExpected(spend=0.00858, input_cost=0.004048, output_cost=0.004532, prompt_tokens=1840, completion_tokens=412)
|
||||
"gemini-4-argon", ExactExpected(spend=0.00858, input_cost=0.004048, output_cost=0.004532, prompt_tokens=1840, completion_tokens=412),
|
||||
usage_fields={"candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 412}]},
|
||||
)
|
||||
GEMINI_GEMINI_4_ARGON_STREAM: Final = _gemini_case(
|
||||
GEMINI_GEMINI_3_8_FLASH_STREAM, "gemini-gemini-4-argon-stream", "gemini/gemini-4-argon",
|
||||
"gemini-4-argon", ExactExpected(spend=0.00858, input_cost=0.004048, output_cost=0.004532, prompt_tokens=1840, completion_tokens=412)
|
||||
"gemini-4-argon", ExactExpected(spend=0.00858, input_cost=0.004048, output_cost=0.004532, prompt_tokens=1840, completion_tokens=412),
|
||||
usage_fields={"candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 412}]},
|
||||
)
|
||||
GEMINI_GEMINI_4_ARGON_CACHE_READ: Final = _gemini_case(
|
||||
GEMINI_GEMINI_3_8_FLASH_CACHE_READ, "gemini-gemini-4-argon-cache_read", "gemini/gemini-4-argon",
|
||||
"gemini-4-argon", ExactExpected(spend=0.00693968, input_cost=0.00275968, output_cost=0.00418, prompt_tokens=12928, completion_tokens=380)
|
||||
"gemini-4-argon", ExactExpected(spend=0.00693968, input_cost=0.00275968, output_cost=0.00418, prompt_tokens=12928, completion_tokens=380),
|
||||
usage_fields={"candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 380}]},
|
||||
)
|
||||
GEMINI_GEMINI_4_ARGON_REASONING: Final = _gemini_case(
|
||||
GEMINI_GEMINI_3_8_FLASH_REASONING, "gemini-gemini-4-argon-reasoning", "gemini/gemini-4-argon",
|
||||
"gemini-4-argon", ExactExpected(spend=0.047168, input_cost=0.002728, output_cost=0.04444, prompt_tokens=1240, completion_tokens=4040)
|
||||
"gemini-4-argon", ExactExpected(spend=0.047168, input_cost=0.002728, output_cost=0.04444, prompt_tokens=1240, completion_tokens=4040),
|
||||
usage_fields={"candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 560}]},
|
||||
)
|
||||
GEMINI_GEMINI_4_ARGON_SERVICE_TIER_FLEX: Final = _gemini_case(
|
||||
GEMINI_GEMINI_3_8_FLASH_SERVICE_TIER_FLEX,
|
||||
"gemini-gemini-4-argon-service_tier_flex",
|
||||
"gemini/gemini-4-argon",
|
||||
"gemini-4-argon",
|
||||
ExactExpected(spend=0.00429, input_cost=0.002024, output_cost=0.002266, prompt_tokens=1840, completion_tokens=412),
|
||||
)
|
||||
GEMINI_GEMINI_4_ARGON_SERVICE_TIER_PRIORITY: Final = _gemini_case(
|
||||
GEMINI_GEMINI_3_8_FLASH_SERVICE_TIER_PRIORITY,
|
||||
"gemini-gemini-4-argon-service_tier_priority",
|
||||
"gemini/gemini-4-argon",
|
||||
"gemini-4-argon",
|
||||
ExactExpected(spend=0.015444, input_cost=0.0072864, output_cost=0.0081576, prompt_tokens=1840, completion_tokens=412),
|
||||
)
|
||||
GEMINI_GEMINI_3_1_PRO_PREVIEW_SERVICE_TIER_PRIORITY: Final = _gemini_case(
|
||||
GEMINI_GEMINI_3_8_FLASH_SERVICE_TIER_PRIORITY,
|
||||
"gemini-gemini-3.1-pro-preview-service_tier_priority",
|
||||
"gemini/gemini-3.1-pro-preview",
|
||||
"gemini-3.1-pro-preview",
|
||||
ExactExpected(
|
||||
spend=0.02018016, input_cost=0.0086112, output_cost=0.01156896, prompt_tokens=1840, completion_tokens=412
|
||||
),
|
||||
)
|
||||
GEMINI_GEMINI_4_ARGON_TOOL_CALL: Final = _gemini_case(
|
||||
GEMINI_GEMINI_3_8_FLASH_TOOL_CALL, "gemini-gemini-4-argon-tool_call", "gemini/gemini-4-argon",
|
||||
|
|
@ -3296,28 +3377,60 @@ GEMINI_GEMINI_4_ARGON_TOOL_CALL: Final = _gemini_case(
|
|||
GEMINI_GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT: Final = _gemini_case(
|
||||
GEMINI_GEMINI_3_8_FLASH_INPUT_TEXT, "gemini-gemini-3.1-pro-preview-input_text",
|
||||
"gemini/gemini-3.1-pro-preview", "gemini-3.1-pro-preview",
|
||||
ExactExpected(spend=0.0112112, input_cost=0.004784, output_cost=0.0064272, prompt_tokens=1840, completion_tokens=412)
|
||||
ExactExpected(spend=0.0112112, input_cost=0.004784, output_cost=0.0064272, prompt_tokens=1840, completion_tokens=412),
|
||||
usage_fields={"serviceTier": "standard"},
|
||||
)
|
||||
GEMINI_GEMINI_3_1_PRO_PREVIEW_STREAM: Final = _gemini_case(
|
||||
GEMINI_GEMINI_3_8_FLASH_STREAM, "gemini-gemini-3.1-pro-preview-stream",
|
||||
"gemini/gemini-3.1-pro-preview", "gemini-3.1-pro-preview",
|
||||
ExactExpected(spend=0.0112112, input_cost=0.004784, output_cost=0.0064272, prompt_tokens=1840, completion_tokens=412)
|
||||
ExactExpected(spend=0.0112112, input_cost=0.004784, output_cost=0.0064272, prompt_tokens=1840, completion_tokens=412),
|
||||
usage_fields={"serviceTier": "standard"},
|
||||
)
|
||||
GEMINI_GEMINI_3_1_PRO_PREVIEW_CACHE_READ: Final = _gemini_case(
|
||||
GEMINI_GEMINI_3_8_FLASH_CACHE_READ, "gemini-gemini-3.1-pro-preview-cache_read",
|
||||
"gemini/gemini-3.1-pro-preview", "gemini-3.1-pro-preview",
|
||||
ExactExpected(spend=0.01078688, input_cost=0.00485888, output_cost=0.005928, prompt_tokens=12928, completion_tokens=380)
|
||||
ExactExpected(spend=0.01078688, input_cost=0.00485888, output_cost=0.005928, prompt_tokens=12928, completion_tokens=380),
|
||||
usage_fields={"serviceTier": "standard"},
|
||||
)
|
||||
GEMINI_GEMINI_3_1_PRO_PREVIEW_REASONING: Final = _gemini_case(
|
||||
GEMINI_GEMINI_3_8_FLASH_REASONING, "gemini-gemini-3.1-pro-preview-reasoning",
|
||||
"gemini/gemini-3.1-pro-preview", "gemini-3.1-pro-preview",
|
||||
ExactExpected(spend=0.066248, input_cost=0.003224, output_cost=0.063024, prompt_tokens=1240, completion_tokens=4040)
|
||||
ExactExpected(spend=0.066248, input_cost=0.003224, output_cost=0.063024, prompt_tokens=1240, completion_tokens=4040),
|
||||
usage_fields={"serviceTier": "standard"},
|
||||
)
|
||||
GEMINI_GEMINI_3_1_PRO_PREVIEW_TOOL_CALL: Final = _gemini_case(
|
||||
GEMINI_GEMINI_3_8_FLASH_TOOL_CALL, "gemini-gemini-3.1-pro-preview-tool_call",
|
||||
"gemini/gemini-3.1-pro-preview", "gemini-3.1-pro-preview",
|
||||
ExactExpected(spend=0.0112112, input_cost=0.004784, output_cost=0.0064272, prompt_tokens=1840, completion_tokens=412)
|
||||
)
|
||||
GEMINI_GEMINI_3_1_PRO_PREVIEW_TIERED_INPUT_ABOVE_200K: Final = _gemini_case(
|
||||
GEMINI_GEMINI_3_1_PRO_TIERED_INPUT_ABOVE_200K,
|
||||
"gemini-gemini-3.1-pro-preview-tiered_input_above_200k",
|
||||
"gemini/gemini-3.1-pro-preview",
|
||||
"gemini-3.1-pro-preview",
|
||||
ExactExpected(
|
||||
spend=1.079468, input_cost=1.06496, output_cost=0.014508, prompt_tokens=204800, completion_tokens=620
|
||||
),
|
||||
usage_fields={"serviceTier": "standard"},
|
||||
)
|
||||
GEMINI_GEMINI_3_1_PRO_PREVIEW_TIERED_CACHE_READ_ABOVE_200K: Final = _gemini_case(
|
||||
GEMINI_GEMINI_3_1_PRO_TIERED_CACHE_READ_ABOVE_200K,
|
||||
"gemini-gemini-3.1-pro-preview-tiered_cache_read_above_200k",
|
||||
"gemini/gemini-3.1-pro-preview",
|
||||
"gemini-3.1-pro-preview",
|
||||
ExactExpected(
|
||||
spend=0.13742976, input_cost=0.12619776, output_cost=0.011232, prompt_tokens=205824, completion_tokens=480
|
||||
),
|
||||
usage_fields={"serviceTier": "standard"},
|
||||
)
|
||||
GEMINI_GEMINI_4_ARGON_ROUTE_BRIDGES: Final = bridge_pairs(
|
||||
(GEMINI_GEMINI_4_ARGON_INPUT_TEXT, GEMINI_GEMINI_4_ARGON_STREAM),
|
||||
("/v1/responses", "/v1/messages"),
|
||||
)
|
||||
GEMINI_GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES: Final = bridge_pairs(
|
||||
(GEMINI_GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT, GEMINI_GEMINI_3_1_PRO_PREVIEW_STREAM),
|
||||
("/v1/responses", "/v1/messages"),
|
||||
)
|
||||
|
||||
CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
||||
GEMINI_GEMINI_3_1_PRO_INPUT_TEXT,
|
||||
|
|
@ -3379,12 +3492,19 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
GEMINI_GEMINI_4_ARGON_STREAM,
|
||||
GEMINI_GEMINI_4_ARGON_CACHE_READ,
|
||||
GEMINI_GEMINI_4_ARGON_REASONING,
|
||||
GEMINI_GEMINI_4_ARGON_SERVICE_TIER_FLEX,
|
||||
GEMINI_GEMINI_4_ARGON_SERVICE_TIER_PRIORITY,
|
||||
GEMINI_GEMINI_4_ARGON_TOOL_CALL,
|
||||
GEMINI_GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT,
|
||||
GEMINI_GEMINI_3_1_PRO_PREVIEW_STREAM,
|
||||
GEMINI_GEMINI_3_1_PRO_PREVIEW_CACHE_READ,
|
||||
GEMINI_GEMINI_3_1_PRO_PREVIEW_TIERED_INPUT_ABOVE_200K,
|
||||
GEMINI_GEMINI_3_1_PRO_PREVIEW_TIERED_CACHE_READ_ABOVE_200K,
|
||||
GEMINI_GEMINI_3_1_PRO_PREVIEW_REASONING,
|
||||
GEMINI_GEMINI_3_1_PRO_PREVIEW_TOOL_CALL,
|
||||
GEMINI_GEMINI_3_1_PRO_PREVIEW_SERVICE_TIER_PRIORITY,
|
||||
*(pair.bridged for pair in GEMINI_GEMINI_4_ARGON_ROUTE_BRIDGES),
|
||||
*(pair.bridged for pair in GEMINI_GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES),
|
||||
)
|
||||
|
||||
PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
||||
|
|
@ -3410,6 +3530,22 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
|||
StreamParityTestCase(
|
||||
plain=GEMINI_GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT, streamed=GEMINI_GEMINI_3_1_PRO_PREVIEW_STREAM
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=GEMINI_GEMINI_4_ARGON_ROUTE_BRIDGES[0].bridged,
|
||||
streamed=GEMINI_GEMINI_4_ARGON_ROUTE_BRIDGES[2].bridged,
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=GEMINI_GEMINI_4_ARGON_ROUTE_BRIDGES[1].bridged,
|
||||
streamed=GEMINI_GEMINI_4_ARGON_ROUTE_BRIDGES[3].bridged,
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=GEMINI_GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES[0].bridged,
|
||||
streamed=GEMINI_GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES[2].bridged,
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=GEMINI_GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES[1].bridged,
|
||||
streamed=GEMINI_GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES[3].bridged,
|
||||
),
|
||||
)
|
||||
|
||||
BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = (
|
||||
|
|
@ -3422,6 +3558,6 @@ BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = (
|
|||
),
|
||||
("/v1/responses", "/v1/messages"),
|
||||
),
|
||||
*bridge_pairs((GEMINI_GEMINI_4_ARGON_INPUT_TEXT,), ("/v1/responses",)),
|
||||
*bridge_pairs((GEMINI_GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT,), ("/v1/responses",)),
|
||||
*GEMINI_GEMINI_4_ARGON_ROUTE_BRIDGES,
|
||||
*GEMINI_GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -19,13 +19,17 @@ from integration.cost_calculation.cost_tracking_case import (
|
|||
RecountRates,
|
||||
SseResponse,
|
||||
WavUpload,
|
||||
with_usage,
|
||||
)
|
||||
from integration.cost_calculation.stream_parity.case import StreamParityTestCase, sse_frames
|
||||
from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs
|
||||
from integration.cost_calculation.cases.anthropic import (
|
||||
CLAUDE_FABLE_5_1_CACHE_READ,
|
||||
CLAUDE_FABLE_5_1_CACHE_WRITE_1H,
|
||||
CLAUDE_FABLE_5_1_CACHE_WRITE_5M,
|
||||
CLAUDE_FABLE_5_1_INPUT_TEXT,
|
||||
CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT,
|
||||
CLAUDE_FABLE_5_1_MESSAGES_STREAM,
|
||||
CLAUDE_FABLE_5_1_STREAM,
|
||||
CLAUDE_FABLE_5_1_THINKING,
|
||||
CLAUDE_FABLE_5_1_TOOL_CALL,
|
||||
|
|
@ -33,12 +37,30 @@ from integration.cost_calculation.cases.anthropic import (
|
|||
CLAUDE_HAIKU_5_5_CACHE_WRITE_1H,
|
||||
CLAUDE_HAIKU_5_5_CACHE_WRITE_5M,
|
||||
CLAUDE_HAIKU_5_5_INPUT_TEXT,
|
||||
CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT,
|
||||
CLAUDE_HAIKU_5_5_MESSAGES_STREAM,
|
||||
CLAUDE_HAIKU_5_5_STREAM,
|
||||
CLAUDE_HAIKU_5_5_THINKING,
|
||||
CLAUDE_HAIKU_5_5_TOOL_CALL,
|
||||
)
|
||||
|
||||
|
||||
def _azure_ai_claude_messages_case(
|
||||
response_case: CostTrackingTestCase,
|
||||
request_case: CostTrackingTestCase,
|
||||
name: str,
|
||||
model: str,
|
||||
) -> CostTrackingTestCase:
|
||||
return response_case.model_copy(
|
||||
update={
|
||||
"name": name,
|
||||
"model": model,
|
||||
"endpoint": "/v1/messages",
|
||||
"request": request_case.request,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _azure_ai_claude_usage(
|
||||
input_tokens: int,
|
||||
output_tokens: int,
|
||||
|
|
@ -2027,7 +2049,66 @@ AZURE_AI_CLAUDE_HAIKU_5_5_TOOL_CALL: Final = _azure_ai_claude_shape_case(
|
|||
),
|
||||
)
|
||||
|
||||
|
||||
AZURE_AI_CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT: Final = _azure_ai_claude_messages_case(
|
||||
AZURE_AI_CLAUDE_FABLE_5_1_INPUT_TEXT,
|
||||
CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT,
|
||||
"azure_ai/claude-fable-5-1-messages_input_text",
|
||||
"azure_ai/claude-fable-5-1",
|
||||
)
|
||||
AZURE_AI_CLAUDE_FABLE_5_1_MESSAGES_STREAM: Final = _azure_ai_claude_messages_case(
|
||||
AZURE_AI_CLAUDE_FABLE_5_1_STREAM,
|
||||
CLAUDE_FABLE_5_1_MESSAGES_STREAM,
|
||||
"azure_ai/claude-fable-5-1-messages_stream",
|
||||
"azure_ai/claude-fable-5-1",
|
||||
)
|
||||
AZURE_AI_CLAUDE_FABLE_5_1_ROUTE_BRIDGES: Final = bridge_pairs(
|
||||
(AZURE_AI_CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT, AZURE_AI_CLAUDE_FABLE_5_1_MESSAGES_STREAM),
|
||||
("/v1/responses",),
|
||||
)
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT: Final = _azure_ai_claude_messages_case(
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT,
|
||||
CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT,
|
||||
"azure_ai/claude-haiku-5-5-messages_input_text",
|
||||
"azure_ai/claude-haiku-5-5",
|
||||
)
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_MESSAGES_STREAM: Final = _azure_ai_claude_messages_case(
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_STREAM,
|
||||
CLAUDE_HAIKU_5_5_MESSAGES_STREAM,
|
||||
"azure_ai/claude-haiku-5-5-messages_stream",
|
||||
"azure_ai/claude-haiku-5-5",
|
||||
)
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_ROUTE_BRIDGES: Final = bridge_pairs(
|
||||
(AZURE_AI_CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT, AZURE_AI_CLAUDE_HAIKU_5_5_MESSAGES_STREAM),
|
||||
("/v1/responses",),
|
||||
)
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_TIERED_INPUT_ABOVE_100K: Final = with_usage(
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT,
|
||||
"azure_ai/claude-haiku-5-5-tiered_input_above_100k",
|
||||
{
|
||||
"input_tokens": 150000,
|
||||
"output_tokens": 480,
|
||||
"cache_read_input_tokens": 0,
|
||||
"cache_creation_input_tokens": 0,
|
||||
"cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0},
|
||||
},
|
||||
ExactExpected(
|
||||
spend=0.14478, input_cost=0.1425, output_cost=0.00228, prompt_tokens=150000, completion_tokens=480
|
||||
),
|
||||
)
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_TIERED_CACHE_READ_ABOVE_100K: Final = with_usage(
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_CACHE_READ,
|
||||
"azure_ai/claude-haiku-5-5-tiered_cache_read_above_100k",
|
||||
{
|
||||
"input_tokens": 4096,
|
||||
"output_tokens": 480,
|
||||
"cache_read_input_tokens": 110592,
|
||||
"cache_creation_input_tokens": 0,
|
||||
"cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0},
|
||||
},
|
||||
ExactExpected(
|
||||
spend=0.01667744, input_cost=0.01439744, output_cost=0.00228, prompt_tokens=114688, completion_tokens=480
|
||||
),
|
||||
)
|
||||
CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
||||
PERPLEXITY_SONAR_NEXT_OFF_PEAK,
|
||||
NOVA_NEXT_TRANSCRIPTIONS_PER_SECOND,
|
||||
|
|
@ -2088,13 +2169,21 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
AZURE_AI_CLAUDE_FABLE_5_1_CACHE_WRITE_1H,
|
||||
AZURE_AI_CLAUDE_FABLE_5_1_THINKING,
|
||||
AZURE_AI_CLAUDE_FABLE_5_1_TOOL_CALL,
|
||||
AZURE_AI_CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT,
|
||||
AZURE_AI_CLAUDE_FABLE_5_1_MESSAGES_STREAM,
|
||||
*(pair.bridged for pair in AZURE_AI_CLAUDE_FABLE_5_1_ROUTE_BRIDGES),
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT,
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_STREAM,
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_CACHE_READ,
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_TIERED_INPUT_ABOVE_100K,
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_TIERED_CACHE_READ_ABOVE_100K,
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_CACHE_WRITE_5M,
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_CACHE_WRITE_1H,
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_THINKING,
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_TOOL_CALL,
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT,
|
||||
AZURE_AI_CLAUDE_HAIKU_5_5_MESSAGES_STREAM,
|
||||
*(pair.bridged for pair in AZURE_AI_CLAUDE_HAIKU_5_5_ROUTE_BRIDGES),
|
||||
)
|
||||
|
||||
PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
||||
|
|
@ -2105,4 +2194,25 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
|||
StreamParityTestCase(
|
||||
plain=AZURE_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT, streamed=AZURE_AI_CLAUDE_HAIKU_5_5_STREAM
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=AZURE_AI_CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT,
|
||||
streamed=AZURE_AI_CLAUDE_FABLE_5_1_MESSAGES_STREAM,
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=AZURE_AI_CLAUDE_FABLE_5_1_ROUTE_BRIDGES[0].bridged,
|
||||
streamed=AZURE_AI_CLAUDE_FABLE_5_1_ROUTE_BRIDGES[1].bridged,
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=AZURE_AI_CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT,
|
||||
streamed=AZURE_AI_CLAUDE_HAIKU_5_5_MESSAGES_STREAM,
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=AZURE_AI_CLAUDE_HAIKU_5_5_ROUTE_BRIDGES[0].bridged,
|
||||
streamed=AZURE_AI_CLAUDE_HAIKU_5_5_ROUTE_BRIDGES[1].bridged,
|
||||
),
|
||||
)
|
||||
|
||||
BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = (
|
||||
*AZURE_AI_CLAUDE_FABLE_5_1_ROUTE_BRIDGES,
|
||||
*AZURE_AI_CLAUDE_HAIKU_5_5_ROUTE_BRIDGES,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -34,6 +34,256 @@ def _stream_for_model(response: SseResponse, model: str) -> SseResponse:
|
|||
return response.model_copy(update={"frames": tuple(frame.replace("gpt-6-luna", model) for frame in response.frames)})
|
||||
|
||||
|
||||
def _responses_json_body(
|
||||
model: str,
|
||||
*,
|
||||
include_content_filters: bool = False,
|
||||
content_filters: JsonValue = None,
|
||||
include_reasoning: bool = True,
|
||||
) -> dict[str, JsonValue]:
|
||||
reasoning_item: Final[dict[str, JsonValue]] = {
|
||||
"content": [],
|
||||
"encrypted_content": "$ENCRYPTED_CONTENT",
|
||||
"id": "rs_$REQUEST_ID",
|
||||
"summary": [],
|
||||
"type": "reasoning",
|
||||
}
|
||||
message_item: Final[dict[str, JsonValue]] = {
|
||||
"content": [
|
||||
{
|
||||
"annotations": [],
|
||||
"logprobs": [],
|
||||
"text": "scripted response",
|
||||
"type": "output_text",
|
||||
}
|
||||
],
|
||||
"id": "msg_$REQUEST_ID",
|
||||
"phase": "final_answer",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"type": "message",
|
||||
}
|
||||
output: Final = [reasoning_item, message_item] if include_reasoning else [message_item]
|
||||
body: Final[dict[str, JsonValue]] = {
|
||||
"access_programs": [],
|
||||
"background": False,
|
||||
"completed_at": 1791855453,
|
||||
"created_at": 1791855452,
|
||||
"error": None,
|
||||
"frequency_penalty": None,
|
||||
"id": "resp_$REQUEST_ID",
|
||||
"incomplete_details": None,
|
||||
"instructions": None,
|
||||
"max_output_tokens": None,
|
||||
"max_tool_calls": None,
|
||||
"metadata": {},
|
||||
"moderation": None,
|
||||
"model": model,
|
||||
"object": "response",
|
||||
"output": output,
|
||||
"parallel_tool_calls": True,
|
||||
"presence_penalty": None,
|
||||
"previous_response_id": None,
|
||||
"prompt_cache_key": None,
|
||||
"prompt_cache_retention": None,
|
||||
"reasoning": {"effort": "medium"},
|
||||
"safety_identifier": None,
|
||||
"service_tier": "default",
|
||||
"status": "completed",
|
||||
"store": True,
|
||||
"temperature": 1.0,
|
||||
"text": None,
|
||||
"tool_choice": "auto",
|
||||
"tool_usage": None,
|
||||
"tools": [],
|
||||
"top_logprobs": None,
|
||||
"top_p": 1.0,
|
||||
"truncation": "disabled",
|
||||
"usage": {
|
||||
"input_tokens": 1840,
|
||||
"input_tokens_details": {"cache_write_tokens": 0, "cached_tokens": 0},
|
||||
"output_tokens": 412,
|
||||
"output_tokens_details": {"reasoning_tokens": 0},
|
||||
"total_tokens": 2252,
|
||||
},
|
||||
"user": None,
|
||||
}
|
||||
return {**body, "content_filters": content_filters} if include_content_filters else body
|
||||
|
||||
|
||||
def _responses_stream_response(
|
||||
model: str,
|
||||
delta_count: int,
|
||||
*,
|
||||
include_content_filters: bool = False,
|
||||
content_filters: JsonValue = None,
|
||||
include_reasoning: bool = True,
|
||||
) -> SseResponse:
|
||||
response_body: Final = _responses_json_body(
|
||||
model,
|
||||
include_content_filters=include_content_filters,
|
||||
content_filters=content_filters,
|
||||
include_reasoning=include_reasoning,
|
||||
)
|
||||
in_progress_body: Final = {
|
||||
**response_body,
|
||||
"output": [],
|
||||
"status": "in_progress",
|
||||
"usage": None,
|
||||
}
|
||||
reasoning_item: Final = {
|
||||
"content": [],
|
||||
"encrypted_content": "$ENCRYPTED_CONTENT",
|
||||
"id": "rs_$REQUEST_ID",
|
||||
"summary": [],
|
||||
"type": "reasoning",
|
||||
}
|
||||
message_item: Final = {
|
||||
"content": [],
|
||||
"id": "msg_$REQUEST_ID",
|
||||
"role": "assistant",
|
||||
"status": "in_progress",
|
||||
"type": "message",
|
||||
}
|
||||
delta_events: Final = tuple(
|
||||
{
|
||||
"content_index": 0,
|
||||
"delta": "scripted response" if index == 0 else "",
|
||||
"item_id": "msg_$REQUEST_ID",
|
||||
"logprobs": [],
|
||||
"obfuscation": None,
|
||||
"output_index": 1 if include_reasoning else 0,
|
||||
"sequence_number": (6 if include_reasoning else 4) + index,
|
||||
"type": "response.output_text.delta",
|
||||
}
|
||||
for index in range(delta_count)
|
||||
)
|
||||
delta_start: Final = 6 if include_reasoning else 4
|
||||
message_output_index: Final = 1 if include_reasoning else 0
|
||||
reasoning_events: Final = (
|
||||
(
|
||||
{
|
||||
"item": reasoning_item,
|
||||
"output_index": 0,
|
||||
"sequence_number": 2,
|
||||
"type": "response.output_item.added",
|
||||
},
|
||||
{
|
||||
"item": reasoning_item,
|
||||
"output_index": 0,
|
||||
"sequence_number": 3,
|
||||
"type": "response.output_item.done",
|
||||
},
|
||||
)
|
||||
if include_reasoning
|
||||
else ()
|
||||
)
|
||||
completed_message_item: Final = {
|
||||
"content": [
|
||||
{
|
||||
"annotations": [],
|
||||
"logprobs": [],
|
||||
"text": "scripted response",
|
||||
"type": "output_text",
|
||||
}
|
||||
],
|
||||
"id": "msg_$REQUEST_ID",
|
||||
"phase": "final_answer",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"type": "message",
|
||||
}
|
||||
return SseResponse(
|
||||
content_type="text/event-stream",
|
||||
frames=sse_frames(
|
||||
{
|
||||
"response": in_progress_body,
|
||||
"sequence_number": 0,
|
||||
"type": "response.created",
|
||||
},
|
||||
{
|
||||
"response": in_progress_body,
|
||||
"sequence_number": 1,
|
||||
"type": "response.in_progress",
|
||||
},
|
||||
*reasoning_events,
|
||||
{
|
||||
"item": message_item,
|
||||
"output_index": message_output_index,
|
||||
"sequence_number": 4 if include_reasoning else 2,
|
||||
"type": "response.output_item.added",
|
||||
},
|
||||
{
|
||||
"content_index": 0,
|
||||
"item_id": "msg_$REQUEST_ID",
|
||||
"output_index": message_output_index,
|
||||
"part": {"annotations": [], "logprobs": [], "text": "", "type": "output_text"},
|
||||
"sequence_number": 5 if include_reasoning else 3,
|
||||
"type": "response.content_part.added",
|
||||
},
|
||||
*delta_events,
|
||||
{
|
||||
"content_index": 0,
|
||||
"item_id": "msg_$REQUEST_ID",
|
||||
"logprobs": [],
|
||||
"output_index": message_output_index,
|
||||
"sequence_number": delta_start + delta_count,
|
||||
"text": "scripted response",
|
||||
"type": "response.output_text.done",
|
||||
},
|
||||
{
|
||||
"content_index": 0,
|
||||
"item_id": "msg_$REQUEST_ID",
|
||||
"output_index": message_output_index,
|
||||
"part": {
|
||||
"annotations": [],
|
||||
"logprobs": [],
|
||||
"text": "scripted response",
|
||||
"type": "output_text",
|
||||
},
|
||||
"sequence_number": delta_start + delta_count + 1,
|
||||
"type": "response.content_part.done",
|
||||
},
|
||||
{
|
||||
"item": completed_message_item,
|
||||
"output_index": message_output_index,
|
||||
"sequence_number": delta_start + delta_count + 2,
|
||||
"type": "response.output_item.done",
|
||||
},
|
||||
{
|
||||
"response": response_body,
|
||||
"sequence_number": delta_start + delta_count + 3,
|
||||
"type": "response.completed",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _responses_stream_case(
|
||||
base: CostTrackingTestCase,
|
||||
name: str,
|
||||
response_model: str,
|
||||
delta_count: int,
|
||||
*,
|
||||
include_content_filters: bool = False,
|
||||
content_filters: JsonValue = None,
|
||||
include_reasoning: bool = True,
|
||||
) -> CostTrackingTestCase:
|
||||
return base.model_copy(
|
||||
update={
|
||||
"name": name,
|
||||
"request": {**base.request, "stream": True},
|
||||
"response": _responses_stream_response(
|
||||
response_model,
|
||||
delta_count,
|
||||
include_content_filters=include_content_filters,
|
||||
content_filters=content_filters,
|
||||
include_reasoning=include_reasoning,
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# gpt-5.3-codex
|
||||
GPT_5_3_CODEX_INPUT_TEXT: Final = CostTrackingTestCase(
|
||||
name="gpt-5.3-codex-input_text",
|
||||
|
|
@ -11356,6 +11606,31 @@ GPT_5_6_TERRA_TOOL_CALL: Final = GPT_6_LUNA_TOOL_CALL.model_copy(
|
|||
}
|
||||
)
|
||||
|
||||
GPT_5_6_TERRA_TIERED_INPUT_ABOVE_272K: Final = with_usage(
|
||||
GPT_5_6_TERRA_INPUT_TEXT,
|
||||
"gpt-5.6-terra-tiered_input_above_272k",
|
||||
{
|
||||
"prompt_tokens": 300000,
|
||||
"total_tokens": 300412,
|
||||
"prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 0},
|
||||
},
|
||||
ExactExpected(
|
||||
spend=1.5696408, input_cost=1.56, output_cost=0.0096408, prompt_tokens=300000, completion_tokens=412
|
||||
),
|
||||
)
|
||||
GPT_5_6_TERRA_TIERED_CACHE_READ_ABOVE_272K: Final = with_usage(
|
||||
GPT_5_6_TERRA_INPUT_TEXT,
|
||||
"gpt-5.6-terra-tiered_cache_read_above_272k",
|
||||
{
|
||||
"prompt_tokens": 300000,
|
||||
"total_tokens": 300412,
|
||||
"prompt_tokens_details": {"audio_tokens": 0, "cache_write_tokens": 0, "cached_tokens": 200000},
|
||||
},
|
||||
ExactExpected(
|
||||
spend=0.6336408, input_cost=0.624, output_cost=0.0096408, prompt_tokens=300000, completion_tokens=412
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
GPT_5_6_SLASH_ALIAS: Final = GPT_5_6_INPUT_TEXT.model_copy(
|
||||
update={"name": "gpt-5.6-slash_alias", "alias": "openai/custom/gpt-5.6-$MARKER"}
|
||||
|
|
@ -11446,6 +11721,63 @@ GPT_5_6_RESPONSES_IMAGE_GENERATION_TOOL_TOKENS: Final = CostTrackingTestCase(
|
|||
),
|
||||
)
|
||||
|
||||
GPT_6_1_SOL_RESPONSES_NATIVE_STREAM: Final = _responses_stream_case(
|
||||
GPT_6_1_SOL_RESPONSES_NATIVE_JSON,
|
||||
"gpt-6.1-sol-responses_native_stream",
|
||||
"gpt-6.1-sol",
|
||||
1,
|
||||
include_reasoning=False,
|
||||
)
|
||||
GPT_6_LUNA_RESPONSES_NATIVE_STREAM: Final = _responses_stream_case(
|
||||
GPT_6_LUNA_RESPONSES_NATIVE_JSON,
|
||||
"gpt-6-luna-responses_native_stream",
|
||||
"gpt-6-luna",
|
||||
45,
|
||||
)
|
||||
GPT_6_ASTRA_RESPONSES_NATIVE_STREAM: Final = _responses_stream_case(
|
||||
GPT_6_ASTRA_RESPONSES_NATIVE_JSON,
|
||||
"gpt-6-astra-responses_native_stream",
|
||||
"gpt-6-astra",
|
||||
36,
|
||||
)
|
||||
GPT_5_6_TERRA_RESPONSES_NATIVE_STREAM: Final = _responses_stream_case(
|
||||
GPT_5_6_TERRA_RESPONSES_NATIVE_JSON,
|
||||
"gpt-5.6-terra-responses_native_stream",
|
||||
"gpt-5.6-terra",
|
||||
95,
|
||||
)
|
||||
GPT_5_6_TERRA_MESSAGES_BRIDGED_JSON: Final = bridge_pairs(
|
||||
(GPT_5_6_TERRA_RESPONSES_NATIVE_JSON,), ("/v1/messages",)
|
||||
)[0].bridged.model_copy(update={"name": "gpt-5.6-terra-messages_bridged_json"})
|
||||
GPT_6_1_SOL_MESSAGES_BRIDGED_STREAM: Final = GPT_6_1_SOL_MESSAGES_BRIDGED_JSON.model_copy(
|
||||
update={
|
||||
"name": "gpt-6.1-sol-messages_bridged_stream",
|
||||
"request": {**GPT_6_1_SOL_MESSAGES_BRIDGED_JSON.request, "stream": True},
|
||||
"response": GPT_6_1_SOL_RESPONSES_NATIVE_STREAM.response,
|
||||
}
|
||||
)
|
||||
GPT_6_LUNA_MESSAGES_BRIDGED_STREAM: Final = GPT_6_LUNA_MESSAGES_BRIDGED_JSON.model_copy(
|
||||
update={
|
||||
"name": "gpt-6-luna-messages_bridged_stream",
|
||||
"request": {**GPT_6_LUNA_MESSAGES_BRIDGED_JSON.request, "stream": True},
|
||||
"response": GPT_6_LUNA_RESPONSES_NATIVE_STREAM.response,
|
||||
}
|
||||
)
|
||||
GPT_6_ASTRA_MESSAGES_BRIDGED_STREAM: Final = GPT_6_ASTRA_MESSAGES_BRIDGED_JSON.model_copy(
|
||||
update={
|
||||
"name": "gpt-6-astra-messages_bridged_stream",
|
||||
"request": {**GPT_6_ASTRA_MESSAGES_BRIDGED_JSON.request, "stream": True},
|
||||
"response": GPT_6_ASTRA_RESPONSES_NATIVE_STREAM.response,
|
||||
}
|
||||
)
|
||||
GPT_5_6_TERRA_MESSAGES_BRIDGED_STREAM: Final = GPT_5_6_TERRA_MESSAGES_BRIDGED_JSON.model_copy(
|
||||
update={
|
||||
"name": "gpt-5.6-terra-messages_bridged_stream",
|
||||
"request": {**GPT_5_6_TERRA_MESSAGES_BRIDGED_JSON.request, "stream": True},
|
||||
"response": GPT_5_6_TERRA_RESPONSES_NATIVE_STREAM.response,
|
||||
}
|
||||
)
|
||||
|
||||
CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
||||
GPT_5_6_SLASH_ALIAS,
|
||||
GPT_5_6_SLASH_ALIAS_STREAM,
|
||||
|
|
@ -11659,6 +11991,8 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
GPT_6_LUNA_CACHE_WRITE_5M,
|
||||
GPT_6_ASTRA_CACHE_WRITE_5M,
|
||||
GPT_5_6_TERRA_INPUT_TEXT,
|
||||
GPT_5_6_TERRA_TIERED_INPUT_ABOVE_272K,
|
||||
GPT_5_6_TERRA_TIERED_CACHE_READ_ABOVE_272K,
|
||||
GPT_5_6_TERRA_STREAM,
|
||||
GPT_5_6_TERRA_RESPONSES_NATIVE_JSON,
|
||||
GPT_5_6_TERRA_RESPONSES_DEPLOYMENT_JSON,
|
||||
|
|
@ -11669,6 +12003,15 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
GPT_5_6_TERRA_SERVICE_TIER_FLEX,
|
||||
GPT_5_6_TERRA_SERVICE_TIER_PRIORITY,
|
||||
GPT_5_6_TERRA_TOOL_CALL,
|
||||
GPT_6_1_SOL_RESPONSES_NATIVE_STREAM,
|
||||
GPT_6_LUNA_RESPONSES_NATIVE_STREAM,
|
||||
GPT_6_ASTRA_RESPONSES_NATIVE_STREAM,
|
||||
GPT_5_6_TERRA_RESPONSES_NATIVE_STREAM,
|
||||
GPT_5_6_TERRA_MESSAGES_BRIDGED_JSON,
|
||||
GPT_6_1_SOL_MESSAGES_BRIDGED_STREAM,
|
||||
GPT_6_LUNA_MESSAGES_BRIDGED_STREAM,
|
||||
GPT_6_ASTRA_MESSAGES_BRIDGED_STREAM,
|
||||
GPT_5_6_TERRA_MESSAGES_BRIDGED_STREAM,
|
||||
)
|
||||
|
||||
PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
||||
|
|
@ -11698,6 +12041,16 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
|||
StreamParityTestCase(plain=GPT_6_ASTRA_INPUT_TEXT, streamed=GPT_6_ASTRA_STREAM),
|
||||
StreamParityTestCase(plain=GPT_6_1_SOL_INPUT_TEXT, streamed=GPT_6_1_SOL_STREAM),
|
||||
StreamParityTestCase(plain=GPT_5_6_TERRA_INPUT_TEXT, streamed=GPT_5_6_TERRA_STREAM),
|
||||
StreamParityTestCase(plain=GPT_6_1_SOL_RESPONSES_NATIVE_JSON, streamed=GPT_6_1_SOL_RESPONSES_NATIVE_STREAM),
|
||||
StreamParityTestCase(plain=GPT_6_LUNA_RESPONSES_NATIVE_JSON, streamed=GPT_6_LUNA_RESPONSES_NATIVE_STREAM),
|
||||
StreamParityTestCase(plain=GPT_6_ASTRA_RESPONSES_NATIVE_JSON, streamed=GPT_6_ASTRA_RESPONSES_NATIVE_STREAM),
|
||||
StreamParityTestCase(plain=GPT_5_6_TERRA_RESPONSES_NATIVE_JSON, streamed=GPT_5_6_TERRA_RESPONSES_NATIVE_STREAM),
|
||||
StreamParityTestCase(plain=GPT_6_1_SOL_MESSAGES_BRIDGED_JSON, streamed=GPT_6_1_SOL_MESSAGES_BRIDGED_STREAM),
|
||||
StreamParityTestCase(plain=GPT_6_LUNA_MESSAGES_BRIDGED_JSON, streamed=GPT_6_LUNA_MESSAGES_BRIDGED_STREAM),
|
||||
StreamParityTestCase(plain=GPT_6_ASTRA_MESSAGES_BRIDGED_JSON, streamed=GPT_6_ASTRA_MESSAGES_BRIDGED_STREAM),
|
||||
StreamParityTestCase(
|
||||
plain=GPT_5_6_TERRA_MESSAGES_BRIDGED_JSON, streamed=GPT_5_6_TERRA_MESSAGES_BRIDGED_STREAM
|
||||
),
|
||||
)
|
||||
|
||||
BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = (
|
||||
|
|
@ -11726,6 +12079,13 @@ BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = (
|
|||
BridgeCostParityTestCase(native=GPT_6_LUNA_RESPONSES_NATIVE_JSON, bridged=GPT_6_LUNA_MESSAGES_BRIDGED_JSON),
|
||||
BridgeCostParityTestCase(native=GPT_6_ASTRA_RESPONSES_NATIVE_JSON, bridged=GPT_6_ASTRA_MESSAGES_BRIDGED_JSON),
|
||||
BridgeCostParityTestCase(native=GPT_6_1_SOL_RESPONSES_NATIVE_JSON, bridged=GPT_6_1_SOL_MESSAGES_BRIDGED_JSON),
|
||||
BridgeCostParityTestCase(native=GPT_5_6_TERRA_RESPONSES_NATIVE_JSON, bridged=GPT_5_6_TERRA_MESSAGES_BRIDGED_JSON),
|
||||
BridgeCostParityTestCase(native=GPT_6_1_SOL_RESPONSES_NATIVE_STREAM, bridged=GPT_6_1_SOL_MESSAGES_BRIDGED_STREAM),
|
||||
BridgeCostParityTestCase(native=GPT_6_LUNA_RESPONSES_NATIVE_STREAM, bridged=GPT_6_LUNA_MESSAGES_BRIDGED_STREAM),
|
||||
BridgeCostParityTestCase(native=GPT_6_ASTRA_RESPONSES_NATIVE_STREAM, bridged=GPT_6_ASTRA_MESSAGES_BRIDGED_STREAM),
|
||||
BridgeCostParityTestCase(
|
||||
native=GPT_5_6_TERRA_RESPONSES_NATIVE_STREAM, bridged=GPT_5_6_TERRA_MESSAGES_BRIDGED_STREAM
|
||||
),
|
||||
BridgeCostParityTestCase(native=GPT_6_SOL_RESPONSES_DEPLOYMENT_JSON, bridged=GPT_6_SOL_CHAT_BRIDGED_JSON),
|
||||
BridgeCostParityTestCase(native=GPT_6_LUNA_RESPONSES_DEPLOYMENT_JSON, bridged=GPT_6_LUNA_CHAT_BRIDGED_JSON),
|
||||
BridgeCostParityTestCase(native=GPT_6_ASTRA_RESPONSES_DEPLOYMENT_JSON, bridged=GPT_6_ASTRA_CHAT_BRIDGED_JSON),
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
PARITY pairs the streamed case with its plain twin where both bill the same row."""
|
||||
|
||||
import json
|
||||
from typing import Final, cast
|
||||
|
||||
from pydantic import JsonValue
|
||||
|
|
@ -11,13 +12,19 @@ from integration.cost_calculation.cases.anthropic import (
|
|||
CLAUDE_FABLE_5_1_CACHE_WRITE_1H,
|
||||
CLAUDE_FABLE_5_1_CACHE_WRITE_5M,
|
||||
CLAUDE_FABLE_5_1_INPUT_TEXT,
|
||||
CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT,
|
||||
CLAUDE_FABLE_5_1_MESSAGES_STREAM,
|
||||
CLAUDE_FABLE_5_1_STREAM,
|
||||
CLAUDE_FABLE_5_1_THINKING,
|
||||
CLAUDE_FABLE_5_1_TOOL_CALL,
|
||||
CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT,
|
||||
CLAUDE_HAIKU_5_5_MESSAGES_STREAM,
|
||||
CLAUDE_OPUS_5_5_CACHE_READ,
|
||||
CLAUDE_OPUS_5_5_CACHE_WRITE_1H,
|
||||
CLAUDE_OPUS_5_5_CACHE_WRITE_5M,
|
||||
CLAUDE_OPUS_5_5_INPUT_TEXT,
|
||||
CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT,
|
||||
CLAUDE_OPUS_5_5_MESSAGES_STREAM,
|
||||
CLAUDE_OPUS_5_5_STREAM,
|
||||
CLAUDE_OPUS_5_5_THINKING,
|
||||
CLAUDE_OPUS_5_5_TOOL_CALL,
|
||||
|
|
@ -25,6 +32,8 @@ from integration.cost_calculation.cases.anthropic import (
|
|||
CLAUDE_SONNET_5_5_CACHE_WRITE_1H,
|
||||
CLAUDE_SONNET_5_5_CACHE_WRITE_5M,
|
||||
CLAUDE_SONNET_5_5_INPUT_TEXT,
|
||||
CLAUDE_SONNET_5_5_MESSAGES_INPUT_TEXT,
|
||||
CLAUDE_SONNET_5_5_MESSAGES_STREAM,
|
||||
CLAUDE_SONNET_5_5_STREAM,
|
||||
CLAUDE_SONNET_5_5_THINKING,
|
||||
CLAUDE_SONNET_5_5_TOOL_CALL,
|
||||
|
|
@ -41,26 +50,72 @@ from integration.cost_calculation.cost_tracking_case import (
|
|||
from integration.cost_calculation.stream_parity.case import StreamParityTestCase, sse_frames
|
||||
|
||||
|
||||
def _gemini_case_response(response: JsonResponse | SseResponse, model_version: str) -> JsonResponse | SseResponse:
|
||||
def _gemini_case_response(
|
||||
response: JsonResponse | SseResponse,
|
||||
model_version: str,
|
||||
usage_fields: dict[str, JsonValue] | None = None,
|
||||
) -> JsonResponse | SseResponse:
|
||||
match response:
|
||||
case JsonResponse():
|
||||
return response.model_copy(update={"body": {**response.body, "modelVersion": model_version}})
|
||||
usage: Final = response.body.get("usageMetadata")
|
||||
body: Final = {**response.body, "modelVersion": model_version}
|
||||
return response.model_copy(
|
||||
update={
|
||||
"body": {
|
||||
**body,
|
||||
**(
|
||||
{
|
||||
"usageMetadata": {
|
||||
**cast(dict[str, JsonValue], usage),
|
||||
**usage_fields,
|
||||
}
|
||||
}
|
||||
if usage_fields is not None and isinstance(usage, dict)
|
||||
else {}
|
||||
),
|
||||
}
|
||||
}
|
||||
)
|
||||
case SseResponse():
|
||||
return response.model_copy(
|
||||
update={
|
||||
"frames": tuple(frame.replace("gemini-3.8-flash", model_version) for frame in response.frames)
|
||||
"frames": tuple(
|
||||
_gemini_case_frame(frame, model_version, usage_fields) for frame in response.frames
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _gemini_case_frame(frame: str, model_version: str, usage_fields: dict[str, JsonValue] | None) -> str:
|
||||
replaced: Final = frame.replace("gemini-3.8-flash", model_version)
|
||||
if usage_fields is None:
|
||||
return replaced
|
||||
prefix, separator, payload = replaced.partition("data: ")
|
||||
if not separator or payload == "[DONE]":
|
||||
return replaced
|
||||
event: Final = cast(dict[str, JsonValue], json.loads(payload))
|
||||
usage: Final = event.get("usageMetadata")
|
||||
if not isinstance(usage, dict):
|
||||
return replaced
|
||||
return f"{prefix}data: {json.dumps({**event, 'usageMetadata': {**usage, **usage_fields}})}"
|
||||
|
||||
|
||||
def _gemini_case(
|
||||
sibling: CostTrackingTestCase, name: str, model: str, expected: ExactExpected
|
||||
sibling: CostTrackingTestCase,
|
||||
name: str,
|
||||
model: str,
|
||||
expected: ExactExpected,
|
||||
*,
|
||||
usage_fields: dict[str, JsonValue] | None = None,
|
||||
model_version: str = "gemini-3.1-pro-preview",
|
||||
) -> CostTrackingTestCase:
|
||||
return sibling.model_copy(
|
||||
update={
|
||||
"name": name,
|
||||
"model": model,
|
||||
"response": _gemini_case_response(cast(JsonResponse | SseResponse, sibling.response), "gemini-3.1-pro-preview"),
|
||||
"response": _gemini_case_response(
|
||||
cast(JsonResponse | SseResponse, sibling.response), model_version, usage_fields
|
||||
),
|
||||
"expected": expected,
|
||||
}
|
||||
)
|
||||
|
|
@ -97,6 +152,7 @@ def _vertex_claude_shape_case(
|
|||
expected: ExactExpected,
|
||||
response_content: list[dict[str, JsonValue]] | None = None,
|
||||
adaptive_thinking: bool = False,
|
||||
thinking_effort: str = "medium",
|
||||
) -> CostTrackingTestCase:
|
||||
response: Final = cast(JsonResponse, sibling.response)
|
||||
request: Final = (
|
||||
|
|
@ -107,7 +163,7 @@ def _vertex_claude_shape_case(
|
|||
if key not in {"reasoning_effort", "allowed_openai_params"}
|
||||
},
|
||||
"thinking": {"type": "adaptive"},
|
||||
"output_config": {"effort": "medium"},
|
||||
"output_config": {"effort": thinking_effort},
|
||||
"allowed_openai_params": ["thinking", "output_config"],
|
||||
}
|
||||
if adaptive_thinking
|
||||
|
|
@ -1716,6 +1772,8 @@ GEMINI_3_8_FLASH_INPUT_TEXT: Final = CostTrackingTestCase(
|
|||
"candidatesTokenCount": 412,
|
||||
"totalTokenCount": 2252,
|
||||
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 1840}],
|
||||
"candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 412}],
|
||||
"trafficType": "ON_DEMAND",
|
||||
},
|
||||
"modelVersion": "gemini-3.8-flash",
|
||||
},
|
||||
|
|
@ -1770,6 +1828,8 @@ GEMINI_3_8_FLASH_CACHE_READ: Final = CostTrackingTestCase(
|
|||
"totalTokenCount": 13308,
|
||||
"cachedContentTokenCount": 12288,
|
||||
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 12928}],
|
||||
"candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 380}],
|
||||
"trafficType": "ON_DEMAND",
|
||||
},
|
||||
"modelVersion": "gemini-3.8-flash",
|
||||
},
|
||||
|
|
@ -2117,6 +2177,8 @@ GEMINI_3_8_FLASH_STREAM: Final = CostTrackingTestCase(
|
|||
"candidatesTokenCount": 412,
|
||||
"totalTokenCount": 2252,
|
||||
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 1840}],
|
||||
"candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 412}],
|
||||
"trafficType": "ON_DEMAND",
|
||||
},
|
||||
"modelVersion": "gemini-3.8-flash",
|
||||
},
|
||||
|
|
@ -3122,28 +3184,258 @@ VERTEX_AI_CLAUDE_FABLE_5_1_TOOL_CALL: Final = _vertex_claude_shape_case(
|
|||
),
|
||||
)
|
||||
|
||||
VERTEX_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT: Final = _vertex_claude_shape_case(
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_INPUT_TEXT,
|
||||
"vertex_ai/claude-haiku-5-5-input_text",
|
||||
"vertex_ai/claude-haiku-5-5",
|
||||
_vertex_claude_usage(input_tokens=1840, output_tokens=412),
|
||||
ExactExpected(spend=0.00507, input_cost=0.002392, output_cost=0.002678, prompt_tokens=1840, completion_tokens=412),
|
||||
)
|
||||
VERTEX_AI_CLAUDE_HAIKU_5_5_STREAM: Final = VERTEX_AI_CLAUDE_SONNET_5_5_STREAM.model_copy(
|
||||
update={
|
||||
"name": "vertex_ai/claude-haiku-5-5-stream",
|
||||
"model": "vertex_ai/claude-haiku-5-5",
|
||||
"response": _vertex_claude_stream_response("claude-haiku-5-5", "end_turn", ("scripted answer",)),
|
||||
"expected": VERTEX_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT.expected,
|
||||
}
|
||||
)
|
||||
VERTEX_AI_CLAUDE_HAIKU_5_5_CACHE_READ: Final = _vertex_claude_shape_case(
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_CACHE_READ,
|
||||
"vertex_ai/claude-haiku-5-5-cache_read",
|
||||
"vertex_ai/claude-haiku-5-5",
|
||||
_vertex_claude_usage(input_tokens=14, output_tokens=6, cache_read_input_tokens=27003),
|
||||
ExactExpected(
|
||||
spend=0.00356759,
|
||||
input_cost=0.00352859,
|
||||
output_cost=0.000039,
|
||||
prompt_tokens=27017,
|
||||
completion_tokens=6,
|
||||
cache_read_cost=0.00351039,
|
||||
),
|
||||
)
|
||||
VERTEX_AI_CLAUDE_HAIKU_5_5_CACHE_WRITE_5M: Final = _vertex_claude_shape_case(
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_CACHE_WRITE_5M,
|
||||
"vertex_ai/claude-haiku-5-5-cache_write_5m",
|
||||
"vertex_ai/claude-haiku-5-5",
|
||||
_vertex_claude_usage(
|
||||
input_tokens=14, output_tokens=6, cache_creation_input_tokens=27011, ephemeral_5m_input_tokens=27011
|
||||
),
|
||||
ExactExpected(
|
||||
spend=0.043950075,
|
||||
input_cost=0.043911075,
|
||||
output_cost=0.000039,
|
||||
prompt_tokens=27025,
|
||||
completion_tokens=6,
|
||||
cache_creation_cost=0.043892875,
|
||||
),
|
||||
)
|
||||
VERTEX_AI_CLAUDE_HAIKU_5_5_CACHE_WRITE_1H: Final = _vertex_claude_shape_case(
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_CACHE_WRITE_1H,
|
||||
"vertex_ai/claude-haiku-5-5-cache_write_1h",
|
||||
"vertex_ai/claude-haiku-5-5",
|
||||
_vertex_claude_usage(
|
||||
input_tokens=14, output_tokens=6, cache_creation_input_tokens=27011, ephemeral_1h_input_tokens=27011
|
||||
),
|
||||
ExactExpected(
|
||||
spend=0.0702858,
|
||||
input_cost=0.0702468,
|
||||
output_cost=0.000039,
|
||||
prompt_tokens=27025,
|
||||
completion_tokens=6,
|
||||
cache_creation_cost=0.0702286,
|
||||
),
|
||||
)
|
||||
VERTEX_AI_CLAUDE_HAIKU_5_5_THINKING: Final = _vertex_claude_shape_case(
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_THINKING,
|
||||
"vertex_ai/claude-haiku-5-5-thinking",
|
||||
"vertex_ai/claude-haiku-5-5",
|
||||
_vertex_claude_usage(input_tokens=18, output_tokens=144, thinking_tokens=37),
|
||||
ExactExpected(
|
||||
spend=0.0009594, input_cost=0.0000234, output_cost=0.000936, prompt_tokens=18, completion_tokens=144
|
||||
),
|
||||
adaptive_thinking=True,
|
||||
thinking_effort="low",
|
||||
)
|
||||
VERTEX_AI_CLAUDE_HAIKU_5_5_TOOL_CALL: Final = _vertex_claude_shape_case(
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_TOOL_CALL,
|
||||
"vertex_ai/claude-haiku-5-5-tool_call",
|
||||
"vertex_ai/claude-haiku-5-5",
|
||||
_vertex_claude_usage(input_tokens=385, output_tokens=50),
|
||||
ExactExpected(
|
||||
spend=0.0008255, input_cost=0.0005005, output_cost=0.000325, prompt_tokens=385, completion_tokens=50
|
||||
),
|
||||
)
|
||||
VERTEX_AI_CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT: Final = CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT.model_copy(
|
||||
update={
|
||||
"name": "vertex_ai/claude-haiku-5-5-messages_input_text",
|
||||
"model": "vertex_ai/claude-haiku-5-5",
|
||||
"endpoint": "/v1/messages",
|
||||
"response": VERTEX_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT.response,
|
||||
"expected": VERTEX_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT.expected,
|
||||
}
|
||||
)
|
||||
VERTEX_AI_CLAUDE_HAIKU_5_5_MESSAGES_STREAM: Final = CLAUDE_HAIKU_5_5_MESSAGES_STREAM.model_copy(
|
||||
update={
|
||||
"name": "vertex_ai/claude-haiku-5-5-messages_stream",
|
||||
"model": "vertex_ai/claude-haiku-5-5",
|
||||
"endpoint": "/v1/messages",
|
||||
"response": VERTEX_AI_CLAUDE_HAIKU_5_5_STREAM.response,
|
||||
"expected": VERTEX_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT.expected,
|
||||
}
|
||||
)
|
||||
|
||||
VERTEX_AI_CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT: Final = CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT.model_copy(
|
||||
update={
|
||||
"name": "vertex_ai/claude-fable-5-1-messages_input_text",
|
||||
"model": "vertex_ai/claude-fable-5-1",
|
||||
"endpoint": "/v1/messages",
|
||||
"response": VERTEX_AI_CLAUDE_FABLE_5_1_INPUT_TEXT.response,
|
||||
"expected": VERTEX_AI_CLAUDE_FABLE_5_1_INPUT_TEXT.expected,
|
||||
}
|
||||
)
|
||||
VERTEX_AI_CLAUDE_FABLE_5_1_MESSAGES_STREAM: Final = CLAUDE_FABLE_5_1_MESSAGES_STREAM.model_copy(
|
||||
update={
|
||||
"name": "vertex_ai/claude-fable-5-1-messages_stream",
|
||||
"model": "vertex_ai/claude-fable-5-1",
|
||||
"endpoint": "/v1/messages",
|
||||
"response": VERTEX_AI_CLAUDE_FABLE_5_1_STREAM.response,
|
||||
"expected": VERTEX_AI_CLAUDE_FABLE_5_1_INPUT_TEXT.expected,
|
||||
}
|
||||
)
|
||||
VERTEX_AI_CLAUDE_FABLE_5_1_BRIDGE_PAIRS: Final = bridge_pairs(
|
||||
(VERTEX_AI_CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT, VERTEX_AI_CLAUDE_FABLE_5_1_MESSAGES_STREAM),
|
||||
("/v1/responses",),
|
||||
)
|
||||
VERTEX_AI_CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT: Final = CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT.model_copy(
|
||||
update={
|
||||
"name": "vertex_ai/claude-opus-5-5-messages_input_text",
|
||||
"model": "vertex_ai/claude-opus-5-5",
|
||||
"endpoint": "/v1/messages",
|
||||
"response": VERTEX_AI_CLAUDE_OPUS_5_5_INPUT_TEXT.response,
|
||||
"expected": VERTEX_AI_CLAUDE_OPUS_5_5_INPUT_TEXT.expected,
|
||||
}
|
||||
)
|
||||
VERTEX_AI_CLAUDE_OPUS_5_5_MESSAGES_STREAM: Final = CLAUDE_OPUS_5_5_MESSAGES_STREAM.model_copy(
|
||||
update={
|
||||
"name": "vertex_ai/claude-opus-5-5-messages_stream",
|
||||
"model": "vertex_ai/claude-opus-5-5",
|
||||
"endpoint": "/v1/messages",
|
||||
"response": VERTEX_AI_CLAUDE_OPUS_5_5_STREAM.response,
|
||||
"expected": VERTEX_AI_CLAUDE_OPUS_5_5_INPUT_TEXT.expected,
|
||||
}
|
||||
)
|
||||
VERTEX_AI_CLAUDE_OPUS_5_5_BRIDGE_PAIRS: Final = bridge_pairs(
|
||||
(VERTEX_AI_CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT, VERTEX_AI_CLAUDE_OPUS_5_5_MESSAGES_STREAM),
|
||||
("/v1/responses",),
|
||||
)
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_INPUT_TEXT: Final = CLAUDE_SONNET_5_5_MESSAGES_INPUT_TEXT.model_copy(
|
||||
update={
|
||||
"name": "vertex_ai/claude-sonnet-5-5-messages_input_text",
|
||||
"model": "vertex_ai/claude-sonnet-5-5",
|
||||
"endpoint": "/v1/messages",
|
||||
"response": VERTEX_AI_CLAUDE_SONNET_5_5_INPUT_TEXT.response,
|
||||
"expected": VERTEX_AI_CLAUDE_SONNET_5_5_INPUT_TEXT.expected,
|
||||
}
|
||||
)
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_STREAM: Final = CLAUDE_SONNET_5_5_MESSAGES_STREAM.model_copy(
|
||||
update={
|
||||
"name": "vertex_ai/claude-sonnet-5-5-messages_stream",
|
||||
"model": "vertex_ai/claude-sonnet-5-5",
|
||||
"endpoint": "/v1/messages",
|
||||
"response": VERTEX_AI_CLAUDE_SONNET_5_5_STREAM.response,
|
||||
"expected": VERTEX_AI_CLAUDE_SONNET_5_5_INPUT_TEXT.expected,
|
||||
}
|
||||
)
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_BRIDGE_PAIRS: Final = bridge_pairs(
|
||||
(VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_INPUT_TEXT, VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_STREAM),
|
||||
("/v1/responses",),
|
||||
)
|
||||
VERTEX_AI_CLAUDE_HAIKU_5_5_BRIDGE_PAIRS: Final = bridge_pairs(
|
||||
(VERTEX_AI_CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT, VERTEX_AI_CLAUDE_HAIKU_5_5_MESSAGES_STREAM),
|
||||
("/v1/responses",),
|
||||
)
|
||||
|
||||
|
||||
GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT: Final = _gemini_case(
|
||||
GEMINI_3_8_FLASH_INPUT_TEXT, "vertex_ai/gemini-3.1-pro-preview-input_text", "vertex_ai/gemini-3.1-pro-preview",
|
||||
ExactExpected(spend=0.0094864, input_cost=0.004048, output_cost=0.0054384, prompt_tokens=1840, completion_tokens=412)
|
||||
ExactExpected(spend=0.0094864, input_cost=0.004048, output_cost=0.0054384, prompt_tokens=1840, completion_tokens=412),
|
||||
usage_fields={
|
||||
"candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 412}],
|
||||
"trafficType": "ON_DEMAND",
|
||||
},
|
||||
)
|
||||
GEMINI_3_1_PRO_PREVIEW_STREAM: Final = _gemini_case(
|
||||
GEMINI_3_8_FLASH_STREAM, "vertex_ai/gemini-3.1-pro-preview-stream", "vertex_ai/gemini-3.1-pro-preview",
|
||||
ExactExpected(spend=0.0094864, input_cost=0.004048, output_cost=0.0054384, prompt_tokens=1840, completion_tokens=412)
|
||||
ExactExpected(spend=0.0094864, input_cost=0.004048, output_cost=0.0054384, prompt_tokens=1840, completion_tokens=412),
|
||||
usage_fields={
|
||||
"candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 412}],
|
||||
"trafficType": "ON_DEMAND",
|
||||
},
|
||||
)
|
||||
GEMINI_3_1_PRO_PREVIEW_CACHE_READ: Final = _gemini_case(
|
||||
GEMINI_3_8_FLASH_CACHE_READ, "vertex_ai/gemini-3.1-pro-preview-cache_read", "vertex_ai/gemini-3.1-pro-preview",
|
||||
ExactExpected(spend=0.00912736, input_cost=0.00411136, output_cost=0.005016, prompt_tokens=12928, completion_tokens=380)
|
||||
ExactExpected(spend=0.00912736, input_cost=0.00411136, output_cost=0.005016, prompt_tokens=12928, completion_tokens=380),
|
||||
usage_fields={
|
||||
"candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 380}],
|
||||
"trafficType": "ON_DEMAND",
|
||||
},
|
||||
)
|
||||
GEMINI_3_1_PRO_PREVIEW_REASONING: Final = _gemini_case(
|
||||
GEMINI_3_1_PRO_REASONING, "vertex_ai/gemini-3.1-pro-preview-reasoning", "vertex_ai/gemini-3.1-pro-preview",
|
||||
ExactExpected(spend=0.056056, input_cost=0.002728, output_cost=0.053328, prompt_tokens=1240, completion_tokens=4040)
|
||||
ExactExpected(spend=0.056056, input_cost=0.002728, output_cost=0.053328, prompt_tokens=1240, completion_tokens=4040),
|
||||
usage_fields={
|
||||
"candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 560}],
|
||||
"trafficType": "ON_DEMAND",
|
||||
},
|
||||
)
|
||||
GEMINI_3_8_FLASH_REASONING: Final = _gemini_case(
|
||||
GEMINI_3_1_PRO_REASONING,
|
||||
"vertex_ai/gemini-3.8-flash-reasoning",
|
||||
"vertex_ai/gemini-3.8-flash",
|
||||
ExactExpected(
|
||||
spend=0.01144,
|
||||
input_cost=0.0006448,
|
||||
output_cost=0.0107952,
|
||||
prompt_tokens=1240,
|
||||
completion_tokens=4040,
|
||||
),
|
||||
usage_fields={
|
||||
"candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 560}],
|
||||
"trafficType": "ON_DEMAND",
|
||||
},
|
||||
model_version="gemini-3.8-flash",
|
||||
)
|
||||
GEMINI_3_1_PRO_PREVIEW_TOOL_CALL: Final = _gemini_case(
|
||||
GEMINI_3_8_FLASH_TOOL_CALL, "vertex_ai/gemini-3.1-pro-preview-tool_call", "vertex_ai/gemini-3.1-pro-preview",
|
||||
ExactExpected(spend=0.0094864, input_cost=0.004048, output_cost=0.0054384, prompt_tokens=1840, completion_tokens=412)
|
||||
)
|
||||
|
||||
GEMINI_3_1_PRO_PREVIEW_TIERED_INPUT_ABOVE_200K: Final = _gemini_case(
|
||||
GEMINI_3_1_PRO_TIERED_INPUT_ABOVE_200K,
|
||||
"vertex_ai/gemini-3.1-pro-preview-tiered_input_above_200k",
|
||||
"vertex_ai/gemini-3.1-pro-preview",
|
||||
ExactExpected(
|
||||
spend=0.913396, input_cost=0.90112, output_cost=0.012276, prompt_tokens=204800, completion_tokens=620
|
||||
),
|
||||
usage_fields={
|
||||
"candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 620}],
|
||||
"trafficType": "ON_DEMAND",
|
||||
},
|
||||
)
|
||||
GEMINI_3_1_PRO_PREVIEW_TIERED_CACHE_READ_ABOVE_200K: Final = _gemini_case(
|
||||
GEMINI_3_1_PRO_TIERED_CACHE_READ_ABOVE_200K,
|
||||
"vertex_ai/gemini-3.1-pro-preview-tiered_cache_read_above_200k",
|
||||
"vertex_ai/gemini-3.1-pro-preview",
|
||||
ExactExpected(
|
||||
spend=0.11628672, input_cost=0.10678272, output_cost=0.009504, prompt_tokens=205824, completion_tokens=480
|
||||
),
|
||||
usage_fields={
|
||||
"candidatesTokensDetails": [{"modality": "TEXT", "tokenCount": 480}],
|
||||
"trafficType": "ON_DEMAND",
|
||||
"cacheTokensDetails": [{"modality": "TEXT", "tokenCount": 201728}],
|
||||
},
|
||||
)
|
||||
|
||||
GEMINI_4_ARGON_INPUT_TEXT: Final = GEMINI_3_8_FLASH_INPUT_TEXT.model_copy(
|
||||
update={
|
||||
"name": "vertex_ai/gemini-4-argon-input_text",
|
||||
|
|
@ -3301,6 +3593,15 @@ GEMINI_4_ARGON_TOOL_CALL: Final = GEMINI_3_8_FLASH_TOOL_CALL.model_copy(
|
|||
}
|
||||
)
|
||||
|
||||
GEMINI_4_ARGON_ROUTE_BRIDGES: Final = bridge_pairs(
|
||||
(GEMINI_4_ARGON_INPUT_TEXT, GEMINI_4_ARGON_STREAM),
|
||||
("/v1/responses", "/v1/messages"),
|
||||
)
|
||||
GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES: Final = bridge_pairs(
|
||||
(GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT, GEMINI_3_1_PRO_PREVIEW_STREAM),
|
||||
("/v1/responses", "/v1/messages"),
|
||||
)
|
||||
|
||||
CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
||||
GEMINI_3_1_PRO_INPUT_TEXT,
|
||||
GEMINI_3_1_PRO_CACHE_READ,
|
||||
|
|
@ -3326,6 +3627,7 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
GEMINI_3_1_PRO_STREAM_TOOL_CALL,
|
||||
GEMINI_3_1_PRO_STREAM_FULL_USAGE,
|
||||
GEMINI_3_8_FLASH_INPUT_TEXT,
|
||||
GEMINI_3_8_FLASH_REASONING,
|
||||
GEMINI_3_8_FLASH_CACHE_READ,
|
||||
GEMINI_3_8_FLASH_SERVICE_TIER_FLEX,
|
||||
GEMINI_3_8_FLASH_SERVICE_TIER_PRIORITY,
|
||||
|
|
@ -3367,9 +3669,30 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
VERTEX_AI_CLAUDE_FABLE_5_1_CACHE_WRITE_1H,
|
||||
VERTEX_AI_CLAUDE_FABLE_5_1_THINKING,
|
||||
VERTEX_AI_CLAUDE_FABLE_5_1_TOOL_CALL,
|
||||
VERTEX_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT,
|
||||
VERTEX_AI_CLAUDE_HAIKU_5_5_STREAM,
|
||||
VERTEX_AI_CLAUDE_HAIKU_5_5_CACHE_READ,
|
||||
VERTEX_AI_CLAUDE_HAIKU_5_5_CACHE_WRITE_5M,
|
||||
VERTEX_AI_CLAUDE_HAIKU_5_5_CACHE_WRITE_1H,
|
||||
VERTEX_AI_CLAUDE_HAIKU_5_5_THINKING,
|
||||
VERTEX_AI_CLAUDE_HAIKU_5_5_TOOL_CALL,
|
||||
VERTEX_AI_CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT,
|
||||
VERTEX_AI_CLAUDE_HAIKU_5_5_MESSAGES_STREAM,
|
||||
*(pair.bridged for pair in VERTEX_AI_CLAUDE_HAIKU_5_5_BRIDGE_PAIRS),
|
||||
VERTEX_AI_CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT,
|
||||
VERTEX_AI_CLAUDE_FABLE_5_1_MESSAGES_STREAM,
|
||||
*(pair.bridged for pair in VERTEX_AI_CLAUDE_FABLE_5_1_BRIDGE_PAIRS),
|
||||
VERTEX_AI_CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT,
|
||||
VERTEX_AI_CLAUDE_OPUS_5_5_MESSAGES_STREAM,
|
||||
*(pair.bridged for pair in VERTEX_AI_CLAUDE_OPUS_5_5_BRIDGE_PAIRS),
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_INPUT_TEXT,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_STREAM,
|
||||
*(pair.bridged for pair in VERTEX_AI_CLAUDE_SONNET_5_5_BRIDGE_PAIRS),
|
||||
GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT,
|
||||
GEMINI_3_1_PRO_PREVIEW_STREAM,
|
||||
GEMINI_3_1_PRO_PREVIEW_CACHE_READ,
|
||||
GEMINI_3_1_PRO_PREVIEW_TIERED_INPUT_ABOVE_200K,
|
||||
GEMINI_3_1_PRO_PREVIEW_TIERED_CACHE_READ_ABOVE_200K,
|
||||
GEMINI_3_1_PRO_PREVIEW_REASONING,
|
||||
GEMINI_3_1_PRO_PREVIEW_TOOL_CALL,
|
||||
GEMINI_4_ARGON_INPUT_TEXT,
|
||||
|
|
@ -3377,6 +3700,8 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
GEMINI_4_ARGON_CACHE_READ,
|
||||
GEMINI_4_ARGON_REASONING,
|
||||
GEMINI_4_ARGON_TOOL_CALL,
|
||||
*(pair.bridged for pair in GEMINI_4_ARGON_ROUTE_BRIDGES),
|
||||
*(pair.bridged for pair in GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES),
|
||||
)
|
||||
|
||||
PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
||||
|
|
@ -3397,8 +3722,59 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = (
|
|||
),
|
||||
StreamParityTestCase(plain=VERTEX_AI_CLAUDE_OPUS_5_5_INPUT_TEXT, streamed=VERTEX_AI_CLAUDE_OPUS_5_5_STREAM),
|
||||
StreamParityTestCase(plain=VERTEX_AI_CLAUDE_FABLE_5_1_INPUT_TEXT, streamed=VERTEX_AI_CLAUDE_FABLE_5_1_STREAM),
|
||||
StreamParityTestCase(
|
||||
plain=VERTEX_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT, streamed=VERTEX_AI_CLAUDE_HAIKU_5_5_STREAM
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=VERTEX_AI_CLAUDE_HAIKU_5_5_MESSAGES_INPUT_TEXT,
|
||||
streamed=VERTEX_AI_CLAUDE_HAIKU_5_5_MESSAGES_STREAM,
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=VERTEX_AI_CLAUDE_HAIKU_5_5_BRIDGE_PAIRS[0].bridged,
|
||||
streamed=VERTEX_AI_CLAUDE_HAIKU_5_5_BRIDGE_PAIRS[1].bridged,
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=VERTEX_AI_CLAUDE_FABLE_5_1_MESSAGES_INPUT_TEXT,
|
||||
streamed=VERTEX_AI_CLAUDE_FABLE_5_1_MESSAGES_STREAM,
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=VERTEX_AI_CLAUDE_FABLE_5_1_BRIDGE_PAIRS[0].bridged,
|
||||
streamed=VERTEX_AI_CLAUDE_FABLE_5_1_BRIDGE_PAIRS[1].bridged,
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=VERTEX_AI_CLAUDE_OPUS_5_5_MESSAGES_INPUT_TEXT,
|
||||
streamed=VERTEX_AI_CLAUDE_OPUS_5_5_MESSAGES_STREAM,
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=VERTEX_AI_CLAUDE_OPUS_5_5_BRIDGE_PAIRS[0].bridged,
|
||||
streamed=VERTEX_AI_CLAUDE_OPUS_5_5_BRIDGE_PAIRS[1].bridged,
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_INPUT_TEXT,
|
||||
streamed=VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_STREAM,
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=VERTEX_AI_CLAUDE_SONNET_5_5_BRIDGE_PAIRS[0].bridged,
|
||||
streamed=VERTEX_AI_CLAUDE_SONNET_5_5_BRIDGE_PAIRS[1].bridged,
|
||||
),
|
||||
StreamParityTestCase(plain=GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT, streamed=GEMINI_3_1_PRO_PREVIEW_STREAM),
|
||||
StreamParityTestCase(plain=GEMINI_4_ARGON_INPUT_TEXT, streamed=GEMINI_4_ARGON_STREAM),
|
||||
StreamParityTestCase(
|
||||
plain=GEMINI_4_ARGON_ROUTE_BRIDGES[0].bridged,
|
||||
streamed=GEMINI_4_ARGON_ROUTE_BRIDGES[2].bridged,
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=GEMINI_4_ARGON_ROUTE_BRIDGES[1].bridged,
|
||||
streamed=GEMINI_4_ARGON_ROUTE_BRIDGES[3].bridged,
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES[0].bridged,
|
||||
streamed=GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES[2].bridged,
|
||||
),
|
||||
StreamParityTestCase(
|
||||
plain=GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES[1].bridged,
|
||||
streamed=GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES[3].bridged,
|
||||
),
|
||||
)
|
||||
|
||||
BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = (
|
||||
|
|
@ -3406,6 +3782,10 @@ BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = (
|
|||
(GEMINI_3_8_FLASH_INPUT_TEXT, GEMINI_3_8_FLASH_CACHE_READ, GEMINI_3_8_FLASH_STREAM),
|
||||
("/v1/responses", "/v1/messages"),
|
||||
),
|
||||
*bridge_pairs((GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT,), ("/v1/responses",)),
|
||||
*bridge_pairs((GEMINI_4_ARGON_INPUT_TEXT,), ("/v1/responses",)),
|
||||
*GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES,
|
||||
*GEMINI_4_ARGON_ROUTE_BRIDGES,
|
||||
*VERTEX_AI_CLAUDE_HAIKU_5_5_BRIDGE_PAIRS,
|
||||
*VERTEX_AI_CLAUDE_FABLE_5_1_BRIDGE_PAIRS,
|
||||
*VERTEX_AI_CLAUDE_OPUS_5_5_BRIDGE_PAIRS,
|
||||
*VERTEX_AI_CLAUDE_SONNET_5_5_BRIDGE_PAIRS,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -889,6 +889,26 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = {
|
|||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
},
|
||||
"vertex_ai/claude-haiku-5-5": {
|
||||
"litellm_provider": "vertex_ai-anthropic_models",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 1.3e-06,
|
||||
"input_cost_per_token_above_100k_tokens": 6.5e-06,
|
||||
"output_cost_per_token": 6.5e-06,
|
||||
"output_cost_per_token_above_100k_tokens": 3.25e-05,
|
||||
"cache_creation_input_token_cost": 1.625e-06,
|
||||
"cache_creation_input_token_cost_above_1hr": 2.6e-06,
|
||||
"cache_creation_input_token_cost_above_100k_tokens": 8.125e-06,
|
||||
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.3e-05,
|
||||
"cache_read_input_token_cost": 1.3e-07,
|
||||
"cache_read_input_token_cost_above_100k_tokens": 6.5e-07,
|
||||
"supports_function_calling": True,
|
||||
"supports_prompt_caching": True,
|
||||
"supports_reasoning": True,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
},
|
||||
"groq/qwen/qwen3.8-27b": {
|
||||
"litellm_provider": "groq",
|
||||
"mode": "chat",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue