diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 8c0c463c217..8db825aa986 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -70,6 +70,11 @@ def pytest_configure(config: pytest.Config) -> None: config.addinivalue_line("markers", "integration: owned real-service integration contracts") config.addinivalue_line("markers", "covers(*ids): legacy contract IDs kept for existing tests, not enforced") config.addinivalue_line("markers", "shares_worker_state: owned proxies here share the worker DB and Redis") + config.addinivalue_line( + "markers", + "billed_against_real_provider_by_cost_tracking_reconciliation_automation: " + "the daily automation also sends this case to the real provider and checks the bill", + ) config.stash[REPORTS] = [] config.stash[SHARD_FILES] = _shard_files(SHARD_PATH.validate_python(config.getoption("integration_shard_files"))) config.pluginmanager.register(IntegrationReportPlugin(config)) diff --git a/tests/integration/cost_calculation/bridge_cost_parity/test_basic.py b/tests/integration/cost_calculation/bridge_cost_parity/test_basic.py index b9b735cc119..7a5efdff187 100644 --- a/tests/integration/cost_calculation/bridge_cost_parity/test_basic.py +++ b/tests/integration/cost_calculation/bridge_cost_parity/test_basic.py @@ -12,12 +12,24 @@ import pytest from integration._support.client import Gateway from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase from integration.cost_calculation.bridge_cost_parity.runner import assert_bridge_cost_parity -from integration.cost_calculation.catalog import BRIDGE_COST_PARITY_CASES +from integration.cost_calculation.catalog import ( + BILLED_AGAINST_REAL_PROVIDER_BY_COST_TRACKING_RECONCILIATION_AUTOMATION, + BRIDGE_COST_PARITY_CASES, +) def _marks(case: BridgeCostParityTestCase) -> tuple[pytest.MarkDecorator, ...]: """A known bridge bug is intermittent (it billed the native row in 1 of 4 local runs), so the xfail is not strict.""" - return (pytest.mark.xfail(strict=False, reason=f"BUG: {case.known_bug}"),) if case.known_bug is not None else () + bug: Final = ( + (pytest.mark.xfail(strict=False, reason=f"BUG: {case.known_bug}"),) if case.known_bug is not None else () + ) + legs: Final = frozenset((case.native.name, case.bridged.name)) + billed: Final = ( + (pytest.mark.billed_against_real_provider_by_cost_tracking_reconciliation_automation,) + if legs & BILLED_AGAINST_REAL_PROVIDER_BY_COST_TRACKING_RECONCILIATION_AUTOMATION + else () + ) + return (*bug, *billed) _CASES: Final = tuple(pytest.param(case, marks=_marks(case), id=case.id) for case in BRIDGE_COST_PARITY_CASES) diff --git a/tests/integration/cost_calculation/cases/batch_realtime.py b/tests/integration/cost_calculation/cases/batch_realtime.py index 52e2021dc7d..a365938e949 100644 --- a/tests/integration/cost_calculation/cases/batch_realtime.py +++ b/tests/integration/cost_calculation/cases/batch_realtime.py @@ -184,6 +184,32 @@ GPT_REALTIME_MINI_2025_12_15_REALTIME_SESSION_WITHOUT_TURNS_ZERO_SPEND: Final = ), ) +GPT_REALTIME_2_1_REALTIME_SINGLE_TURN_TEXT: Final = RealtimeCostCase( + name="gpt-realtime-2.1-realtime-single_turn_text", + covers="quota_management.spend_tracking.realtime_costs.single_turn", + model="gpt-realtime-2.1", + litellm_model="openai/gpt-realtime-2.1", + turns=( + RealtimeTurn( + input_tokens=129, + output_tokens=17, + input_text_tokens=129, + input_audio_tokens=0, + input_cached_tokens=0, + output_text_tokens=17, + output_audio_tokens=0, + ), + ), + expected=ExactExpected( + spend=0.000924, + input_cost=0.000516, + output_cost=0.000408, + prompt_tokens=129, + completion_tokens=17, + cost_header=False, + ), +) + BATCH_CASES: Final[tuple[BatchCostCase, ...]] = ( GPT_5_6_BATCH_HALVED_RATES_WHEN_MAP_HAS_NO_BATCH_KEYS, @@ -197,4 +223,8 @@ REALTIME_CASES: Final[tuple[RealtimeCostCase, ...]] = ( GPT_REALTIME_MINI_2025_12_15_REALTIME_TWO_TURNS_SUMMED_INTO_ONE_ROW, GPT_REALTIME_MINI_2025_12_15_REALTIME_PRICED_FROM_SESSION_CREATED_MODEL, GPT_REALTIME_MINI_2025_12_15_REALTIME_SESSION_WITHOUT_TURNS_ZERO_SPEND, + GPT_REALTIME_2_1_REALTIME_SINGLE_TURN_TEXT, +) +BILLED_AGAINST_REAL_PROVIDER_BY_COST_TRACKING_RECONCILIATION_AUTOMATION: Final[tuple[RealtimeCostCase, ...]] = ( + GPT_REALTIME_2_1_REALTIME_SINGLE_TURN_TEXT, ) diff --git a/tests/integration/cost_calculation/cases/openai.py b/tests/integration/cost_calculation/cases/openai.py index a3d5527daad..a662677f804 100644 --- a/tests/integration/cost_calculation/cases/openai.py +++ b/tests/integration/cost_calculation/cases/openai.py @@ -8470,6 +8470,19 @@ GPT_5_6_CHAT_REQUEST_TO_EMBEDDING_ENTRY: Final = CostTrackingTestCase( # gpt-6.1-sol +_PROVIDER_CACHEABLE_PROMPT: Final = ( + "The pricing harness keeps one stable instruction block so the provider can cache it. " * 220 +) +_GPT_6_PROMPT_CACHE_REQUEST: Final[Mapping[str, JsonValue]] = { + "model": "$MODEL", + "messages": [ + {"role": "system", "content": _PROVIDER_CACHEABLE_PROMPT}, + {"role": "user", "content": "Reply with OK."}, + ], + "reasoning_effort": "low", + "stream": False, +} + GPT_6_1_SOL_INPUT_TEXT: Final = CostTrackingTestCase( name="gpt-6.1-sol-input_text", covers="quota_management.spend_tracking.cost_matrix.logs_cost", @@ -8511,31 +8524,7 @@ GPT_6_1_SOL_CACHE_READ: Final = GPT_5_6_CACHE_READ.model_copy( update={ "name": "gpt-6.1-sol-cache_read", "model": "gpt-6.1-sol", - "request": { - "model": "$MODEL", - "messages": [ - { - "role": "system", - "content": [ - { - "type": "text", - "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", - } - ], - }, - { - "role": "user", - "content": [ - { - "type": "text", - "text": "b770f5df16a2 summarize the attached material in one line and name the city weather", - } - ], - }, - ], - "stream": False, - "allowed_openai_params": [], - }, + "request": _GPT_6_PROMPT_CACHE_REQUEST, "response": JsonResponse( content_type="application/json", body={ @@ -8585,31 +8574,7 @@ GPT_6_LUNA_CACHE_READ: Final = GPT_5_6_CACHE_READ.model_copy( update={ "name": "gpt-6-luna-cache_read", "model": "gpt-6-luna", - "request": { - "model": "$MODEL", - "messages": [ - { - "role": "system", - "content": [ - { - "type": "text", - "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", - } - ], - }, - { - "role": "user", - "content": [ - { - "type": "text", - "text": "a880e9da100c summarize the attached material in one line and name the city weather", - } - ], - }, - ], - "stream": False, - "allowed_openai_params": [], - }, + "request": _GPT_6_PROMPT_CACHE_REQUEST, "response": JsonResponse( content_type="application/json", body={ @@ -8659,31 +8624,7 @@ GPT_6_ASTRA_CACHE_READ: Final = GPT_5_6_CACHE_READ.model_copy( update={ "name": "gpt-6-astra-cache_read", "model": "gpt-6-astra", - "request": { - "model": "$MODEL", - "messages": [ - { - "role": "system", - "content": [ - { - "type": "text", - "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", - } - ], - }, - { - "role": "user", - "content": [ - { - "type": "text", - "text": "d45eefac915d summarize the attached material in one line and name the city weather", - } - ], - }, - ], - "stream": False, - "allowed_openai_params": [], - }, + "request": _GPT_6_PROMPT_CACHE_REQUEST, "response": JsonResponse( content_type="application/json", body={ @@ -12089,7 +12030,489 @@ GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED_STREAM: Final = ( ) ) +def _gpt_6_on_model(base: CostTrackingTestCase, model: str, shape: str, expected: ExactExpected) -> CostTrackingTestCase: + response: Final = ( + _stream_for_model(base.response, model) + if isinstance(base.response, SseResponse) + else _response_for_model(cast(JsonResponse, base.response), model) + ) + return base.model_copy( + update={"name": f"{model}-{shape}", "model": model, "response": response, "expected": expected} + ) + + +def _gpt_6_with_real_usage( + base: CostTrackingTestCase, + model: str, + shape: str, + request: Mapping[str, JsonValue], + usage: Mapping[str, JsonValue], + expected: ExactExpected, +) -> CostTrackingTestCase: + body: Final = {**cast(JsonResponse, base.response).body, "model": model, "usage": dict(usage)} + return base.model_copy( + update={ + "name": f"{model}-{shape}", + "model": model, + "request": dict(request), + "response": cast(JsonResponse, base.response).model_copy(update={"body": body}), + "expected": expected, + } + ) + + +def _gpt_6_tool_call_served_by_responses_api(model: str, expected: ExactExpected) -> CostTrackingTestCase: + body: Final = cast(JsonResponse, GPT_6_LUNA_RESPONSES_NATIVE_JSON.response).body + function_call: Final[dict[str, JsonValue]] = { + "type": "function_call", + "id": "fc_$REQUEST_ID", + "call_id": "call_$REQUEST_ID", + "name": "get_weather", + "arguments": '{"city": "Lisbon"}', + "status": "completed", + } + return GPT_6_LUNA_TOOL_CALL.model_copy( + update={ + "name": f"{model}-tool_call", + "model": model, + "request": {**GPT_6_LUNA_TOOL_CALL.request, "reasoning_effort": "low"}, + "response": JsonResponse( + content_type="application/json", body={**body, "model": model, "output": [function_call]} + ), + "expected": expected, + } + ) + + +_GPT_6_RESPONSES_CACHE_WRITE_12H_REQUEST: Final[Mapping[str, JsonValue]] = { + "model": "$MODEL", + "input": f"{_PROVIDER_CACHEABLE_PROMPT} Reply with OK.", + "prompt_cache_options": {"mode": "implicit", "ttl": "12h"}, + "reasoning": {"effort": "low"}, +} +_GPT_6_RESPONSES_CACHE_WRITE_30M_REQUEST: Final[Mapping[str, JsonValue]] = { + "model": "$MODEL", + "input": f"{_PROVIDER_CACHEABLE_PROMPT} Reply with OK.", + "reasoning": {"effort": "low"}, +} +_GPT_6_RESPONSES_WEB_SEARCH_REQUEST: Final[Mapping[str, JsonValue]] = { + "model": "$MODEL", + "input": "What is the population of Lisbon? Search the web and answer in one line.", + "tools": [{"type": "web_search", "search_context_size": "medium"}], + "reasoning": {"effort": "low"}, +} + +GPT_6_1_TERRA_INPUT_TEXT: Final = _gpt_6_on_model( + GPT_6_LUNA_INPUT_TEXT, + "gpt-6.1-terra", + "input_text", + ExactExpected(spend=0.004017, input_cost=0.0018952, output_cost=0.0021218, prompt_tokens=1840, completion_tokens=412), +) +GPT_6_1_TERRA_STREAM: Final = _gpt_6_on_model( + GPT_6_LUNA_STREAM, + "gpt-6.1-terra", + "stream", + ExactExpected(spend=0.004017, input_cost=0.0018952, output_cost=0.0021218, prompt_tokens=1840, completion_tokens=412), +) +GPT_6_1_TERRA_REASONING: Final = _gpt_6_on_model( + GPT_6_LUNA_REASONING, + "gpt-6.1-terra", + "reasoning", + ExactExpected(spend=0.0220832, input_cost=0.0012772, output_cost=0.020806, prompt_tokens=1240, completion_tokens=4040), +) +GPT_6_1_TERRA_TOOL_CALL: Final = _gpt_6_on_model( + GPT_6_LUNA_TOOL_CALL, + "gpt-6.1-terra", + "tool_call", + ExactExpected(spend=0.004017, input_cost=0.0018952, output_cost=0.0021218, prompt_tokens=1840, completion_tokens=412), +) +GPT_6_1_TERRA_SERVICE_TIER_FLEX: Final = _gpt_6_on_model( + GPT_6_LUNA_SERVICE_TIER_FLEX, + "gpt-6.1-terra", + "service_tier_flex", + ExactExpected(spend=0.0020085, input_cost=0.0009476, output_cost=0.0010609, prompt_tokens=1840, completion_tokens=412), +) +GPT_6_1_TERRA_SERVICE_TIER_PRIORITY: Final = _gpt_6_on_model( + GPT_6_LUNA_SERVICE_TIER_PRIORITY, + "gpt-6.1-terra", + "service_tier_priority", + ExactExpected(spend=0.008034, input_cost=0.0037904, output_cost=0.0042436, prompt_tokens=1840, completion_tokens=412), +) +GPT_6_1_TERRA_RESPONSES_NATIVE_JSON: Final = _gpt_6_on_model( + GPT_6_LUNA_RESPONSES_NATIVE_JSON, + "gpt-6.1-terra", + "responses_native_json", + ExactExpected(spend=0.004017, input_cost=0.0018952, output_cost=0.0021218, prompt_tokens=1840, completion_tokens=412), +) +GPT_6_ASTRA_TOOL_CALL: Final = _gpt_6_tool_call_served_by_responses_api( + "gpt-6-astra", + ExactExpected(spend=0.0468, input_cost=0.02208, output_cost=0.02472, prompt_tokens=1840, completion_tokens=412), +) +GPT_6_1_SOL_TOOL_CALL: Final = _gpt_6_tool_call_served_by_responses_api( + "gpt-6.1-sol", + ExactExpected(spend=0.0078, input_cost=0.00368, output_cost=0.00412, prompt_tokens=1840, completion_tokens=412), +) +GPT_6_LUNA_PROMPT_CACHE_WRITE: Final = _gpt_6_with_real_usage( + GPT_6_LUNA_CACHE_READ, + "gpt-6-luna", + "prompt_cache_write", + _GPT_6_PROMPT_CACHE_REQUEST, + {"prompt_tokens": 3334, "completion_tokens": 4, "total_tokens": 3338, "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 3331, "audio_tokens": 0}, "completion_tokens_details": {"reasoning_tokens": 0, "audio_tokens": 0, "accepted_prediction_tokens": 0, "rejected_prediction_tokens": 0}}, + ExactExpected(spend=0.000418675, input_cost=0.000416675, output_cost=2e-06, prompt_tokens=3334, completion_tokens=4, cache_creation_cost=0.000416375), +) +GPT_6_LUNA_PROMPT_CACHE_READ: Final = _gpt_6_with_real_usage( + GPT_6_LUNA_CACHE_READ, + "gpt-6-luna", + "prompt_cache_read", + _GPT_6_PROMPT_CACHE_REQUEST, + {"prompt_tokens": 3334, "completion_tokens": 4, "total_tokens": 3338, "prompt_tokens_details": {"cached_tokens": 3331, "cache_write_tokens": 0, "audio_tokens": 0}, "completion_tokens_details": {"reasoning_tokens": 0, "audio_tokens": 0, "accepted_prediction_tokens": 0, "rejected_prediction_tokens": 0}}, + ExactExpected(spend=3.561e-05, input_cost=3.361e-05, output_cost=2e-06, prompt_tokens=3334, completion_tokens=4, cache_read_cost=3.331e-05), +) +GPT_6_LUNA_RESPONSES_PROMPT_CACHE_WRITE_12H: Final = _gpt_6_with_real_usage( + GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE, + "gpt-6-luna", + "responses_prompt_cache_write_12h", + _GPT_6_RESPONSES_CACHE_WRITE_12H_REQUEST, + {"input_tokens": 3334, "input_tokens_details": {"cache_write_tokens": 3331, "cache_write_tokens_details": {"ttl_12h_tokens": 3331, "ttl_30m_tokens": 0}, "cached_tokens": 0}, "output_tokens": 5, "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": 3339}, + ExactExpected(spend=0.000419175, input_cost=0.000416675, output_cost=2.5e-06, prompt_tokens=3334, completion_tokens=5, cache_creation_cost=0.000416375), +) +GPT_6_LUNA_RESPONSES_WEB_SEARCH_MEDIUM: Final = _gpt_6_with_real_usage( + GPT_5_6_RESPONSES_WEB_SEARCH_MEDIUM, + "gpt-6-luna", + "responses_web_search_medium", + _GPT_6_RESPONSES_WEB_SEARCH_REQUEST, + {"input_tokens": 8636, "input_tokens_details": {"cache_write_tokens": 20, "cache_write_tokens_details": {"ttl_12h_tokens": 0, "ttl_30m_tokens": 20}, "cached_tokens": 4435}, "output_tokens": 81, "output_tokens_details": {"reasoning_tokens": 43}, "total_tokens": 8717}, + ExactExpected(spend=0.01050545, input_cost=0.00046495, output_cost=4.05e-05, prompt_tokens=8636, completion_tokens=81, tool_usage_cost=0.01, cache_read_cost=4.435e-05, cache_creation_cost=2.5e-06), +) +GPT_6_SOL_PROMPT_CACHE_WRITE: Final = _gpt_6_with_real_usage( + GPT_6_LUNA_CACHE_READ, + "gpt-6-sol", + "prompt_cache_write", + _GPT_6_PROMPT_CACHE_REQUEST, + {"prompt_tokens": 3336, "completion_tokens": 4, "total_tokens": 3340, "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 3333, "audio_tokens": 0}, "completion_tokens_details": {"reasoning_tokens": 0, "audio_tokens": 0, "accepted_prediction_tokens": 0, "rejected_prediction_tokens": 0}}, + ExactExpected(spend=0.00921635, input_cost=0.00917235, output_cost=4.4e-05, prompt_tokens=3336, completion_tokens=4, cache_creation_cost=0.00916575), +) +GPT_6_SOL_PROMPT_CACHE_READ: Final = _gpt_6_with_real_usage( + GPT_6_LUNA_CACHE_READ, + "gpt-6-sol", + "prompt_cache_read", + _GPT_6_PROMPT_CACHE_REQUEST, + {"prompt_tokens": 3336, "completion_tokens": 4, "total_tokens": 3340, "prompt_tokens_details": {"cached_tokens": 3333, "cache_write_tokens": 0, "audio_tokens": 0}, "completion_tokens_details": {"reasoning_tokens": 0, "audio_tokens": 0, "accepted_prediction_tokens": 0, "rejected_prediction_tokens": 0}}, + ExactExpected(spend=0.00078386, input_cost=0.00073986, output_cost=4.4e-05, prompt_tokens=3336, completion_tokens=4, cache_read_cost=0.00073326), +) +GPT_6_SOL_RESPONSES_PROMPT_CACHE_WRITE_12H: Final = _gpt_6_with_real_usage( + GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE, + "gpt-6-sol", + "responses_prompt_cache_write_12h", + _GPT_6_RESPONSES_CACHE_WRITE_12H_REQUEST, + {"input_tokens": 3332, "input_tokens_details": {"cache_write_tokens": 3329, "cache_write_tokens_details": {"ttl_12h_tokens": 3329, "ttl_30m_tokens": 0}, "cached_tokens": 0}, "output_tokens": 6, "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": 3338}, + ExactExpected(spend=0.0183821, input_cost=0.0183161, output_cost=6.6e-05, prompt_tokens=3332, completion_tokens=6, cache_creation_cost=0.0183095), +) +GPT_6_SOL_RESPONSES_WEB_SEARCH_MEDIUM: Final = _gpt_6_with_real_usage( + GPT_5_6_RESPONSES_WEB_SEARCH_MEDIUM, + "gpt-6-sol", + "responses_web_search_medium", + _GPT_6_RESPONSES_WEB_SEARCH_REQUEST, + {"input_tokens": 8763, "input_tokens_details": {"cache_write_tokens": 4455, "cache_write_tokens_details": {"ttl_12h_tokens": 0, "ttl_30m_tokens": 4455}, "cached_tokens": 0}, "output_tokens": 94, "output_tokens_details": {"reasoning_tokens": 52}, "total_tokens": 8857}, + ExactExpected(spend=0.03276285, input_cost=0.02172885, output_cost=0.001034, prompt_tokens=8763, completion_tokens=94, tool_usage_cost=0.01, cache_creation_cost=0.01225125), +) +GPT_6_ASTRA_PROMPT_CACHE_WRITE: Final = _gpt_6_with_real_usage( + GPT_6_LUNA_CACHE_READ, + "gpt-6-astra", + "prompt_cache_write", + _GPT_6_PROMPT_CACHE_REQUEST, + {"prompt_tokens": 3333, "completion_tokens": 4, "total_tokens": 3337, "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 3330, "audio_tokens": 0}, "completion_tokens_details": {"reasoning_tokens": 0, "audio_tokens": 0, "accepted_prediction_tokens": 0, "rejected_prediction_tokens": 0}}, + ExactExpected(spend=0.050226, input_cost=0.049986, output_cost=0.00024, prompt_tokens=3333, completion_tokens=4, cache_creation_cost=0.04995), +) +GPT_6_ASTRA_PROMPT_CACHE_READ: Final = _gpt_6_with_real_usage( + GPT_6_LUNA_CACHE_READ, + "gpt-6-astra", + "prompt_cache_read", + _GPT_6_PROMPT_CACHE_REQUEST, + {"prompt_tokens": 3333, "completion_tokens": 4, "total_tokens": 3337, "prompt_tokens_details": {"cached_tokens": 3330, "cache_write_tokens": 0, "audio_tokens": 0}, "completion_tokens_details": {"reasoning_tokens": 0, "audio_tokens": 0, "accepted_prediction_tokens": 0, "rejected_prediction_tokens": 0}}, + ExactExpected(spend=0.004272, input_cost=0.004032, output_cost=0.00024, prompt_tokens=3333, completion_tokens=4, cache_read_cost=0.003996), +) +GPT_6_ASTRA_RESPONSES_PROMPT_CACHE_WRITE_12H: Final = _gpt_6_with_real_usage( + GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE, + "gpt-6-astra", + "responses_prompt_cache_write_12h", + _GPT_6_RESPONSES_CACHE_WRITE_12H_REQUEST, + {"input_tokens": 3329, "input_tokens_details": {"cache_write_tokens": 3326, "cache_write_tokens_details": {"ttl_12h_tokens": 3326, "ttl_30m_tokens": 0}, "cached_tokens": 0}, "output_tokens": 5, "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": 3334}, + ExactExpected(spend=0.100116, input_cost=0.099816, output_cost=0.0003, prompt_tokens=3329, completion_tokens=5, cache_creation_cost=0.09978), +) +GPT_6_ASTRA_RESPONSES_WEB_SEARCH_MEDIUM: Final = _gpt_6_with_real_usage( + GPT_5_6_RESPONSES_WEB_SEARCH_MEDIUM, + "gpt-6-astra", + "responses_web_search_medium", + _GPT_6_RESPONSES_WEB_SEARCH_REQUEST, + {"input_tokens": 8580, "input_tokens_details": {"cache_write_tokens": 4455, "cache_write_tokens_details": {"ttl_12h_tokens": 0, "ttl_30m_tokens": 4455}, "cached_tokens": 0}, "output_tokens": 81, "output_tokens_details": {"reasoning_tokens": 38}, "total_tokens": 8661}, + ExactExpected(spend=0.131185, input_cost=0.116325, output_cost=0.00486, prompt_tokens=8580, completion_tokens=81, tool_usage_cost=0.01, cache_creation_cost=0.066825), +) +GPT_6_1_SOL_PROMPT_CACHE_WRITE: Final = _gpt_6_with_real_usage( + GPT_6_LUNA_CACHE_READ, + "gpt-6.1-sol", + "prompt_cache_write", + _GPT_6_PROMPT_CACHE_REQUEST, + {"prompt_tokens": 3333, "completion_tokens": 4, "total_tokens": 3337, "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 3330, "audio_tokens": 0}, "completion_tokens_details": {"reasoning_tokens": 0, "audio_tokens": 0, "accepted_prediction_tokens": 0, "rejected_prediction_tokens": 0}}, + ExactExpected(spend=0.008371, input_cost=0.008331, output_cost=4e-05, prompt_tokens=3333, completion_tokens=4, cache_creation_cost=0.008325), +) +GPT_6_1_SOL_PROMPT_CACHE_READ: Final = _gpt_6_with_real_usage( + GPT_6_LUNA_CACHE_READ, + "gpt-6.1-sol", + "prompt_cache_read", + _GPT_6_PROMPT_CACHE_REQUEST, + {"prompt_tokens": 3333, "completion_tokens": 4, "total_tokens": 3337, "prompt_tokens_details": {"cached_tokens": 3330, "cache_write_tokens": 0, "audio_tokens": 0}, "completion_tokens_details": {"reasoning_tokens": 0, "audio_tokens": 0, "accepted_prediction_tokens": 0, "rejected_prediction_tokens": 0}}, + ExactExpected(spend=0.000379, input_cost=0.000339, output_cost=4e-05, prompt_tokens=3333, completion_tokens=4, cache_read_cost=0.000333), +) +GPT_6_1_SOL_RESPONSES_PROMPT_CACHE_WRITE_12H: Final = _gpt_6_with_real_usage( + GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE, + "gpt-6.1-sol", + "responses_prompt_cache_write_12h", + _GPT_6_RESPONSES_CACHE_WRITE_12H_REQUEST, + {"input_tokens": 3328, "input_tokens_details": {"cache_write_tokens": 3325, "cache_write_tokens_details": {"ttl_12h_tokens": 3325, "ttl_30m_tokens": 0}, "cached_tokens": 0}, "output_tokens": 5, "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": 3333}, + ExactExpected(spend=0.016681, input_cost=0.016631, output_cost=5e-05, prompt_tokens=3328, completion_tokens=5, cache_creation_cost=0.016625), +) +GPT_6_1_SOL_RESPONSES_WEB_SEARCH_MEDIUM: Final = _gpt_6_with_real_usage( + GPT_5_6_RESPONSES_WEB_SEARCH_MEDIUM, + "gpt-6.1-sol", + "responses_web_search_medium", + _GPT_6_RESPONSES_WEB_SEARCH_REQUEST, + {"input_tokens": 8617, "input_tokens_details": {"cache_write_tokens": 4455, "cache_write_tokens_details": {"ttl_12h_tokens": 0, "ttl_30m_tokens": 4455}, "cached_tokens": 0}, "output_tokens": 83, "output_tokens_details": {"reasoning_tokens": 38}, "total_tokens": 8700}, + ExactExpected(spend=0.0302915, input_cost=0.0194615, output_cost=0.00083, prompt_tokens=8617, completion_tokens=83, tool_usage_cost=0.01, cache_creation_cost=0.0111375), +) +GPT_6_1_TERRA_PROMPT_CACHE_WRITE: Final = _gpt_6_with_real_usage( + GPT_6_LUNA_CACHE_READ, + "gpt-6.1-terra", + "prompt_cache_write", + _GPT_6_PROMPT_CACHE_REQUEST, + {"prompt_tokens": 3333, "completion_tokens": 4, "total_tokens": 3337, "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 3330, "audio_tokens": 0}, "completion_tokens_details": {"reasoning_tokens": 0, "audio_tokens": 0, "accepted_prediction_tokens": 0, "rejected_prediction_tokens": 0}}, + ExactExpected(spend=0.004311065, input_cost=0.004290465, output_cost=2.06e-05, prompt_tokens=3333, completion_tokens=4, cache_creation_cost=0.004287375), +) +GPT_6_1_TERRA_PROMPT_CACHE_READ: Final = _gpt_6_with_real_usage( + GPT_6_LUNA_CACHE_READ, + "gpt-6.1-terra", + "prompt_cache_read", + _GPT_6_PROMPT_CACHE_REQUEST, + {"prompt_tokens": 3333, "completion_tokens": 4, "total_tokens": 3337, "prompt_tokens_details": {"cached_tokens": 3330, "cache_write_tokens": 0, "audio_tokens": 0}, "completion_tokens_details": {"reasoning_tokens": 0, "audio_tokens": 0, "accepted_prediction_tokens": 0, "rejected_prediction_tokens": 0}}, + ExactExpected(spend=0.000195185, input_cost=0.000174585, output_cost=2.06e-05, prompt_tokens=3333, completion_tokens=4, cache_read_cost=0.000171495), +) +GPT_6_1_TERRA_RESPONSES_PROMPT_CACHE_WRITE_12H: Final = _gpt_6_with_real_usage( + GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE, + "gpt-6.1-terra", + "responses_prompt_cache_write_12h", + _GPT_6_RESPONSES_CACHE_WRITE_12H_REQUEST, + {"input_tokens": 3332, "input_tokens_details": {"cache_write_tokens": 3329, "cache_write_tokens_details": {"ttl_12h_tokens": 3329, "ttl_30m_tokens": 0}, "cached_tokens": 0}, "output_tokens": 5, "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": 3337}, + ExactExpected(spend=0.008601015, input_cost=0.008575265, output_cost=2.575e-05, prompt_tokens=3332, completion_tokens=5, cache_creation_cost=0.008572175), +) +GPT_6_1_TERRA_RESPONSES_WEB_SEARCH_MEDIUM: Final = _gpt_6_with_real_usage( + GPT_5_6_RESPONSES_WEB_SEARCH_MEDIUM, + "gpt-6.1-terra", + "responses_web_search_medium", + _GPT_6_RESPONSES_WEB_SEARCH_REQUEST, + {"input_tokens": 8657, "input_tokens_details": {"cache_write_tokens": 4455, "cache_write_tokens_details": {"ttl_12h_tokens": 0, "ttl_30m_tokens": 4455}, "cached_tokens": 0}, "output_tokens": 77, "output_tokens_details": {"reasoning_tokens": 36}, "total_tokens": 8734}, + ExactExpected(spend=0.0204604225, input_cost=0.0100638725, output_cost=0.00039655, prompt_tokens=8657, completion_tokens=77, tool_usage_cost=0.01, cache_creation_cost=0.0057358125), +) +GPT_6_LUNA_RESPONSES_PROMPT_CACHE_WRITE_30M: Final = _gpt_6_with_real_usage( + GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE, + "gpt-6-luna", + "responses_prompt_cache_write_30m", + _GPT_6_RESPONSES_CACHE_WRITE_30M_REQUEST, + {"input_tokens": 3329, "input_tokens_details": {"cache_write_tokens": 3326, "cache_write_tokens_details": {"ttl_12h_tokens": 0, "ttl_30m_tokens": 3326}, "cached_tokens": 0}, "output_tokens": 5, "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": 3334}, + ExactExpected(spend=0.00041855, input_cost=0.00041605, output_cost=2.5e-06, prompt_tokens=3329, completion_tokens=5, cache_creation_cost=0.00041575), +) +GPT_6_SOL_RESPONSES_PROMPT_CACHE_WRITE_30M: Final = _gpt_6_with_real_usage( + GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE, + "gpt-6-sol", + "responses_prompt_cache_write_30m", + _GPT_6_RESPONSES_CACHE_WRITE_30M_REQUEST, + {"input_tokens": 3328, "input_tokens_details": {"cache_write_tokens": 3325, "cache_write_tokens_details": {"ttl_12h_tokens": 0, "ttl_30m_tokens": 3325}, "cached_tokens": 0}, "output_tokens": 6, "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": 3334}, + ExactExpected(spend=0.00921635, input_cost=0.00915035, output_cost=6.6e-05, prompt_tokens=3328, completion_tokens=6, cache_creation_cost=0.00914375), +) +GPT_6_ASTRA_RESPONSES_PROMPT_CACHE_WRITE_30M: Final = _gpt_6_with_real_usage( + GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE, + "gpt-6-astra", + "responses_prompt_cache_write_30m", + _GPT_6_RESPONSES_CACHE_WRITE_30M_REQUEST, + {"input_tokens": 3327, "input_tokens_details": {"cache_write_tokens": 3324, "cache_write_tokens_details": {"ttl_12h_tokens": 0, "ttl_30m_tokens": 3324}, "cached_tokens": 0}, "output_tokens": 5, "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": 3332}, + ExactExpected(spend=0.050196, input_cost=0.049896, output_cost=0.0003, prompt_tokens=3327, completion_tokens=5, cache_creation_cost=0.04986), +) +GPT_6_1_SOL_RESPONSES_PROMPT_CACHE_WRITE_30M: Final = _gpt_6_with_real_usage( + GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE, + "gpt-6.1-sol", + "responses_prompt_cache_write_30m", + _GPT_6_RESPONSES_CACHE_WRITE_30M_REQUEST, + {"input_tokens": 3329, "input_tokens_details": {"cache_write_tokens": 3326, "cache_write_tokens_details": {"ttl_12h_tokens": 0, "ttl_30m_tokens": 3326}, "cached_tokens": 0}, "output_tokens": 5, "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": 3334}, + ExactExpected(spend=0.008371, input_cost=0.008321, output_cost=5e-05, prompt_tokens=3329, completion_tokens=5, cache_creation_cost=0.008315), +) +GPT_6_1_TERRA_RESPONSES_PROMPT_CACHE_WRITE_30M: Final = _gpt_6_with_real_usage( + GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE, + "gpt-6.1-terra", + "responses_prompt_cache_write_30m", + _GPT_6_RESPONSES_CACHE_WRITE_30M_REQUEST, + {"input_tokens": 3329, "input_tokens_details": {"cache_write_tokens": 3326, "cache_write_tokens_details": {"ttl_12h_tokens": 0, "ttl_30m_tokens": 3326}, "cached_tokens": 0}, "output_tokens": 5, "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": 3334}, + ExactExpected(spend=0.004311065, input_cost=0.004285315, output_cost=2.575e-05, prompt_tokens=3329, completion_tokens=5, cache_creation_cost=0.004282225), +) +TEXT_EMBEDDING_3_LARGE_DIMENSIONS: Final = CostTrackingTestCase( + name="text-embedding-3-large-dimensions", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="text-embedding-3-large", + endpoint="/v1/embeddings", + request={"model": "$MODEL", "input": "price this embedding request", "dimensions": 256}, + response=JsonResponse( + content_type="application/json", + body={ + "object": "list", + "data": [{"object": "embedding", "embedding": [-0.060546875, -0.0176849365234375], "index": 0}], + "model": "text-embedding-3-large", + "usage": {"prompt_tokens": 4, "total_tokens": 4}, + }, + ), + expected=ExactExpected(spend=5.2e-07, input_cost=5.2e-07, output_cost=0.0, prompt_tokens=4, completion_tokens=0), +) + +def _gpt_image_2_6_sunburst_body(usage: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + return { + "created": 1794891118, + "background": "opaque", + "data": [{"b64_json": "AA=="}], + "output_format": "png", + "quality": "low", + "size": "1024x1024", + "usage": dict(usage), + } + + +GPT_IMAGE_2_6_SUNBURST_IMAGES_LOW: Final = CostTrackingTestCase( + name="gpt-image-2.6-sunburst-images-low", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gpt-image-2.6-sunburst", + endpoint="/v1/images/generations", + deployment=Deployment(model="openai/gpt-image-2.6-sunburst"), + request={"prompt": "a plain red square", "size": "1024x1024", "quality": "low", "n": 1}, + response=JsonResponse( + content_type="application/json", + body=_gpt_image_2_6_sunburst_body( + { + "input_tokens": 10, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 10}, + "output_tokens": 824, + "output_tokens_details": {"image_tokens": 824, "text_tokens": 0}, + "total_tokens": 834, + } + ), + ), + expected=ExactExpected( + spend=0.0247702, + input_cost=5.02e-05, + output_cost=0.02472, + prompt_tokens=10, + completion_tokens=824, + breakdown_persisted=False, + ), +) +GPT_IMAGE_2_6_SUNBURST_IMAGES_EDIT: Final = CostTrackingTestCase( + name="gpt-image-2.6-sunburst-images-edit", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gpt-image-2.6-sunburst", + endpoint="/v1/images/edits", + deployment=Deployment(model="openai/gpt-image-2.6-sunburst"), + upload=PngUpload(kind="png"), + request={"prompt": "make the square blue", "size": "1024x1024", "quality": "low", "n": 1}, + response=JsonResponse( + content_type="application/json", + body=_gpt_image_2_6_sunburst_body( + { + "input_tokens": 1044, + "input_tokens_details": {"image_tokens": 1024, "text_tokens": 20}, + "output_tokens": 824, + "output_tokens_details": {"image_tokens": 824, "text_tokens": 0}, + "total_tokens": 1868, + } + ), + ), + expected=ExactExpected( + spend=0.0330124, + input_cost=0.0082924, + output_cost=0.02472, + prompt_tokens=1044, + completion_tokens=824, + breakdown_persisted=False, + ), +) +GPT_4O_TRANSCRIBE_TRANSCRIPTIONS_TOKENS: Final = CostTrackingTestCase( + name="gpt-4o-transcribe-transcriptions-tokens", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gpt-4o-transcribe", + endpoint="/v1/audio/transcriptions", + upload=WavUpload(kind="wav", seconds=3.0), + request={"response_format": "json"}, + response=JsonResponse( + content_type="application/json", + body={ + "text": "The pricing harness says hello.", + "usage": { + "type": "tokens", + "total_tokens": 38, + "input_tokens": 30, + "input_token_details": {"text_tokens": 0, "audio_tokens": 30}, + "output_tokens": 8, + }, + }, + ), + expected=ExactExpected(spend=0.00026, input_cost=0.00018, output_cost=8e-05, prompt_tokens=30, completion_tokens=8), +) +GPT_4O_MINI_TTS_SPEECH: Final = CostTrackingTestCase( + name="gpt-4o-mini-tts-speech", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gpt-4o-mini-tts", + endpoint="/v1/audio/speech", + request={"input": "The pricing harness says hello.", "voice": "alloy", "response_format": "mp3"}, + response=BinaryResponse(content_type="audio/mpeg", length=2048), + expected=ExactExpected( + spend=3.66e-06, input_cost=3.66e-06, output_cost=0.0, prompt_tokens=0, completion_tokens=0 + ), +) + CASES: Final[tuple[CostTrackingTestCase, ...]] = ( + GPT_6_LUNA_RESPONSES_PROMPT_CACHE_WRITE_30M, + GPT_6_SOL_RESPONSES_PROMPT_CACHE_WRITE_30M, + GPT_6_ASTRA_RESPONSES_PROMPT_CACHE_WRITE_30M, + GPT_6_1_SOL_RESPONSES_PROMPT_CACHE_WRITE_30M, + GPT_6_1_TERRA_RESPONSES_PROMPT_CACHE_WRITE_30M, + GPT_IMAGE_2_6_SUNBURST_IMAGES_LOW, + GPT_IMAGE_2_6_SUNBURST_IMAGES_EDIT, + GPT_4O_TRANSCRIBE_TRANSCRIPTIONS_TOKENS, + GPT_4O_MINI_TTS_SPEECH, + GPT_6_1_TERRA_INPUT_TEXT, + GPT_6_1_TERRA_STREAM, + GPT_6_1_TERRA_REASONING, + GPT_6_1_TERRA_TOOL_CALL, + GPT_6_1_TERRA_SERVICE_TIER_FLEX, + GPT_6_1_TERRA_SERVICE_TIER_PRIORITY, + GPT_6_1_TERRA_RESPONSES_NATIVE_JSON, + GPT_6_ASTRA_TOOL_CALL, + GPT_6_1_SOL_TOOL_CALL, + GPT_6_LUNA_PROMPT_CACHE_WRITE, + GPT_6_LUNA_PROMPT_CACHE_READ, + GPT_6_LUNA_RESPONSES_PROMPT_CACHE_WRITE_12H, + GPT_6_LUNA_RESPONSES_WEB_SEARCH_MEDIUM, + GPT_6_SOL_PROMPT_CACHE_WRITE, + GPT_6_SOL_PROMPT_CACHE_READ, + GPT_6_SOL_RESPONSES_PROMPT_CACHE_WRITE_12H, + GPT_6_SOL_RESPONSES_WEB_SEARCH_MEDIUM, + GPT_6_ASTRA_PROMPT_CACHE_WRITE, + GPT_6_ASTRA_PROMPT_CACHE_READ, + GPT_6_ASTRA_RESPONSES_PROMPT_CACHE_WRITE_12H, + GPT_6_ASTRA_RESPONSES_WEB_SEARCH_MEDIUM, + GPT_6_1_SOL_PROMPT_CACHE_WRITE, + GPT_6_1_SOL_PROMPT_CACHE_READ, + GPT_6_1_SOL_RESPONSES_PROMPT_CACHE_WRITE_12H, + GPT_6_1_SOL_RESPONSES_WEB_SEARCH_MEDIUM, + GPT_6_1_TERRA_PROMPT_CACHE_WRITE, + GPT_6_1_TERRA_PROMPT_CACHE_READ, + GPT_6_1_TERRA_RESPONSES_PROMPT_CACHE_WRITE_12H, + GPT_6_1_TERRA_RESPONSES_WEB_SEARCH_MEDIUM, + TEXT_EMBEDDING_3_LARGE_DIMENSIONS, GATEWAY_REPORTED_COST, GATEWAY_REPORTED_COST_STREAM, GATEWAY_REPORTED_COST_OPERATOR_PRICED, @@ -12343,6 +12766,74 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GPT_5_6_TERRA_MESSAGES_BRIDGED_STREAM, ) +BILLED_AGAINST_REAL_PROVIDER_BY_COST_TRACKING_RECONCILIATION_AUTOMATION: Final[tuple[CostTrackingTestCase, ...]] = ( + GPT_6_LUNA_RESPONSES_PROMPT_CACHE_WRITE_30M, + GPT_6_SOL_RESPONSES_PROMPT_CACHE_WRITE_30M, + GPT_6_ASTRA_RESPONSES_PROMPT_CACHE_WRITE_30M, + GPT_6_1_SOL_RESPONSES_PROMPT_CACHE_WRITE_30M, + GPT_6_1_TERRA_RESPONSES_PROMPT_CACHE_WRITE_30M, + GPT_IMAGE_2_6_SUNBURST_IMAGES_LOW, + GPT_IMAGE_2_6_SUNBURST_IMAGES_EDIT, + GPT_4O_TRANSCRIBE_TRANSCRIPTIONS_TOKENS, + GPT_4O_MINI_TTS_SPEECH, + GPT_6_1_TERRA_INPUT_TEXT, + GPT_6_1_TERRA_STREAM, + GPT_6_1_TERRA_REASONING, + GPT_6_1_TERRA_TOOL_CALL, + GPT_6_1_TERRA_SERVICE_TIER_FLEX, + GPT_6_1_TERRA_SERVICE_TIER_PRIORITY, + GPT_6_1_TERRA_RESPONSES_NATIVE_JSON, + GPT_6_ASTRA_TOOL_CALL, + GPT_6_1_SOL_TOOL_CALL, + GPT_6_LUNA_PROMPT_CACHE_WRITE, + GPT_6_LUNA_PROMPT_CACHE_READ, + GPT_6_LUNA_RESPONSES_PROMPT_CACHE_WRITE_12H, + GPT_6_LUNA_RESPONSES_WEB_SEARCH_MEDIUM, + GPT_6_SOL_PROMPT_CACHE_WRITE, + GPT_6_SOL_PROMPT_CACHE_READ, + GPT_6_SOL_RESPONSES_PROMPT_CACHE_WRITE_12H, + GPT_6_SOL_RESPONSES_WEB_SEARCH_MEDIUM, + GPT_6_ASTRA_PROMPT_CACHE_WRITE, + GPT_6_ASTRA_PROMPT_CACHE_READ, + GPT_6_ASTRA_RESPONSES_PROMPT_CACHE_WRITE_12H, + GPT_6_ASTRA_RESPONSES_WEB_SEARCH_MEDIUM, + GPT_6_1_SOL_PROMPT_CACHE_WRITE, + GPT_6_1_SOL_PROMPT_CACHE_READ, + GPT_6_1_SOL_RESPONSES_PROMPT_CACHE_WRITE_12H, + GPT_6_1_SOL_RESPONSES_WEB_SEARCH_MEDIUM, + GPT_6_1_TERRA_PROMPT_CACHE_WRITE, + GPT_6_1_TERRA_PROMPT_CACHE_READ, + GPT_6_1_TERRA_RESPONSES_PROMPT_CACHE_WRITE_12H, + GPT_6_1_TERRA_RESPONSES_WEB_SEARCH_MEDIUM, + TEXT_EMBEDDING_3_LARGE_DIMENSIONS, + GPT_6_LUNA_INPUT_TEXT, + GPT_6_LUNA_STREAM, + GPT_6_LUNA_REASONING, + GPT_6_LUNA_TOOL_CALL, + GPT_6_LUNA_SERVICE_TIER_FLEX, + GPT_6_LUNA_SERVICE_TIER_PRIORITY, + GPT_6_LUNA_RESPONSES_NATIVE_JSON, + GPT_6_SOL_INPUT_TEXT, + GPT_6_SOL_STREAM, + GPT_6_SOL_REASONING, + GPT_6_SOL_TOOL_CALL, + GPT_6_SOL_SERVICE_TIER_FLEX, + GPT_6_SOL_SERVICE_TIER_PRIORITY, + GPT_6_SOL_RESPONSES_NATIVE_JSON, + GPT_6_ASTRA_INPUT_TEXT, + GPT_6_ASTRA_STREAM, + GPT_6_ASTRA_REASONING, + GPT_6_ASTRA_SERVICE_TIER_FLEX, + GPT_6_ASTRA_SERVICE_TIER_PRIORITY, + GPT_6_ASTRA_RESPONSES_NATIVE_JSON, + GPT_6_1_SOL_INPUT_TEXT, + GPT_6_1_SOL_STREAM, + GPT_6_1_SOL_REASONING, + GPT_6_1_SOL_SERVICE_TIER_FLEX, + GPT_6_1_SOL_SERVICE_TIER_PRIORITY, + GPT_6_1_SOL_RESPONSES_NATIVE_JSON, +) + PARITY: Final[tuple[StreamParityTestCase, ...]] = ( StreamParityTestCase(plain=GATEWAY_REPORTED_COST, streamed=GATEWAY_REPORTED_COST_STREAM), StreamParityTestCase( @@ -12386,6 +12877,7 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = ( StreamParityTestCase(plain=GPT_6_ASTRA_INPUT_TEXT, streamed=GPT_6_ASTRA_STREAM), StreamParityTestCase(plain=GPT_6_1_SOL_INPUT_TEXT, streamed=GPT_6_1_SOL_STREAM), StreamParityTestCase(plain=GPT_5_6_TERRA_INPUT_TEXT, streamed=GPT_5_6_TERRA_STREAM), + StreamParityTestCase(plain=GPT_6_1_TERRA_INPUT_TEXT, streamed=GPT_6_1_TERRA_STREAM), StreamParityTestCase(plain=GPT_6_1_SOL_RESPONSES_NATIVE_JSON, streamed=GPT_6_1_SOL_RESPONSES_NATIVE_STREAM), StreamParityTestCase(plain=GPT_6_LUNA_RESPONSES_NATIVE_JSON, streamed=GPT_6_LUNA_RESPONSES_NATIVE_STREAM), StreamParityTestCase(plain=GPT_6_ASTRA_RESPONSES_NATIVE_JSON, streamed=GPT_6_ASTRA_RESPONSES_NATIVE_STREAM), diff --git a/tests/integration/cost_calculation/catalog.py b/tests/integration/cost_calculation/catalog.py index 2c1d9d9e22c..9b13928c397 100644 --- a/tests/integration/cost_calculation/catalog.py +++ b/tests/integration/cost_calculation/catalog.py @@ -52,6 +52,13 @@ _PROVIDER_MODULES: Final = ( xai, ) CASES: Final[tuple[CostTrackingTestCase, ...]] = tuple(case for module in _PROVIDER_MODULES for case in module.CASES) +BILLED_AGAINST_REAL_PROVIDER_BY_COST_TRACKING_RECONCILIATION_AUTOMATION: Final[frozenset[str]] = frozenset( + case.name + for case in ( + *openai.BILLED_AGAINST_REAL_PROVIDER_BY_COST_TRACKING_RECONCILIATION_AUTOMATION, + *batch_realtime.BILLED_AGAINST_REAL_PROVIDER_BY_COST_TRACKING_RECONCILIATION_AUTOMATION, + ) +) PARITY_CASES: Final[tuple[StreamParityTestCase, ...]] = tuple( pair for module in _PROVIDER_MODULES for pair in module.PARITY ) diff --git a/tests/integration/cost_calculation/cost_map.py b/tests/integration/cost_calculation/cost_map.py index 9e15e79c0f7..2abb3103086 100644 --- a/tests/integration/cost_calculation/cost_map.py +++ b/tests/integration/cost_calculation/cost_map.py @@ -73,7 +73,30 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "supports_prompt_caching": True, "supports_reasoning": True, }, + "gpt-6.1-terra": { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1.03e-06, + "output_cost_per_token": 5.15e-06, + "input_cost_per_token_flex": 5.15e-07, + "output_cost_per_token_flex": 2.575e-06, + "input_cost_per_token_priority": 2.06e-06, + "output_cost_per_token_priority": 1.03e-05, + "cache_read_input_token_cost": 5.15e-08, + "cache_creation_input_token_cost": 1.2875e-06, + "cache_creation_input_token_cost_above_12hr": 2.575e-06, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01, + }, + }, "gpt-6-luna": { + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01, + }, "output_cost_per_token_priority": 1e-06, "input_cost_per_token_priority": 2e-07, "output_cost_per_token_flex": 2.5e-07, @@ -966,6 +989,27 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "input_cost_per_token": 1.51e-06, "output_cost_per_token": 7.51e-06, }, + "gpt-image-2.6-sunburst": { + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_token": 5.02e-06, + "input_cost_per_image_token": 8e-06, + "output_cost_per_image_token": 3e-05, + }, + "gpt-4o-mini-tts": { + "litellm_provider": "openai", + "mode": "audio_speech", + "input_cost_per_token": 6.1e-07, + "output_cost_per_audio_token": 1.2e-05, + "output_cost_per_second": 0.00025, + }, + "gpt-4o-transcribe": { + "litellm_provider": "openai", + "mode": "audio_transcription", + "input_cost_per_token": 2.52e-06, + "input_cost_per_audio_token": 6e-06, + "output_cost_per_token": 1e-05, + }, "text-embedding-3-large": {"litellm_provider": "openai", "mode": "embedding", "input_cost_per_token": 1.3e-07}, "gpt-5.4": { "litellm_provider": "openai", @@ -1324,6 +1368,12 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "supports_function_calling": True, }, "gpt-6.1-sol": { + "cache_creation_input_token_cost_above_12hr": 5e-06, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01, + }, "output_cost_per_token_priority": 2e-05, "input_cost_per_token_priority": 4e-06, "output_cost_per_token_flex": 5e-06, @@ -1530,6 +1580,13 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "supports_function_calling": True, }, "gpt-6-sol": { + "cache_creation_input_token_cost": 2.75e-06, + "cache_creation_input_token_cost_above_12hr": 5.5e-06, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01, + }, "cache_read_input_token_cost": 2.2e-07, "input_cost_per_token": 2.2e-06, "input_cost_per_token_flex": 1.1e-06, @@ -1552,6 +1609,12 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "cache_read_input_token_cost_above_272k_tokens": 4.4e-07, }, "gpt-6-astra": { + "cache_creation_input_token_cost_above_12hr": 3e-05, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01, + "search_context_size_high": 0.01, + }, "cache_read_input_token_cost": 1.2e-06, "cache_creation_input_token_cost": 1.5e-05, "cache_creation_input_token_cost_above_272k_tokens": 3e-05, diff --git a/tests/integration/cost_calculation/stream_parity/test_basic.py b/tests/integration/cost_calculation/stream_parity/test_basic.py index d26c58a35f2..f1a3c52b638 100644 --- a/tests/integration/cost_calculation/stream_parity/test_basic.py +++ b/tests/integration/cost_calculation/stream_parity/test_basic.py @@ -14,7 +14,10 @@ from typing import Final import pytest from integration._support.client import Gateway -from integration.cost_calculation.catalog import PARITY_CASES +from integration.cost_calculation.catalog import ( + BILLED_AGAINST_REAL_PROVIDER_BY_COST_TRACKING_RECONCILIATION_AUTOMATION, + PARITY_CASES, +) from integration.cost_calculation.stream_parity.bases import anthropic, bedrock, openai, openai_responses, xai from integration.cost_calculation.stream_parity.case import StreamParityTestCase from integration.cost_calculation.stream_parity.runner import assert_stream_parity @@ -34,7 +37,19 @@ BRIDGES: Final = ( CASES: Final = (*NATIVE, *BRIDGES, *PARITY_CASES) -@pytest.mark.parametrize("case", CASES, ids=[case.id for case in CASES]) +def _marks(case: StreamParityTestCase) -> tuple[pytest.MarkDecorator, ...]: + legs: Final = frozenset((case.plain.name, case.streamed.name)) + return ( + (pytest.mark.billed_against_real_provider_by_cost_tracking_reconciliation_automation,) + if legs & BILLED_AGAINST_REAL_PROVIDER_BY_COST_TRACKING_RECONCILIATION_AUTOMATION + else () + ) + + +_PARAMS: Final = tuple(pytest.param(case, marks=_marks(case), id=case.id) for case in CASES) + + +@pytest.mark.parametrize("case", _PARAMS) @pytest.mark.timeout(180) def test_streamed_request_bills_the_same_row_as_the_non_streamed_request( case: StreamParityTestCase, gateway: Gateway diff --git a/tests/integration/cost_calculation/test_batch_realtime_cost.py b/tests/integration/cost_calculation/test_batch_realtime_cost.py index 0e7de18a9a7..d82246fa57e 100644 --- a/tests/integration/cost_calculation/test_batch_realtime_cost.py +++ b/tests/integration/cost_calculation/test_batch_realtime_cost.py @@ -15,7 +15,11 @@ from integration._support.client import JSON_OBJECT, Gateway, Scenario, object_v from integration._support.upstream import delete_scenario, register_scenario from integration.cost_calculation.assertions import assert_exact from integration.cost_calculation.conftest import poll_rows, poll_rows_where, read_rows_now -from integration.cost_calculation.catalog import BATCH_CASES, REALTIME_CASES +from integration.cost_calculation.catalog import ( + BATCH_CASES, + BILLED_AGAINST_REAL_PROVIDER_BY_COST_TRACKING_RECONCILIATION_AUTOMATION, + REALTIME_CASES, +) from integration.cost_calculation.cost_tracking_case import ( BatchCostCase, JsonResponse, @@ -252,9 +256,18 @@ async def _run_realtime(url: str, key: str, model_name: str, turn_count: int) -> return session +def _realtime_marks(case: RealtimeCostCase) -> tuple[pytest.MarkDecorator, ...]: + billed: Final = ( + (pytest.mark.billed_against_real_provider_by_cost_tracking_reconciliation_automation,) + if case.name in BILLED_AGAINST_REAL_PROVIDER_BY_COST_TRACKING_RECONCILIATION_AUTOMATION + else () + ) + return (pytest.mark.covers(case.covers), *billed) + + @pytest.mark.parametrize( "case", - tuple(pytest.param(case, marks=pytest.mark.covers(case.covers), id=case.name) for case in REALTIME_CASES), + tuple(pytest.param(case, marks=_realtime_marks(case), id=case.name) for case in REALTIME_CASES), ) def test_realtime_costs(gateway: Gateway, case: RealtimeCostCase) -> None: with gateway.scenario() as scenario: diff --git a/tests/integration/cost_calculation/test_cost_tracking.py b/tests/integration/cost_calculation/test_cost_tracking.py index d54c59fbbaa..d8ca4ab31c3 100644 --- a/tests/integration/cost_calculation/test_cost_tracking.py +++ b/tests/integration/cost_calculation/test_cost_tracking.py @@ -28,7 +28,11 @@ from integration.cost_calculation.conftest import ( read_rows_now, register_scenario_deployment, ) -from integration.cost_calculation.catalog import STANDALONE_CASES, data_errors +from integration.cost_calculation.catalog import ( + BILLED_AGAINST_REAL_PROVIDER_BY_COST_TRACKING_RECONCILIATION_AUTOMATION, + STANDALONE_CASES, + data_errors, +) from integration.cost_calculation.stream_parity.runner import flush_spend_logs from integration.cost_calculation.cost_tracking_case import ( BinaryResponse, @@ -55,7 +59,12 @@ def _marks(case: CostTrackingTestCase) -> tuple[pytest.MarkDecorator, ...]: bug: Final = ( (pytest.mark.xfail(strict=True, reason=f"BUG: {case.known_bug}"),) if case.known_bug is not None else () ) - return (pytest.mark.covers(case.covers), *bug) + billed: Final = ( + (pytest.mark.billed_against_real_provider_by_cost_tracking_reconciliation_automation,) + if case.name in BILLED_AGAINST_REAL_PROVIDER_BY_COST_TRACKING_RECONCILIATION_AUTOMATION + else () + ) + return (pytest.mark.covers(case.covers), *bug, *billed) _CASES: Final = tuple(pytest.param(case, marks=_marks(case), id=case.name) for case in STANDALONE_CASES)