test(cost): mark the cost tracking cases the reconciliation automation bills against the real provider (#52226)

* test(cost): mark cost tracking cases that the reconciliation automation bills against the real provider

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(cost): add real-provider GPT-6, image, audio, embedding and realtime cost cases for the reconciliation automation

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(cost): make the GPT-6 prompt cache cases cacheable by the real provider and add the 30 minute Responses cache write

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* chore: drop handle_jwt.py that was re-added by mistake

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Kerry Lu 2026-11-16 23:01:32 -08:00 • committed by GitHub
parent f5e911afd0
commit fd12cba0b2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 729 additions and 83 deletions

View file

@ -70,6 +70,11 @@ def pytest_configure(config: pytest.Config) -> None:
config.addinivalue_line("markers", "integration: owned real-service integration contracts")
config.addinivalue_line("markers", "covers(*ids): legacy contract IDs kept for existing tests, not enforced")
config.addinivalue_line("markers", "shares_worker_state: owned proxies here share the worker DB and Redis")
config.addinivalue_line(
"markers",
"billed_against_real_provider_by_cost_tracking_reconciliation_automation: "
"the daily automation also sends this case to the real provider and checks the bill",
)
config.stash[REPORTS] = []
config.stash[SHARD_FILES] = _shard_files(SHARD_PATH.validate_python(config.getoption("integration_shard_files")))
config.pluginmanager.register(IntegrationReportPlugin(config))

View file

@ -12,12 +12,24 @@ import pytest
from integration._support.client import Gateway
from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase
from integration.cost_calculation.bridge_cost_parity.runner import assert_bridge_cost_parity
from integration.cost_calculation.catalog import BRIDGE_COST_PARITY_CASES
from integration.cost_calculation.catalog import (
BILLED_AGAINST_REAL_PROVIDER_BY_COST_TRACKING_RECONCILIATION_AUTOMATION,
BRIDGE_COST_PARITY_CASES,
)
def _marks(case: BridgeCostParityTestCase) -> tuple[pytest.MarkDecorator, ...]:
"""A known bridge bug is intermittent (it billed the native row in 1 of 4 local runs), so the xfail is not strict."""
return (pytest.mark.xfail(strict=False, reason=f"BUG: {case.known_bug}"),) if case.known_bug is not None else ()
bug: Final = (
(pytest.mark.xfail(strict=False, reason=f"BUG: {case.known_bug}"),) if case.known_bug is not None else ()
)
legs: Final = frozenset((case.native.name, case.bridged.name))
billed: Final = (
(pytest.mark.billed_against_real_provider_by_cost_tracking_reconciliation_automation,)
if legs & BILLED_AGAINST_REAL_PROVIDER_BY_COST_TRACKING_RECONCILIATION_AUTOMATION
else ()
)
return (*bug, *billed)
_CASES: Final = tuple(pytest.param(case, marks=_marks(case), id=case.id) for case in BRIDGE_COST_PARITY_CASES)

View file

@ -184,6 +184,32 @@ GPT_REALTIME_MINI_2025_12_15_REALTIME_SESSION_WITHOUT_TURNS_ZERO_SPEND: Final =
),
)
GPT_REALTIME_2_1_REALTIME_SINGLE_TURN_TEXT: Final = RealtimeCostCase(
name="gpt-realtime-2.1-realtime-single_turn_text",
covers="quota_management.spend_tracking.realtime_costs.single_turn",
model="gpt-realtime-2.1",
litellm_model="openai/gpt-realtime-2.1",
turns=(
RealtimeTurn(
input_tokens=129,
output_tokens=17,
input_text_tokens=129,
input_audio_tokens=0,
input_cached_tokens=0,
output_text_tokens=17,
output_audio_tokens=0,
),
),
expected=ExactExpected(
spend=0.000924,
input_cost=0.000516,
output_cost=0.000408,
prompt_tokens=129,
completion_tokens=17,
cost_header=False,
),
)
BATCH_CASES: Final[tuple[BatchCostCase, ...]] = (
GPT_5_6_BATCH_HALVED_RATES_WHEN_MAP_HAS_NO_BATCH_KEYS,
@ -197,4 +223,8 @@ REALTIME_CASES: Final[tuple[RealtimeCostCase, ...]] = (
GPT_REALTIME_MINI_2025_12_15_REALTIME_TWO_TURNS_SUMMED_INTO_ONE_ROW,
GPT_REALTIME_MINI_2025_12_15_REALTIME_PRICED_FROM_SESSION_CREATED_MODEL,
GPT_REALTIME_MINI_2025_12_15_REALTIME_SESSION_WITHOUT_TURNS_ZERO_SPEND,
GPT_REALTIME_2_1_REALTIME_SINGLE_TURN_TEXT,
)
BILLED_AGAINST_REAL_PROVIDER_BY_COST_TRACKING_RECONCILIATION_AUTOMATION: Final[tuple[RealtimeCostCase, ...]] = (
GPT_REALTIME_2_1_REALTIME_SINGLE_TURN_TEXT,
)

View file

@ -8470,6 +8470,19 @@ GPT_5_6_CHAT_REQUEST_TO_EMBEDDING_ENTRY: Final = CostTrackingTestCase(
# gpt-6.1-sol
_PROVIDER_CACHEABLE_PROMPT: Final = (
"The pricing harness keeps one stable instruction block so the provider can cache it. " * 220
)
_GPT_6_PROMPT_CACHE_REQUEST: Final[Mapping[str, JsonValue]] = {
"model": "$MODEL",
"messages": [
{"role": "system", "content": _PROVIDER_CACHEABLE_PROMPT},
{"role": "user", "content": "Reply with OK."},
],
"reasoning_effort": "low",
"stream": False,
}
GPT_6_1_SOL_INPUT_TEXT: Final = CostTrackingTestCase(
name="gpt-6.1-sol-input_text",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
@ -8511,31 +8524,7 @@ GPT_6_1_SOL_CACHE_READ: Final = GPT_5_6_CACHE_READ.model_copy(
update={
"name": "gpt-6.1-sol-cache_read",
"model": "gpt-6.1-sol",
"request": {
"model": "$MODEL",
"messages": [
{
"role": "system",
"content": [
{
"type": "text",
"text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.",
}
],
},
{
"role": "user",
"content": [
{
"type": "text",
"text": "b770f5df16a2 summarize the attached material in one line and name the city weather",
}
],
},
],
"stream": False,
"allowed_openai_params": [],
},
"request": _GPT_6_PROMPT_CACHE_REQUEST,
"response": JsonResponse(
content_type="application/json",
body={
@ -8585,31 +8574,7 @@ GPT_6_LUNA_CACHE_READ: Final = GPT_5_6_CACHE_READ.model_copy(
update={
"name": "gpt-6-luna-cache_read",
"model": "gpt-6-luna",
"request": {
"model": "$MODEL",
"messages": [
{
"role": "system",
"content": [
{
"type": "text",
"text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.",
}
],
},
{
"role": "user",
"content": [
{
"type": "text",
"text": "a880e9da100c summarize the attached material in one line and name the city weather",
}
],
},
],
"stream": False,
"allowed_openai_params": [],
},
"request": _GPT_6_PROMPT_CACHE_REQUEST,
"response": JsonResponse(
content_type="application/json",
body={
@ -8659,31 +8624,7 @@ GPT_6_ASTRA_CACHE_READ: Final = GPT_5_6_CACHE_READ.model_copy(
update={
"name": "gpt-6-astra-cache_read",
"model": "gpt-6-astra",
"request": {
"model": "$MODEL",
"messages": [
{
"role": "system",
"content": [
{
"type": "text",
"text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.",
}
],
},
{
"role": "user",
"content": [
{
"type": "text",
"text": "d45eefac915d summarize the attached material in one line and name the city weather",
}
],
},
],
"stream": False,
"allowed_openai_params": [],
},
"request": _GPT_6_PROMPT_CACHE_REQUEST,
"response": JsonResponse(
content_type="application/json",
body={
@ -12089,7 +12030,489 @@ GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED_STREAM: Final = (
)
)
def _gpt_6_on_model(base: CostTrackingTestCase, model: str, shape: str, expected: ExactExpected) -> CostTrackingTestCase:
response: Final = (
_stream_for_model(base.response, model)
if isinstance(base.response, SseResponse)
else _response_for_model(cast(JsonResponse, base.response), model)
)
return base.model_copy(
update={"name": f"{model}-{shape}", "model": model, "response": response, "expected": expected}
)
def _gpt_6_with_real_usage(
base: CostTrackingTestCase,
model: str,
shape: str,
request: Mapping[str, JsonValue],
usage: Mapping[str, JsonValue],
expected: ExactExpected,
) -> CostTrackingTestCase:
body: Final = {**cast(JsonResponse, base.response).body, "model": model, "usage": dict(usage)}
return base.model_copy(
update={
"name": f"{model}-{shape}",
"model": model,
"request": dict(request),
"response": cast(JsonResponse, base.response).model_copy(update={"body": body}),
"expected": expected,
}
)
def _gpt_6_tool_call_served_by_responses_api(model: str, expected: ExactExpected) -> CostTrackingTestCase:
body: Final = cast(JsonResponse, GPT_6_LUNA_RESPONSES_NATIVE_JSON.response).body
function_call: Final[dict[str, JsonValue]] = {
"type": "function_call",
"id": "fc_$REQUEST_ID",
"call_id": "call_$REQUEST_ID",
"name": "get_weather",
"arguments": '{"city": "Lisbon"}',
"status": "completed",
}
return GPT_6_LUNA_TOOL_CALL.model_copy(
update={
"name": f"{model}-tool_call",
"model": model,
"request": {**GPT_6_LUNA_TOOL_CALL.request, "reasoning_effort": "low"},
"response": JsonResponse(
content_type="application/json", body={**body, "model": model, "output": [function_call]}
),
"expected": expected,
}
)
_GPT_6_RESPONSES_CACHE_WRITE_12H_REQUEST: Final[Mapping[str, JsonValue]] = {
"model": "$MODEL",
"input": f"{_PROVIDER_CACHEABLE_PROMPT} Reply with OK.",
"prompt_cache_options": {"mode": "implicit", "ttl": "12h"},
"reasoning": {"effort": "low"},
}
_GPT_6_RESPONSES_CACHE_WRITE_30M_REQUEST: Final[Mapping[str, JsonValue]] = {
"model": "$MODEL",
"input": f"{_PROVIDER_CACHEABLE_PROMPT} Reply with OK.",
"reasoning": {"effort": "low"},
}
_GPT_6_RESPONSES_WEB_SEARCH_REQUEST: Final[Mapping[str, JsonValue]] = {
"model": "$MODEL",
"input": "What is the population of Lisbon? Search the web and answer in one line.",
"tools": [{"type": "web_search", "search_context_size": "medium"}],
"reasoning": {"effort": "low"},
}
GPT_6_1_TERRA_INPUT_TEXT: Final = _gpt_6_on_model(
GPT_6_LUNA_INPUT_TEXT,
"gpt-6.1-terra",
"input_text",
ExactExpected(spend=0.004017, input_cost=0.0018952, output_cost=0.0021218, prompt_tokens=1840, completion_tokens=412),
)
GPT_6_1_TERRA_STREAM: Final = _gpt_6_on_model(
GPT_6_LUNA_STREAM,
"gpt-6.1-terra",
"stream",
ExactExpected(spend=0.004017, input_cost=0.0018952, output_cost=0.0021218, prompt_tokens=1840, completion_tokens=412),
)
GPT_6_1_TERRA_REASONING: Final = _gpt_6_on_model(
GPT_6_LUNA_REASONING,
"gpt-6.1-terra",
"reasoning",
ExactExpected(spend=0.0220832, input_cost=0.0012772, output_cost=0.020806, prompt_tokens=1240, completion_tokens=4040),
)
GPT_6_1_TERRA_TOOL_CALL: Final = _gpt_6_on_model(
GPT_6_LUNA_TOOL_CALL,
"gpt-6.1-terra",
"tool_call",
ExactExpected(spend=0.004017, input_cost=0.0018952, output_cost=0.0021218, prompt_tokens=1840, completion_tokens=412),
)
GPT_6_1_TERRA_SERVICE_TIER_FLEX: Final = _gpt_6_on_model(
GPT_6_LUNA_SERVICE_TIER_FLEX,
"gpt-6.1-terra",
"service_tier_flex",
ExactExpected(spend=0.0020085, input_cost=0.0009476, output_cost=0.0010609, prompt_tokens=1840, completion_tokens=412),
)
GPT_6_1_TERRA_SERVICE_TIER_PRIORITY: Final = _gpt_6_on_model(
GPT_6_LUNA_SERVICE_TIER_PRIORITY,
"gpt-6.1-terra",
"service_tier_priority",
ExactExpected(spend=0.008034, input_cost=0.0037904, output_cost=0.0042436, prompt_tokens=1840, completion_tokens=412),
)
GPT_6_1_TERRA_RESPONSES_NATIVE_JSON: Final = _gpt_6_on_model(
GPT_6_LUNA_RESPONSES_NATIVE_JSON,
"gpt-6.1-terra",
"responses_native_json",
ExactExpected(spend=0.004017, input_cost=0.0018952, output_cost=0.0021218, prompt_tokens=1840, completion_tokens=412),
)
GPT_6_ASTRA_TOOL_CALL: Final = _gpt_6_tool_call_served_by_responses_api(
"gpt-6-astra",
ExactExpected(spend=0.0468, input_cost=0.02208, output_cost=0.02472, prompt_tokens=1840, completion_tokens=412),
)
GPT_6_1_SOL_TOOL_CALL: Final = _gpt_6_tool_call_served_by_responses_api(
"gpt-6.1-sol",
ExactExpected(spend=0.0078, input_cost=0.00368, output_cost=0.00412, prompt_tokens=1840, completion_tokens=412),
)
GPT_6_LUNA_PROMPT_CACHE_WRITE: Final = _gpt_6_with_real_usage(
GPT_6_LUNA_CACHE_READ,
"gpt-6-luna",
"prompt_cache_write",
_GPT_6_PROMPT_CACHE_REQUEST,
{"prompt_tokens": 3334, "completion_tokens": 4, "total_tokens": 3338, "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 3331, "audio_tokens": 0}, "completion_tokens_details": {"reasoning_tokens": 0, "audio_tokens": 0, "accepted_prediction_tokens": 0, "rejected_prediction_tokens": 0}},
ExactExpected(spend=0.000418675, input_cost=0.000416675, output_cost=2e-06, prompt_tokens=3334, completion_tokens=4, cache_creation_cost=0.000416375),
)
GPT_6_LUNA_PROMPT_CACHE_READ: Final = _gpt_6_with_real_usage(
GPT_6_LUNA_CACHE_READ,
"gpt-6-luna",
"prompt_cache_read",
_GPT_6_PROMPT_CACHE_REQUEST,
{"prompt_tokens": 3334, "completion_tokens": 4, "total_tokens": 3338, "prompt_tokens_details": {"cached_tokens": 3331, "cache_write_tokens": 0, "audio_tokens": 0}, "completion_tokens_details": {"reasoning_tokens": 0, "audio_tokens": 0, "accepted_prediction_tokens": 0, "rejected_prediction_tokens": 0}},
ExactExpected(spend=3.561e-05, input_cost=3.361e-05, output_cost=2e-06, prompt_tokens=3334, completion_tokens=4, cache_read_cost=3.331e-05),
)
GPT_6_LUNA_RESPONSES_PROMPT_CACHE_WRITE_12H: Final = _gpt_6_with_real_usage(
GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE,
"gpt-6-luna",
"responses_prompt_cache_write_12h",
_GPT_6_RESPONSES_CACHE_WRITE_12H_REQUEST,
{"input_tokens": 3334, "input_tokens_details": {"cache_write_tokens": 3331, "cache_write_tokens_details": {"ttl_12h_tokens": 3331, "ttl_30m_tokens": 0}, "cached_tokens": 0}, "output_tokens": 5, "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": 3339},
ExactExpected(spend=0.000419175, input_cost=0.000416675, output_cost=2.5e-06, prompt_tokens=3334, completion_tokens=5, cache_creation_cost=0.000416375),
)
GPT_6_LUNA_RESPONSES_WEB_SEARCH_MEDIUM: Final = _gpt_6_with_real_usage(
GPT_5_6_RESPONSES_WEB_SEARCH_MEDIUM,
"gpt-6-luna",
"responses_web_search_medium",
_GPT_6_RESPONSES_WEB_SEARCH_REQUEST,
{"input_tokens": 8636, "input_tokens_details": {"cache_write_tokens": 20, "cache_write_tokens_details": {"ttl_12h_tokens": 0, "ttl_30m_tokens": 20}, "cached_tokens": 4435}, "output_tokens": 81, "output_tokens_details": {"reasoning_tokens": 43}, "total_tokens": 8717},
ExactExpected(spend=0.01050545, input_cost=0.00046495, output_cost=4.05e-05, prompt_tokens=8636, completion_tokens=81, tool_usage_cost=0.01, cache_read_cost=4.435e-05, cache_creation_cost=2.5e-06),
)
GPT_6_SOL_PROMPT_CACHE_WRITE: Final = _gpt_6_with_real_usage(
GPT_6_LUNA_CACHE_READ,
"gpt-6-sol",
"prompt_cache_write",
_GPT_6_PROMPT_CACHE_REQUEST,
{"prompt_tokens": 3336, "completion_tokens": 4, "total_tokens": 3340, "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 3333, "audio_tokens": 0}, "completion_tokens_details": {"reasoning_tokens": 0, "audio_tokens": 0, "accepted_prediction_tokens": 0, "rejected_prediction_tokens": 0}},
ExactExpected(spend=0.00921635, input_cost=0.00917235, output_cost=4.4e-05, prompt_tokens=3336, completion_tokens=4, cache_creation_cost=0.00916575),
)
GPT_6_SOL_PROMPT_CACHE_READ: Final = _gpt_6_with_real_usage(
GPT_6_LUNA_CACHE_READ,
"gpt-6-sol",
"prompt_cache_read",
_GPT_6_PROMPT_CACHE_REQUEST,
{"prompt_tokens": 3336, "completion_tokens": 4, "total_tokens": 3340, "prompt_tokens_details": {"cached_tokens": 3333, "cache_write_tokens": 0, "audio_tokens": 0}, "completion_tokens_details": {"reasoning_tokens": 0, "audio_tokens": 0, "accepted_prediction_tokens": 0, "rejected_prediction_tokens": 0}},
ExactExpected(spend=0.00078386, input_cost=0.00073986, output_cost=4.4e-05, prompt_tokens=3336, completion_tokens=4, cache_read_cost=0.00073326),
)
GPT_6_SOL_RESPONSES_PROMPT_CACHE_WRITE_12H: Final = _gpt_6_with_real_usage(
GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE,
"gpt-6-sol",
"responses_prompt_cache_write_12h",
_GPT_6_RESPONSES_CACHE_WRITE_12H_REQUEST,
{"input_tokens": 3332, "input_tokens_details": {"cache_write_tokens": 3329, "cache_write_tokens_details": {"ttl_12h_tokens": 3329, "ttl_30m_tokens": 0}, "cached_tokens": 0}, "output_tokens": 6, "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": 3338},
ExactExpected(spend=0.0183821, input_cost=0.0183161, output_cost=6.6e-05, prompt_tokens=3332, completion_tokens=6, cache_creation_cost=0.0183095),
)
GPT_6_SOL_RESPONSES_WEB_SEARCH_MEDIUM: Final = _gpt_6_with_real_usage(
GPT_5_6_RESPONSES_WEB_SEARCH_MEDIUM,
"gpt-6-sol",
"responses_web_search_medium",
_GPT_6_RESPONSES_WEB_SEARCH_REQUEST,
{"input_tokens": 8763, "input_tokens_details": {"cache_write_tokens": 4455, "cache_write_tokens_details": {"ttl_12h_tokens": 0, "ttl_30m_tokens": 4455}, "cached_tokens": 0}, "output_tokens": 94, "output_tokens_details": {"reasoning_tokens": 52}, "total_tokens": 8857},
ExactExpected(spend=0.03276285, input_cost=0.02172885, output_cost=0.001034, prompt_tokens=8763, completion_tokens=94, tool_usage_cost=0.01, cache_creation_cost=0.01225125),
)
GPT_6_ASTRA_PROMPT_CACHE_WRITE: Final = _gpt_6_with_real_usage(
GPT_6_LUNA_CACHE_READ,
"gpt-6-astra",
"prompt_cache_write",
_GPT_6_PROMPT_CACHE_REQUEST,
{"prompt_tokens": 3333, "completion_tokens": 4, "total_tokens": 3337, "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 3330, "audio_tokens": 0}, "completion_tokens_details": {"reasoning_tokens": 0, "audio_tokens": 0, "accepted_prediction_tokens": 0, "rejected_prediction_tokens": 0}},
ExactExpected(spend=0.050226, input_cost=0.049986, output_cost=0.00024, prompt_tokens=3333, completion_tokens=4, cache_creation_cost=0.04995),
)
GPT_6_ASTRA_PROMPT_CACHE_READ: Final = _gpt_6_with_real_usage(
GPT_6_LUNA_CACHE_READ,
"gpt-6-astra",
"prompt_cache_read",
_GPT_6_PROMPT_CACHE_REQUEST,
{"prompt_tokens": 3333, "completion_tokens": 4, "total_tokens": 3337, "prompt_tokens_details": {"cached_tokens": 3330, "cache_write_tokens": 0, "audio_tokens": 0}, "completion_tokens_details": {"reasoning_tokens": 0, "audio_tokens": 0, "accepted_prediction_tokens": 0, "rejected_prediction_tokens": 0}},
ExactExpected(spend=0.004272, input_cost=0.004032, output_cost=0.00024, prompt_tokens=3333, completion_tokens=4, cache_read_cost=0.003996),
)
GPT_6_ASTRA_RESPONSES_PROMPT_CACHE_WRITE_12H: Final = _gpt_6_with_real_usage(
GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE,
"gpt-6-astra",
"responses_prompt_cache_write_12h",
_GPT_6_RESPONSES_CACHE_WRITE_12H_REQUEST,
{"input_tokens": 3329, "input_tokens_details": {"cache_write_tokens": 3326, "cache_write_tokens_details": {"ttl_12h_tokens": 3326, "ttl_30m_tokens": 0}, "cached_tokens": 0}, "output_tokens": 5, "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": 3334},
ExactExpected(spend=0.100116, input_cost=0.099816, output_cost=0.0003, prompt_tokens=3329, completion_tokens=5, cache_creation_cost=0.09978),
)
GPT_6_ASTRA_RESPONSES_WEB_SEARCH_MEDIUM: Final = _gpt_6_with_real_usage(
GPT_5_6_RESPONSES_WEB_SEARCH_MEDIUM,
"gpt-6-astra",
"responses_web_search_medium",
_GPT_6_RESPONSES_WEB_SEARCH_REQUEST,
{"input_tokens": 8580, "input_tokens_details": {"cache_write_tokens": 4455, "cache_write_tokens_details": {"ttl_12h_tokens": 0, "ttl_30m_tokens": 4455}, "cached_tokens": 0}, "output_tokens": 81, "output_tokens_details": {"reasoning_tokens": 38}, "total_tokens": 8661},
ExactExpected(spend=0.131185, input_cost=0.116325, output_cost=0.00486, prompt_tokens=8580, completion_tokens=81, tool_usage_cost=0.01, cache_creation_cost=0.066825),
)
GPT_6_1_SOL_PROMPT_CACHE_WRITE: Final = _gpt_6_with_real_usage(
GPT_6_LUNA_CACHE_READ,
"gpt-6.1-sol",
"prompt_cache_write",
_GPT_6_PROMPT_CACHE_REQUEST,
{"prompt_tokens": 3333, "completion_tokens": 4, "total_tokens": 3337, "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 3330, "audio_tokens": 0}, "completion_tokens_details": {"reasoning_tokens": 0, "audio_tokens": 0, "accepted_prediction_tokens": 0, "rejected_prediction_tokens": 0}},
ExactExpected(spend=0.008371, input_cost=0.008331, output_cost=4e-05, prompt_tokens=3333, completion_tokens=4, cache_creation_cost=0.008325),
)
GPT_6_1_SOL_PROMPT_CACHE_READ: Final = _gpt_6_with_real_usage(
GPT_6_LUNA_CACHE_READ,
"gpt-6.1-sol",
"prompt_cache_read",
_GPT_6_PROMPT_CACHE_REQUEST,
{"prompt_tokens": 3333, "completion_tokens": 4, "total_tokens": 3337, "prompt_tokens_details": {"cached_tokens": 3330, "cache_write_tokens": 0, "audio_tokens": 0}, "completion_tokens_details": {"reasoning_tokens": 0, "audio_tokens": 0, "accepted_prediction_tokens": 0, "rejected_prediction_tokens": 0}},
ExactExpected(spend=0.000379, input_cost=0.000339, output_cost=4e-05, prompt_tokens=3333, completion_tokens=4, cache_read_cost=0.000333),
)
GPT_6_1_SOL_RESPONSES_PROMPT_CACHE_WRITE_12H: Final = _gpt_6_with_real_usage(
GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE,
"gpt-6.1-sol",
"responses_prompt_cache_write_12h",
_GPT_6_RESPONSES_CACHE_WRITE_12H_REQUEST,
{"input_tokens": 3328, "input_tokens_details": {"cache_write_tokens": 3325, "cache_write_tokens_details": {"ttl_12h_tokens": 3325, "ttl_30m_tokens": 0}, "cached_tokens": 0}, "output_tokens": 5, "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": 3333},
ExactExpected(spend=0.016681, input_cost=0.016631, output_cost=5e-05, prompt_tokens=3328, completion_tokens=5, cache_creation_cost=0.016625),
)
GPT_6_1_SOL_RESPONSES_WEB_SEARCH_MEDIUM: Final = _gpt_6_with_real_usage(
GPT_5_6_RESPONSES_WEB_SEARCH_MEDIUM,
"gpt-6.1-sol",
"responses_web_search_medium",
_GPT_6_RESPONSES_WEB_SEARCH_REQUEST,
{"input_tokens": 8617, "input_tokens_details": {"cache_write_tokens": 4455, "cache_write_tokens_details": {"ttl_12h_tokens": 0, "ttl_30m_tokens": 4455}, "cached_tokens": 0}, "output_tokens": 83, "output_tokens_details": {"reasoning_tokens": 38}, "total_tokens": 8700},
ExactExpected(spend=0.0302915, input_cost=0.0194615, output_cost=0.00083, prompt_tokens=8617, completion_tokens=83, tool_usage_cost=0.01, cache_creation_cost=0.0111375),
)
GPT_6_1_TERRA_PROMPT_CACHE_WRITE: Final = _gpt_6_with_real_usage(
GPT_6_LUNA_CACHE_READ,
"gpt-6.1-terra",
"prompt_cache_write",
_GPT_6_PROMPT_CACHE_REQUEST,
{"prompt_tokens": 3333, "completion_tokens": 4, "total_tokens": 3337, "prompt_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 3330, "audio_tokens": 0}, "completion_tokens_details": {"reasoning_tokens": 0, "audio_tokens": 0, "accepted_prediction_tokens": 0, "rejected_prediction_tokens": 0}},
ExactExpected(spend=0.004311065, input_cost=0.004290465, output_cost=2.06e-05, prompt_tokens=3333, completion_tokens=4, cache_creation_cost=0.004287375),
)
GPT_6_1_TERRA_PROMPT_CACHE_READ: Final = _gpt_6_with_real_usage(
GPT_6_LUNA_CACHE_READ,
"gpt-6.1-terra",
"prompt_cache_read",
_GPT_6_PROMPT_CACHE_REQUEST,
{"prompt_tokens": 3333, "completion_tokens": 4, "total_tokens": 3337, "prompt_tokens_details": {"cached_tokens": 3330, "cache_write_tokens": 0, "audio_tokens": 0}, "completion_tokens_details": {"reasoning_tokens": 0, "audio_tokens": 0, "accepted_prediction_tokens": 0, "rejected_prediction_tokens": 0}},
ExactExpected(spend=0.000195185, input_cost=0.000174585, output_cost=2.06e-05, prompt_tokens=3333, completion_tokens=4, cache_read_cost=0.000171495),
)
GPT_6_1_TERRA_RESPONSES_PROMPT_CACHE_WRITE_12H: Final = _gpt_6_with_real_usage(
GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE,
"gpt-6.1-terra",
"responses_prompt_cache_write_12h",
_GPT_6_RESPONSES_CACHE_WRITE_12H_REQUEST,
{"input_tokens": 3332, "input_tokens_details": {"cache_write_tokens": 3329, "cache_write_tokens_details": {"ttl_12h_tokens": 3329, "ttl_30m_tokens": 0}, "cached_tokens": 0}, "output_tokens": 5, "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": 3337},
ExactExpected(spend=0.008601015, input_cost=0.008575265, output_cost=2.575e-05, prompt_tokens=3332, completion_tokens=5, cache_creation_cost=0.008572175),
)
GPT_6_1_TERRA_RESPONSES_WEB_SEARCH_MEDIUM: Final = _gpt_6_with_real_usage(
GPT_5_6_RESPONSES_WEB_SEARCH_MEDIUM,
"gpt-6.1-terra",
"responses_web_search_medium",
_GPT_6_RESPONSES_WEB_SEARCH_REQUEST,
{"input_tokens": 8657, "input_tokens_details": {"cache_write_tokens": 4455, "cache_write_tokens_details": {"ttl_12h_tokens": 0, "ttl_30m_tokens": 4455}, "cached_tokens": 0}, "output_tokens": 77, "output_tokens_details": {"reasoning_tokens": 36}, "total_tokens": 8734},
ExactExpected(spend=0.0204604225, input_cost=0.0100638725, output_cost=0.00039655, prompt_tokens=8657, completion_tokens=77, tool_usage_cost=0.01, cache_creation_cost=0.0057358125),
)
GPT_6_LUNA_RESPONSES_PROMPT_CACHE_WRITE_30M: Final = _gpt_6_with_real_usage(
GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE,
"gpt-6-luna",
"responses_prompt_cache_write_30m",
_GPT_6_RESPONSES_CACHE_WRITE_30M_REQUEST,
{"input_tokens": 3329, "input_tokens_details": {"cache_write_tokens": 3326, "cache_write_tokens_details": {"ttl_12h_tokens": 0, "ttl_30m_tokens": 3326}, "cached_tokens": 0}, "output_tokens": 5, "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": 3334},
ExactExpected(spend=0.00041855, input_cost=0.00041605, output_cost=2.5e-06, prompt_tokens=3329, completion_tokens=5, cache_creation_cost=0.00041575),
)
GPT_6_SOL_RESPONSES_PROMPT_CACHE_WRITE_30M: Final = _gpt_6_with_real_usage(
GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE,
"gpt-6-sol",
"responses_prompt_cache_write_30m",
_GPT_6_RESPONSES_CACHE_WRITE_30M_REQUEST,
{"input_tokens": 3328, "input_tokens_details": {"cache_write_tokens": 3325, "cache_write_tokens_details": {"ttl_12h_tokens": 0, "ttl_30m_tokens": 3325}, "cached_tokens": 0}, "output_tokens": 6, "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": 3334},
ExactExpected(spend=0.00921635, input_cost=0.00915035, output_cost=6.6e-05, prompt_tokens=3328, completion_tokens=6, cache_creation_cost=0.00914375),
)
GPT_6_ASTRA_RESPONSES_PROMPT_CACHE_WRITE_30M: Final = _gpt_6_with_real_usage(
GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE,
"gpt-6-astra",
"responses_prompt_cache_write_30m",
_GPT_6_RESPONSES_CACHE_WRITE_30M_REQUEST,
{"input_tokens": 3327, "input_tokens_details": {"cache_write_tokens": 3324, "cache_write_tokens_details": {"ttl_12h_tokens": 0, "ttl_30m_tokens": 3324}, "cached_tokens": 0}, "output_tokens": 5, "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": 3332},
ExactExpected(spend=0.050196, input_cost=0.049896, output_cost=0.0003, prompt_tokens=3327, completion_tokens=5, cache_creation_cost=0.04986),
)
GPT_6_1_SOL_RESPONSES_PROMPT_CACHE_WRITE_30M: Final = _gpt_6_with_real_usage(
GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE,
"gpt-6.1-sol",
"responses_prompt_cache_write_30m",
_GPT_6_RESPONSES_CACHE_WRITE_30M_REQUEST,
{"input_tokens": 3329, "input_tokens_details": {"cache_write_tokens": 3326, "cache_write_tokens_details": {"ttl_12h_tokens": 0, "ttl_30m_tokens": 3326}, "cached_tokens": 0}, "output_tokens": 5, "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": 3334},
ExactExpected(spend=0.008371, input_cost=0.008321, output_cost=5e-05, prompt_tokens=3329, completion_tokens=5, cache_creation_cost=0.008315),
)
GPT_6_1_TERRA_RESPONSES_PROMPT_CACHE_WRITE_30M: Final = _gpt_6_with_real_usage(
GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE,
"gpt-6.1-terra",
"responses_prompt_cache_write_30m",
_GPT_6_RESPONSES_CACHE_WRITE_30M_REQUEST,
{"input_tokens": 3329, "input_tokens_details": {"cache_write_tokens": 3326, "cache_write_tokens_details": {"ttl_12h_tokens": 0, "ttl_30m_tokens": 3326}, "cached_tokens": 0}, "output_tokens": 5, "output_tokens_details": {"reasoning_tokens": 0}, "total_tokens": 3334},
ExactExpected(spend=0.004311065, input_cost=0.004285315, output_cost=2.575e-05, prompt_tokens=3329, completion_tokens=5, cache_creation_cost=0.004282225),
)
TEXT_EMBEDDING_3_LARGE_DIMENSIONS: Final = CostTrackingTestCase(
name="text-embedding-3-large-dimensions",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="text-embedding-3-large",
endpoint="/v1/embeddings",
request={"model": "$MODEL", "input": "price this embedding request", "dimensions": 256},
response=JsonResponse(
content_type="application/json",
body={
"object": "list",
"data": [{"object": "embedding", "embedding": [-0.060546875, -0.0176849365234375], "index": 0}],
"model": "text-embedding-3-large",
"usage": {"prompt_tokens": 4, "total_tokens": 4},
},
),
expected=ExactExpected(spend=5.2e-07, input_cost=5.2e-07, output_cost=0.0, prompt_tokens=4, completion_tokens=0),
)
def _gpt_image_2_6_sunburst_body(usage: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
return {
"created": 1794891118,
"background": "opaque",
"data": [{"b64_json": "AA=="}],
"output_format": "png",
"quality": "low",
"size": "1024x1024",
"usage": dict(usage),
}
GPT_IMAGE_2_6_SUNBURST_IMAGES_LOW: Final = CostTrackingTestCase(
name="gpt-image-2.6-sunburst-images-low",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="gpt-image-2.6-sunburst",
endpoint="/v1/images/generations",
deployment=Deployment(model="openai/gpt-image-2.6-sunburst"),
request={"prompt": "a plain red square", "size": "1024x1024", "quality": "low", "n": 1},
response=JsonResponse(
content_type="application/json",
body=_gpt_image_2_6_sunburst_body(
{
"input_tokens": 10,
"input_tokens_details": {"image_tokens": 0, "text_tokens": 10},
"output_tokens": 824,
"output_tokens_details": {"image_tokens": 824, "text_tokens": 0},
"total_tokens": 834,
}
),
),
expected=ExactExpected(
spend=0.0247702,
input_cost=5.02e-05,
output_cost=0.02472,
prompt_tokens=10,
completion_tokens=824,
breakdown_persisted=False,
),
)
GPT_IMAGE_2_6_SUNBURST_IMAGES_EDIT: Final = CostTrackingTestCase(
name="gpt-image-2.6-sunburst-images-edit",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="gpt-image-2.6-sunburst",
endpoint="/v1/images/edits",
deployment=Deployment(model="openai/gpt-image-2.6-sunburst"),
upload=PngUpload(kind="png"),
request={"prompt": "make the square blue", "size": "1024x1024", "quality": "low", "n": 1},
response=JsonResponse(
content_type="application/json",
body=_gpt_image_2_6_sunburst_body(
{
"input_tokens": 1044,
"input_tokens_details": {"image_tokens": 1024, "text_tokens": 20},
"output_tokens": 824,
"output_tokens_details": {"image_tokens": 824, "text_tokens": 0},
"total_tokens": 1868,
}
),
),
expected=ExactExpected(
spend=0.0330124,
input_cost=0.0082924,
output_cost=0.02472,
prompt_tokens=1044,
completion_tokens=824,
breakdown_persisted=False,
),
)
GPT_4O_TRANSCRIBE_TRANSCRIPTIONS_TOKENS: Final = CostTrackingTestCase(
name="gpt-4o-transcribe-transcriptions-tokens",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="gpt-4o-transcribe",
endpoint="/v1/audio/transcriptions",
upload=WavUpload(kind="wav", seconds=3.0),
request={"response_format": "json"},
response=JsonResponse(
content_type="application/json",
body={
"text": "The pricing harness says hello.",
"usage": {
"type": "tokens",
"total_tokens": 38,
"input_tokens": 30,
"input_token_details": {"text_tokens": 0, "audio_tokens": 30},
"output_tokens": 8,
},
},
),
expected=ExactExpected(spend=0.00026, input_cost=0.00018, output_cost=8e-05, prompt_tokens=30, completion_tokens=8),
)
GPT_4O_MINI_TTS_SPEECH: Final = CostTrackingTestCase(
name="gpt-4o-mini-tts-speech",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="gpt-4o-mini-tts",
endpoint="/v1/audio/speech",
request={"input": "The pricing harness says hello.", "voice": "alloy", "response_format": "mp3"},
response=BinaryResponse(content_type="audio/mpeg", length=2048),
expected=ExactExpected(
spend=3.66e-06, input_cost=3.66e-06, output_cost=0.0, prompt_tokens=0, completion_tokens=0
),
)
CASES: Final[tuple[CostTrackingTestCase, ...]] = (
GPT_6_LUNA_RESPONSES_PROMPT_CACHE_WRITE_30M,
GPT_6_SOL_RESPONSES_PROMPT_CACHE_WRITE_30M,
GPT_6_ASTRA_RESPONSES_PROMPT_CACHE_WRITE_30M,
GPT_6_1_SOL_RESPONSES_PROMPT_CACHE_WRITE_30M,
GPT_6_1_TERRA_RESPONSES_PROMPT_CACHE_WRITE_30M,
GPT_IMAGE_2_6_SUNBURST_IMAGES_LOW,
GPT_IMAGE_2_6_SUNBURST_IMAGES_EDIT,
GPT_4O_TRANSCRIBE_TRANSCRIPTIONS_TOKENS,
GPT_4O_MINI_TTS_SPEECH,
GPT_6_1_TERRA_INPUT_TEXT,
GPT_6_1_TERRA_STREAM,
GPT_6_1_TERRA_REASONING,
GPT_6_1_TERRA_TOOL_CALL,
GPT_6_1_TERRA_SERVICE_TIER_FLEX,
GPT_6_1_TERRA_SERVICE_TIER_PRIORITY,
GPT_6_1_TERRA_RESPONSES_NATIVE_JSON,
GPT_6_ASTRA_TOOL_CALL,
GPT_6_1_SOL_TOOL_CALL,
GPT_6_LUNA_PROMPT_CACHE_WRITE,
GPT_6_LUNA_PROMPT_CACHE_READ,
GPT_6_LUNA_RESPONSES_PROMPT_CACHE_WRITE_12H,
GPT_6_LUNA_RESPONSES_WEB_SEARCH_MEDIUM,
GPT_6_SOL_PROMPT_CACHE_WRITE,
GPT_6_SOL_PROMPT_CACHE_READ,
GPT_6_SOL_RESPONSES_PROMPT_CACHE_WRITE_12H,
GPT_6_SOL_RESPONSES_WEB_SEARCH_MEDIUM,
GPT_6_ASTRA_PROMPT_CACHE_WRITE,
GPT_6_ASTRA_PROMPT_CACHE_READ,
GPT_6_ASTRA_RESPONSES_PROMPT_CACHE_WRITE_12H,
GPT_6_ASTRA_RESPONSES_WEB_SEARCH_MEDIUM,
GPT_6_1_SOL_PROMPT_CACHE_WRITE,
GPT_6_1_SOL_PROMPT_CACHE_READ,
GPT_6_1_SOL_RESPONSES_PROMPT_CACHE_WRITE_12H,
GPT_6_1_SOL_RESPONSES_WEB_SEARCH_MEDIUM,
GPT_6_1_TERRA_PROMPT_CACHE_WRITE,
GPT_6_1_TERRA_PROMPT_CACHE_READ,
GPT_6_1_TERRA_RESPONSES_PROMPT_CACHE_WRITE_12H,
GPT_6_1_TERRA_RESPONSES_WEB_SEARCH_MEDIUM,
TEXT_EMBEDDING_3_LARGE_DIMENSIONS,
GATEWAY_REPORTED_COST,
GATEWAY_REPORTED_COST_STREAM,
GATEWAY_REPORTED_COST_OPERATOR_PRICED,
@ -12343,6 +12766,74 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
GPT_5_6_TERRA_MESSAGES_BRIDGED_STREAM,
)
BILLED_AGAINST_REAL_PROVIDER_BY_COST_TRACKING_RECONCILIATION_AUTOMATION: Final[tuple[CostTrackingTestCase, ...]] = (
GPT_6_LUNA_RESPONSES_PROMPT_CACHE_WRITE_30M,
GPT_6_SOL_RESPONSES_PROMPT_CACHE_WRITE_30M,
GPT_6_ASTRA_RESPONSES_PROMPT_CACHE_WRITE_30M,
GPT_6_1_SOL_RESPONSES_PROMPT_CACHE_WRITE_30M,
GPT_6_1_TERRA_RESPONSES_PROMPT_CACHE_WRITE_30M,
GPT_IMAGE_2_6_SUNBURST_IMAGES_LOW,
GPT_IMAGE_2_6_SUNBURST_IMAGES_EDIT,
GPT_4O_TRANSCRIBE_TRANSCRIPTIONS_TOKENS,
GPT_4O_MINI_TTS_SPEECH,
GPT_6_1_TERRA_INPUT_TEXT,
GPT_6_1_TERRA_STREAM,
GPT_6_1_TERRA_REASONING,
GPT_6_1_TERRA_TOOL_CALL,
GPT_6_1_TERRA_SERVICE_TIER_FLEX,
GPT_6_1_TERRA_SERVICE_TIER_PRIORITY,
GPT_6_1_TERRA_RESPONSES_NATIVE_JSON,
GPT_6_ASTRA_TOOL_CALL,
GPT_6_1_SOL_TOOL_CALL,
GPT_6_LUNA_PROMPT_CACHE_WRITE,
GPT_6_LUNA_PROMPT_CACHE_READ,
GPT_6_LUNA_RESPONSES_PROMPT_CACHE_WRITE_12H,
GPT_6_LUNA_RESPONSES_WEB_SEARCH_MEDIUM,
GPT_6_SOL_PROMPT_CACHE_WRITE,
GPT_6_SOL_PROMPT_CACHE_READ,
GPT_6_SOL_RESPONSES_PROMPT_CACHE_WRITE_12H,
GPT_6_SOL_RESPONSES_WEB_SEARCH_MEDIUM,
GPT_6_ASTRA_PROMPT_CACHE_WRITE,
GPT_6_ASTRA_PROMPT_CACHE_READ,
GPT_6_ASTRA_RESPONSES_PROMPT_CACHE_WRITE_12H,
GPT_6_ASTRA_RESPONSES_WEB_SEARCH_MEDIUM,
GPT_6_1_SOL_PROMPT_CACHE_WRITE,
GPT_6_1_SOL_PROMPT_CACHE_READ,
GPT_6_1_SOL_RESPONSES_PROMPT_CACHE_WRITE_12H,
GPT_6_1_SOL_RESPONSES_WEB_SEARCH_MEDIUM,
GPT_6_1_TERRA_PROMPT_CACHE_WRITE,
GPT_6_1_TERRA_PROMPT_CACHE_READ,
GPT_6_1_TERRA_RESPONSES_PROMPT_CACHE_WRITE_12H,
GPT_6_1_TERRA_RESPONSES_WEB_SEARCH_MEDIUM,
TEXT_EMBEDDING_3_LARGE_DIMENSIONS,
GPT_6_LUNA_INPUT_TEXT,
GPT_6_LUNA_STREAM,
GPT_6_LUNA_REASONING,
GPT_6_LUNA_TOOL_CALL,
GPT_6_LUNA_SERVICE_TIER_FLEX,
GPT_6_LUNA_SERVICE_TIER_PRIORITY,
GPT_6_LUNA_RESPONSES_NATIVE_JSON,
GPT_6_SOL_INPUT_TEXT,
GPT_6_SOL_STREAM,
GPT_6_SOL_REASONING,
GPT_6_SOL_TOOL_CALL,
GPT_6_SOL_SERVICE_TIER_FLEX,
GPT_6_SOL_SERVICE_TIER_PRIORITY,
GPT_6_SOL_RESPONSES_NATIVE_JSON,
GPT_6_ASTRA_INPUT_TEXT,
GPT_6_ASTRA_STREAM,
GPT_6_ASTRA_REASONING,
GPT_6_ASTRA_SERVICE_TIER_FLEX,
GPT_6_ASTRA_SERVICE_TIER_PRIORITY,
GPT_6_ASTRA_RESPONSES_NATIVE_JSON,
GPT_6_1_SOL_INPUT_TEXT,
GPT_6_1_SOL_STREAM,
GPT_6_1_SOL_REASONING,
GPT_6_1_SOL_SERVICE_TIER_FLEX,
GPT_6_1_SOL_SERVICE_TIER_PRIORITY,
GPT_6_1_SOL_RESPONSES_NATIVE_JSON,
)
PARITY: Final[tuple[StreamParityTestCase, ...]] = (
StreamParityTestCase(plain=GATEWAY_REPORTED_COST, streamed=GATEWAY_REPORTED_COST_STREAM),
StreamParityTestCase(
@ -12386,6 +12877,7 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = (
StreamParityTestCase(plain=GPT_6_ASTRA_INPUT_TEXT, streamed=GPT_6_ASTRA_STREAM),
StreamParityTestCase(plain=GPT_6_1_SOL_INPUT_TEXT, streamed=GPT_6_1_SOL_STREAM),
StreamParityTestCase(plain=GPT_5_6_TERRA_INPUT_TEXT, streamed=GPT_5_6_TERRA_STREAM),
StreamParityTestCase(plain=GPT_6_1_TERRA_INPUT_TEXT, streamed=GPT_6_1_TERRA_STREAM),
StreamParityTestCase(plain=GPT_6_1_SOL_RESPONSES_NATIVE_JSON, streamed=GPT_6_1_SOL_RESPONSES_NATIVE_STREAM),
StreamParityTestCase(plain=GPT_6_LUNA_RESPONSES_NATIVE_JSON, streamed=GPT_6_LUNA_RESPONSES_NATIVE_STREAM),
StreamParityTestCase(plain=GPT_6_ASTRA_RESPONSES_NATIVE_JSON, streamed=GPT_6_ASTRA_RESPONSES_NATIVE_STREAM),

View file

@ -52,6 +52,13 @@ _PROVIDER_MODULES: Final = (
xai,
)
CASES: Final[tuple[CostTrackingTestCase, ...]] = tuple(case for module in _PROVIDER_MODULES for case in module.CASES)
BILLED_AGAINST_REAL_PROVIDER_BY_COST_TRACKING_RECONCILIATION_AUTOMATION: Final[frozenset[str]] = frozenset(
case.name
for case in (
*openai.BILLED_AGAINST_REAL_PROVIDER_BY_COST_TRACKING_RECONCILIATION_AUTOMATION,
*batch_realtime.BILLED_AGAINST_REAL_PROVIDER_BY_COST_TRACKING_RECONCILIATION_AUTOMATION,
)
)
PARITY_CASES: Final[tuple[StreamParityTestCase, ...]] = tuple(
pair for module in _PROVIDER_MODULES for pair in module.PARITY
)

View file

@ -73,7 +73,30 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = {
"supports_prompt_caching": True,
"supports_reasoning": True,
},
"gpt-6.1-terra": {
"litellm_provider": "openai",
"mode": "chat",
"input_cost_per_token": 1.03e-06,
"output_cost_per_token": 5.15e-06,
"input_cost_per_token_flex": 5.15e-07,
"output_cost_per_token_flex": 2.575e-06,
"input_cost_per_token_priority": 2.06e-06,
"output_cost_per_token_priority": 1.03e-05,
"cache_read_input_token_cost": 5.15e-08,
"cache_creation_input_token_cost": 1.2875e-06,
"cache_creation_input_token_cost_above_12hr": 2.575e-06,
"search_context_cost_per_query": {
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01,
"search_context_size_high": 0.01,
},
},
"gpt-6-luna": {
"search_context_cost_per_query": {
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01,
"search_context_size_high": 0.01,
},
"output_cost_per_token_priority": 1e-06,
"input_cost_per_token_priority": 2e-07,
"output_cost_per_token_flex": 2.5e-07,
@ -966,6 +989,27 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = {
"input_cost_per_token": 1.51e-06,
"output_cost_per_token": 7.51e-06,
},
"gpt-image-2.6-sunburst": {
"litellm_provider": "openai",
"mode": "image_generation",
"input_cost_per_token": 5.02e-06,
"input_cost_per_image_token": 8e-06,
"output_cost_per_image_token": 3e-05,
},
"gpt-4o-mini-tts": {
"litellm_provider": "openai",
"mode": "audio_speech",
"input_cost_per_token": 6.1e-07,
"output_cost_per_audio_token": 1.2e-05,
"output_cost_per_second": 0.00025,
},
"gpt-4o-transcribe": {
"litellm_provider": "openai",
"mode": "audio_transcription",
"input_cost_per_token": 2.52e-06,
"input_cost_per_audio_token": 6e-06,
"output_cost_per_token": 1e-05,
},
"text-embedding-3-large": {"litellm_provider": "openai", "mode": "embedding", "input_cost_per_token": 1.3e-07},
"gpt-5.4": {
"litellm_provider": "openai",
@ -1324,6 +1368,12 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = {
"supports_function_calling": True,
},
"gpt-6.1-sol": {
"cache_creation_input_token_cost_above_12hr": 5e-06,
"search_context_cost_per_query": {
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01,
"search_context_size_high": 0.01,
},
"output_cost_per_token_priority": 2e-05,
"input_cost_per_token_priority": 4e-06,
"output_cost_per_token_flex": 5e-06,
@ -1530,6 +1580,13 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = {
"supports_function_calling": True,
},
"gpt-6-sol": {
"cache_creation_input_token_cost": 2.75e-06,
"cache_creation_input_token_cost_above_12hr": 5.5e-06,
"search_context_cost_per_query": {
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01,
"search_context_size_high": 0.01,
},
"cache_read_input_token_cost": 2.2e-07,
"input_cost_per_token": 2.2e-06,
"input_cost_per_token_flex": 1.1e-06,
@ -1552,6 +1609,12 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = {
"cache_read_input_token_cost_above_272k_tokens": 4.4e-07,
},
"gpt-6-astra": {
"cache_creation_input_token_cost_above_12hr": 3e-05,
"search_context_cost_per_query": {
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01,
"search_context_size_high": 0.01,
},
"cache_read_input_token_cost": 1.2e-06,
"cache_creation_input_token_cost": 1.5e-05,
"cache_creation_input_token_cost_above_272k_tokens": 3e-05,

View file

@ -14,7 +14,10 @@ from typing import Final
import pytest
from integration._support.client import Gateway
from integration.cost_calculation.catalog import PARITY_CASES
from integration.cost_calculation.catalog import (
BILLED_AGAINST_REAL_PROVIDER_BY_COST_TRACKING_RECONCILIATION_AUTOMATION,
PARITY_CASES,
)
from integration.cost_calculation.stream_parity.bases import anthropic, bedrock, openai, openai_responses, xai
from integration.cost_calculation.stream_parity.case import StreamParityTestCase
from integration.cost_calculation.stream_parity.runner import assert_stream_parity
@ -34,7 +37,19 @@ BRIDGES: Final = (
CASES: Final = (*NATIVE, *BRIDGES, *PARITY_CASES)
@pytest.mark.parametrize("case", CASES, ids=[case.id for case in CASES])
def _marks(case: StreamParityTestCase) -> tuple[pytest.MarkDecorator, ...]:
legs: Final = frozenset((case.plain.name, case.streamed.name))
return (
(pytest.mark.billed_against_real_provider_by_cost_tracking_reconciliation_automation,)
if legs & BILLED_AGAINST_REAL_PROVIDER_BY_COST_TRACKING_RECONCILIATION_AUTOMATION
else ()
)
_PARAMS: Final = tuple(pytest.param(case, marks=_marks(case), id=case.id) for case in CASES)
@pytest.mark.parametrize("case", _PARAMS)
@pytest.mark.timeout(180)
def test_streamed_request_bills_the_same_row_as_the_non_streamed_request(
case: StreamParityTestCase, gateway: Gateway

View file

@ -15,7 +15,11 @@ from integration._support.client import JSON_OBJECT, Gateway, Scenario, object_v
from integration._support.upstream import delete_scenario, register_scenario
from integration.cost_calculation.assertions import assert_exact
from integration.cost_calculation.conftest import poll_rows, poll_rows_where, read_rows_now
from integration.cost_calculation.catalog import BATCH_CASES, REALTIME_CASES
from integration.cost_calculation.catalog import (
BATCH_CASES,
BILLED_AGAINST_REAL_PROVIDER_BY_COST_TRACKING_RECONCILIATION_AUTOMATION,
REALTIME_CASES,
)
from integration.cost_calculation.cost_tracking_case import (
BatchCostCase,
JsonResponse,
@ -252,9 +256,18 @@ async def _run_realtime(url: str, key: str, model_name: str, turn_count: int) ->
return session
def _realtime_marks(case: RealtimeCostCase) -> tuple[pytest.MarkDecorator, ...]:
billed: Final = (
(pytest.mark.billed_against_real_provider_by_cost_tracking_reconciliation_automation,)
if case.name in BILLED_AGAINST_REAL_PROVIDER_BY_COST_TRACKING_RECONCILIATION_AUTOMATION
else ()
)
return (pytest.mark.covers(case.covers), *billed)
@pytest.mark.parametrize(
"case",
tuple(pytest.param(case, marks=pytest.mark.covers(case.covers), id=case.name) for case in REALTIME_CASES),
tuple(pytest.param(case, marks=_realtime_marks(case), id=case.name) for case in REALTIME_CASES),
)
def test_realtime_costs(gateway: Gateway, case: RealtimeCostCase) -> None:
with gateway.scenario() as scenario:

View file

@ -28,7 +28,11 @@ from integration.cost_calculation.conftest import (
read_rows_now,
register_scenario_deployment,
)
from integration.cost_calculation.catalog import STANDALONE_CASES, data_errors
from integration.cost_calculation.catalog import (
BILLED_AGAINST_REAL_PROVIDER_BY_COST_TRACKING_RECONCILIATION_AUTOMATION,
STANDALONE_CASES,
data_errors,
)
from integration.cost_calculation.stream_parity.runner import flush_spend_logs
from integration.cost_calculation.cost_tracking_case import (
BinaryResponse,
@ -55,7 +59,12 @@ def _marks(case: CostTrackingTestCase) -> tuple[pytest.MarkDecorator, ...]:
bug: Final = (
(pytest.mark.xfail(strict=True, reason=f"BUG: {case.known_bug}"),) if case.known_bug is not None else ()
)
return (pytest.mark.covers(case.covers), *bug)
billed: Final = (
(pytest.mark.billed_against_real_provider_by_cost_tracking_reconciliation_automation,)
if case.name in BILLED_AGAINST_REAL_PROVIDER_BY_COST_TRACKING_RECONCILIATION_AUTOMATION
else ()
)
return (pytest.mark.covers(case.covers), *bug, *billed)
_CASES: Final = tuple(pytest.param(case, marks=_marks(case), id=case.name) for case in STANDALONE_CASES)