diff --git a/tests/integration/_support/upstream.py b/tests/integration/_support/upstream.py index 1ad02b6a3f2..a289589b2dc 100644 --- a/tests/integration/_support/upstream.py +++ b/tests/integration/_support/upstream.py @@ -210,6 +210,7 @@ class Provider: "$REQUEST_ID", scenario_id ).encode(), media_type=response.content_type, + status_code=response.status, ) case SseResponse(): stream_body: Final = ("\n\n".join(response.frames) + "\n\n").replace( diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index 365456c0cec..5a3a1135144 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -397,6 +397,15 @@ "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_full_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_native_json]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-upstream_500_zero_spend]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-upstream_429_zero_spend]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-input_text]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], @@ -1327,6 +1336,69 @@ "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_full_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_reasoning]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_stream_cache_read]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_incomplete]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_previous_response_id]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_web_search_medium]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-responses_file_search]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_service_tier_flex]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-responses_service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_cache_write_5m]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_cache_write_1h]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_web_search]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_stream_cache_read]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-messages_tiered_input_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-messages_input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-messages_input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-messages_cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], "tests/integration/mcp/test_mcp_lifecycle.py::test_health_intersects_route_restricted_key_grants_in_both_management_modes": [ "other.mcp.health.restricted_keys_intersect_grants_in_both_modes" ], diff --git a/tests/integration/cost_calculation/conftest.py b/tests/integration/cost_calculation/conftest.py index f1b8901d626..166488e36a5 100644 --- a/tests/integration/cost_calculation/conftest.py +++ b/tests/integration/cost_calculation/conftest.py @@ -50,6 +50,15 @@ class CostRow(BaseModel): return self.metadata.cost_breakdown +class FailureRow(BaseModel): + model_config = ConfigDict(extra="ignore") + + spend: float + status: str + prompt_tokens: int | None = None + completion_tokens: int | None = None + + def approx_equal(actual: float, expected: float) -> bool: return abs(actual - expected) <= max(1e-9, abs(expected) * 1e-2) @@ -92,6 +101,28 @@ def poll_cost_row(key: str) -> CostRow: return result +def poll_failure_row(key: str) -> FailureRow: + digest: Final = sha256(key.encode()).hexdigest() + + def read() -> FailureRow | None: + rows: Final = read_rows( + 'SELECT spend, status, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" WHERE api_key=%s', + (digest,), + ) + return next( + ( + parsed + for row in rows + if (parsed := FailureRow.model_validate(row)).status == "failure" + ), + None, + ) + + result: Final = eventually(read, lambda row: row is not None, seconds=60) + assert result is not None + return result + + @functools.cache def _vertex_private_key_pem() -> str: return rsa.generate_private_key(public_exponent=65537, key_size=2048).private_bytes( diff --git a/tests/integration/cost_calculation/cost_tracking_case.py b/tests/integration/cost_calculation/cost_tracking_case.py index 6af95f995ff..baee004c062 100644 --- a/tests/integration/cost_calculation/cost_tracking_case.py +++ b/tests/integration/cost_calculation/cost_tracking_case.py @@ -71,6 +71,7 @@ class JsonResponse(BaseModel): content_type: Literal["application/json"] body: dict[str, JsonValue] + status: int = 200 class SseResponse(BaseModel): @@ -108,6 +109,10 @@ class ExactExpected(BaseModel): output_cost: float prompt_tokens: int completion_tokens: int + cache_read_cost: float | None = None + cache_creation_cost: float | None = None + reasoning_cost: float | None = None + tool_usage_cost: float | None = None class RecountRates(BaseModel): @@ -123,7 +128,19 @@ class RecountExpected(BaseModel): recount: RecountRates -Expected: TypeAlias = ExactExpected | RecountExpected +class FailureDetails(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + status: int + + +class FailureExpected(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + failure: FailureDetails + + +Expected: TypeAlias = ExactExpected | RecountExpected | FailureExpected class CostTrackingTestCase(BaseModel): @@ -132,6 +149,15 @@ class CostTrackingTestCase(BaseModel): name: str covers: str model: str + endpoint: Literal[ + "/v1/chat/completions", + "/v1/responses", + "/v1/messages", + "/v1/embeddings", + "/v1/rerank", + "/v1/completions", + "/v1/moderations", + ] = "/v1/chat/completions" deployment: Deployment | None = None request: dict[str, JsonValue] response: StoredResponse @@ -146,7 +172,7 @@ class CostTrackingTestCase(BaseModel): provider: Final = self.rates.litellm_provider prefix: Final = ( "openai" - if provider == "openai" and self.rates.mode == "chat" + if provider == "openai" and (self.rates.mode == "chat" or self.endpoint == "/v1/responses") else "openai/responses" if provider == "openai" else _PROVIDER_PREFIXES.get(provider) @@ -240,6 +266,48 @@ def data_errors() -> tuple[str, ...]: or case.expected.recount.output_cost_per_token != (COST_MAP[case.model].output_cost_per_token or 0.0) ) ) + component_mismatches: Final = sorted( + case.name + for case in CASES + if isinstance(case.expected, ExactExpected) + and any( + component is not None + for component in ( + case.expected.cache_read_cost, + case.expected.cache_creation_cost, + case.expected.reasoning_cost, + case.expected.tool_usage_cost, + ) + ) + and ( + (case.expected.cache_read_cost or 0.0) + (case.expected.cache_creation_cost or 0.0) + > case.expected.input_cost + or (case.expected.reasoning_cost or 0.0) > case.expected.output_cost + or not _approx_equal( + case.expected.input_cost + + case.expected.output_cost + + (case.expected.tool_usage_cost or 0.0), + case.expected.spend, + ) + ) + ) + failure_response_mismatches: Final = sorted( + case.name + for case in CASES + if ( + isinstance(case.expected, FailureExpected) + and ( + not isinstance(case.response, JsonResponse) + or not 400 <= case.response.status <= 599 + or not 400 <= case.expected.failure.status <= 599 + ) + ) + or ( + not isinstance(case.expected, FailureExpected) + and isinstance(case.response, JsonResponse) + and case.response.status != 200 + ) + ) return tuple( message for message in ( @@ -248,6 +316,14 @@ def data_errors() -> tuple[str, ...]: f"duplicate case names: {duplicate_names}" if duplicate_names else None, f"cost-map entries share input_cost_per_token: {shared_input_rates}" if shared_input_rates else None, f"recount rates differ from cost-map rates: {recount_mismatches}" if recount_mismatches else None, + f"breakdown components are inconsistent: {component_mismatches}" if component_mismatches else None, + f"failure response statuses are inconsistent: {failure_response_mismatches}" + if failure_response_mismatches + else None, ) if message is not None ) + + +def _approx_equal(actual: float, expected: float) -> bool: + return abs(actual - expected) <= max(1e-9, abs(expected) * 1e-2) diff --git a/tests/integration/cost_calculation/cost_tracking_cases.json b/tests/integration/cost_calculation/cost_tracking_cases.json index d8b9be3a558..ac4c6c125fb 100644 --- a/tests/integration/cost_calculation/cost_tracking_cases.json +++ b/tests/integration/cost_calculation/cost_tracking_cases.json @@ -534,7 +534,8 @@ "input_cost": 0.00616704, "output_cost": 0.00627, "prompt_tokens": 12928, - "completion_tokens": 380 + "completion_tokens": 380, + "cache_read_cost": 0.00405504 } }, { @@ -605,7 +606,8 @@ "input_cost": 0.0397056, "output_cost": 0.005775, "prompt_tokens": 9728, - "completion_tokens": 350 + "completion_tokens": 350, + "cache_creation_cost": 0.038016 } }, { @@ -681,7 +683,8 @@ "input_cost": 0.0574464, "output_cost": 0.005775, "prompt_tokens": 9728, - "completion_tokens": 350 + "completion_tokens": 350, + "cache_creation_cost": 0.0557568 } }, { @@ -3417,7 +3420,8 @@ "input_cost": 0.002232, "output_cost": 0.065484, "prompt_tokens": 1240, - "completion_tokens": 4040 + "completion_tokens": 4040, + "reasoning_cost": 0.05742 } }, { @@ -3638,7 +3642,8 @@ "input_cost": 0.003312, "output_cost": 0.0059328, "prompt_tokens": 1840, - "completion_tokens": 412 + "completion_tokens": 412, + "tool_usage_cost": 0.0125 } }, { @@ -4494,7 +4499,8 @@ "input_cost": 0.0018688, "output_cost": 0.0019, "prompt_tokens": 12928, - "completion_tokens": 380 + "completion_tokens": 380, + "cache_read_cost": 0.0012288 } }, { @@ -8011,6 +8017,7 @@ "spend": 0.0012456, "input_cost": 0.0010176, "output_cost": 0.000228, + "cache_read_cost": 0.0009216, "prompt_tokens": 12928, "completion_tokens": 380 } @@ -16955,7 +16962,8 @@ "input_cost": 0.00276, "output_cost": 0.004944, "prompt_tokens": 1840, - "completion_tokens": 412 + "completion_tokens": 412, + "tool_usage_cost": 0.0025 } }, { @@ -21797,6 +21805,122 @@ "completion_tokens": 1592 } }, + { + "name": "gpt-5.6-responses_native_json", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.6", + "endpoint": "/v1/responses", + "request": { + "model": "$MODEL", + "input": "responses native fixture", + "stream": false + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "status": "completed", + "created_at": 1700000000, + "model": "gpt-5.6", + "output": [ + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted response", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 11, + "output_tokens": 7, + "total_tokens": 18, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + } + }, + "expected": { + "spend": 0.00011725, + "input_cost": 1.925e-05, + "output_cost": 9.8e-05, + "prompt_tokens": 11, + "completion_tokens": 7 + } + }, + { + "name": "gpt-5.6-upstream_500_zero_spend", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "user", + "content": "scripted upstream failure 500" + } + ], + "stream": false + }, + "response": { + "content_type": "application/json", + "status": 500, + "body": { + "error": { + "message": "scripted upstream failure", + "type": "server_error", + "code": "500" + } + } + }, + "expected": { + "failure": { + "status": 500 + } + } + }, + { + "name": "gpt-5.6-upstream_429_zero_spend", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "user", + "content": "scripted upstream failure 429" + } + ], + "stream": false + }, + "response": { + "content_type": "application/json", + "status": 429, + "body": { + "error": { + "message": "scripted upstream failure", + "type": "rate_limit_error", + "code": "429" + } + } + }, + "expected": { + "failure": { + "status": 429 + } + } + }, { "name": "meta.llama4-maverick-17b-instruct-v1:0-input_text", "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", @@ -25653,6 +25777,1092 @@ "prompt_tokens": 11056, "completion_tokens": 412 } + }, + { + "name": "gpt-5.6-responses_cache_read", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.6", + "endpoint": "/v1/responses", + "request": { + "model": "$MODEL", + "input": "summarize this text" + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "gpt-5.6", + "output": [ + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted response", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 12928, + "output_tokens": 380, + "total_tokens": 13308, + "input_tokens_details": { + "cached_tokens": 12288 + }, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + } + }, + "expected": { + "spend": 0.0085904, + "input_cost": 0.0032704, + "output_cost": 0.00532, + "prompt_tokens": 12928, + "completion_tokens": 380, + "cache_read_cost": 0.0021504 + } + }, + { + "name": "gpt-5.6-responses_reasoning", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.6", + "endpoint": "/v1/responses", + "request": { + "model": "$MODEL", + "input": "reason about this text" + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "gpt-5.6", + "output": [ + { + "type": "reasoning", + "id": "rs_$REQUEST_ID", + "status": "completed", + "summary": [] + }, + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted response", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 1240, + "output_tokens": 4040, + "total_tokens": 5280, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens_details": { + "reasoning_tokens": 3480 + } + } + } + }, + "expected": { + "spend": 0.06569, + "input_cost": 0.00217, + "output_cost": 0.06352, + "prompt_tokens": 1240, + "completion_tokens": 4040, + "reasoning_cost": 0.05568 + } + }, + { + "name": "gpt-5.6-responses_stream", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.6", + "endpoint": "/v1/responses", + "request": { + "model": "$MODEL", + "input": "stream this text", + "stream": true + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_$REQUEST_ID\",\"object\":\"response\",\"created_at\":1700000000,\"status\":\"in_progress\",\"model\":\"gpt-5.6\",\"output\":[{\"type\":\"message\",\"id\":\"msg_$REQUEST_ID\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"scripted response\",\"annotations\":[]}]}],\"usage\":null}}", + "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"type\":\"message\",\"id\":\"msg_$REQUEST_ID\",\"status\":\"in_progress\",\"role\":\"assistant\",\"content\":[]}}", + "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"item_id\":\"msg_$REQUEST_ID\",\"output_index\":0,\"content_index\":0,\"part\":{\"type\":\"content_part\",\"text\":\"\"}}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"item_id\":\"msg_$REQUEST_ID\",\"output_index\":0,\"content_index\":0,\"delta\":\"scripted \"}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"item_id\":\"msg_$REQUEST_ID\",\"output_index\":0,\"content_index\":0,\"delta\":\"response\"}", + "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"item_id\":\"msg_$REQUEST_ID\",\"output_index\":0,\"content_index\":0,\"text\":\"scripted response\"}", + "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"item_id\":\"msg_$REQUEST_ID\",\"output_index\":0,\"content_index\":0,\"part\":{\"type\":\"output_text\",\"text\":\"scripted response\",\"annotations\":[]}}", + "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"type\":\"message\",\"id\":\"msg_$REQUEST_ID\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"scripted response\",\"annotations\":[]}]}}", + "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_$REQUEST_ID\",\"object\":\"response\",\"created_at\":1700000000,\"status\":\"completed\",\"model\":\"gpt-5.6\",\"output\":[{\"type\":\"message\",\"id\":\"msg_$REQUEST_ID\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"scripted response\",\"annotations\":[]}]}],\"usage\":{\"input_tokens\":1840,\"output_tokens\":412,\"total_tokens\":2252,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens_details\":{\"reasoning_tokens\":0}}}}" + ] + }, + "expected": { + "spend": 0.008988, + "input_cost": 0.00322, + "output_cost": 0.005768, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.6-responses_stream_cache_read", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.6", + "endpoint": "/v1/responses", + "request": { + "model": "$MODEL", + "input": "stream cached text", + "stream": true + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_$REQUEST_ID\",\"object\":\"response\",\"created_at\":1700000000,\"status\":\"in_progress\",\"model\":\"gpt-5.6\",\"output\":[{\"type\":\"message\",\"id\":\"msg_$REQUEST_ID\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"scripted response\",\"annotations\":[]}]}],\"usage\":null}}", + "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"type\":\"message\",\"id\":\"msg_$REQUEST_ID\",\"status\":\"in_progress\",\"role\":\"assistant\",\"content\":[]}}", + "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"item_id\":\"msg_$REQUEST_ID\",\"output_index\":0,\"content_index\":0,\"part\":{\"type\":\"content_part\",\"text\":\"\"}}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"item_id\":\"msg_$REQUEST_ID\",\"output_index\":0,\"content_index\":0,\"delta\":\"scripted \"}", + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"item_id\":\"msg_$REQUEST_ID\",\"output_index\":0,\"content_index\":0,\"delta\":\"response\"}", + "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"item_id\":\"msg_$REQUEST_ID\",\"output_index\":0,\"content_index\":0,\"text\":\"scripted response\"}", + "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"item_id\":\"msg_$REQUEST_ID\",\"output_index\":0,\"content_index\":0,\"part\":{\"type\":\"output_text\",\"text\":\"scripted response\",\"annotations\":[]}}", + "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"type\":\"message\",\"id\":\"msg_$REQUEST_ID\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"scripted response\",\"annotations\":[]}]}}", + "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_$REQUEST_ID\",\"object\":\"response\",\"created_at\":1700000000,\"status\":\"completed\",\"model\":\"gpt-5.6\",\"output\":[{\"type\":\"message\",\"id\":\"msg_$REQUEST_ID\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[{\"type\":\"output_text\",\"text\":\"scripted response\",\"annotations\":[]}]}],\"usage\":{\"input_tokens\":12928,\"output_tokens\":380,\"total_tokens\":13308,\"input_tokens_details\":{\"cached_tokens\":12288},\"output_tokens_details\":{\"reasoning_tokens\":0}}}}" + ] + }, + "expected": { + "spend": 0.0085904, + "input_cost": 0.0032704, + "output_cost": 0.00532, + "prompt_tokens": 12928, + "completion_tokens": 380, + "cache_read_cost": 0.0021504 + } + }, + { + "name": "gpt-5.6-responses_incomplete", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.6", + "endpoint": "/v1/responses", + "request": { + "model": "$MODEL", + "input": "truncate this text", + "max_output_tokens": 100 + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1700000000, + "status": "incomplete", + "model": "gpt-5.6", + "output": [ + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted response", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 1840, + "output_tokens": 100, + "total_tokens": 1940, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens_details": { + "reasoning_tokens": 0 + } + }, + "incomplete_details": { + "reason": "max_output_tokens" + } + } + }, + "expected": { + "spend": 0.00462, + "input_cost": 0.00322, + "output_cost": 0.0014, + "prompt_tokens": 1840, + "completion_tokens": 100 + } + }, + { + "name": "gpt-5.6-responses_previous_response_id", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.6", + "endpoint": "/v1/responses", + "request": { + "model": "$MODEL", + "input": "continue this text", + "previous_response_id": "resp_scripted_prior" + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "gpt-5.6", + "output": [ + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted response", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "total_tokens": 2252, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + } + }, + "expected": { + "spend": 0.008988, + "input_cost": 0.00322, + "output_cost": 0.005768, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.6-responses_web_search_medium", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.6", + "endpoint": "/v1/responses", + "request": { + "model": "$MODEL", + "input": "search this text", + "tools": [ + { + "type": "web_search_preview", + "search_context_size": "medium" + } + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "gpt-5.6", + "output": [ + { + "type": "web_search_call", + "id": "ws_$REQUEST_ID", + "status": "completed", + "action": { + "type": "search", + "query": "scripted query" + } + }, + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted response", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "total_tokens": 2252, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + } + }, + "expected": { + "spend": 0.021488, + "input_cost": 0.00322, + "output_cost": 0.005768, + "prompt_tokens": 1840, + "completion_tokens": 412, + "tool_usage_cost": 0.0125 + } + }, + { + "name": "gpt-5.3-codex-responses_file_search", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.3-codex", + "endpoint": "/v1/responses", + "request": { + "model": "$MODEL", + "input": "search files", + "tools": [ + { + "type": "file_search", + "vector_store_ids": [ + "vs_scripted" + ] + } + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "gpt-5.3-codex", + "output": [ + { + "type": "file_search_call", + "id": "fs_$REQUEST_ID", + "status": "completed", + "queries": [ + "scripted query" + ], + "results": [] + }, + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted response", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "total_tokens": 2252, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens_details": { + "reasoning_tokens": 0 + } + } + } + }, + "expected": { + "spend": 0.010204, + "input_cost": 0.00276, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412, + "tool_usage_cost": 0.0025 + } + }, + { + "name": "gpt-5.6-responses_service_tier_flex", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.6", + "endpoint": "/v1/responses", + "request": { + "model": "$MODEL", + "input": "flex text", + "service_tier": "flex" + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "gpt-5.6", + "output": [ + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted response", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "total_tokens": 2252, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "service_tier": "flex" + }, + "service_tier": "flex" + } + }, + "expected": { + "spend": 0.004494, + "input_cost": 0.00161, + "output_cost": 0.002884, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.6-responses_service_tier_priority", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.6", + "endpoint": "/v1/responses", + "request": { + "model": "$MODEL", + "input": "priority text", + "service_tier": "priority" + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "gpt-5.6", + "output": [ + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted response", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "total_tokens": 2252, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "service_tier": "priority" + }, + "service_tier": "priority" + } + }, + "expected": { + "spend": 0.017976, + "input_cost": 0.00644, + "output_cost": 0.011536, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-sonnet-5-messages_input_text", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-sonnet-5", + "endpoint": "/v1/messages", + "request": { + "model": "$MODEL", + "max_tokens": 412, + "messages": [ + { + "role": "user", + "content": "summarize the attached material in one line" + } + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [ + { + "type": "text", + "text": "scripted response" + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 1840, + "output_tokens": 412 + } + } + }, + "expected": { + "spend": 0.0117, + "input_cost": 0.00552, + "output_cost": 0.00618, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-sonnet-5-messages_cache_read", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-sonnet-5", + "endpoint": "/v1/messages", + "request": { + "model": "$MODEL", + "max_tokens": 412, + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "cached text", + "cache_control": { + "type": "ephemeral" + } + } + ] + } + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [ + { + "type": "text", + "text": "scripted response" + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 640, + "output_tokens": 380, + "cache_read_input_tokens": 12288 + } + } + }, + "expected": { + "spend": 0.0113064, + "input_cost": 0.0056064, + "output_cost": 0.0057, + "prompt_tokens": 12928, + "completion_tokens": 380, + "cache_read_cost": 0.0036864 + } + }, + { + "name": "claude-sonnet-5-messages_cache_write_5m", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-sonnet-5", + "endpoint": "/v1/messages", + "request": { + "model": "$MODEL", + "max_tokens": 350, + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "cache this text", + "cache_control": { + "type": "ephemeral" + } + } + ] + } + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [ + { + "type": "text", + "text": "scripted response" + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 512, + "output_tokens": 350, + "cache_creation_input_tokens": 9216, + "cache_creation": { + "ephemeral_5m_input_tokens": 9216, + "ephemeral_1h_input_tokens": 0 + } + } + } + }, + "expected": { + "spend": 0.041346, + "input_cost": 0.036096, + "output_cost": 0.00525, + "prompt_tokens": 9728, + "completion_tokens": 350, + "cache_creation_cost": 0.03456 + } + }, + { + "name": "claude-sonnet-5-messages_cache_write_1h", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-sonnet-5", + "endpoint": "/v1/messages", + "request": { + "model": "$MODEL", + "max_tokens": 350, + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "cache this text for an hour", + "cache_control": { + "type": "ephemeral", + "ttl": "1h" + } + } + ] + } + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [ + { + "type": "text", + "text": "scripted response" + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 512, + "output_tokens": 350, + "cache_creation_input_tokens": 9216, + "cache_creation": { + "ephemeral_5m_input_tokens": 2048, + "ephemeral_1h_input_tokens": 7168 + } + } + } + }, + "expected": { + "spend": 0.057474, + "input_cost": 0.052224, + "output_cost": 0.00525, + "prompt_tokens": 9728, + "completion_tokens": 350, + "cache_creation_cost": 0.050688 + } + }, + { + "name": "claude-sonnet-5-messages_web_search", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-sonnet-5", + "endpoint": "/v1/messages", + "request": { + "model": "$MODEL", + "max_tokens": 412, + "messages": [ + { + "role": "user", + "content": "summarize the attached material in one line" + } + ], + "tools": [ + { + "type": "web_search_20250305", + "name": "web_search" + } + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [ + { + "type": "server_tool_use", + "id": "srv_$REQUEST_ID", + "name": "web_search", + "input": { + "query": "scripted query" + } + }, + { + "type": "web_search_tool_result", + "tool_use_id": "srv_$REQUEST_ID", + "content": [ + { + "type": "web_search_result", + "title": "scripted result", + "url": "https://scripted.example" + } + ] + }, + { + "type": "text", + "text": "scripted response" + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "server_tool_use": { + "web_search_requests": 2 + } + } + } + }, + "expected": { + "spend": 0.0317, + "input_cost": 0.00552, + "output_cost": 0.00618, + "prompt_tokens": 1840, + "completion_tokens": 412, + "tool_usage_cost": 0.02 + } + }, + { + "name": "claude-sonnet-5-messages_stream", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-sonnet-5", + "endpoint": "/v1/messages", + "request": { + "model": "$MODEL", + "max_tokens": 412, + "messages": [ + { + "role": "user", + "content": "summarize the attached material in one line" + } + ], + "stream": true + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_$REQUEST_ID\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"claude-sonnet-5\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"usage\":{\"input_tokens\":1840}}}", + "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"scripted \"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"response\"}}", + "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", + "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":412}}", + "event: message_stop\ndata: {\"type\":\"message_stop\"}" + ] + }, + "expected": { + "spend": 0.0117, + "input_cost": 0.00552, + "output_cost": 0.00618, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-sonnet-5-messages_stream_cache_read", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-sonnet-5", + "endpoint": "/v1/messages", + "request": { + "model": "$MODEL", + "max_tokens": 380, + "messages": [ + { + "role": "user", + "content": "summarize the attached material in one line" + } + ], + "stream": true + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_$REQUEST_ID\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"claude-sonnet-5\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"usage\":{\"input_tokens\":640,\"cache_read_input_tokens\":12288}}}", + "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"scripted \"}}", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"response\"}}", + "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", + "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":380}}", + "event: message_stop\ndata: {\"type\":\"message_stop\"}" + ] + }, + "expected": { + "spend": 0.0113064, + "input_cost": 0.0056064, + "output_cost": 0.0057, + "prompt_tokens": 12928, + "completion_tokens": 380, + "cache_read_cost": 0.0036864 + } + }, + { + "name": "claude-sonnet-5-messages_tiered_input_above_200k", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-sonnet-5", + "endpoint": "/v1/messages", + "request": { + "model": "$MODEL", + "max_tokens": 620, + "messages": [ + { + "role": "user", + "content": "summarize the attached material in one line" + } + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [ + { + "type": "text", + "text": "scripted response" + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 210000, + "output_tokens": 620 + } + } + }, + "expected": { + "spend": 1.27395, + "input_cost": 1.26, + "output_cost": 0.01395, + "prompt_tokens": 210000, + "completion_tokens": 620 + } + }, + { + "name": "claude-haiku-4-5-messages_input_text", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-haiku-4-5", + "endpoint": "/v1/messages", + "request": { + "model": "$MODEL", + "max_tokens": 412, + "messages": [ + { + "role": "user", + "content": "summarize the attached material in one line" + } + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [ + { + "type": "text", + "text": "scripted response" + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 1840, + "output_tokens": 412 + } + } + }, + "expected": { + "spend": 0.0039, + "input_cost": 0.00184, + "output_cost": 0.00206, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "us.anthropic.claude-opus-5-v1:0-messages_input_text", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "us.anthropic.claude-opus-5-v1:0", + "endpoint": "/v1/messages", + "request": { + "model": "$MODEL", + "max_tokens": 412, + "messages": [ + { + "role": "user", + "content": "summarize the attached material in one line" + } + ] + }, + "response": { + "content_type": "application/json", + "body": { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "text": "scripted response" + } + ] + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 1840, + "outputTokens": 412 + }, + "metrics": { + "latencyMs": 42 + } + } + }, + "expected": { + "spend": 0.02145, + "input_cost": 0.01012, + "output_cost": 0.01133, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "anthropic.claude-sonnet-5-v1:0-messages_cache_read", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "anthropic.claude-sonnet-5-v1:0", + "endpoint": "/v1/messages", + "request": { + "model": "$MODEL", + "max_tokens": 412, + "messages": [ + { + "role": "user", + "content": "summarize the attached material in one line" + } + ] + }, + "response": { + "content_type": "application/json", + "body": { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "text": "scripted response" + } + ] + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 640, + "outputTokens": 380, + "totalTokens": 13308, + "cacheReadInputTokens": 12288 + }, + "metrics": { + "latencyMs": 42 + } + } + }, + "expected": { + "spend": 0.01243704, + "input_cost": 0.00616704, + "output_cost": 0.00627, + "prompt_tokens": 12928, + "completion_tokens": 380, + "cache_read_cost": 0.00405504 + } } ] } diff --git a/tests/integration/cost_calculation/test_cost_tracking.py b/tests/integration/cost_calculation/test_cost_tracking.py index a8a56fbfbbd..fd18c400b4e 100644 --- a/tests/integration/cost_calculation/test_cost_tracking.py +++ b/tests/integration/cost_calculation/test_cost_tracking.py @@ -12,12 +12,14 @@ from integration.cost_calculation.conftest import ( approx_equal, assert_total_is_sum_of_components, poll_cost_row, + poll_failure_row, register_scenario_deployment, ) from integration.cost_calculation.cost_tracking_case import ( CASES, CostTrackingTestCase, ExactExpected, + FailureExpected, RecountExpected, data_errors, ) @@ -40,7 +42,9 @@ def _assert_stream_has_no_error(response_text: str) -> None: if payload == "[DONE]": continue parsed = JSON_OBJECT.validate_json(payload) - assert "error" not in parsed, f"stream carried an error event: {parsed}" + assert ( + "error" not in parsed and parsed.get("type") not in {"error", "response.failed"} + ), f"stream carried an error event: {parsed}" @pytest.mark.parametrize("case", _CASES) @@ -51,10 +55,22 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) model_name: Final = register_scenario_deployment(scenario, case, marker, key) response: Final = gateway.request( "POST", - "/v1/chat/completions", + case.endpoint, {**case.request, "model": model_name}, key=key, ) + if isinstance(case.expected, FailureExpected): + assert response.status_code == case.expected.failure.status, ( + f"{case.name}: proxy returned {response.status_code}, expected {case.expected.failure.status}: " + f"{response.text[:400]}" + ) + response_cost: Final = response.headers.get("x-litellm-response-cost") + assert response_cost is None or approx_equal(float(response_cost), 0.0), ( + f"{case.name}: failure response cost was {response_cost}" + ) + row: Final = poll_failure_row(key) + assert row.spend == 0, f"{case.name}: failure spend was {row.spend}" + return assert response.is_success, f"{case.name}: proxy returned {response.status_code}: {response.text[:400]}" if case.response.content_type == "text/event-stream": _assert_stream_has_no_error(response.text) @@ -92,6 +108,62 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) assert breakdown.output_cost is not None and approx_equal(breakdown.output_cost, expected.output_cost), ( f"{case.name}: output_cost {breakdown.output_cost} != expected {expected.output_cost}" ) + for field, header_name, actual_component, expected_component in ( + ( + "cache_read_cost", + "x-litellm-response-cost-cache-read", + breakdown.cache_read_cost, + expected.cache_read_cost, + ), + ( + "cache_creation_cost", + "x-litellm-response-cost-cache-creation", + breakdown.cache_creation_cost, + expected.cache_creation_cost, + ), + ( + "reasoning_cost", + "x-litellm-response-cost-reasoning", + breakdown.reasoning_cost, + expected.reasoning_cost, + ), + ( + "tool_usage_cost", + "x-litellm-response-cost-tool-usage", + breakdown.tool_usage_cost, + expected.tool_usage_cost, + ), + ): + if expected_component is None: + continue + assert actual_component is not None and approx_equal(actual_component, expected_component), ( + f"{case.name}: {field} {actual_component} != expected {expected_component}" + ) + if case.response.content_type == "application/json": + header: Final = response.headers.get(header_name) + assert header is not None and approx_equal(float(header), expected_component), ( + f"{case.name}: {header_name} {header} != expected {expected_component}" + ) + if case.response.content_type == "application/json" and any( + component is not None + for component in ( + expected.cache_read_cost, + expected.cache_creation_cost, + expected.reasoning_cost, + expected.tool_usage_cost, + ) + ): + input_header: Final = response.headers.get("x-litellm-response-cost-input") + output_header: Final = response.headers.get("x-litellm-response-cost-output") + expected_input_header: Final = expected.input_cost - ( + expected.cache_read_cost or 0.0 + ) - (expected.cache_creation_cost or 0.0) + assert input_header is not None and approx_equal(float(input_header), expected_input_header), ( + f"{case.name}: x-litellm-response-cost-input {input_header} != expected {expected_input_header}" + ) + assert output_header is not None and approx_equal(float(output_header), expected.output_cost), ( + f"{case.name}: x-litellm-response-cost-output {output_header} != expected {expected.output_cost}" + ) assert row.prompt_tokens == expected.prompt_tokens, ( f"{case.name}: prompt_tokens {row.prompt_tokens} != expected {expected.prompt_tokens}" )