From 09876dedb701423565bc10e49002710bbd811c12 Mon Sep 17 00:00:00 2001 From: kerry Date: Thu, 1 Oct 2026 01:59:57 +0000 Subject: [PATCH] test(pricing): tighten tier matrix assertions from review Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_service_tier_pricing_chaos.py | 77 ++++++++++++++----- .../test_service_tier_pricing_matrix.py | 71 ++++++++++------- 2 files changed, 100 insertions(+), 48 deletions(-) diff --git a/tests/integration/pricing/test_service_tier_pricing_chaos.py b/tests/integration/pricing/test_service_tier_pricing_chaos.py index 52d0911d3c6..a0d52148ea5 100644 --- a/tests/integration/pricing/test_service_tier_pricing_chaos.py +++ b/tests/integration/pricing/test_service_tier_pricing_chaos.py @@ -71,7 +71,11 @@ def _start_scripted_upstream(port: int) -> subprocess.Popen[bytes]: except httpx.TransportError: return 0 - assert eventually(ready_status, lambda status: status == 200, seconds=30) == 200 + try: + assert eventually(ready_status, lambda status: status == 200, seconds=30) == 200 + except BaseException: + _stop_scripted_upstream(process, force=True) + raise return process @@ -89,6 +93,14 @@ def _stop_scripted_upstream(process: subprocess.Popen[bytes], *, force: bool = F process.wait(timeout=5) +def _accumulate_observations( + upstream_url: str, + observations: list[dict[str, JsonValue]], +) -> tuple[dict[str, JsonValue], ...]: + observations.extend(matrix._observations_at(upstream_url)) # mutable-ok: retain destructive queue drains + return tuple(observations) + + def _register_owned_scenario( upstream_url: str, identifier: str, @@ -217,21 +229,25 @@ def _completed_success(outcome: matrix.MatrixOutcome | None) -> bool: def _assert_cost_rows( key: str, outcomes: tuple[matrix.MatrixOutcome | None, ...], + requests: tuple[MatrixRequest, ...], ) -> None: - successful: Final = tuple(outcome for outcome in outcomes if _completed_success(outcome)) - failed: Final = tuple(outcome for outcome in outcomes if not _completed_success(outcome)) + paired_outcomes: Final = tuple(zip(outcomes, requests, strict=True)) + successful: Final = tuple(outcome for outcome, _ in paired_outcomes if _completed_success(outcome)) + successful_requests: Final = tuple(request for outcome, request in paired_outcomes if _completed_success(outcome)) + failed: Final = tuple(outcome for outcome, _ in paired_outcomes if not _completed_success(outcome)) expected_cost: Final = matrix._expected_cost( "priority", matrix.MATRIX_PROMPT_TOKENS, matrix.MATRIX_COMPLETION_TOKENS, custom_standard=False, ) - for outcome in successful: + for outcome, request in zip(successful, successful_requests, strict=True): matrix._assert_billing( outcome, expected_cost=expected_cost, prompt_tokens=matrix.MATRIX_PROMPT_TOKENS, completion_tokens=matrix.MATRIX_COMPLETION_TOKENS, + allow_missing_header=request.stream and request.surface == "chat", ) assert all(outcome.request_id is not None for outcome in successful), successful successful_ids: Final = tuple(cast(str, outcome.request_id) for outcome in successful) @@ -247,6 +263,23 @@ def _assert_cost_rows( outcome.request_id for outcome in failed if outcome is not None and outcome.request_id is not None ) assert all(row["status"] != "success" or str(row["request_id"]) not in failed_ids for row in rows), rows + failed_rows: Final = tuple(row for row in rows if row["status"] != "success") + expected_failure_costs: Final = tuple( + matrix._expected_cost( + "priority", + int(str(row["prompt_tokens"] or 0)), + int(str(row["completion_tokens"] or 0)), + custom_standard=False, + ) + for row in failed_rows + ) + assert all( + float(str(row["spend"])) == pytest.approx(expected_cost, rel=1e-6) + for row, expected_cost in zip(failed_rows, expected_failure_costs, strict=True) + ), tuple(zip(failed_rows, expected_failure_costs, strict=True)) + total_spend: Final = sum(float(str(row["spend"])) for row in rows) + expected_total_spend: Final = len(successful) * expected_cost + sum(expected_failure_costs) + assert total_spend == pytest.approx(expected_total_spend, rel=1e-6), rows def _uvicorn_workers(process: subprocess.Popen[bytes]) -> tuple[psutil.Process, ...]: @@ -293,12 +326,13 @@ def test_upstream_restart_mid_mixed_burst_preserves_exact_tier_billing( ) first_response: Final = futures[0].result(timeout=30) assert _completed_success(first_response), first_response - observed_before_stop: Final = eventually( - lambda: matrix._observations_at(upstream_url), + observations: Final[list[dict[str, JsonValue]]] = [] # mutable-ok: keep earlier queue drains + eventually( + lambda: _accumulate_observations(upstream_url, observations), lambda calls: any(object_value(call["body"]).get("stream") is True for call in calls), seconds=30, ) - observed_at_stop: Final = matrix._observations_at(upstream_url) + _accumulate_observations(upstream_url, observations) _stop_scripted_upstream(first_upstream, force=True) burst_outcomes: Final = tuple(future.result(timeout=90) for future in futures) assert any(not _completed_success(outcome) for outcome in burst_outcomes), burst_outcomes @@ -313,19 +347,19 @@ def test_upstream_restart_mid_mixed_burst_preserves_exact_tier_billing( _capture_http(candidate, key, request) for request in recovery_requests ) assert all(_completed_success(outcome) for outcome in recovery_outcomes), recovery_outcomes - recovery_observations: Final = matrix._observations_at(upstream_url) - observed_requests: Final = ( - *observed_before_stop, - *observed_at_stop, - *recovery_observations, - ) + _accumulate_observations(upstream_url, observations) + observed_requests: Final = tuple(observations) successful_count: Final = sum(_completed_success(outcome) for outcome in burst_outcomes) assert len(observed_requests) >= successful_count + len(recovery_outcomes), ( len(observed_requests), successful_count, ) _assert_all_forwarded(observed_requests) - _assert_cost_rows(key, (*burst_outcomes, *recovery_outcomes)) + _assert_cost_rows( + key, + (*burst_outcomes, *recovery_outcomes), + (*requests, *recovery_requests), + ) finally: _stop_scripted_upstream(recovery_upstream) finally: @@ -365,8 +399,9 @@ def test_two_worker_proxy_survives_worker_kill_and_keeps_tier_billing( futures: Final = tuple( executor.submit(_capture_http, candidate, key, request) for request in requests ) - initial_observations: Final = eventually( - lambda: matrix._observations_at(upstream_url), + observations: Final[list[dict[str, JsonValue]]] = [] # mutable-ok: keep earlier queue drains + eventually( + lambda: _accumulate_observations(upstream_url, observations), lambda calls: len(calls) >= 4, seconds=30, ) @@ -393,14 +428,18 @@ def test_two_worker_proxy_survives_worker_kill_and_keeps_tier_billing( _capture_http(candidate, key, request) for request in recovery_requests ) assert all(_completed_success(outcome) for outcome in recovery_outcomes), recovery_outcomes - recovery_observations: Final = matrix._observations_at(upstream_url) - observed_requests: Final = (*initial_observations, *recovery_observations) + _accumulate_observations(upstream_url, observations) + observed_requests: Final = tuple(observations) successful_count: Final = sum(_completed_success(outcome) for outcome in burst_outcomes) assert len(observed_requests) >= successful_count + len(recovery_outcomes), ( len(observed_requests), successful_count, ) _assert_all_forwarded(observed_requests) - _assert_cost_rows(key, (*burst_outcomes, *recovery_outcomes)) + _assert_cost_rows( + key, + (*burst_outcomes, *recovery_outcomes), + (*requests, *recovery_requests), + ) finally: _stop_scripted_upstream(upstream_process) diff --git a/tests/integration/pricing/test_service_tier_pricing_matrix.py b/tests/integration/pricing/test_service_tier_pricing_matrix.py index 9d390fb510a..0d00e9ccfbc 100644 --- a/tests/integration/pricing/test_service_tier_pricing_matrix.py +++ b/tests/integration/pricing/test_service_tier_pricing_matrix.py @@ -532,7 +532,7 @@ def _assert_forwarded_body( assert "service_tier" not in body, body -def _assert_upstream( +def _assert_upstream_request( gateway: Gateway, *, expected_count: int = 1, @@ -541,11 +541,31 @@ def _assert_upstream( ) -> dict[str, JsonValue]: requests: Final = _observations(gateway) assert len(requests) == expected_count, requests - latest: Final = object_value(requests[-1]["body"]) - _assert_forwarded_body(latest, expected_tier=expected_tier, tier_present=tier_present) + latest: Final = requests[-1] + _assert_forwarded_body( + object_value(latest["body"]), + expected_tier=expected_tier, + tier_present=tier_present, + ) return latest +def _assert_upstream( + gateway: Gateway, + *, + expected_count: int = 1, + expected_tier: JsonValue | None = None, + tier_present: bool = False, +) -> dict[str, JsonValue]: + request: Final = _assert_upstream_request( + gateway, + expected_count=expected_count, + expected_tier=expected_tier, + tier_present=tier_present, + ) + return object_value(request["body"]) + + def _assert_azure_upstream(gateway: Gateway, *, scenario_id: str) -> None: observations: Final = _observations(gateway) requests: Final = tuple( @@ -660,7 +680,7 @@ def _assert_cost_header( outcome: MatrixOutcome, *, expected_cost: float, - allow_missing_header: bool = True, + allow_missing_header: bool = False, ) -> None: assert outcome.status == 200, outcome.text assert outcome.request_id is not None, outcome @@ -676,7 +696,7 @@ def _assert_billing( expected_cost: float, prompt_tokens: int, completion_tokens: int, - allow_missing_header: bool = True, + allow_missing_header: bool = False, ) -> dict[str, object]: _assert_cost_header(outcome, expected_cost=expected_cost, allow_missing_header=allow_missing_header) request_id: Final = outcome.request_id @@ -889,6 +909,7 @@ def test_openai_sdk_streaming_priority_chat_consumes_usage_and_logs_spend(gatewa expected_cost=_expected_cost("priority", MATRIX_PROMPT_TOKENS, MATRIX_COMPLETION_TOKENS), prompt_tokens=MATRIX_PROMPT_TOKENS, completion_tokens=MATRIX_COMPLETION_TOKENS, + allow_missing_header=True, ) _assert_upstream(gateway, expected_tier="priority", tier_present=True) @@ -956,6 +977,7 @@ def test_openai_sdk_streaming_responses_priority_consumes_full_stream(gateway: G _assert_cost_header( outcome, expected_cost=expected_cost, + allow_missing_header=True, ) rows: Final = eventually( lambda: _spend_rows_for_key(key), @@ -971,7 +993,7 @@ def test_openai_sdk_streaming_responses_priority_consumes_full_stream(gateway: G @pytest.mark.parametrize("stream", (False, True), ids=("non-streaming", "streaming")) -def test_messages_endpoint_preserves_path_and_custom_standard_billing( +def test_messages_endpoint_routes_to_responses_upstream_and_bills_custom_standard( gateway: Gateway, stream: bool, ) -> None: @@ -1007,8 +1029,12 @@ def test_messages_endpoint_preserves_path_and_custom_standard_billing( expected_cost=_expected_cost(None, MATRIX_PROMPT_TOKENS, MATRIX_COMPLETION_TOKENS), prompt_tokens=MATRIX_PROMPT_TOKENS, completion_tokens=MATRIX_COMPLETION_TOKENS, + allow_missing_header=stream, ) - observed: Final = _assert_upstream(gateway) + observed_request: Final = _assert_upstream_request(gateway) + observed_path: Final = string_value(observed_request["path"]) + assert observed_path == f"/{scenario_id}/responses", observed_request + observed: Final = object_value(observed_request["body"]) assert "service_tier" not in observed @@ -1389,6 +1415,7 @@ def test_ptu_deployment_keeps_zero_per_request_cost_with_attribution_enabled( expected_cost=0.0, prompt_tokens=MATRIX_PROMPT_TOKENS, completion_tokens=MATRIX_COMPLETION_TOKENS, + allow_missing_header=True, ) _assert_upstream(gateway, expected_tier="priority", tier_present=True) @@ -1761,14 +1788,8 @@ def test_three_repeated_priority_requests_create_three_identical_cost_rows(gatew ) rows: Final = tuple(_spend_rows(cast(str, request_id))[0] for request_id in ids) assert len(rows) == 3, rows - assert all( - outcome.cost_header is None or float(str(row["spend"])) == pytest.approx(outcome.cost_header, rel=1e-6) - for outcome, row in zip(outcomes, rows, strict=True) - ), (outcomes, rows) - assert all( - outcome.cost_header is None or outcome.cost_header == pytest.approx(expected_cost, rel=1e-6) - for outcome in outcomes - ), outcomes + for outcome in outcomes: + _assert_cost_header(outcome, expected_cost=expected_cost) assert all(float(str(row["spend"])) == pytest.approx(expected_cost, rel=1e-6) for row in rows), rows assert all(int(str(row["prompt_tokens"])) == MATRIX_PROMPT_TOKENS for row in rows), rows assert all(int(str(row["completion_tokens"])) == MATRIX_COMPLETION_TOKENS for row in rows), rows @@ -1827,10 +1848,8 @@ def test_priority_burst_uses_catalog_rates_during_standard_price_update(gateway: outcomes: Final = tuple(future.result(timeout=90) for future in futures) expected_cost: Final = _expected_cost("priority", MATRIX_PROMPT_TOKENS, MATRIX_COMPLETION_TOKENS) assert all(outcome.status == 200 for outcome in outcomes), outcomes - assert all( - outcome.cost_header is None or outcome.cost_header == pytest.approx(expected_cost, rel=1e-6) - for outcome in outcomes - ), outcomes + for outcome in outcomes: + _assert_cost_header(outcome, expected_cost=expected_cost, allow_missing_header=True) ids: Final = tuple(outcome.request_id for outcome in outcomes) assert None not in ids, outcomes rows: Final = tuple(_spend_rows(cast(str, request_id))[0] for request_id in ids) @@ -1890,8 +1909,8 @@ def test_cache_hit_keeps_the_observed_second_request_billing(gateway: Gateway) - assert first.request_id is not None and second.request_id is not None, (first, second) assert first.request_id == second.request_id, (first, second) expected_first_cost: Final = control_cost - assert first.cost_header is None or first.cost_header == pytest.approx(expected_first_cost, rel=1e-6), first - assert second.cost_header == pytest.approx(expected_first_cost, rel=1e-6), second + _assert_cost_header(first, expected_cost=expected_first_cost) + _assert_cost_header(second, expected_cost=expected_first_cost) rows: Final = _spend_rows(first.request_id) assert float(str(rows[0]["spend"])) == pytest.approx(expected_first_cost, rel=1e-6), rows cache_hit_rows: Final = _cache_hit_spend_rows(second.request_id) @@ -1955,12 +1974,6 @@ def test_response_priority_matches_catalog_priced_rule(gateway: Gateway) -> None MATRIX_COMPLETION_TOKENS, custom_standard=False, ) - expected_custom_cost: Final = ( - catalog_cost - if priority_outcome.cost_header == pytest.approx(catalog_cost, rel=1e-6) - else CUSTOM_STANDARD_INPUT_RATE * MATRIX_PROMPT_TOKENS - + CUSTOM_STANDARD_OUTPUT_RATE * MATRIX_COMPLETION_TOKENS - ) assert custom_outcome.body.get("service_tier") == "priority", custom_outcome.body assert catalog_outcome.body.get("service_tier") == "priority", catalog_outcome.body assert priority_outcome.body.get("service_tier") == "priority", priority_outcome.body @@ -1970,7 +1983,7 @@ def test_response_priority_matches_catalog_priced_rule(gateway: Gateway) -> None ), (custom_observed, catalog_observed, priority_observed) _assert_billing( custom_outcome, - expected_cost=expected_custom_cost, + expected_cost=catalog_cost, prompt_tokens=MATRIX_PROMPT_TOKENS, completion_tokens=MATRIX_COMPLETION_TOKENS, ) @@ -1982,7 +1995,7 @@ def test_response_priority_matches_catalog_priced_rule(gateway: Gateway) -> None ) _assert_billing( priority_outcome, - expected_cost=expected_custom_cost, + expected_cost=catalog_cost, prompt_tokens=MATRIX_PROMPT_TOKENS, completion_tokens=MATRIX_COMPLETION_TOKENS, )