test(pricing): tighten tier matrix assertions from review

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
kerry 2026-10-01 01:59:57 +00:00
parent dd7001c5ec
commit 09876dedb7
2 changed files with 100 additions and 48 deletions

View file

@ -71,7 +71,11 @@ def _start_scripted_upstream(port: int) -> subprocess.Popen[bytes]:
except httpx.TransportError:
return 0
assert eventually(ready_status, lambda status: status == 200, seconds=30) == 200
try:
assert eventually(ready_status, lambda status: status == 200, seconds=30) == 200
except BaseException:
_stop_scripted_upstream(process, force=True)
raise
return process
@ -89,6 +93,14 @@ def _stop_scripted_upstream(process: subprocess.Popen[bytes], *, force: bool = F
process.wait(timeout=5)
def _accumulate_observations(
upstream_url: str,
observations: list[dict[str, JsonValue]],
) -> tuple[dict[str, JsonValue], ...]:
observations.extend(matrix._observations_at(upstream_url)) # mutable-ok: retain destructive queue drains
return tuple(observations)
def _register_owned_scenario(
upstream_url: str,
identifier: str,
@ -217,21 +229,25 @@ def _completed_success(outcome: matrix.MatrixOutcome | None) -> bool:
def _assert_cost_rows(
key: str,
outcomes: tuple[matrix.MatrixOutcome | None, ...],
requests: tuple[MatrixRequest, ...],
) -> None:
successful: Final = tuple(outcome for outcome in outcomes if _completed_success(outcome))
failed: Final = tuple(outcome for outcome in outcomes if not _completed_success(outcome))
paired_outcomes: Final = tuple(zip(outcomes, requests, strict=True))
successful: Final = tuple(outcome for outcome, _ in paired_outcomes if _completed_success(outcome))
successful_requests: Final = tuple(request for outcome, request in paired_outcomes if _completed_success(outcome))
failed: Final = tuple(outcome for outcome, _ in paired_outcomes if not _completed_success(outcome))
expected_cost: Final = matrix._expected_cost(
"priority",
matrix.MATRIX_PROMPT_TOKENS,
matrix.MATRIX_COMPLETION_TOKENS,
custom_standard=False,
)
for outcome in successful:
for outcome, request in zip(successful, successful_requests, strict=True):
matrix._assert_billing(
outcome,
expected_cost=expected_cost,
prompt_tokens=matrix.MATRIX_PROMPT_TOKENS,
completion_tokens=matrix.MATRIX_COMPLETION_TOKENS,
allow_missing_header=request.stream and request.surface == "chat",
)
assert all(outcome.request_id is not None for outcome in successful), successful
successful_ids: Final = tuple(cast(str, outcome.request_id) for outcome in successful)
@ -247,6 +263,23 @@ def _assert_cost_rows(
outcome.request_id for outcome in failed if outcome is not None and outcome.request_id is not None
)
assert all(row["status"] != "success" or str(row["request_id"]) not in failed_ids for row in rows), rows
failed_rows: Final = tuple(row for row in rows if row["status"] != "success")
expected_failure_costs: Final = tuple(
matrix._expected_cost(
"priority",
int(str(row["prompt_tokens"] or 0)),
int(str(row["completion_tokens"] or 0)),
custom_standard=False,
)
for row in failed_rows
)
assert all(
float(str(row["spend"])) == pytest.approx(expected_cost, rel=1e-6)
for row, expected_cost in zip(failed_rows, expected_failure_costs, strict=True)
), tuple(zip(failed_rows, expected_failure_costs, strict=True))
total_spend: Final = sum(float(str(row["spend"])) for row in rows)
expected_total_spend: Final = len(successful) * expected_cost + sum(expected_failure_costs)
assert total_spend == pytest.approx(expected_total_spend, rel=1e-6), rows
def _uvicorn_workers(process: subprocess.Popen[bytes]) -> tuple[psutil.Process, ...]:
@ -293,12 +326,13 @@ def test_upstream_restart_mid_mixed_burst_preserves_exact_tier_billing(
)
first_response: Final = futures[0].result(timeout=30)
assert _completed_success(first_response), first_response
observed_before_stop: Final = eventually(
lambda: matrix._observations_at(upstream_url),
observations: Final[list[dict[str, JsonValue]]] = [] # mutable-ok: keep earlier queue drains
eventually(
lambda: _accumulate_observations(upstream_url, observations),
lambda calls: any(object_value(call["body"]).get("stream") is True for call in calls),
seconds=30,
)
observed_at_stop: Final = matrix._observations_at(upstream_url)
_accumulate_observations(upstream_url, observations)
_stop_scripted_upstream(first_upstream, force=True)
burst_outcomes: Final = tuple(future.result(timeout=90) for future in futures)
assert any(not _completed_success(outcome) for outcome in burst_outcomes), burst_outcomes
@ -313,19 +347,19 @@ def test_upstream_restart_mid_mixed_burst_preserves_exact_tier_billing(
_capture_http(candidate, key, request) for request in recovery_requests
)
assert all(_completed_success(outcome) for outcome in recovery_outcomes), recovery_outcomes
recovery_observations: Final = matrix._observations_at(upstream_url)
observed_requests: Final = (
*observed_before_stop,
*observed_at_stop,
*recovery_observations,
)
_accumulate_observations(upstream_url, observations)
observed_requests: Final = tuple(observations)
successful_count: Final = sum(_completed_success(outcome) for outcome in burst_outcomes)
assert len(observed_requests) >= successful_count + len(recovery_outcomes), (
len(observed_requests),
successful_count,
)
_assert_all_forwarded(observed_requests)
_assert_cost_rows(key, (*burst_outcomes, *recovery_outcomes))
_assert_cost_rows(
key,
(*burst_outcomes, *recovery_outcomes),
(*requests, *recovery_requests),
)
finally:
_stop_scripted_upstream(recovery_upstream)
finally:
@ -365,8 +399,9 @@ def test_two_worker_proxy_survives_worker_kill_and_keeps_tier_billing(
futures: Final = tuple(
executor.submit(_capture_http, candidate, key, request) for request in requests
)
initial_observations: Final = eventually(
lambda: matrix._observations_at(upstream_url),
observations: Final[list[dict[str, JsonValue]]] = [] # mutable-ok: keep earlier queue drains
eventually(
lambda: _accumulate_observations(upstream_url, observations),
lambda calls: len(calls) >= 4,
seconds=30,
)
@ -393,14 +428,18 @@ def test_two_worker_proxy_survives_worker_kill_and_keeps_tier_billing(
_capture_http(candidate, key, request) for request in recovery_requests
)
assert all(_completed_success(outcome) for outcome in recovery_outcomes), recovery_outcomes
recovery_observations: Final = matrix._observations_at(upstream_url)
observed_requests: Final = (*initial_observations, *recovery_observations)
_accumulate_observations(upstream_url, observations)
observed_requests: Final = tuple(observations)
successful_count: Final = sum(_completed_success(outcome) for outcome in burst_outcomes)
assert len(observed_requests) >= successful_count + len(recovery_outcomes), (
len(observed_requests),
successful_count,
)
_assert_all_forwarded(observed_requests)
_assert_cost_rows(key, (*burst_outcomes, *recovery_outcomes))
_assert_cost_rows(
key,
(*burst_outcomes, *recovery_outcomes),
(*requests, *recovery_requests),
)
finally:
_stop_scripted_upstream(upstream_process)

View file

@ -532,7 +532,7 @@ def _assert_forwarded_body(
assert "service_tier" not in body, body
def _assert_upstream(
def _assert_upstream_request(
gateway: Gateway,
*,
expected_count: int = 1,
@ -541,11 +541,31 @@ def _assert_upstream(
) -> dict[str, JsonValue]:
requests: Final = _observations(gateway)
assert len(requests) == expected_count, requests
latest: Final = object_value(requests[-1]["body"])
_assert_forwarded_body(latest, expected_tier=expected_tier, tier_present=tier_present)
latest: Final = requests[-1]
_assert_forwarded_body(
object_value(latest["body"]),
expected_tier=expected_tier,
tier_present=tier_present,
)
return latest
def _assert_upstream(
gateway: Gateway,
*,
expected_count: int = 1,
expected_tier: JsonValue | None = None,
tier_present: bool = False,
) -> dict[str, JsonValue]:
request: Final = _assert_upstream_request(
gateway,
expected_count=expected_count,
expected_tier=expected_tier,
tier_present=tier_present,
)
return object_value(request["body"])
def _assert_azure_upstream(gateway: Gateway, *, scenario_id: str) -> None:
observations: Final = _observations(gateway)
requests: Final = tuple(
@ -660,7 +680,7 @@ def _assert_cost_header(
outcome: MatrixOutcome,
*,
expected_cost: float,
allow_missing_header: bool = True,
allow_missing_header: bool = False,
) -> None:
assert outcome.status == 200, outcome.text
assert outcome.request_id is not None, outcome
@ -676,7 +696,7 @@ def _assert_billing(
expected_cost: float,
prompt_tokens: int,
completion_tokens: int,
allow_missing_header: bool = True,
allow_missing_header: bool = False,
) -> dict[str, object]:
_assert_cost_header(outcome, expected_cost=expected_cost, allow_missing_header=allow_missing_header)
request_id: Final = outcome.request_id
@ -889,6 +909,7 @@ def test_openai_sdk_streaming_priority_chat_consumes_usage_and_logs_spend(gatewa
expected_cost=_expected_cost("priority", MATRIX_PROMPT_TOKENS, MATRIX_COMPLETION_TOKENS),
prompt_tokens=MATRIX_PROMPT_TOKENS,
completion_tokens=MATRIX_COMPLETION_TOKENS,
allow_missing_header=True,
)
_assert_upstream(gateway, expected_tier="priority", tier_present=True)
@ -956,6 +977,7 @@ def test_openai_sdk_streaming_responses_priority_consumes_full_stream(gateway: G
_assert_cost_header(
outcome,
expected_cost=expected_cost,
allow_missing_header=True,
)
rows: Final = eventually(
lambda: _spend_rows_for_key(key),
@ -971,7 +993,7 @@ def test_openai_sdk_streaming_responses_priority_consumes_full_stream(gateway: G
@pytest.mark.parametrize("stream", (False, True), ids=("non-streaming", "streaming"))
def test_messages_endpoint_preserves_path_and_custom_standard_billing(
def test_messages_endpoint_routes_to_responses_upstream_and_bills_custom_standard(
gateway: Gateway,
stream: bool,
) -> None:
@ -1007,8 +1029,12 @@ def test_messages_endpoint_preserves_path_and_custom_standard_billing(
expected_cost=_expected_cost(None, MATRIX_PROMPT_TOKENS, MATRIX_COMPLETION_TOKENS),
prompt_tokens=MATRIX_PROMPT_TOKENS,
completion_tokens=MATRIX_COMPLETION_TOKENS,
allow_missing_header=stream,
)
observed: Final = _assert_upstream(gateway)
observed_request: Final = _assert_upstream_request(gateway)
observed_path: Final = string_value(observed_request["path"])
assert observed_path == f"/{scenario_id}/responses", observed_request
observed: Final = object_value(observed_request["body"])
assert "service_tier" not in observed
@ -1389,6 +1415,7 @@ def test_ptu_deployment_keeps_zero_per_request_cost_with_attribution_enabled(
expected_cost=0.0,
prompt_tokens=MATRIX_PROMPT_TOKENS,
completion_tokens=MATRIX_COMPLETION_TOKENS,
allow_missing_header=True,
)
_assert_upstream(gateway, expected_tier="priority", tier_present=True)
@ -1761,14 +1788,8 @@ def test_three_repeated_priority_requests_create_three_identical_cost_rows(gatew
)
rows: Final = tuple(_spend_rows(cast(str, request_id))[0] for request_id in ids)
assert len(rows) == 3, rows
assert all(
outcome.cost_header is None or float(str(row["spend"])) == pytest.approx(outcome.cost_header, rel=1e-6)
for outcome, row in zip(outcomes, rows, strict=True)
), (outcomes, rows)
assert all(
outcome.cost_header is None or outcome.cost_header == pytest.approx(expected_cost, rel=1e-6)
for outcome in outcomes
), outcomes
for outcome in outcomes:
_assert_cost_header(outcome, expected_cost=expected_cost)
assert all(float(str(row["spend"])) == pytest.approx(expected_cost, rel=1e-6) for row in rows), rows
assert all(int(str(row["prompt_tokens"])) == MATRIX_PROMPT_TOKENS for row in rows), rows
assert all(int(str(row["completion_tokens"])) == MATRIX_COMPLETION_TOKENS for row in rows), rows
@ -1827,10 +1848,8 @@ def test_priority_burst_uses_catalog_rates_during_standard_price_update(gateway:
outcomes: Final = tuple(future.result(timeout=90) for future in futures)
expected_cost: Final = _expected_cost("priority", MATRIX_PROMPT_TOKENS, MATRIX_COMPLETION_TOKENS)
assert all(outcome.status == 200 for outcome in outcomes), outcomes
assert all(
outcome.cost_header is None or outcome.cost_header == pytest.approx(expected_cost, rel=1e-6)
for outcome in outcomes
), outcomes
for outcome in outcomes:
_assert_cost_header(outcome, expected_cost=expected_cost, allow_missing_header=True)
ids: Final = tuple(outcome.request_id for outcome in outcomes)
assert None not in ids, outcomes
rows: Final = tuple(_spend_rows(cast(str, request_id))[0] for request_id in ids)
@ -1890,8 +1909,8 @@ def test_cache_hit_keeps_the_observed_second_request_billing(gateway: Gateway) -
assert first.request_id is not None and second.request_id is not None, (first, second)
assert first.request_id == second.request_id, (first, second)
expected_first_cost: Final = control_cost
assert first.cost_header is None or first.cost_header == pytest.approx(expected_first_cost, rel=1e-6), first
assert second.cost_header == pytest.approx(expected_first_cost, rel=1e-6), second
_assert_cost_header(first, expected_cost=expected_first_cost)
_assert_cost_header(second, expected_cost=expected_first_cost)
rows: Final = _spend_rows(first.request_id)
assert float(str(rows[0]["spend"])) == pytest.approx(expected_first_cost, rel=1e-6), rows
cache_hit_rows: Final = _cache_hit_spend_rows(second.request_id)
@ -1955,12 +1974,6 @@ def test_response_priority_matches_catalog_priced_rule(gateway: Gateway) -> None
MATRIX_COMPLETION_TOKENS,
custom_standard=False,
)
expected_custom_cost: Final = (
catalog_cost
if priority_outcome.cost_header == pytest.approx(catalog_cost, rel=1e-6)
else CUSTOM_STANDARD_INPUT_RATE * MATRIX_PROMPT_TOKENS
+ CUSTOM_STANDARD_OUTPUT_RATE * MATRIX_COMPLETION_TOKENS
)
assert custom_outcome.body.get("service_tier") == "priority", custom_outcome.body
assert catalog_outcome.body.get("service_tier") == "priority", catalog_outcome.body
assert priority_outcome.body.get("service_tier") == "priority", priority_outcome.body
@ -1970,7 +1983,7 @@ def test_response_priority_matches_catalog_priced_rule(gateway: Gateway) -> None
), (custom_observed, catalog_observed, priority_observed)
_assert_billing(
custom_outcome,
expected_cost=expected_custom_cost,
expected_cost=catalog_cost,
prompt_tokens=MATRIX_PROMPT_TOKENS,
completion_tokens=MATRIX_COMPLETION_TOKENS,
)
@ -1982,7 +1995,7 @@ def test_response_priority_matches_catalog_priced_rule(gateway: Gateway) -> None
)
_assert_billing(
priority_outcome,
expected_cost=expected_custom_cost,
expected_cost=catalog_cost,
prompt_tokens=MATRIX_PROMPT_TOKENS,
completion_tokens=MATRIX_COMPLETION_TOKENS,
)