mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
test(pricing): tighten tier matrix assertions from review
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
dd7001c5ec
commit
09876dedb7
2 changed files with 100 additions and 48 deletions
|
|
@ -71,7 +71,11 @@ def _start_scripted_upstream(port: int) -> subprocess.Popen[bytes]:
|
|||
except httpx.TransportError:
|
||||
return 0
|
||||
|
||||
assert eventually(ready_status, lambda status: status == 200, seconds=30) == 200
|
||||
try:
|
||||
assert eventually(ready_status, lambda status: status == 200, seconds=30) == 200
|
||||
except BaseException:
|
||||
_stop_scripted_upstream(process, force=True)
|
||||
raise
|
||||
return process
|
||||
|
||||
|
||||
|
|
@ -89,6 +93,14 @@ def _stop_scripted_upstream(process: subprocess.Popen[bytes], *, force: bool = F
|
|||
process.wait(timeout=5)
|
||||
|
||||
|
||||
def _accumulate_observations(
|
||||
upstream_url: str,
|
||||
observations: list[dict[str, JsonValue]],
|
||||
) -> tuple[dict[str, JsonValue], ...]:
|
||||
observations.extend(matrix._observations_at(upstream_url)) # mutable-ok: retain destructive queue drains
|
||||
return tuple(observations)
|
||||
|
||||
|
||||
def _register_owned_scenario(
|
||||
upstream_url: str,
|
||||
identifier: str,
|
||||
|
|
@ -217,21 +229,25 @@ def _completed_success(outcome: matrix.MatrixOutcome | None) -> bool:
|
|||
def _assert_cost_rows(
|
||||
key: str,
|
||||
outcomes: tuple[matrix.MatrixOutcome | None, ...],
|
||||
requests: tuple[MatrixRequest, ...],
|
||||
) -> None:
|
||||
successful: Final = tuple(outcome for outcome in outcomes if _completed_success(outcome))
|
||||
failed: Final = tuple(outcome for outcome in outcomes if not _completed_success(outcome))
|
||||
paired_outcomes: Final = tuple(zip(outcomes, requests, strict=True))
|
||||
successful: Final = tuple(outcome for outcome, _ in paired_outcomes if _completed_success(outcome))
|
||||
successful_requests: Final = tuple(request for outcome, request in paired_outcomes if _completed_success(outcome))
|
||||
failed: Final = tuple(outcome for outcome, _ in paired_outcomes if not _completed_success(outcome))
|
||||
expected_cost: Final = matrix._expected_cost(
|
||||
"priority",
|
||||
matrix.MATRIX_PROMPT_TOKENS,
|
||||
matrix.MATRIX_COMPLETION_TOKENS,
|
||||
custom_standard=False,
|
||||
)
|
||||
for outcome in successful:
|
||||
for outcome, request in zip(successful, successful_requests, strict=True):
|
||||
matrix._assert_billing(
|
||||
outcome,
|
||||
expected_cost=expected_cost,
|
||||
prompt_tokens=matrix.MATRIX_PROMPT_TOKENS,
|
||||
completion_tokens=matrix.MATRIX_COMPLETION_TOKENS,
|
||||
allow_missing_header=request.stream and request.surface == "chat",
|
||||
)
|
||||
assert all(outcome.request_id is not None for outcome in successful), successful
|
||||
successful_ids: Final = tuple(cast(str, outcome.request_id) for outcome in successful)
|
||||
|
|
@ -247,6 +263,23 @@ def _assert_cost_rows(
|
|||
outcome.request_id for outcome in failed if outcome is not None and outcome.request_id is not None
|
||||
)
|
||||
assert all(row["status"] != "success" or str(row["request_id"]) not in failed_ids for row in rows), rows
|
||||
failed_rows: Final = tuple(row for row in rows if row["status"] != "success")
|
||||
expected_failure_costs: Final = tuple(
|
||||
matrix._expected_cost(
|
||||
"priority",
|
||||
int(str(row["prompt_tokens"] or 0)),
|
||||
int(str(row["completion_tokens"] or 0)),
|
||||
custom_standard=False,
|
||||
)
|
||||
for row in failed_rows
|
||||
)
|
||||
assert all(
|
||||
float(str(row["spend"])) == pytest.approx(expected_cost, rel=1e-6)
|
||||
for row, expected_cost in zip(failed_rows, expected_failure_costs, strict=True)
|
||||
), tuple(zip(failed_rows, expected_failure_costs, strict=True))
|
||||
total_spend: Final = sum(float(str(row["spend"])) for row in rows)
|
||||
expected_total_spend: Final = len(successful) * expected_cost + sum(expected_failure_costs)
|
||||
assert total_spend == pytest.approx(expected_total_spend, rel=1e-6), rows
|
||||
|
||||
|
||||
def _uvicorn_workers(process: subprocess.Popen[bytes]) -> tuple[psutil.Process, ...]:
|
||||
|
|
@ -293,12 +326,13 @@ def test_upstream_restart_mid_mixed_burst_preserves_exact_tier_billing(
|
|||
)
|
||||
first_response: Final = futures[0].result(timeout=30)
|
||||
assert _completed_success(first_response), first_response
|
||||
observed_before_stop: Final = eventually(
|
||||
lambda: matrix._observations_at(upstream_url),
|
||||
observations: Final[list[dict[str, JsonValue]]] = [] # mutable-ok: keep earlier queue drains
|
||||
eventually(
|
||||
lambda: _accumulate_observations(upstream_url, observations),
|
||||
lambda calls: any(object_value(call["body"]).get("stream") is True for call in calls),
|
||||
seconds=30,
|
||||
)
|
||||
observed_at_stop: Final = matrix._observations_at(upstream_url)
|
||||
_accumulate_observations(upstream_url, observations)
|
||||
_stop_scripted_upstream(first_upstream, force=True)
|
||||
burst_outcomes: Final = tuple(future.result(timeout=90) for future in futures)
|
||||
assert any(not _completed_success(outcome) for outcome in burst_outcomes), burst_outcomes
|
||||
|
|
@ -313,19 +347,19 @@ def test_upstream_restart_mid_mixed_burst_preserves_exact_tier_billing(
|
|||
_capture_http(candidate, key, request) for request in recovery_requests
|
||||
)
|
||||
assert all(_completed_success(outcome) for outcome in recovery_outcomes), recovery_outcomes
|
||||
recovery_observations: Final = matrix._observations_at(upstream_url)
|
||||
observed_requests: Final = (
|
||||
*observed_before_stop,
|
||||
*observed_at_stop,
|
||||
*recovery_observations,
|
||||
)
|
||||
_accumulate_observations(upstream_url, observations)
|
||||
observed_requests: Final = tuple(observations)
|
||||
successful_count: Final = sum(_completed_success(outcome) for outcome in burst_outcomes)
|
||||
assert len(observed_requests) >= successful_count + len(recovery_outcomes), (
|
||||
len(observed_requests),
|
||||
successful_count,
|
||||
)
|
||||
_assert_all_forwarded(observed_requests)
|
||||
_assert_cost_rows(key, (*burst_outcomes, *recovery_outcomes))
|
||||
_assert_cost_rows(
|
||||
key,
|
||||
(*burst_outcomes, *recovery_outcomes),
|
||||
(*requests, *recovery_requests),
|
||||
)
|
||||
finally:
|
||||
_stop_scripted_upstream(recovery_upstream)
|
||||
finally:
|
||||
|
|
@ -365,8 +399,9 @@ def test_two_worker_proxy_survives_worker_kill_and_keeps_tier_billing(
|
|||
futures: Final = tuple(
|
||||
executor.submit(_capture_http, candidate, key, request) for request in requests
|
||||
)
|
||||
initial_observations: Final = eventually(
|
||||
lambda: matrix._observations_at(upstream_url),
|
||||
observations: Final[list[dict[str, JsonValue]]] = [] # mutable-ok: keep earlier queue drains
|
||||
eventually(
|
||||
lambda: _accumulate_observations(upstream_url, observations),
|
||||
lambda calls: len(calls) >= 4,
|
||||
seconds=30,
|
||||
)
|
||||
|
|
@ -393,14 +428,18 @@ def test_two_worker_proxy_survives_worker_kill_and_keeps_tier_billing(
|
|||
_capture_http(candidate, key, request) for request in recovery_requests
|
||||
)
|
||||
assert all(_completed_success(outcome) for outcome in recovery_outcomes), recovery_outcomes
|
||||
recovery_observations: Final = matrix._observations_at(upstream_url)
|
||||
observed_requests: Final = (*initial_observations, *recovery_observations)
|
||||
_accumulate_observations(upstream_url, observations)
|
||||
observed_requests: Final = tuple(observations)
|
||||
successful_count: Final = sum(_completed_success(outcome) for outcome in burst_outcomes)
|
||||
assert len(observed_requests) >= successful_count + len(recovery_outcomes), (
|
||||
len(observed_requests),
|
||||
successful_count,
|
||||
)
|
||||
_assert_all_forwarded(observed_requests)
|
||||
_assert_cost_rows(key, (*burst_outcomes, *recovery_outcomes))
|
||||
_assert_cost_rows(
|
||||
key,
|
||||
(*burst_outcomes, *recovery_outcomes),
|
||||
(*requests, *recovery_requests),
|
||||
)
|
||||
finally:
|
||||
_stop_scripted_upstream(upstream_process)
|
||||
|
|
|
|||
|
|
@ -532,7 +532,7 @@ def _assert_forwarded_body(
|
|||
assert "service_tier" not in body, body
|
||||
|
||||
|
||||
def _assert_upstream(
|
||||
def _assert_upstream_request(
|
||||
gateway: Gateway,
|
||||
*,
|
||||
expected_count: int = 1,
|
||||
|
|
@ -541,11 +541,31 @@ def _assert_upstream(
|
|||
) -> dict[str, JsonValue]:
|
||||
requests: Final = _observations(gateway)
|
||||
assert len(requests) == expected_count, requests
|
||||
latest: Final = object_value(requests[-1]["body"])
|
||||
_assert_forwarded_body(latest, expected_tier=expected_tier, tier_present=tier_present)
|
||||
latest: Final = requests[-1]
|
||||
_assert_forwarded_body(
|
||||
object_value(latest["body"]),
|
||||
expected_tier=expected_tier,
|
||||
tier_present=tier_present,
|
||||
)
|
||||
return latest
|
||||
|
||||
|
||||
def _assert_upstream(
|
||||
gateway: Gateway,
|
||||
*,
|
||||
expected_count: int = 1,
|
||||
expected_tier: JsonValue | None = None,
|
||||
tier_present: bool = False,
|
||||
) -> dict[str, JsonValue]:
|
||||
request: Final = _assert_upstream_request(
|
||||
gateway,
|
||||
expected_count=expected_count,
|
||||
expected_tier=expected_tier,
|
||||
tier_present=tier_present,
|
||||
)
|
||||
return object_value(request["body"])
|
||||
|
||||
|
||||
def _assert_azure_upstream(gateway: Gateway, *, scenario_id: str) -> None:
|
||||
observations: Final = _observations(gateway)
|
||||
requests: Final = tuple(
|
||||
|
|
@ -660,7 +680,7 @@ def _assert_cost_header(
|
|||
outcome: MatrixOutcome,
|
||||
*,
|
||||
expected_cost: float,
|
||||
allow_missing_header: bool = True,
|
||||
allow_missing_header: bool = False,
|
||||
) -> None:
|
||||
assert outcome.status == 200, outcome.text
|
||||
assert outcome.request_id is not None, outcome
|
||||
|
|
@ -676,7 +696,7 @@ def _assert_billing(
|
|||
expected_cost: float,
|
||||
prompt_tokens: int,
|
||||
completion_tokens: int,
|
||||
allow_missing_header: bool = True,
|
||||
allow_missing_header: bool = False,
|
||||
) -> dict[str, object]:
|
||||
_assert_cost_header(outcome, expected_cost=expected_cost, allow_missing_header=allow_missing_header)
|
||||
request_id: Final = outcome.request_id
|
||||
|
|
@ -889,6 +909,7 @@ def test_openai_sdk_streaming_priority_chat_consumes_usage_and_logs_spend(gatewa
|
|||
expected_cost=_expected_cost("priority", MATRIX_PROMPT_TOKENS, MATRIX_COMPLETION_TOKENS),
|
||||
prompt_tokens=MATRIX_PROMPT_TOKENS,
|
||||
completion_tokens=MATRIX_COMPLETION_TOKENS,
|
||||
allow_missing_header=True,
|
||||
)
|
||||
_assert_upstream(gateway, expected_tier="priority", tier_present=True)
|
||||
|
||||
|
|
@ -956,6 +977,7 @@ def test_openai_sdk_streaming_responses_priority_consumes_full_stream(gateway: G
|
|||
_assert_cost_header(
|
||||
outcome,
|
||||
expected_cost=expected_cost,
|
||||
allow_missing_header=True,
|
||||
)
|
||||
rows: Final = eventually(
|
||||
lambda: _spend_rows_for_key(key),
|
||||
|
|
@ -971,7 +993,7 @@ def test_openai_sdk_streaming_responses_priority_consumes_full_stream(gateway: G
|
|||
|
||||
|
||||
@pytest.mark.parametrize("stream", (False, True), ids=("non-streaming", "streaming"))
|
||||
def test_messages_endpoint_preserves_path_and_custom_standard_billing(
|
||||
def test_messages_endpoint_routes_to_responses_upstream_and_bills_custom_standard(
|
||||
gateway: Gateway,
|
||||
stream: bool,
|
||||
) -> None:
|
||||
|
|
@ -1007,8 +1029,12 @@ def test_messages_endpoint_preserves_path_and_custom_standard_billing(
|
|||
expected_cost=_expected_cost(None, MATRIX_PROMPT_TOKENS, MATRIX_COMPLETION_TOKENS),
|
||||
prompt_tokens=MATRIX_PROMPT_TOKENS,
|
||||
completion_tokens=MATRIX_COMPLETION_TOKENS,
|
||||
allow_missing_header=stream,
|
||||
)
|
||||
observed: Final = _assert_upstream(gateway)
|
||||
observed_request: Final = _assert_upstream_request(gateway)
|
||||
observed_path: Final = string_value(observed_request["path"])
|
||||
assert observed_path == f"/{scenario_id}/responses", observed_request
|
||||
observed: Final = object_value(observed_request["body"])
|
||||
assert "service_tier" not in observed
|
||||
|
||||
|
||||
|
|
@ -1389,6 +1415,7 @@ def test_ptu_deployment_keeps_zero_per_request_cost_with_attribution_enabled(
|
|||
expected_cost=0.0,
|
||||
prompt_tokens=MATRIX_PROMPT_TOKENS,
|
||||
completion_tokens=MATRIX_COMPLETION_TOKENS,
|
||||
allow_missing_header=True,
|
||||
)
|
||||
_assert_upstream(gateway, expected_tier="priority", tier_present=True)
|
||||
|
||||
|
|
@ -1761,14 +1788,8 @@ def test_three_repeated_priority_requests_create_three_identical_cost_rows(gatew
|
|||
)
|
||||
rows: Final = tuple(_spend_rows(cast(str, request_id))[0] for request_id in ids)
|
||||
assert len(rows) == 3, rows
|
||||
assert all(
|
||||
outcome.cost_header is None or float(str(row["spend"])) == pytest.approx(outcome.cost_header, rel=1e-6)
|
||||
for outcome, row in zip(outcomes, rows, strict=True)
|
||||
), (outcomes, rows)
|
||||
assert all(
|
||||
outcome.cost_header is None or outcome.cost_header == pytest.approx(expected_cost, rel=1e-6)
|
||||
for outcome in outcomes
|
||||
), outcomes
|
||||
for outcome in outcomes:
|
||||
_assert_cost_header(outcome, expected_cost=expected_cost)
|
||||
assert all(float(str(row["spend"])) == pytest.approx(expected_cost, rel=1e-6) for row in rows), rows
|
||||
assert all(int(str(row["prompt_tokens"])) == MATRIX_PROMPT_TOKENS for row in rows), rows
|
||||
assert all(int(str(row["completion_tokens"])) == MATRIX_COMPLETION_TOKENS for row in rows), rows
|
||||
|
|
@ -1827,10 +1848,8 @@ def test_priority_burst_uses_catalog_rates_during_standard_price_update(gateway:
|
|||
outcomes: Final = tuple(future.result(timeout=90) for future in futures)
|
||||
expected_cost: Final = _expected_cost("priority", MATRIX_PROMPT_TOKENS, MATRIX_COMPLETION_TOKENS)
|
||||
assert all(outcome.status == 200 for outcome in outcomes), outcomes
|
||||
assert all(
|
||||
outcome.cost_header is None or outcome.cost_header == pytest.approx(expected_cost, rel=1e-6)
|
||||
for outcome in outcomes
|
||||
), outcomes
|
||||
for outcome in outcomes:
|
||||
_assert_cost_header(outcome, expected_cost=expected_cost, allow_missing_header=True)
|
||||
ids: Final = tuple(outcome.request_id for outcome in outcomes)
|
||||
assert None not in ids, outcomes
|
||||
rows: Final = tuple(_spend_rows(cast(str, request_id))[0] for request_id in ids)
|
||||
|
|
@ -1890,8 +1909,8 @@ def test_cache_hit_keeps_the_observed_second_request_billing(gateway: Gateway) -
|
|||
assert first.request_id is not None and second.request_id is not None, (first, second)
|
||||
assert first.request_id == second.request_id, (first, second)
|
||||
expected_first_cost: Final = control_cost
|
||||
assert first.cost_header is None or first.cost_header == pytest.approx(expected_first_cost, rel=1e-6), first
|
||||
assert second.cost_header == pytest.approx(expected_first_cost, rel=1e-6), second
|
||||
_assert_cost_header(first, expected_cost=expected_first_cost)
|
||||
_assert_cost_header(second, expected_cost=expected_first_cost)
|
||||
rows: Final = _spend_rows(first.request_id)
|
||||
assert float(str(rows[0]["spend"])) == pytest.approx(expected_first_cost, rel=1e-6), rows
|
||||
cache_hit_rows: Final = _cache_hit_spend_rows(second.request_id)
|
||||
|
|
@ -1955,12 +1974,6 @@ def test_response_priority_matches_catalog_priced_rule(gateway: Gateway) -> None
|
|||
MATRIX_COMPLETION_TOKENS,
|
||||
custom_standard=False,
|
||||
)
|
||||
expected_custom_cost: Final = (
|
||||
catalog_cost
|
||||
if priority_outcome.cost_header == pytest.approx(catalog_cost, rel=1e-6)
|
||||
else CUSTOM_STANDARD_INPUT_RATE * MATRIX_PROMPT_TOKENS
|
||||
+ CUSTOM_STANDARD_OUTPUT_RATE * MATRIX_COMPLETION_TOKENS
|
||||
)
|
||||
assert custom_outcome.body.get("service_tier") == "priority", custom_outcome.body
|
||||
assert catalog_outcome.body.get("service_tier") == "priority", catalog_outcome.body
|
||||
assert priority_outcome.body.get("service_tier") == "priority", priority_outcome.body
|
||||
|
|
@ -1970,7 +1983,7 @@ def test_response_priority_matches_catalog_priced_rule(gateway: Gateway) -> None
|
|||
), (custom_observed, catalog_observed, priority_observed)
|
||||
_assert_billing(
|
||||
custom_outcome,
|
||||
expected_cost=expected_custom_cost,
|
||||
expected_cost=catalog_cost,
|
||||
prompt_tokens=MATRIX_PROMPT_TOKENS,
|
||||
completion_tokens=MATRIX_COMPLETION_TOKENS,
|
||||
)
|
||||
|
|
@ -1982,7 +1995,7 @@ def test_response_priority_matches_catalog_priced_rule(gateway: Gateway) -> None
|
|||
)
|
||||
_assert_billing(
|
||||
priority_outcome,
|
||||
expected_cost=expected_custom_cost,
|
||||
expected_cost=catalog_cost,
|
||||
prompt_tokens=MATRIX_PROMPT_TOKENS,
|
||||
completion_tokens=MATRIX_COMPLETION_TOKENS,
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue