mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
5f17261534
CLIENT PAUSE ALL for the length of the chaos phase instead of CLIENT PAUSE WRITE, so every Redis touchpoint on the request path times out rather than just the writes. The pause is sized to the phase because it freezes the control connection too; teardown's CLIENT UNPAUSE is a safety net for a phase that overran Latency, RSS and CPU are now budgeted as chaos-over-baseline ratios (p50/p90/p99 for latency and RSS, CPU seconds per request once) through a small phase_budget module, replacing the machine-shaped absolutes. The Redis timeout rate is reported but no longer asserted The final /metrics scrape waits for litellm_deployment_failure_responses_total to stop moving, since that counter is bumped from the async logging queue and lagged the load generator by thousands of increments. The model group carries a unique marker so a deployment left behind by an aborted run cannot absorb this run's retries Co-Authored-By: Claude Code <noreply@anthropic.com>
71 lines
3 KiB
Python
71 lines
3 KiB
Python
from __future__ import annotations
|
|
|
|
from typing import Final
|
|
|
|
from proxy_usage import UsageSample, UsageWindow
|
|
|
|
_MB: Final = 2**20
|
|
|
|
|
|
def _window(*points: tuple[float, int, float]) -> UsageWindow:
|
|
return UsageWindow(
|
|
samples=tuple(
|
|
UsageSample(elapsed_seconds=elapsed, rss_bytes=rss, cpu_seconds=cpu) for elapsed, rss, cpu in points
|
|
)
|
|
)
|
|
|
|
|
|
class TestRssPercentiles:
|
|
def test_the_tail_percentiles_reach_the_peak_the_median_hides(self) -> None:
|
|
# 100 one-second samples: 89 flat, 10 elevated, 1 spike. The median stays flat, p90 sees the
|
|
# elevated plateau, and only the max reaches the spike.
|
|
window: Final = _window(
|
|
*((float(i), 100 * _MB, float(i)) for i in range(89)),
|
|
*((float(89 + i), 300 * _MB, float(89 + i)) for i in range(10)),
|
|
(99.0, 900 * _MB, 99.0),
|
|
)
|
|
|
|
assert window.rss_percentile(0.5) == 100 * _MB
|
|
assert window.rss_percentile(0.9) == 300 * _MB
|
|
assert window.rss_percentile(0.99) == 300 * _MB
|
|
assert window.rss_percentile(1.0) == 900 * _MB
|
|
|
|
def test_an_empty_window_reports_zero_rather_than_indexing_nothing(self) -> None:
|
|
assert _window().rss_percentile(0.5) == 0
|
|
|
|
|
|
class TestCpuUtilization:
|
|
def test_utilization_is_the_counter_delta_over_the_interval_not_the_counter_itself(self) -> None:
|
|
# The counter climbs 0.5 CPU seconds per second, then 4.0 per second: half a core, then four.
|
|
window: Final = _window((0.0, _MB, 0.0), (1.0, _MB, 0.5), (2.0, _MB, 1.0), (3.0, _MB, 5.0))
|
|
|
|
p50, p90, p99 = window.cpu_utilization_percentiles()
|
|
|
|
assert (p50, p90, p99) == (0.5, 4.0, 4.0)
|
|
assert window.cpu_seconds_consumed() == 5.0
|
|
|
|
def test_a_single_sample_has_no_interval_and_reports_zero(self) -> None:
|
|
window: Final = _window((0.0, _MB, 3.0))
|
|
|
|
assert window.cpu_utilization_percentiles() == (0.0, 0.0, 0.0)
|
|
assert window.cpu_seconds_consumed() == 0.0
|
|
|
|
def test_cost_per_request_separates_runs_that_cores_busy_reports_identically(self) -> None:
|
|
# Both windows pin 4 cores for 10 seconds, so utilization cannot tell them apart. The
|
|
# second one served a tenth of the traffic for the same CPU, which is the regression shape.
|
|
window: Final = _window(*((float(i), _MB, 4.0 * i) for i in range(11)))
|
|
|
|
assert window.cpu_utilization_percentiles()[0] == 4.0
|
|
assert window.cpu_seconds_per_request(4000) == 0.01
|
|
assert window.cpu_seconds_per_request(400) == 0.1
|
|
|
|
def test_no_requests_reports_zero_cost_rather_than_dividing_by_zero(self) -> None:
|
|
assert _window((0.0, _MB, 0.0), (1.0, _MB, 1.0)).cpu_seconds_per_request(0) == 0.0
|
|
|
|
def test_summary_reports_every_percentile_in_human_units(self) -> None:
|
|
window: Final = _window((0.0, 200 * _MB, 0.0), (1.0, 200 * _MB, 1.5), (2.0, 200 * _MB, 3.0))
|
|
|
|
assert window.summary() == (
|
|
"RSS p50 200 MB, p90 200 MB, p99 200 MB; "
|
|
"CPU cores busy p50 1.50, p90 1.50, p99 1.50; 3.0 CPU seconds consumed"
|
|
)
|