diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 0e6ddddf897..f362d4cc6e5 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -1273,19 +1273,12 @@ class TagListResponse(RootModel[list[TagListEntry]]): class ProcessMemory(BaseModel): - """The `memory` block of GET /debug/memory/summary: the serving worker's resident - set in MB, or `error` when the proxy has no psutil to read it with.""" - ram_usage_mb: float | None = None system_memory_percent: float | None = None error: str | None = None class MemorySummaryResponse(BaseModel): - """GET /debug/memory/summary (master key). One worker's resident memory, keyed by - its hostname (the pod name on Kubernetes) and pid so readings behind a load - balancer can be told apart per worker; older proxies omit the hostname.""" - worker_pid: int hostname: str | None = None status: str diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index b7865667cd5..2e0a23ce58b 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -469,10 +469,6 @@ class ProxyClient: ).info def memory_summary_everywhere(self) -> Mapping[str, Result[MemorySummaryResponse]]: - """GET /debug/memory/summary under the master key on every replica in - PROXY_REPLICA_URLS (the data-plane URL alone when the stack exports no - per-gateway addresses). Each read reports the pid of the worker that answered, - so a single address in front of several pods still tells its readings apart.""" return { url: transport.get( "/debug/memory/summary", diff --git a/tests/e2e/router/reliability_support.py b/tests/e2e/router/reliability_support.py index af1ed1ec64c..df2ff03aa4a 100644 --- a/tests/e2e/router/reliability_support.py +++ b/tests/e2e/router/reliability_support.py @@ -52,8 +52,6 @@ def create_bad_base_deployment(proxy: ProxyClient, name: str) -> str: def create_never_benched_refusing_deployment(proxy: ProxyClient, name: str) -> str: - """cooldown_time 0 keeps the router retrying this deployment instead of benching it - after allowed_fails, which would skip the retry loop the memory test measures.""" return proxy.create_model( name, LiteLLMParamsBody(model=REAL_MODEL, api_key=REAL_KEY, api_base="http://127.0.0.1:9/v1", cooldown_time=0), diff --git a/tests/e2e/router/test_reliability_memory_e2e.py b/tests/e2e/router/test_reliability_memory_e2e.py index c343a59d529..17e3e1a1996 100644 --- a/tests/e2e/router/test_reliability_memory_e2e.py +++ b/tests/e2e/router/test_reliability_memory_e2e.py @@ -219,44 +219,45 @@ def _stored_request_kb(proxy: ProxyClient, call: FailedCall) -> float: return len(json.dumps(snapshot).encode()) / 1024 -@pytest.mark.covers("reliability.perf.memory.under_slo") -def test_failing_requests_do_not_grow_rss_or_stored_request( - client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str -) -> None: - marker: Final = unique_marker() - primary: Final = f"reliability-memory-{marker}" - fallback: Final = f"reliability-memory-fb-{marker}" - _register_refusing_group(client.proxy, resources, primary) - _register_refusing_group(client.proxy, resources, fallback) - override: Final = RouterSettingsOverride( - num_retries=MEMORY_RETRIES_PER_REQUEST, fallbacks=[{primary: [fallback]}] - ) +class TestReliabilityMemory: + @pytest.mark.covers("reliability.perf.memory.under_slo") + def test_failing_requests_do_not_grow_rss_or_stored_request( + self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str + ) -> None: + marker: Final = unique_marker() + primary: Final = f"reliability-memory-{marker}" + fallback: Final = f"reliability-memory-fb-{marker}" + _register_refusing_group(client.proxy, resources, primary) + _register_refusing_group(client.proxy, resources, fallback) + override: Final = RouterSettingsOverride( + num_retries=MEMORY_RETRIES_PER_REQUEST, fallbacks=[{primary: [fallback]}] + ) - probe: Final = _fail_once(client.proxy, scoped_key, primary, override) - _assert_every_call_failed_through_fallback((probe,), fallback) - stored_kb: Final = _stored_request_kb(client.proxy, probe) - assert stored_kb <= MEMORY_STORED_REQUEST_BUDGET_KB, ( - f"the spend log of one failing request stored a {stored_kb:.0f} KB request body, past the " - f"{MEMORY_STORED_REQUEST_BUDGET_KB:.0f} KB budget for a {len(TRANSCRIPT)}-message transcript with " - f"{MEMORY_RETRIES_PER_REQUEST} retries and a fallback; the retry breadcrumbs are copying the whole " - f"request into the stored snapshot the way the v1.100.0 ones did" - ) + probe: Final = _fail_once(client.proxy, scoped_key, primary, override) + _assert_every_call_failed_through_fallback((probe,), fallback) + stored_kb: Final = _stored_request_kb(client.proxy, probe) + assert stored_kb <= MEMORY_STORED_REQUEST_BUDGET_KB, ( + f"the spend log of one failing request stored a {stored_kb:.0f} KB request body, past the " + f"{MEMORY_STORED_REQUEST_BUDGET_KB:.0f} KB budget for a {len(TRANSCRIPT)}-message transcript with " + f"{MEMORY_RETRIES_PER_REQUEST} retries and a fallback; the retry breadcrumbs are copying the whole " + f"request into the stored snapshot the way the v1.100.0 ones did" + ) - warmup: Final = _fail_many(client.proxy, scoped_key, primary, override) - _assert_every_call_failed_through_fallback(warmup, fallback) - warm: Final = _settled_rss_per_worker(client.proxy) + warmup: Final = _fail_many(client.proxy, scoped_key, primary, override) + _assert_every_call_failed_through_fallback(warmup, fallback) + warm: Final = _settled_rss_per_worker(client.proxy) - measured: Final = _fail_many(client.proxy, scoped_key, primary, override) - _assert_every_call_failed_through_fallback(measured, fallback) - after: Final = _settled_rss_per_worker(client.proxy) + measured: Final = _fail_many(client.proxy, scoped_key, primary, override) + _assert_every_call_failed_through_fallback(measured, fallback) + after: Final = _settled_rss_per_worker(client.proxy) - heaviest: Final = _heaviest_worker_growth(warm, after) - assert heaviest.growth_mb <= MEMORY_RSS_BUDGET_MB, ( - f"proxy RSS grew {heaviest.growth_mb:.1f} MB over a second batch of {MEMORY_REQUESTS_PER_PHASE} failing " - f"requests ({MEMORY_RETRIES_PER_REQUEST} retries each plus a fallback) after an identical warmup batch, " - f"past the {MEMORY_RSS_BUDGET_MB:.0f} MB budget: worker pid {heaviest.warm.worker_pid} on " - f"{heaviest.warm.hostname or 'an unnamed host'} behind {heaviest.warm.replica} settled at " - f"{heaviest.warm.ram_usage_mb:.1f} MB warm and " - f"{heaviest.after.ram_usage_mb:.1f} MB after; failing requests are leaking memory the way the " - f"v1.100.0 retry breadcrumbs did" - ) + heaviest: Final = _heaviest_worker_growth(warm, after) + assert heaviest.growth_mb <= MEMORY_RSS_BUDGET_MB, ( + f"proxy RSS grew {heaviest.growth_mb:.1f} MB over a second batch of {MEMORY_REQUESTS_PER_PHASE} failing " + f"requests ({MEMORY_RETRIES_PER_REQUEST} retries each plus a fallback) after an identical warmup batch, " + f"past the {MEMORY_RSS_BUDGET_MB:.0f} MB budget: worker pid {heaviest.warm.worker_pid} on " + f"{heaviest.warm.hostname or 'an unnamed host'} behind {heaviest.warm.replica} settled at " + f"{heaviest.warm.ram_usage_mb:.1f} MB warm and " + f"{heaviest.after.ram_usage_mb:.1f} MB after; failing requests are leaking memory the way the " + f"v1.100.0 retry breadcrumbs did" + )