From f7ae9efad2a6db8d588a201f1d644d706944580e Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:47:14 -0700 Subject: [PATCH] test(e2e): hold every worker under an idle RSS budget before any traffic (#42552) * test(e2e): hold every worker under an idle RSS budget before any traffic The harness reads /debug/memory/summary on every replica once at collection time, right after the readiness gate and before this pytest process sends any traffic, and the memory suite's first test fails when any worker idles past E2E_MEMORY_IDLE_RSS_BUDGET_MB (768 MB by default) or gives no reading at all. A v1.100.x worker with a database idled at 836-886 MB where v1.101.0rc1 idled at 544 MB on the same database: prisma-client-py's default recursive type depth generated 91k TypedDict classes that v1.101.0's recursive_type_depth = -1 cut to 19k. The budget starts at the rc1 reading plus headroom. * test(e2e): read idle RSS only when the idle budget test is selected Gate the collection-time /debug/memory/summary read on a selected test using the idle_rss fixture and skip it under --collect-only, so sessions that never run the idle budget test pay no round trip. Drop the markerless unit test file the e2e guide bans and assert live that every configured replica was measured * test(e2e): take the idle RSS read after collection settles Read every replica's RSS from a tryfirst pytest_collection_finish hook so -k and -m deselection has already run, and only when a selected test still asks for the idle_rss fixture and the run is not --collect-only * test(e2e): record the heaviest idle RSS reading as junit properties * test(e2e): attach the idle RSS properties from the harness's setup hook --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- tests/e2e/AGENTS.md | 4 +- tests/e2e/conftest.py | 41 +++++++++- tests/e2e/coverage_registry/reliability.yaml | 1 + tests/e2e/e2e_config.py | 1 + tests/e2e/memory_readings.py | 78 +++++++++++++++++++ tests/e2e/proxy_client.py | 5 +- .../e2e/router/test_reliability_memory_e2e.py | 63 ++++++++++----- 7 files changed, 167 insertions(+), 26 deletions(-) create mode 100644 tests/e2e/memory_readings.py diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index ad1e0322787..6212a172fe3 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -43,7 +43,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `mcp/` - the MCP server surface over api_key auth against the real Datadog remote MCP server (see "MCP suite: real Datadog only" below); plus the gateway-managed OAuth (authorization_code) path exercised through `/chat/completions` in `test_mcp_chat_completion_oauth_e2e.py` and direct MCP protocol operations in `test_mcp_oauth_happy_path_e2e.py`, the one behavior Datadog's static-header auth cannot reach, seeding the per-user upstream token via the interactive authorize dance driven with the mcp SDK's own OAuth client (headless-browser consent from a saved session) and asserting the completion or protocol call lists and executes the server's tools with the stored per-user token - `logging/` - logging-integration delivery (datadog and friends) - `security/` - secret handling and log-leak protection -- `router/` - routing and reliability behavior (fallbacks, cooldowns) plus the memory regression test (`test_reliability_memory_e2e.py`: a few hundred failing requests with retries and fallbacks must not grow proxy RSS past a fixed budget nor store a request snapshot past a fixed size, the release-gate check for the v1.100.0 retry-breadcrumb leak) +- `router/` - routing and reliability behavior (fallbacks, cooldowns) plus the memory tests (`test_reliability_memory_e2e.py`: every worker's RSS as read at collection time, before any test traffic, must sit under a fixed idle budget, the release-gate check for a DB-backed boot that idles near the pod limit the way v1.100.x did; and a few hundred failing requests with retries and fallbacks must not grow proxy RSS past a fixed budget nor store a request snapshot past a fixed size, the release-gate check for the v1.100.0 retry-breadcrumb leak) - `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set and excluded from the per-PR selector like the rest of `load/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at; see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests @@ -191,7 +191,7 @@ reliability... behavior : fallback | retry | cooldown | timeout | routing | cache | circuit_breaker | perf variant : 5xx | context_window | content_policy | 429 | timeout simple_shuffle | usage_based | latency_based | cost_based | least_busy - latency | throughput | session_anomaly | memory (perf only; SLO/threshold assertion, not binary) + latency | throughput | session_anomaly | memory | idle_memory (perf only; SLO/threshold assertion, not binary) assertion : routes_to_fallback | succeeds_within_retries | picks_under_tpm | returns_cached | trips_then_recovers | under_slo e.g. reliability.fallback.context_window.routes_to_fallback exercised_on=[chat_completions] diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index a2398ef7c3c..f1f94f2d2b3 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -46,6 +46,7 @@ from fixture_mode import pytest_fixture_setup as pytest_fixture_setup from idp import Identity, Keycloak, keycloak_from_env from junit_properties import attach_result_properties from lifecycle import ProxyClientProvider, ResourceManager +from memory_readings import RssCapture, read_rss_everywhere from models import TeamNewBody, UserNewBody, UserNewResponse from provider_cache_routing import LIVE_PROVIDER_REQUIRED from provider_edge import replay_leftover_error @@ -53,6 +54,9 @@ from proxy_client import ProxyClient, build_proxy_client _E2E_TEST_RAN = pytest.StashKey[bool]() _CALL_PASSED = pytest.StashKey[bool]() +_IDLE_RSS = pytest.StashKey[RssCapture]() + +IDLE_RSS_READ_TIMEOUT_SECONDS: Final = 10.0 OPT_IN_MARKERS: Final = MappingProxyType( { @@ -186,6 +190,16 @@ def _needs_unset_opt_in(item: pytest.Item) -> bool: ) +def _reaches_proxy(item: pytest.Item) -> bool: + """True for a live test that talks to the shared proxy: `e2e`-marked and not a + `migration_startup` test, which boots its own container instead.""" + return item.get_closest_marker("e2e") is not None and item.get_closest_marker("migration_startup") is None + + +def _uses_idle_rss(item: pytest.Item) -> bool: + return isinstance(item, pytest.Function) and "idle_rss" in item.fixturenames + + def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: """Deselect every test behind an opt-in marker whose env var is unset (see OPT_IN_MARKERS): those tests need a proxy configured differently from the @@ -215,6 +229,20 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item items.sort(key=lambda item: item.get_closest_marker("load") is not None) +@pytest.hookimpl(tryfirst=True) +def pytest_collection_finish(session: pytest.Session) -> None: + """When a selected test asks for the `idle_rss` fixture and this is not a + `--collect-only` run, read every replica's RSS once, right here at the end of + collection and before this process sends any traffic. tryfirst keeps the read + ahead of xdist's own collection-finish report, and the controller schedules no + test until every worker has reported, so this is the idle footprint of a stack + that just passed its readiness gate. The fixture hands the capture to the + idle-budget test in router/test_reliability_memory_e2e.py.""" + if session.config.getoption("collectonly") or not any(_uses_idle_rss(item) for item in session.items): + return + session.config.stash[_IDLE_RSS] = read_rss_everywhere(build_proxy_client(), timeout=IDLE_RSS_READ_TIMEOUT_SECONDS) + + def _liveness_reason(label: str, base_url: str) -> str | None: """None if `base_url` answers its liveness probe, else a failure reason.""" try: @@ -246,7 +274,9 @@ def pytest_runtest_setup(item: pytest.Item) -> None: run even when none is up. Never skip for a missing proxy. Replay mode needs the proxy too: only provider-bound traffic replays from the bundle.""" LIVE_PROVIDER_REQUIRED.set(item.get_closest_marker("provider_live") is not None) - if item.get_closest_marker("e2e") is None or item.get_closest_marker("migration_startup") is not None: + if _uses_idle_rss(item): + item.user_properties.extend(item.config.stash[_IDLE_RSS].junit_properties) + if not _reaches_proxy(item): return if isinstance(item, pytest.Function) and "oauth_gateway" in item.fixturenames: return @@ -261,7 +291,7 @@ def pytest_runtest_call(item: pytest.Item) -> None: guard before truncating the spend-log DB. Tests under `tests/e2e/` without the `e2e` marker (pure unit coverage for the harness itself) never hit the proxy, so they must not arm the destructive DB truncate.""" - if item.get_closest_marker("e2e") is None or item.get_closest_marker("migration_startup") is not None: + if not _reaches_proxy(item): return item.session.stash[_E2E_TEST_RAN] = True @@ -325,6 +355,13 @@ def proxy() -> ProxyClient: return build_proxy_client() +@pytest.fixture(scope="session") +def idle_rss(request: pytest.FixtureRequest) -> RssCapture: + """Every replica's RSS as read once at the end of collection, before this process + sent any traffic (see pytest_collection_finish).""" + return request.config.stash[_IDLE_RSS] + + @pytest.fixture def resources(client: ProxyClientProvider) -> Iterator[ResourceManager]: """init -> run -> teardown: create a manager, run the test, release resources. diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml index 9f88f2478c0..5040f5f4dcf 100644 --- a/tests/e2e/coverage_registry/reliability.yaml +++ b/tests/e2e/coverage_registry/reliability.yaml @@ -38,4 +38,5 @@ - {id: reliability.timeout.stream_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: stream_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions], source: "litellm/router.py:551", rationale: "Streaming chunk-delivery timeout"} - {id: reliability.perf.throughput.under_slo, module: reliability, tier: P1, behavior: perf, variant: throughput, assertions: [under_slo], exercised_on: [chat_completions, messages], source: grammar, rationale: "Throughput SLO under load"} - {id: reliability.perf.memory.under_slo, module: reliability, tier: P1, behavior: perf, variant: memory, assertions: [under_slo], exercised_on: [chat_completions], source: grammar, rationale: "Proxy RSS and the stored request snapshot stay within fixed budgets across a few hundred failing requests with retries and fallbacks, the v1.100.0 retry-breadcrumb leak shape (MAT-335)"} +- {id: reliability.perf.idle_memory.under_slo, module: reliability, tier: P1, behavior: perf, variant: idle_memory, assertions: [under_slo], exercised_on: [], source: grammar, rationale: "Every worker's RSS as read right after the readiness gate and before any test traffic stays under a fixed idle budget; a DB-backed v1.100.x worker idled at 886 MB against a 2 GiB pod limit where v1.101.0rc1 idled at 544 MB"} - {id: reliability.perf.session_anomaly.under_slo, module: reliability, tier: P1, behavior: perf, variant: session_anomaly, assertions: [under_slo], exercised_on: [messages], source: grammar, rationale: "Weekly Claude Code-shaped multi-turn session load against real providers; ceilings on error rate, warm-turn cache read/write, p95 turn time, and gateway-recorded spend (LIT-4562)"} diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 77395a066ac..ac26b2a3875 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -173,6 +173,7 @@ MEMORY_CONCURRENCY = int(os.environ.get("E2E_MEMORY_CONCURRENCY", "4")) MEMORY_RSS_SETTLE_SAMPLES = int(os.environ.get("E2E_MEMORY_RSS_SETTLE_SAMPLES", "15")) MEMORY_RSS_SAMPLE_INTERVAL_SECONDS = float(os.environ.get("E2E_MEMORY_RSS_SAMPLE_INTERVAL_SECONDS", "1")) MEMORY_RSS_BUDGET_MB = float(os.environ.get("E2E_MEMORY_RSS_BUDGET_MB", "48")) +MEMORY_IDLE_RSS_BUDGET_MB = float(os.environ.get("E2E_MEMORY_IDLE_RSS_BUDGET_MB", "768")) MEMORY_STORED_REQUEST_BUDGET_KB = float(os.environ.get("E2E_MEMORY_STORED_REQUEST_BUDGET_KB", "64")) diff --git a/tests/e2e/memory_readings.py b/tests/e2e/memory_readings.py new file mode 100644 index 00000000000..37a27c455f8 --- /dev/null +++ b/tests/e2e/memory_readings.py @@ -0,0 +1,78 @@ +"""Per-worker RSS readings of the proxy through /debug/memory/summary. + +One read goes to every configured replica (PROXY_REPLICA_URLS) under the master +key and answers from whichever worker behind that address took the connection; +the release stack runs one worker per gateway replica, so a read per replica is +a read per worker. A reading keys its worker by replica address, hostname, and +pid, since pods in their own pid namespaces report the same pids. A replica that +gives no reading (unreachable, a non-2xx, or a summary without ram_usage_mb) is +kept as a failure reason rather than dropped, so a test can fail on it by name +instead of passing on the replicas that did answer. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final + +from e2e_http import Result, Success +from models import MemorySummaryResponse +from proxy_client import ProxyClient + +WorkerKey = tuple[str, str | None, int] + + +@dataclass(frozen=True, slots=True) +class RssReading: + replica: str + hostname: str | None + worker_pid: int + ram_usage_mb: float + + @property + def worker(self) -> WorkerKey: + return (self.replica, self.hostname, self.worker_pid) + + @property + def where(self) -> str: + return f"worker pid {self.worker_pid} on {self.hostname or 'an unnamed host'} behind {self.replica}" + + +@dataclass(frozen=True, slots=True) +class RssCapture: + readings: tuple[RssReading, ...] + failures: tuple[str, ...] + + @property + def heaviest(self) -> RssReading | None: + return max(self.readings, key=lambda reading: reading.ram_usage_mb, default=None) + + @property + def junit_properties(self) -> tuple[tuple[str, object], ...]: + heaviest: Final = self.heaviest + if heaviest is None: + return () + return (("idle_rss_heaviest_mb", heaviest.ram_usage_mb), ("idle_rss_heaviest_worker", heaviest.where)) + + +def _outcome(replica: str, result: Result[MemorySummaryResponse]) -> RssReading | str: + match result: + case Success(data=body) if body.memory.ram_usage_mb is not None: + return RssReading(replica, body.hostname, body.worker_pid, body.memory.ram_usage_mb) + case Success(data=body): + return f"{replica} answered /debug/memory/summary without ram_usage_mb: {body.memory.error}" + case _: + return f"{replica} gave no /debug/memory/summary reading: {result}" + + +def rss_capture(summaries: Mapping[str, Result[MemorySummaryResponse]]) -> RssCapture: + outcomes: Final = tuple(_outcome(replica, result) for replica, result in summaries.items()) + return RssCapture( + readings=tuple(outcome for outcome in outcomes if isinstance(outcome, RssReading)), + failures=tuple(outcome for outcome in outcomes if isinstance(outcome, str)), + ) + + +def read_rss_everywhere(proxy: ProxyClient, *, timeout: float | None = None) -> RssCapture: + return rss_capture(proxy.memory_summary_everywhere(timeout=timeout)) diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index 66df1c4c106..51ea9fbe7bd 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -508,13 +508,16 @@ class ProxyClient: ) ).info - def memory_summary_everywhere(self) -> Mapping[str, Result[MemorySummaryResponse]]: + def memory_summary_everywhere( + self, *, timeout: float | None = None + ) -> Mapping[str, Result[MemorySummaryResponse]]: return { url: transport.get( "/debug/memory/summary", headers=self.management_headers(transport=transport), params=NoBody(), response_type=MemorySummaryResponse, + timeout=timeout, ) for url, transport in self.replicas.items() } diff --git a/tests/e2e/router/test_reliability_memory_e2e.py b/tests/e2e/router/test_reliability_memory_e2e.py index 17e3e1a1996..2568434a08f 100644 --- a/tests/e2e/router/test_reliability_memory_e2e.py +++ b/tests/e2e/router/test_reliability_memory_e2e.py @@ -38,6 +38,20 @@ size budget: the deterministic catch for a breadcrumb that copies the whole request. It runs before the phases because the leaking writer drops its own rows under the phases' traffic (a queue budget hit, a recursion limit on the nested copies), which would turn the size check into a missing-row check. + +A second, cheaper check holds the idle footprint: every worker's RSS as the harness +read it at collection time, before this pytest process sent any traffic (see +conftest.pytest_collection_finish), must sit under a fixed budget. On the +release gate that is a fresh stack right after its readiness gate, one worker per +gateway replica, so the reading is what a DB-backed boot costs on its own. A +v1.100.x worker with a database idled at 886 MB RSS where v1.101.0rc1 idled at +544 MB on the same database (1.3 GB against 560 MB at the pod level, under a +2 GiB limit): the generated Prisma client at prisma-client-py's default recursive +type depth, 91k TypedDict classes that v1.101.0 cut to 19k with +recursive_type_depth = -1. The budget starts at the rc1 reading plus headroom and +E2E_MEMORY_IDLE_RSS_BUDGET_MB overrides it; a later session on the same stack (the +changed-files workflow's repeat passes, a developer's local loop) measures a proxy +already warmed by traffic, which that headroom also has to cover. """ from __future__ import annotations @@ -55,6 +69,7 @@ import pytest from complexity_router_client import ComplexityRouterClient from e2e_config import ( MEMORY_CONCURRENCY, + MEMORY_IDLE_RSS_BUDGET_MB, MEMORY_REQUESTS_PER_PHASE, MEMORY_RETRIES_PER_REQUEST, MEMORY_RSS_BUDGET_MB, @@ -62,10 +77,11 @@ from e2e_config import ( MEMORY_RSS_SETTLE_SAMPLES, MEMORY_STORED_REQUEST_BUDGET_KB, MEMORY_TRANSCRIPT_TURNS, + PROXY_REPLICA_URLS, unique_marker, ) -from e2e_http import unwrap from lifecycle import ResourceManager +from memory_readings import RssCapture, RssReading, WorkerKey, read_rss_everywhere from models import ChatMessage, RouterSettingsOverride, SpendLogRow from proxy_client import ProxyClient from reliability_support import chat_override, create_never_benched_refusing_deployment @@ -84,21 +100,6 @@ class FailedCall: call_id: str | None -WorkerKey = tuple[str, str | None, int] - - -@dataclass(frozen=True, slots=True) -class RssReading: - replica: str - hostname: str | None - worker_pid: int - ram_usage_mb: float - - @property - def worker(self) -> WorkerKey: - return (self.replica, self.hostname, self.worker_pid) - - @dataclass(frozen=True, slots=True) class WorkerGrowth: warm: RssReading @@ -143,12 +144,12 @@ def _fail_many(proxy: ProxyClient, key: str, model: str, override: RouterSetting def _read_rss_everywhere_after_pause(proxy: ProxyClient) -> tuple[RssReading, ...]: time.sleep(MEMORY_RSS_SAMPLE_INTERVAL_SECONDS) - return tuple( - RssReading(replica, body.hostname, body.worker_pid, body.memory.ram_usage_mb) - for replica, result in proxy.memory_summary_everywhere().items() - for body in (unwrap(result),) - if body.memory.ram_usage_mb is not None + capture: Final = read_rss_everywhere(proxy) + assert not capture.failures, ( + f"{len(capture.failures)} replica(s) gave no RSS reading mid-checkpoint, so their workers cannot be " + f"compared with themselves: {'; '.join(capture.failures)}" ) + return capture.readings def _readings_until_no_new_worker( @@ -220,6 +221,26 @@ def _stored_request_kb(proxy: ProxyClient, call: FailedCall) -> float: class TestReliabilityMemory: + @pytest.mark.covers("reliability.perf.idle_memory.under_slo") + def test_workers_idle_under_rss_budget_before_traffic(self, idle_rss: RssCapture) -> None: + assert not idle_rss.failures, ( + f"{len(idle_rss.failures)} replica(s) gave no RSS reading when the session started, so their idle " + f"footprint went unmeasured: {'; '.join(idle_rss.failures)}" + ) + unmeasured: Final = frozenset(PROXY_REPLICA_URLS) - frozenset(reading.replica for reading in idle_rss.readings) + assert not unmeasured, ( + f"{len(unmeasured)} of {len(PROXY_REPLICA_URLS)} replica(s) gave neither an RSS reading nor a failure " + f"reason when the session started, so their idle footprint went unmeasured: {', '.join(sorted(unmeasured))}" + ) + heaviest: Final = idle_rss.heaviest + assert heaviest is not None, "no replica was configured to read, so nothing was measured" + assert heaviest.ram_usage_mb <= MEMORY_IDLE_RSS_BUDGET_MB, ( + f"{heaviest.where} sat at {heaviest.ram_usage_mb:.0f} MB RSS when the session started, before it sent " + f"any traffic, past the {MEMORY_IDLE_RSS_BUDGET_MB:.0f} MB idle budget; a DB-backed v1.100.x worker idled " + f"at 886 MB where v1.101.0rc1 idled at 544 MB, and at that size the release stack's 2 GiB pod limit " + f"leaves the worker little room for traffic" + ) + @pytest.mark.covers("reliability.perf.memory.under_slo") def test_failing_requests_do_not_grow_rss_or_stored_request( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str