diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index ad1e0322787..6212a172fe3 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -43,7 +43,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `mcp/` - the MCP server surface over api_key auth against the real Datadog remote MCP server (see "MCP suite: real Datadog only" below); plus the gateway-managed OAuth (authorization_code) path exercised through `/chat/completions` in `test_mcp_chat_completion_oauth_e2e.py` and direct MCP protocol operations in `test_mcp_oauth_happy_path_e2e.py`, the one behavior Datadog's static-header auth cannot reach, seeding the per-user upstream token via the interactive authorize dance driven with the mcp SDK's own OAuth client (headless-browser consent from a saved session) and asserting the completion or protocol call lists and executes the server's tools with the stored per-user token - `logging/` - logging-integration delivery (datadog and friends) - `security/` - secret handling and log-leak protection -- `router/` - routing and reliability behavior (fallbacks, cooldowns) plus the memory regression test (`test_reliability_memory_e2e.py`: a few hundred failing requests with retries and fallbacks must not grow proxy RSS past a fixed budget nor store a request snapshot past a fixed size, the release-gate check for the v1.100.0 retry-breadcrumb leak) +- `router/` - routing and reliability behavior (fallbacks, cooldowns) plus the memory tests (`test_reliability_memory_e2e.py`: every worker's RSS as read at collection time, before any test traffic, must sit under a fixed idle budget, the release-gate check for a DB-backed boot that idles near the pod limit the way v1.100.x did; and a few hundred failing requests with retries and fallbacks must not grow proxy RSS past a fixed budget nor store a request snapshot past a fixed size, the release-gate check for the v1.100.0 retry-breadcrumb leak) - `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set and excluded from the per-PR selector like the rest of `load/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at; see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests @@ -191,7 +191,7 @@ reliability... behavior : fallback | retry | cooldown | timeout | routing | cache | circuit_breaker | perf variant : 5xx | context_window | content_policy | 429 | timeout simple_shuffle | usage_based | latency_based | cost_based | least_busy - latency | throughput | session_anomaly | memory (perf only; SLO/threshold assertion, not binary) + latency | throughput | session_anomaly | memory | idle_memory (perf only; SLO/threshold assertion, not binary) assertion : routes_to_fallback | succeeds_within_retries | picks_under_tpm | returns_cached | trips_then_recovers | under_slo e.g. reliability.fallback.context_window.routes_to_fallback exercised_on=[chat_completions] diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index a2398ef7c3c..f1f94f2d2b3 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -46,6 +46,7 @@ from fixture_mode import pytest_fixture_setup as pytest_fixture_setup from idp import Identity, Keycloak, keycloak_from_env from junit_properties import attach_result_properties from lifecycle import ProxyClientProvider, ResourceManager +from memory_readings import RssCapture, read_rss_everywhere from models import TeamNewBody, UserNewBody, UserNewResponse from provider_cache_routing import LIVE_PROVIDER_REQUIRED from provider_edge import replay_leftover_error @@ -53,6 +54,9 @@ from proxy_client import ProxyClient, build_proxy_client _E2E_TEST_RAN = pytest.StashKey[bool]() _CALL_PASSED = pytest.StashKey[bool]() +_IDLE_RSS = pytest.StashKey[RssCapture]() + +IDLE_RSS_READ_TIMEOUT_SECONDS: Final = 10.0 OPT_IN_MARKERS: Final = MappingProxyType( { @@ -186,6 +190,16 @@ def _needs_unset_opt_in(item: pytest.Item) -> bool: ) +def _reaches_proxy(item: pytest.Item) -> bool: + """True for a live test that talks to the shared proxy: `e2e`-marked and not a + `migration_startup` test, which boots its own container instead.""" + return item.get_closest_marker("e2e") is not None and item.get_closest_marker("migration_startup") is None + + +def _uses_idle_rss(item: pytest.Item) -> bool: + return isinstance(item, pytest.Function) and "idle_rss" in item.fixturenames + + def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: """Deselect every test behind an opt-in marker whose env var is unset (see OPT_IN_MARKERS): those tests need a proxy configured differently from the @@ -215,6 +229,20 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item items.sort(key=lambda item: item.get_closest_marker("load") is not None) +@pytest.hookimpl(tryfirst=True) +def pytest_collection_finish(session: pytest.Session) -> None: + """When a selected test asks for the `idle_rss` fixture and this is not a + `--collect-only` run, read every replica's RSS once, right here at the end of + collection and before this process sends any traffic. tryfirst keeps the read + ahead of xdist's own collection-finish report, and the controller schedules no + test until every worker has reported, so this is the idle footprint of a stack + that just passed its readiness gate. The fixture hands the capture to the + idle-budget test in router/test_reliability_memory_e2e.py.""" + if session.config.getoption("collectonly") or not any(_uses_idle_rss(item) for item in session.items): + return + session.config.stash[_IDLE_RSS] = read_rss_everywhere(build_proxy_client(), timeout=IDLE_RSS_READ_TIMEOUT_SECONDS) + + def _liveness_reason(label: str, base_url: str) -> str | None: """None if `base_url` answers its liveness probe, else a failure reason.""" try: @@ -246,7 +274,9 @@ def pytest_runtest_setup(item: pytest.Item) -> None: run even when none is up. Never skip for a missing proxy. Replay mode needs the proxy too: only provider-bound traffic replays from the bundle.""" LIVE_PROVIDER_REQUIRED.set(item.get_closest_marker("provider_live") is not None) - if item.get_closest_marker("e2e") is None or item.get_closest_marker("migration_startup") is not None: + if _uses_idle_rss(item): + item.user_properties.extend(item.config.stash[_IDLE_RSS].junit_properties) + if not _reaches_proxy(item): return if isinstance(item, pytest.Function) and "oauth_gateway" in item.fixturenames: return @@ -261,7 +291,7 @@ def pytest_runtest_call(item: pytest.Item) -> None: guard before truncating the spend-log DB. Tests under `tests/e2e/` without the `e2e` marker (pure unit coverage for the harness itself) never hit the proxy, so they must not arm the destructive DB truncate.""" - if item.get_closest_marker("e2e") is None or item.get_closest_marker("migration_startup") is not None: + if not _reaches_proxy(item): return item.session.stash[_E2E_TEST_RAN] = True @@ -325,6 +355,13 @@ def proxy() -> ProxyClient: return build_proxy_client() +@pytest.fixture(scope="session") +def idle_rss(request: pytest.FixtureRequest) -> RssCapture: + """Every replica's RSS as read once at the end of collection, before this process + sent any traffic (see pytest_collection_finish).""" + return request.config.stash[_IDLE_RSS] + + @pytest.fixture def resources(client: ProxyClientProvider) -> Iterator[ResourceManager]: """init -> run -> teardown: create a manager, run the test, release resources. diff --git a/tests/e2e/coverage_registry/reliability.yaml b/tests/e2e/coverage_registry/reliability.yaml index 9f88f2478c0..5040f5f4dcf 100644 --- a/tests/e2e/coverage_registry/reliability.yaml +++ b/tests/e2e/coverage_registry/reliability.yaml @@ -38,4 +38,5 @@ - {id: reliability.timeout.stream_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: stream_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions], source: "litellm/router.py:551", rationale: "Streaming chunk-delivery timeout"} - {id: reliability.perf.throughput.under_slo, module: reliability, tier: P1, behavior: perf, variant: throughput, assertions: [under_slo], exercised_on: [chat_completions, messages], source: grammar, rationale: "Throughput SLO under load"} - {id: reliability.perf.memory.under_slo, module: reliability, tier: P1, behavior: perf, variant: memory, assertions: [under_slo], exercised_on: [chat_completions], source: grammar, rationale: "Proxy RSS and the stored request snapshot stay within fixed budgets across a few hundred failing requests with retries and fallbacks, the v1.100.0 retry-breadcrumb leak shape (MAT-335)"} +- {id: reliability.perf.idle_memory.under_slo, module: reliability, tier: P1, behavior: perf, variant: idle_memory, assertions: [under_slo], exercised_on: [], source: grammar, rationale: "Every worker's RSS as read right after the readiness gate and before any test traffic stays under a fixed idle budget; a DB-backed v1.100.x worker idled at 886 MB against a 2 GiB pod limit where v1.101.0rc1 idled at 544 MB"} - {id: reliability.perf.session_anomaly.under_slo, module: reliability, tier: P1, behavior: perf, variant: session_anomaly, assertions: [under_slo], exercised_on: [messages], source: grammar, rationale: "Weekly Claude Code-shaped multi-turn session load against real providers; ceilings on error rate, warm-turn cache read/write, p95 turn time, and gateway-recorded spend (LIT-4562)"} diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 77395a066ac..ac26b2a3875 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -173,6 +173,7 @@ MEMORY_CONCURRENCY = int(os.environ.get("E2E_MEMORY_CONCURRENCY", "4")) MEMORY_RSS_SETTLE_SAMPLES = int(os.environ.get("E2E_MEMORY_RSS_SETTLE_SAMPLES", "15")) MEMORY_RSS_SAMPLE_INTERVAL_SECONDS = float(os.environ.get("E2E_MEMORY_RSS_SAMPLE_INTERVAL_SECONDS", "1")) MEMORY_RSS_BUDGET_MB = float(os.environ.get("E2E_MEMORY_RSS_BUDGET_MB", "48")) +MEMORY_IDLE_RSS_BUDGET_MB = float(os.environ.get("E2E_MEMORY_IDLE_RSS_BUDGET_MB", "768")) MEMORY_STORED_REQUEST_BUDGET_KB = float(os.environ.get("E2E_MEMORY_STORED_REQUEST_BUDGET_KB", "64")) diff --git a/tests/e2e/memory_readings.py b/tests/e2e/memory_readings.py new file mode 100644 index 00000000000..37a27c455f8 --- /dev/null +++ b/tests/e2e/memory_readings.py @@ -0,0 +1,78 @@ +"""Per-worker RSS readings of the proxy through /debug/memory/summary. + +One read goes to every configured replica (PROXY_REPLICA_URLS) under the master +key and answers from whichever worker behind that address took the connection; +the release stack runs one worker per gateway replica, so a read per replica is +a read per worker. A reading keys its worker by replica address, hostname, and +pid, since pods in their own pid namespaces report the same pids. A replica that +gives no reading (unreachable, a non-2xx, or a summary without ram_usage_mb) is +kept as a failure reason rather than dropped, so a test can fail on it by name +instead of passing on the replicas that did answer. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final + +from e2e_http import Result, Success +from models import MemorySummaryResponse +from proxy_client import ProxyClient + +WorkerKey = tuple[str, str | None, int] + + +@dataclass(frozen=True, slots=True) +class RssReading: + replica: str + hostname: str | None + worker_pid: int + ram_usage_mb: float + + @property + def worker(self) -> WorkerKey: + return (self.replica, self.hostname, self.worker_pid) + + @property + def where(self) -> str: + return f"worker pid {self.worker_pid} on {self.hostname or 'an unnamed host'} behind {self.replica}" + + +@dataclass(frozen=True, slots=True) +class RssCapture: + readings: tuple[RssReading, ...] + failures: tuple[str, ...] + + @property + def heaviest(self) -> RssReading | None: + return max(self.readings, key=lambda reading: reading.ram_usage_mb, default=None) + + @property + def junit_properties(self) -> tuple[tuple[str, object], ...]: + heaviest: Final = self.heaviest + if heaviest is None: + return () + return (("idle_rss_heaviest_mb", heaviest.ram_usage_mb), ("idle_rss_heaviest_worker", heaviest.where)) + + +def _outcome(replica: str, result: Result[MemorySummaryResponse]) -> RssReading | str: + match result: + case Success(data=body) if body.memory.ram_usage_mb is not None: + return RssReading(replica, body.hostname, body.worker_pid, body.memory.ram_usage_mb) + case Success(data=body): + return f"{replica} answered /debug/memory/summary without ram_usage_mb: {body.memory.error}" + case _: + return f"{replica} gave no /debug/memory/summary reading: {result}" + + +def rss_capture(summaries: Mapping[str, Result[MemorySummaryResponse]]) -> RssCapture: + outcomes: Final = tuple(_outcome(replica, result) for replica, result in summaries.items()) + return RssCapture( + readings=tuple(outcome for outcome in outcomes if isinstance(outcome, RssReading)), + failures=tuple(outcome for outcome in outcomes if isinstance(outcome, str)), + ) + + +def read_rss_everywhere(proxy: ProxyClient, *, timeout: float | None = None) -> RssCapture: + return rss_capture(proxy.memory_summary_everywhere(timeout=timeout)) diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index 66df1c4c106..51ea9fbe7bd 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -508,13 +508,16 @@ class ProxyClient: ) ).info - def memory_summary_everywhere(self) -> Mapping[str, Result[MemorySummaryResponse]]: + def memory_summary_everywhere( + self, *, timeout: float | None = None + ) -> Mapping[str, Result[MemorySummaryResponse]]: return { url: transport.get( "/debug/memory/summary", headers=self.management_headers(transport=transport), params=NoBody(), response_type=MemorySummaryResponse, + timeout=timeout, ) for url, transport in self.replicas.items() } diff --git a/tests/e2e/router/test_reliability_memory_e2e.py b/tests/e2e/router/test_reliability_memory_e2e.py index 17e3e1a1996..2568434a08f 100644 --- a/tests/e2e/router/test_reliability_memory_e2e.py +++ b/tests/e2e/router/test_reliability_memory_e2e.py @@ -38,6 +38,20 @@ size budget: the deterministic catch for a breadcrumb that copies the whole request. It runs before the phases because the leaking writer drops its own rows under the phases' traffic (a queue budget hit, a recursion limit on the nested copies), which would turn the size check into a missing-row check. + +A second, cheaper check holds the idle footprint: every worker's RSS as the harness +read it at collection time, before this pytest process sent any traffic (see +conftest.pytest_collection_finish), must sit under a fixed budget. On the +release gate that is a fresh stack right after its readiness gate, one worker per +gateway replica, so the reading is what a DB-backed boot costs on its own. A +v1.100.x worker with a database idled at 886 MB RSS where v1.101.0rc1 idled at +544 MB on the same database (1.3 GB against 560 MB at the pod level, under a +2 GiB limit): the generated Prisma client at prisma-client-py's default recursive +type depth, 91k TypedDict classes that v1.101.0 cut to 19k with +recursive_type_depth = -1. The budget starts at the rc1 reading plus headroom and +E2E_MEMORY_IDLE_RSS_BUDGET_MB overrides it; a later session on the same stack (the +changed-files workflow's repeat passes, a developer's local loop) measures a proxy +already warmed by traffic, which that headroom also has to cover. """ from __future__ import annotations @@ -55,6 +69,7 @@ import pytest from complexity_router_client import ComplexityRouterClient from e2e_config import ( MEMORY_CONCURRENCY, + MEMORY_IDLE_RSS_BUDGET_MB, MEMORY_REQUESTS_PER_PHASE, MEMORY_RETRIES_PER_REQUEST, MEMORY_RSS_BUDGET_MB, @@ -62,10 +77,11 @@ from e2e_config import ( MEMORY_RSS_SETTLE_SAMPLES, MEMORY_STORED_REQUEST_BUDGET_KB, MEMORY_TRANSCRIPT_TURNS, + PROXY_REPLICA_URLS, unique_marker, ) -from e2e_http import unwrap from lifecycle import ResourceManager +from memory_readings import RssCapture, RssReading, WorkerKey, read_rss_everywhere from models import ChatMessage, RouterSettingsOverride, SpendLogRow from proxy_client import ProxyClient from reliability_support import chat_override, create_never_benched_refusing_deployment @@ -84,21 +100,6 @@ class FailedCall: call_id: str | None -WorkerKey = tuple[str, str | None, int] - - -@dataclass(frozen=True, slots=True) -class RssReading: - replica: str - hostname: str | None - worker_pid: int - ram_usage_mb: float - - @property - def worker(self) -> WorkerKey: - return (self.replica, self.hostname, self.worker_pid) - - @dataclass(frozen=True, slots=True) class WorkerGrowth: warm: RssReading @@ -143,12 +144,12 @@ def _fail_many(proxy: ProxyClient, key: str, model: str, override: RouterSetting def _read_rss_everywhere_after_pause(proxy: ProxyClient) -> tuple[RssReading, ...]: time.sleep(MEMORY_RSS_SAMPLE_INTERVAL_SECONDS) - return tuple( - RssReading(replica, body.hostname, body.worker_pid, body.memory.ram_usage_mb) - for replica, result in proxy.memory_summary_everywhere().items() - for body in (unwrap(result),) - if body.memory.ram_usage_mb is not None + capture: Final = read_rss_everywhere(proxy) + assert not capture.failures, ( + f"{len(capture.failures)} replica(s) gave no RSS reading mid-checkpoint, so their workers cannot be " + f"compared with themselves: {'; '.join(capture.failures)}" ) + return capture.readings def _readings_until_no_new_worker( @@ -220,6 +221,26 @@ def _stored_request_kb(proxy: ProxyClient, call: FailedCall) -> float: class TestReliabilityMemory: + @pytest.mark.covers("reliability.perf.idle_memory.under_slo") + def test_workers_idle_under_rss_budget_before_traffic(self, idle_rss: RssCapture) -> None: + assert not idle_rss.failures, ( + f"{len(idle_rss.failures)} replica(s) gave no RSS reading when the session started, so their idle " + f"footprint went unmeasured: {'; '.join(idle_rss.failures)}" + ) + unmeasured: Final = frozenset(PROXY_REPLICA_URLS) - frozenset(reading.replica for reading in idle_rss.readings) + assert not unmeasured, ( + f"{len(unmeasured)} of {len(PROXY_REPLICA_URLS)} replica(s) gave neither an RSS reading nor a failure " + f"reason when the session started, so their idle footprint went unmeasured: {', '.join(sorted(unmeasured))}" + ) + heaviest: Final = idle_rss.heaviest + assert heaviest is not None, "no replica was configured to read, so nothing was measured" + assert heaviest.ram_usage_mb <= MEMORY_IDLE_RSS_BUDGET_MB, ( + f"{heaviest.where} sat at {heaviest.ram_usage_mb:.0f} MB RSS when the session started, before it sent " + f"any traffic, past the {MEMORY_IDLE_RSS_BUDGET_MB:.0f} MB idle budget; a DB-backed v1.100.x worker idled " + f"at 886 MB where v1.101.0rc1 idled at 544 MB, and at that size the release stack's 2 GiB pod limit " + f"leaves the worker little room for traffic" + ) + @pytest.mark.covers("reliability.perf.memory.under_slo") def test_failing_requests_do_not_grow_rss_or_stored_request( self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str