test(e2e): memory regression test for failing requests on the release gate

tests/e2e/CLAUDE.md

@@ -17,7 +17,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family
 - `mcp/` - the MCP server surface over api_key auth against the real Datadog remote MCP server (see "MCP suite: real Datadog only" below); plus the gateway-managed OAuth (authorization_code) path exercised through `/chat/completions`, the one behavior Datadog's static-header auth cannot reach, seeding the per-user upstream token via the interactive authorize dance driven with the mcp SDK's own OAuth client (headless-browser consent from a saved session) and asserting the completion lists and executes the server's tools with the stored per-user token
 - `logging/` - logging-integration delivery (datadog and friends)
 - `security/` - secret handling and log-leak protection
-- `router/` - routing and reliability behavior (fallbacks, cooldowns)
+- `router/` - routing and reliability behavior (fallbacks, cooldowns) plus the memory regression test (`test_reliability_memory_e2e.py`: a few hundred failing requests with retries and fallbacks must not grow proxy RSS past a fixed budget nor store a request snapshot past a fixed size, the release-gate check for the v1.100.0 retry-breadcrumb leak)
 - `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What remains here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`) and markerless harness unit tests for the Locust/session-anomaly aggregation logic
 - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite
 - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests
@@ -163,7 +163,7 @@ reliability.<behavior>.<variant>.<assertion>
   behavior  : fallback | retry | cooldown | timeout | routing | cache | circuit_breaker | perf
   variant   : <trigger>   5xx | context_window | content_policy | 429 | timeout
               <strategy>  simple_shuffle | usage_based | latency_based | cost_based | least_busy
-              <dimension> latency | throughput | session_anomaly   (perf only; SLO/threshold assertion, not binary)
+              <dimension> latency | throughput | session_anomaly | memory   (perf only; SLO/threshold assertion, not binary)
   assertion : routes_to_fallback | succeeds_within_retries | picks_under_tpm | returns_cached
               | trips_then_recovers | under_slo
   e.g.  reliability.fallback.context_window.routes_to_fallback     exercised_on=[chat_completions]

tests/e2e/coverage_registry/reliability.yaml

@@ -34,4 +34,5 @@
 - {id: reliability.timeout.request_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: request_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions, messages], source: "litellm/router.py:545-551", rationale: "Per-request timeout raises Timeout"}
 - {id: reliability.timeout.stream_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: stream_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions], source: "litellm/router.py:551", rationale: "Streaming chunk-delivery timeout"}
 - {id: reliability.perf.throughput.under_slo, module: reliability, tier: P1, behavior: perf, variant: throughput, assertions: [under_slo], exercised_on: [chat_completions, messages], source: grammar, rationale: "Throughput SLO under load"}
+- {id: reliability.perf.memory.under_slo, module: reliability, tier: P1, behavior: perf, variant: memory, assertions: [under_slo], exercised_on: [chat_completions], source: grammar, rationale: "Proxy RSS and the stored request snapshot stay within fixed budgets across a few hundred failing requests with retries and fallbacks, the v1.100.0 retry-breadcrumb leak shape (MAT-335)"}
 - {id: reliability.perf.session_anomaly.under_slo, module: reliability, tier: P1, behavior: perf, variant: session_anomaly, assertions: [under_slo], exercised_on: [messages], source: grammar, rationale: "Weekly Claude Code-shaped multi-turn session load against real providers; ceilings on error rate, warm-turn cache read/write, p95 turn time, and gateway-recorded spend (LIT-4562)"}

tests/e2e/e2e_config.py

@@ -160,6 +160,14 @@ ANOMALY_MAX_KEY_SPEND_USD = float(
 ANOMALY_SPEND_SETTLE_SECONDS = float(
     os.environ.get("E2E_ANOMALY_SPEND_SETTLE_SECONDS", "75")
 )
+MEMORY_REQUESTS_PER_PHASE = int(os.environ.get("E2E_MEMORY_REQUESTS_PER_PHASE", "300"))
+MEMORY_RETRIES_PER_REQUEST = int(os.environ.get("E2E_MEMORY_RETRIES_PER_REQUEST", "2"))
+MEMORY_TRANSCRIPT_TURNS = int(os.environ.get("E2E_MEMORY_TRANSCRIPT_TURNS", "40"))
+MEMORY_CONCURRENCY = int(os.environ.get("E2E_MEMORY_CONCURRENCY", "4"))
+MEMORY_RSS_SETTLE_SAMPLES = int(os.environ.get("E2E_MEMORY_RSS_SETTLE_SAMPLES", "15"))
+MEMORY_RSS_SAMPLE_INTERVAL_SECONDS = float(os.environ.get("E2E_MEMORY_RSS_SAMPLE_INTERVAL_SECONDS", "1"))
+MEMORY_RSS_BUDGET_MB = float(os.environ.get("E2E_MEMORY_RSS_BUDGET_MB", "48"))
+MEMORY_STORED_REQUEST_BUDGET_KB = float(os.environ.get("E2E_MEMORY_STORED_REQUEST_BUDGET_KB", "64"))
 
 
 def ws_base_url() -> str:

tests/e2e/models.py

@@ -11,7 +11,16 @@ from datetime import datetime
 from typing import Final, Literal
 
 from e2e_http import PartialBody
-from pydantic import AliasChoices, BaseModel, ConfigDict, Field, RootModel, model_serializer, model_validator
+from pydantic import (
+    AliasChoices,
+    BaseModel,
+    ConfigDict,
+    Field,
+    JsonValue,
+    RootModel,
+    model_serializer,
+    model_validator,
+)
 
 # ---------- keys ----------
 
@@ -695,6 +704,7 @@ class SpendLogRow(BaseModel):
     total_tokens: int | None = None
     request_tags: list[str] | None = None
     metadata: SpendLogMetadata | None = None
+    proxy_server_request: JsonValue = None
 
 
 class SpendLogs(RootModel[list[SpendLogRow]]):
@@ -926,6 +936,7 @@ class LiteLLMParamsBody(BaseModel):
     timeout: float | None = None
     tpm: int | None = None
     weight: int | None = None
+    cooldown_time: float | None = None
 
 
 ModelMode = Literal["batch", "realtime", "image_generation"]
@@ -1260,6 +1271,24 @@ class TagListResponse(RootModel[list[TagListEntry]]):
 # ---------- health / lifecycle ----------
 
 
+class ProcessMemory(BaseModel):
+    """The `memory` block of GET /debug/memory/summary: the serving worker's resident
+    set in MB, or `error` when the proxy has no psutil to read it with."""
+
+    ram_usage_mb: float | None = None
+    system_memory_percent: float | None = None
+    error: str | None = None
+
+
+class MemorySummaryResponse(BaseModel):
+    """GET /debug/memory/summary (master key). One worker's resident memory, keyed by
+    its pid so readings behind a load balancer can be told apart per pod."""
+
+    worker_pid: int
+    status: str
+    memory: ProcessMemory
+
+
 class ReadinessResponse(BaseModel):
     """GET /health/readiness (public probe). The low-detail payload a load
     balancer sees: `status` plus the resolved DB state (`connected`,

tests/e2e/proxy_client.py

@@ -63,6 +63,7 @@ from models import (
     ModelNewBody,
     ModelNewResponse,
     ModelsListParams,
+    MemorySummaryResponse,
     ModelsListResponse,
     ModelUpdateBody,
     OcrBody,
@@ -467,6 +468,21 @@ class ProxyClient:
             )
         ).info
 
+    def memory_summary_everywhere(self) -> Mapping[str, Result[MemorySummaryResponse]]:
+        """GET /debug/memory/summary under the master key on every replica in
+        PROXY_REPLICA_URLS (the data-plane URL alone when the stack exports no
+        per-gateway addresses). Each read reports the pid of the worker that answered,
+        so a single address in front of several pods still tells its readings apart."""
+        return {
+            url: transport.get(
+                "/debug/memory/summary",
+                headers=transport.master,
+                params=NoBody(),
+                response_type=MemorySummaryResponse,
+            )
+            for url, transport in self.replicas.items()
+        }
+
     def read_back_everywhere[R: BaseModel](
         self,
         path: str,

tests/e2e/router/reliability_support.py

@@ -11,6 +11,8 @@ body, so a single long-lived proxy serves every reliability behavior.
 
 from __future__ import annotations
 
+from collections.abc import Sequence
+
 from pydantic import ValidationError
 
 from proxy_client import ProxyClient
@@ -49,6 +51,16 @@ def create_bad_base_deployment(proxy: ProxyClient, name: str) -> str:
     )
 
 
+def create_never_benched_refusing_deployment(proxy: ProxyClient, name: str) -> str:
+    """Register a deployment that refuses every call at the socket and opts out of the
+    stack's cooldown policy (cooldown_time 0), so the router keeps retrying it for the
+    whole run instead of benching it after allowed_fails and skipping the retry loop."""
+    return proxy.create_model(
+        name,
+        LiteLLMParamsBody(model=REAL_MODEL, api_key=REAL_KEY, api_base="http://127.0.0.1:9/v1", cooldown_time=0),
+    )
+
+
 def create_timeout_deployment(proxy: ProxyClient, name: str) -> str:
     """Register a deployment with a 1ms deadline the real backend always exceeds."""
     return proxy.create_model(name, LiteLLMParamsBody(model=REAL_MODEL, api_key=REAL_KEY, timeout=0.001))
@@ -109,15 +121,17 @@ def chat_override(
     override: RouterSettingsOverride | None = None,
     stream: bool = False,
     cache: dict[str, bool] | None = {"no-cache": True},
+    history: Sequence[ChatMessage] = (),
 ) -> StreamingResponse:
     """POST /chat/completions with an optional per-request router_settings_override,
-    returning the raw outcome so tests read status, body, and reliability headers."""
+    returning the raw outcome so tests read status, body, and reliability headers.
+    `history` is the conversation sent ahead of the user turn carrying `content`."""
     return proxy.transport.send(
         "/chat/completions",
         headers=proxy.transport.bearer(key),
         json=ReliabilityChatBody(
             model=model,
-            messages=[ChatMessage(role="user", content=content)],
+            messages=[*history, ChatMessage(role="user", content=content)],
             max_tokens=512,
             stream=stream,
             router_settings_override=override,

tests/e2e/router/test_reliability_memory_e2e.py

@@ -0,0 +1,233 @@
+"""Live e2e: a few hundred requests that fail before any provider answers must not
+grow the proxy's resident memory past a fixed budget once the proxy is warm.
+
+The regression this guards shipped in v1.100.0: every retry breadcrumb copied the
+whole request and the copies nested into one router-global list, so a proxy under
+retry-heavy failing traffic grew until it was OOM-killed. The traffic here has that
+shape: a model group whose deployments refuse at the socket (an unreachable base
+URL) with cooldown_time 0 so the router keeps retrying them, per-request retries,
+and a fallback group that refuses the same way, each request carrying a long chat
+transcript so every whole-request copy costs hundreds of containers instead of a
+handful. Under the stack's cooldown policy a
+deployment that fails a handful of times in a row is benched (a bad-credential 401
+included), the router answers "No deployments available" without retrying, and the
+retry loop that leaks stops running; cooldown_time 0 keeps it running.
+
+Two identical phases run back to back. The first is the warmup that grows the
+proxy's caches and allocator arenas to their steady state, the second is the one
+the budget applies to, so a healthy proxy shows the second phase adding roughly
+nothing while a leaking one adds a fixed amount per request. RSS is read through
+/debug/memory/summary on every configured replica; a burst of failing calls leaves
+a transient bulge of garbage that gc reclaims within seconds, so each checkpoint
+samples for a settle window and keeps the lowest reading per worker, and the growth
+is judged per worker (by pid) so a stack serving one address from several pods
+compares each pod with itself.
+
+RSS alone is a coarse gauge: on the release stack (spend logs storing prompts,
+json logs, prometheus and otel callbacks) the same v1.100.0 breadcrumbs grew RSS
+by only about 15 MB per 300 failing requests, while every failing request's stored
+request snapshot carried a copy of the request per failed attempt, over 100 KB on
+the first call and a couple of MB once the copies nested, against tens of KB with
+the fix. So the first check sends one failing request before the phases, reads its
+spend log back through /spend/logs, and holds the stored request body to a fixed
+size budget: the deterministic catch for a breadcrumb that copies the whole
+request. It runs before the phases because the leaking writer drops its own rows
+under the phases' traffic (a queue budget hit, a recursion limit on the nested
+copies), which would turn the size check into a missing-row check.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from collections.abc import Mapping, Sequence
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import Final
+
+import pytest
+
+from complexity_router_client import ComplexityRouterClient
+from e2e_config import (
+    MEMORY_CONCURRENCY,
+    MEMORY_REQUESTS_PER_PHASE,
+    MEMORY_RETRIES_PER_REQUEST,
+    MEMORY_RSS_BUDGET_MB,
+    MEMORY_RSS_SAMPLE_INTERVAL_SECONDS,
+    MEMORY_RSS_SETTLE_SAMPLES,
+    MEMORY_STORED_REQUEST_BUDGET_KB,
+    MEMORY_TRANSCRIPT_TURNS,
+    unique_marker,
+)
+from e2e_http import unwrap
+from lifecycle import ResourceManager
+from models import ChatMessage, RouterSettingsOverride, SpendLogRow
+from proxy_client import ProxyClient
+from reliability_support import chat_override, create_never_benched_refusing_deployment
+
+pytestmark = pytest.mark.e2e
+
+DEPLOYMENTS_PER_GROUP: Final = 2
+
+
+@dataclass(frozen=True, slots=True)
+class FailedCall:
+    status_code: int
+    seconds: float
+    body_head: str
+    call_id: str | None
+
+
+@dataclass(frozen=True, slots=True)
+class RssReading:
+    replica: str
+    worker_pid: int
+    ram_usage_mb: float
+
+
+@dataclass(frozen=True, slots=True)
+class WorkerGrowth:
+    warm: RssReading
+    after: RssReading
+
+    @property
+    def growth_mb(self) -> float:
+        return self.after.ram_usage_mb - self.warm.ram_usage_mb
+
+
+def _register_refusing_group(proxy: ProxyClient, resources: ResourceManager, name: str) -> None:
+    for model_id in tuple(create_never_benched_refusing_deployment(proxy, name) for _ in range(DEPLOYMENTS_PER_GROUP)):
+        resources.defer(lambda model_id=model_id: proxy.delete_model(model_id))
+
+
+def _transcript(turns: int) -> tuple[ChatMessage, ...]:
+    return tuple(
+        ChatMessage(role=role, content=f"turn {turn} {role}")
+        for turn in range(turns)
+        for role in ("user", "assistant")
+    )
+
+
+TRANSCRIPT: Final = _transcript(MEMORY_TRANSCRIPT_TURNS)
+
+
+def _fail_once(proxy: ProxyClient, key: str, model: str, override: RouterSettingsOverride) -> FailedCall:
+    started: Final = time.perf_counter()
+    resp: Final = chat_override(
+        proxy, key, model, f"memory regression {unique_marker()}", override=override, history=TRANSCRIPT
+    )
+    return FailedCall(resp.status_code, time.perf_counter() - started, resp.body[:300], resp.call_id)
+
+
+def _fail_many(proxy: ProxyClient, key: str, model: str, override: RouterSettingsOverride) -> tuple[FailedCall, ...]:
+    with ThreadPoolExecutor(max_workers=MEMORY_CONCURRENCY) as pool:
+        futures: Final = tuple(
+            pool.submit(_fail_once, proxy, key, model, override) for _ in range(MEMORY_REQUESTS_PER_PHASE)
+        )
+        return tuple(future.result() for future in futures)
+
+
+def _read_rss_everywhere_after_pause(proxy: ProxyClient) -> tuple[RssReading, ...]:
+    time.sleep(MEMORY_RSS_SAMPLE_INTERVAL_SECONDS)
+    return tuple(
+        RssReading(replica, body.worker_pid, body.memory.ram_usage_mb)
+        for replica, result in proxy.memory_summary_everywhere().items()
+        for body in (unwrap(result),)
+        if body.memory.ram_usage_mb is not None
+    )
+
+
+def _settled_rss_per_worker(proxy: ProxyClient) -> Mapping[int, RssReading]:
+    readings: Final = tuple(
+        reading for _ in range(MEMORY_RSS_SETTLE_SAMPLES) for reading in _read_rss_everywhere_after_pause(proxy)
+    )
+    assert readings, "no /debug/memory/summary read carried ram_usage_mb, so the proxy cannot report its RSS"
+    return MappingProxyType(
+        {
+            pid: min((reading for reading in readings if reading.worker_pid == pid), key=lambda r: r.ram_usage_mb)
+            for pid in {reading.worker_pid for reading in readings}
+        }
+    )
+
+
+def _heaviest_worker_growth(warm: Mapping[int, RssReading], after: Mapping[int, RssReading]) -> WorkerGrowth:
+    growths: Final = tuple(WorkerGrowth(warm[pid], after[pid]) for pid in warm.keys() & after.keys())
+    assert growths, (
+        f"no worker answered /debug/memory/summary at both checkpoints (warm pids {sorted(warm)}, "
+        f"after pids {sorted(after)}), so no worker can be compared with itself"
+    )
+    return max(growths, key=lambda growth: growth.growth_mb)
+
+
+def _assert_every_call_failed_through_fallback(calls: Sequence[FailedCall], fallback: str) -> None:
+    served: Final = tuple(call for call in calls if call.status_code == 200)
+    assert not served, (
+        f"{len(served)} of {len(calls)} calls came back 200, so they reached a provider and never "
+        f"exercised the retry loop: {served[0].body_head}"
+    )
+    without_fallback: Final = tuple(call for call in calls if fallback not in call.body_head)
+    assert not without_fallback, (
+        f"{len(without_fallback)} of {len(calls)} failures never named the fallback group {fallback}, "
+        f"so the request did not run through retries into the fallback: {without_fallback[0].body_head}"
+    )
+
+
+def _stored_request_kb(proxy: ProxyClient, call: FailedCall) -> float:
+    assert call.call_id, (
+        f"the failing call carried no x-litellm-call-id header, so its spend log cannot be read back: {call.body_head}"
+    )
+    rows: Final[Sequence[SpendLogRow]] = proxy.poll_logs_for_request_id(call.call_id)
+    assert rows, (
+        f"no spend log row appeared for failing call {call.call_id} within the poll window: either the stack "
+        "writes no spend logs or its writer dropped the row, which the v1.100.0 one did once the stored "
+        "request outgrew the writer's queue budget"
+    )
+    snapshot: Final = rows[0].proxy_server_request
+    assert snapshot, (
+        f"spend log {call.call_id} stored no request body, so the stack is not running with "
+        "general_settings.store_prompts_in_spend_logs and the stored-request check would pass vacuously"
+    )
+    return len(json.dumps(snapshot).encode()) / 1024
+
+
+@pytest.mark.covers("reliability.perf.memory.under_slo")
+def test_failing_requests_do_not_grow_rss_or_stored_request(
+    client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str
+) -> None:
+    marker: Final = unique_marker()
+    primary: Final = f"reliability-memory-{marker}"
+    fallback: Final = f"reliability-memory-fb-{marker}"
+    _register_refusing_group(client.proxy, resources, primary)
+    _register_refusing_group(client.proxy, resources, fallback)
+    override: Final = RouterSettingsOverride(
+        num_retries=MEMORY_RETRIES_PER_REQUEST, fallbacks=[{primary: [fallback]}]
+    )
+
+    probe: Final = _fail_once(client.proxy, scoped_key, primary, override)
+    _assert_every_call_failed_through_fallback((probe,), fallback)
+    stored_kb: Final = _stored_request_kb(client.proxy, probe)
+    assert stored_kb <= MEMORY_STORED_REQUEST_BUDGET_KB, (
+        f"the spend log of one failing request stored a {stored_kb:.0f} KB request body, past the "
+        f"{MEMORY_STORED_REQUEST_BUDGET_KB:.0f} KB budget for a {len(TRANSCRIPT)}-message transcript with "
+        f"{MEMORY_RETRIES_PER_REQUEST} retries and a fallback; the retry breadcrumbs are copying the whole "
+        f"request into the stored snapshot the way the v1.100.0 ones did"
+    )
+
+    warmup: Final = _fail_many(client.proxy, scoped_key, primary, override)
+    _assert_every_call_failed_through_fallback(warmup, fallback)
+    warm: Final = _settled_rss_per_worker(client.proxy)
+
+    measured: Final = _fail_many(client.proxy, scoped_key, primary, override)
+    _assert_every_call_failed_through_fallback(measured, fallback)
+    after: Final = _settled_rss_per_worker(client.proxy)
+
+    heaviest: Final = _heaviest_worker_growth(warm, after)
+    assert heaviest.growth_mb <= MEMORY_RSS_BUDGET_MB, (
+        f"proxy RSS grew {heaviest.growth_mb:.1f} MB over a second batch of {MEMORY_REQUESTS_PER_PHASE} failing "
+        f"requests ({MEMORY_RETRIES_PER_REQUEST} retries each plus a fallback) after an identical warmup batch, "
+        f"past the {MEMORY_RSS_BUDGET_MB:.0f} MB budget: worker pid {heaviest.warm.worker_pid} at "
+        f"{heaviest.warm.replica} settled at {heaviest.warm.ram_usage_mb:.1f} MB warm and "
+        f"{heaviest.after.ram_usage_mb:.1f} MB after; failing requests are leaking memory the way the "
+        f"v1.100.0 retry breadcrumbs did"
+    )