From d4e083348ca5edfbd45142fd23dbe08f3f3cb9d0 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Fri, 11 Sep 2026 15:14:19 -0700 Subject: [PATCH] revert: drop the create-release.yml gating and E2E_REDIS_CHAOS opt-in create-release.yml is back to calling the chaos test through no mechanism at all; it never called it. Also drops the E2E_REDIS_CHAOS opt-in gate itself: the redis_chaos marker still exists for -m selection and is still excluded from the per-PR selector by path (tests/e2e/(ui|claude_code|load)/), but the test no longer needs an env var to run once its file is targeted. Co-Authored-By: Claude Code --- .github/workflows/create-release.yml | 41 +++------------------- .github/workflows/test-e2e-redis-chaos.yml | 1 - tests/e2e/CLAUDE.md | 2 +- tests/e2e/CONTRIBUTING.md | 2 +- tests/e2e/conftest.py | 2 +- tests/e2e/e2e_config.py | 1 - tests/e2e/load/conftest.py | 8 ++--- tests/e2e/load/test_redis_chaos_e2e.py | 3 +- tests/e2e/pytest.ini | 2 +- 9 files changed, 11 insertions(+), 51 deletions(-) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 5bf10ef324f..0ad84cd3ceb 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -15,13 +15,11 @@ on: permissions: {} jobs: - # Stable and RC tags are gated on the Redis chaos load test; dev, nightly, alpha and beta - # tags are cut too often to spend a multi-minute chaos run on each one. - run-stable-release-checks: - name: Validate inputs and decide whether this tag is gated + release: + name: Create Release runs-on: ubuntu-latest - outputs: - gated: ${{ steps.decide.outputs.gated }} + permissions: + contents: write steps: - name: Validate inputs env: @@ -37,37 +35,6 @@ jobs: exit 1 fi - - name: Decide - id: decide - env: - TAG: ${{ inputs.tag }} - run: | - if echo "${TAG}" | grep -qiE '(nightly|alpha|beta|[-.]dev)'; then - echo "gated=false" >> "$GITHUB_OUTPUT" - echo "${TAG} is a pre-release that skips the chaos gate" - else - echo "gated=true" >> "$GITHUB_OUTPUT" - echo "${TAG} is a stable or RC tag and must pass the chaos gate" - fi - - redis-chaos-check: - name: Redis Chaos E2E - needs: run-stable-release-checks - if: needs.run-stable-release-checks.outputs.gated == 'true' - permissions: - contents: read - uses: ./.github/workflows/test-e2e-redis-chaos.yml - with: - ref: ${{ inputs.commit_hash }} - - release: - name: Create Release - needs: [run-stable-release-checks, redis-chaos-check] - if: always() && needs.run-stable-release-checks.result == 'success' && (needs.redis-chaos-check.result == 'success' || needs.redis-chaos-check.result == 'skipped') - runs-on: ubuntu-latest - permissions: - contents: write - steps: - name: Create release env: TAG: ${{ inputs.tag }} diff --git a/.github/workflows/test-e2e-redis-chaos.yml b/.github/workflows/test-e2e-redis-chaos.yml index c7412a63334..6b066739c95 100644 --- a/.github/workflows/test-e2e-redis-chaos.yml +++ b/.github/workflows/test-e2e-redis-chaos.yml @@ -91,7 +91,6 @@ jobs: - name: Run the Redis chaos load test env: - E2E_REDIS_CHAOS: "1" LITELLM_PROXY_URL: http://localhost:4000 REDIS_HOST: 127.0.0.1 REDIS_PORT: "6379" diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 20cd0c35465..f9ad73b4508 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -18,7 +18,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `logging/` - logging-integration delivery (datadog and friends) - `security/` - secret handling and log-leak protection - `router/` - routing and reliability behavior (fallbacks, cooldowns) -- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set, driven by `.github/workflows/test-e2e-redis-chaos.yml`, which `create-release.yml` calls to gate stable and RC tags on the commit being released), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic +- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, excluded from the per-PR selector like the rest of `load/`, driven by `.github/workflows/test-e2e-redis-chaos.yml`), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index b6faebed867..a82e8ea43f5 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -65,7 +65,7 @@ A couple of logging destinations are configured on the proxy rather than by the ### The pull request check -Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots when `create-release.yml` calls it to gate a stable or RC release +Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 700dc822d59..72448201f8b 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -61,7 +61,7 @@ def pytest_configure(config: pytest.Config) -> None: config.addinivalue_line( "markers", "redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from " - "gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set", + "gateway/redis_chaos_ci_config.yml on the same host", ) diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 7344a2b6cec..86f0c616c04 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -143,7 +143,6 @@ LOAD_MIN_CONCURRENCY_EFFICIENCY = float(os.environ.get("E2E_LOAD_MIN_CONCURRENCY WEEKLY_ANOMALY_OPT_IN_ENV = "E2E_WEEKLY_ANOMALY" MANAGED_FILES_OPT_IN_ENV = "E2E_MANAGED_FILES_STACK" -REDIS_CHAOS_OPT_IN_ENV = "E2E_REDIS_CHAOS" ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6")) ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6")) ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3")) diff --git a/tests/e2e/load/conftest.py b/tests/e2e/load/conftest.py index e6f3c7aa538..7c749837225 100644 --- a/tests/e2e/load/conftest.py +++ b/tests/e2e/load/conftest.py @@ -3,15 +3,11 @@ from __future__ import annotations import os import pytest - -from e2e_config import REDIS_CHAOS_OPT_IN_ENV, WEEKLY_ANOMALY_OPT_IN_ENV +from e2e_config import WEEKLY_ANOMALY_OPT_IN_ENV from load_client import LoadClient, build_client from proxy_client import ProxyClient -_OPT_IN_MARKERS = ( - ("weekly", WEEKLY_ANOMALY_OPT_IN_ENV), - ("redis_chaos", REDIS_CHAOS_OPT_IN_ENV), -) +_OPT_IN_MARKERS = (("weekly", WEEKLY_ANOMALY_OPT_IN_ENV),) def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: diff --git a/tests/e2e/load/test_redis_chaos_e2e.py b/tests/e2e/load/test_redis_chaos_e2e.py index 9a9e8082e5b..2e1e2a92bd3 100644 --- a/tests/e2e/load/test_redis_chaos_e2e.py +++ b/tests/e2e/load/test_redis_chaos_e2e.py @@ -27,8 +27,7 @@ per-phase RSS, CPU, and log-bytes budgets are here to catch. Needs the proxy on the same host, since RSS and CPU come from psutil on its process tree: a multi-worker proxy serves /metrics from the prometheus multiprocess collector, which drops the process collector's memory and CPU series. Log bytes are read from the file the proxy's -stdout/stderr was redirected to, so the same host requirement covers that too. Deselected -unless E2E_REDIS_CHAOS is set. +stdout/stderr was redirected to, so the same host requirement covers that too. """ from __future__ import annotations diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index 774d9644497..28f7fc011f7 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -9,4 +9,4 @@ markers = load: heavy throughput/load test; collected last so it never perturbs latency-sensitive suites weekly: real-provider anomaly load test that spends real money; deselected unless E2E_WEEKLY_ANOMALY is set managed_files: needs a proxy running with require_managed_files enabled; deselected unless E2E_MANAGED_FILES_STACK is set - redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set + redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from gateway/redis_chaos_ci_config.yml on the same host