From 4a646dd9a0d7acd2ea7c3fbd57de1e17ead7cec8 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Wed, 2 Sep 2026 14:53:40 -0700
Subject: [PATCH 01/16] ci(e2e): run a PR's changed e2e tests three times
 behind a human-approved environment

Adds a required-check candidate that selects the tests/e2e test files a PR added or
modified, boots a stage-mirror stack on the runner (migrations, backend, two gateway
processes behind nginx, Postgres, Jaeger, TLS cluster Valkey), and runs those files
three times with retries off. The run job sits behind the e2e-changed GitHub
environment, so a reviewer approves each run before the OIDC token that reads the
provider keys from AWS Secrets Manager exists. Supersedes #34981
---
 .github/e2e-stack/down.sh                    |  17 ++
 .github/e2e-stack/secrets_to_env.py          |  28 +++
 .github/e2e-stack/up.sh                      | 198 +++++++++++++++++++
 .github/workflows/test-e2e-changed.yml       | 171 ++++++++++++++++
 tests/e2e/CONTRIBUTING.md                    |   4 +
 tests/e2e/gateway/stage_mirror_ci_config.yml |  63 ++++++
 6 files changed, 481 insertions(+)
 create mode 100755 .github/e2e-stack/down.sh
 create mode 100644 .github/e2e-stack/secrets_to_env.py
 create mode 100755 .github/e2e-stack/up.sh
 create mode 100644 .github/workflows/test-e2e-changed.yml
 create mode 100644 tests/e2e/gateway/stage_mirror_ci_config.yml

diff --git a/.github/e2e-stack/down.sh b/.github/e2e-stack/down.sh
new file mode 100755
index 00000000000..9f72f2d6e64
--- /dev/null
+++ b/.github/e2e-stack/down.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -uo pipefail
+
+STACK_DIR="${E2E_STACK_DIR:-${RUNNER_TEMP:-/tmp}/litellm-e2e-stack}"
+
+for pid_file in "${STACK_DIR}"/pids/*.pid; do
+  [[ -f "${pid_file}" ]] || continue
+  pkill -TERM -P "$(cat "${pid_file}")" 2>/dev/null
+  kill -TERM "$(cat "${pid_file}")" 2>/dev/null
+  rm -f "${pid_file}"
+done
+
+for container in e2e-nginx e2e-valkey e2e-jaeger e2e-postgres; do
+  docker rm -f "${container}" >/dev/null 2>&1
+done
+
+exit 0
diff --git a/.github/e2e-stack/secrets_to_env.py b/.github/e2e-stack/secrets_to_env.py
new file mode 100644
index 00000000000..25b22555c29
--- /dev/null
+++ b/.github/e2e-stack/secrets_to_env.py
@@ -0,0 +1,28 @@
+import sys
+from pathlib import Path
+
+from pydantic import TypeAdapter
+
+secrets_adapter: TypeAdapter[dict[str, str]] = TypeAdapter(dict[str, str])
+
+
+def main() -> int:
+    env_path = Path(sys.argv[1])
+    secrets = secrets_adapter.validate_json(sys.stdin.read())
+    unwritable = tuple(
+        key for key, value in secrets.items() if "'" in value or "\n" in value or "\r" in value
+    )
+    if unwritable:
+        _ = sys.stderr.write(f"values contain characters unsafe for both bash and dotenv: {', '.join(unwritable)}\n")
+        return 1
+    lines = tuple(f"{key}='{value}'" for key, value in secrets.items() if value)
+    with env_path.open("a") as handle:
+        _ = handle.write("\n".join(lines) + "\n")
+    for value in secrets.values():
+        if value:
+            _ = sys.stdout.write(f"::add-mask::{value}\n")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/.github/e2e-stack/up.sh b/.github/e2e-stack/up.sh
new file mode 100755
index 00000000000..06fce35a896
--- /dev/null
+++ b/.github/e2e-stack/up.sh
@@ -0,0 +1,198 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+STACK_DIR="${E2E_STACK_DIR:-${RUNNER_TEMP:-/tmp}/litellm-e2e-stack}"
+CERTS_DIR="${STACK_DIR}/certs"
+LOGS_DIR="${STACK_DIR}/logs"
+PIDS_DIR="${STACK_DIR}/pids"
+
+POSTGRES_IMAGE="${E2E_POSTGRES_IMAGE:-postgres:16.6}"
+VALKEY_IMAGE="${E2E_VALKEY_IMAGE:-valkey/valkey:8.1}"
+JAEGER_IMAGE="${E2E_JAEGER_IMAGE:-jaegertracing/jaeger:2.10.0}"
+NGINX_IMAGE="${E2E_NGINX_IMAGE:-nginx:1.29-alpine}"
+
+LB_PORT="${E2E_LB_PORT:-4000}"
+GATEWAY_PORT_1="${E2E_GATEWAY_PORT_1:-4010}"
+GATEWAY_PORT_2="${E2E_GATEWAY_PORT_2:-4011}"
+BACKEND_PORT="${E2E_BACKEND_PORT:-4001}"
+REDIS_PORT="${E2E_REDIS_PORT:-6379}"
+DATABASE_HOST="${E2E_DATABASE_HOST:-127.0.0.1}"
+DATABASE_PORT="${E2E_DATABASE_PORT:-5432}"
+DATABASE_USER="${E2E_DATABASE_USER:-litellm}"
+DATABASE_PASSWORD="${E2E_DATABASE_PASSWORD:-dbpassword9090}"
+DATABASE_NAME="${E2E_DATABASE_NAME:-litellm}"
+JAEGER_OTLP_PORT="${E2E_JAEGER_OTLP_PORT:-4318}"
+JAEGER_QUERY_PORT="${E2E_JAEGER_QUERY_PORT:-16686}"
+
+MASTER_KEY="${LITELLM_MASTER_KEY:-sk-e2e-$(openssl rand -hex 16)}"
+
+mkdir -p "${CERTS_DIR}" "${LOGS_DIR}" "${PIDS_DIR}"
+
+log() { printf 'e2e-stack: %s\n' "$*"; }
+
+port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null; }
+
+wait_for() {
+  local label="$1" check="$2" deadline=$((SECONDS + ${3:-120}))
+  until eval "${check}"; do
+    if ((SECONDS >= deadline)); then
+      log "timed out waiting for ${label}"
+      tail -n 60 "${LOGS_DIR}"/*.log 2>/dev/null || true
+      exit 1
+    fi
+    sleep 2
+  done
+  log "${label} is up"
+}
+
+if [[ -f "${REPO_ROOT}/tests/e2e/.env" ]]; then
+  set -a
+  source "${REPO_ROOT}/tests/e2e/.env"
+  set +a
+fi
+
+if ! port_open "${DATABASE_PORT}"; then
+  docker run -d --name e2e-postgres -p "${DATABASE_PORT}:5432" \
+    -e "POSTGRES_USER=${DATABASE_USER}" -e "POSTGRES_PASSWORD=${DATABASE_PASSWORD}" -e "POSTGRES_DB=${DATABASE_NAME}" \
+    "${POSTGRES_IMAGE}" >/dev/null
+fi
+wait_for "postgres" "port_open ${DATABASE_PORT}"
+
+if ! port_open "${JAEGER_QUERY_PORT}"; then
+  docker run -d --name e2e-jaeger -p "${JAEGER_OTLP_PORT}:4318" -p "${JAEGER_QUERY_PORT}:16686" \
+    "${JAEGER_IMAGE}" >/dev/null
+fi
+wait_for "jaeger" "curl -fs http://127.0.0.1:${JAEGER_QUERY_PORT}/api/services >/dev/null"
+
+openssl genrsa -out "${CERTS_DIR}/ca.key" 2048 2>/dev/null
+openssl req -x509 -new -nodes -key "${CERTS_DIR}/ca.key" -sha256 -days 7 \
+  -subj "/CN=litellm-e2e-ca" \
+  -addext "basicConstraints=critical,CA:TRUE" -addext "keyUsage=critical,keyCertSign,cRLSign" \
+  -out "${CERTS_DIR}/ca.crt" 2>/dev/null
+openssl genrsa -out "${CERTS_DIR}/server.key" 2048 2>/dev/null
+openssl req -new -key "${CERTS_DIR}/server.key" -subj "/CN=localhost" -out "${CERTS_DIR}/server.csr" 2>/dev/null
+openssl x509 -req -in "${CERTS_DIR}/server.csr" -CA "${CERTS_DIR}/ca.crt" -CAkey "${CERTS_DIR}/ca.key" \
+  -CAcreateserial -days 7 -sha256 \
+  -extfile <(printf 'basicConstraints=CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\nsubjectAltName=DNS:localhost,IP:127.0.0.1\n') \
+  -out "${CERTS_DIR}/server.crt" 2>/dev/null
+chmod 644 "${CERTS_DIR}"/*.key "${CERTS_DIR}"/*.crt
+
+CERTIFI_BUNDLE="$(cd "${REPO_ROOT}" && uv run --no-sync python -c 'import certifi; print(certifi.where())')"
+cat "${CERTIFI_BUNDLE}" "${CERTS_DIR}/ca.crt" > "${CERTS_DIR}/ca-bundle.pem"
+
+docker rm -f e2e-valkey >/dev/null 2>&1 || true
+docker run -d --name e2e-valkey -p "${REDIS_PORT}:${REDIS_PORT}" -v "${CERTS_DIR}:/certs:ro" \
+  "${VALKEY_IMAGE}" valkey-server \
+  --cluster-enabled yes --port 0 --tls-port "${REDIS_PORT}" \
+  --tls-cert-file /certs/server.crt --tls-key-file /certs/server.key --tls-ca-cert-file /certs/ca.crt \
+  --tls-auth-clients no --cluster-announce-ip 127.0.0.1 >/dev/null
+VALKEY_CLI="docker exec e2e-valkey valkey-cli --tls --cacert /certs/ca.crt -h 127.0.0.1 -p ${REDIS_PORT}"
+wait_for "valkey" "${VALKEY_CLI} ping 2>/dev/null | grep -q PONG"
+${VALKEY_CLI} cluster addslotsrange 0 16383 >/dev/null
+wait_for "valkey cluster" "${VALKEY_CLI} cluster info 2>/dev/null | grep -q cluster_state:ok"
+
+CONFIG_SOURCE="${REPO_ROOT}/tests/e2e/gateway/stage_mirror_ci_config.yml"
+CONFIG_PATH="${CONFIG_SOURCE}"
+if [[ "${REDIS_PORT}" != "6379" ]]; then
+  CONFIG_PATH="${STACK_DIR}/litellm-config.yml"
+  sed "s/port: 6379/port: ${REDIS_PORT}/" "${CONFIG_SOURCE}" > "${CONFIG_PATH}"
+fi
+
+SERVER_ENV=(
+  "LITELLM_MASTER_KEY=${MASTER_KEY}"
+  "DATABASE_HOST=${DATABASE_HOST}"
+  "DATABASE_PORT=${DATABASE_PORT}"
+  "DATABASE_USER=${DATABASE_USER}"
+  "DATABASE_PASSWORD=${DATABASE_PASSWORD}"
+  "DATABASE_NAME=${DATABASE_NAME}"
+  "DISABLE_SCHEMA_UPDATE=true"
+  "REDIS_HOST=127.0.0.1"
+  "REDIS_PORT=${REDIS_PORT}"
+  "REDIS_CLUSTER_NODES=[{\"host\":\"127.0.0.1\",\"port\":${REDIS_PORT}}]"
+  "CONFIG_FILE_PATH=${CONFIG_PATH}"
+  "STORE_MODEL_IN_DB=True"
+  "OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf"
+  "OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:${JAEGER_OTLP_PORT}"
+  "SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem"
+  "PYTHONPATH=${REPO_ROOT}"
+)
+if [[ -n "${VERTEXAI_CREDENTIALS:-}" ]]; then
+  printf '%s' "${VERTEXAI_CREDENTIALS}" > "${STACK_DIR}/vertex-adc.json"
+  SERVER_ENV+=("GOOGLE_APPLICATION_CREDENTIALS=${STACK_DIR}/vertex-adc.json")
+fi
+
+cd "${REPO_ROOT}"
+
+log "running migrations"
+env "${SERVER_ENV[@]}" uv run --no-sync python migrations/run.py >"${LOGS_DIR}/migrations.log" 2>&1
+
+start_server() {
+  local name="$1"; shift
+  env "${SERVER_ENV[@]}" "$@" >"${LOGS_DIR}/${name}.log" 2>&1 &
+  echo $! > "${PIDS_DIR}/${name}.pid"
+}
+
+start_server backend uv run --no-sync uvicorn backend.main:app --host 0.0.0.0 --port "${BACKEND_PORT}"
+start_server gateway-1 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_1}"
+start_server gateway-2 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_2}"
+
+if [[ "$(uname)" == "Linux" ]]; then
+  NGINX_UPSTREAM_HOST=127.0.0.1
+  NGINX_DOCKER_ARGS=(--network host)
+else
+  NGINX_UPSTREAM_HOST=host.docker.internal
+  NGINX_DOCKER_ARGS=(-p "${LB_PORT}:${LB_PORT}")
+fi
+
+cat > "${STACK_DIR}/nginx.conf" <<EOF
+events {}
+http {
+  map \$http_upgrade \$connection_upgrade {
+    default upgrade;
+    '' close;
+  }
+  upstream litellm_gateways {
+    server ${NGINX_UPSTREAM_HOST}:${GATEWAY_PORT_1};
+    server ${NGINX_UPSTREAM_HOST}:${GATEWAY_PORT_2};
+  }
+  server {
+    listen ${LB_PORT};
+    client_max_body_size 100m;
+    location / {
+      proxy_pass http://litellm_gateways;
+      proxy_http_version 1.1;
+      proxy_set_header Host \$host;
+      proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto \$scheme;
+      proxy_set_header Upgrade \$http_upgrade;
+      proxy_set_header Connection \$connection_upgrade;
+      proxy_buffering off;
+      proxy_read_timeout 600s;
+      proxy_send_timeout 600s;
+    }
+  }
+}
+EOF
+
+docker rm -f e2e-nginx >/dev/null 2>&1 || true
+docker run -d --name e2e-nginx "${NGINX_DOCKER_ARGS[@]}" \
+  -v "${STACK_DIR}/nginx.conf:/etc/nginx/nginx.conf:ro" "${NGINX_IMAGE}" >/dev/null
+
+wait_for "backend" "curl -fs http://127.0.0.1:${BACKEND_PORT}/health/liveliness >/dev/null" 300
+wait_for "gateway-1" "curl -fs http://127.0.0.1:${GATEWAY_PORT_1}/health/liveliness >/dev/null" 300
+wait_for "gateway-2" "curl -fs http://127.0.0.1:${GATEWAY_PORT_2}/health/liveliness >/dev/null" 300
+wait_for "load balancer" "curl -fs http://127.0.0.1:${LB_PORT}/health/liveliness >/dev/null" 60
+
+cat > "${STACK_DIR}/stack.env" <<EOF
+LITELLM_PROXY_URL=http://127.0.0.1:${LB_PORT}
+LITELLM_CONTROL_PLANE_URL=http://127.0.0.1:${BACKEND_PORT}
+LITELLM_MASTER_KEY=${MASTER_KEY}
+REDIS_HOST=127.0.0.1
+REDIS_PORT=${REDIS_PORT}
+E2E_OTEL_QUERY_URL=http://127.0.0.1:${JAEGER_QUERY_PORT}
+SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem
+DATABASE_URL=postgresql://${DATABASE_USER}:${DATABASE_PASSWORD}@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_NAME}
+EOF
+
+log "stack is up; pytest env written to ${STACK_DIR}/stack.env"
diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml
new file mode 100644
index 00000000000..6ad252e7246
--- /dev/null
+++ b/.github/workflows/test-e2e-changed.yml
@@ -0,0 +1,171 @@
+name: e2e-changed-tests
+
+on:
+  pull_request:
+
+concurrency:
+  group: e2e-changed-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+permissions: {}
+
+jobs:
+  detect:
+    name: Detect changed e2e tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      pull-requests: read
+    outputs:
+      tests: ${{ steps.changed.outputs.tests }}
+      any: ${{ steps.changed.outputs.any }}
+    steps:
+      - name: List the e2e test files this PR added or modified
+        id: changed
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          SMOKE_TESTS: tests/e2e/access_control
+        run: |
+          files="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" --paginate \
+            --jq '.[] | select(.status != "removed") | .filename')"
+          tests="$(printf '%s\n' "${files}" \
+            | grep -E '^tests/e2e/([A-Za-z0-9_.-]+/)*test_[A-Za-z0-9_.-]+\.py$' \
+            | grep -vE '^tests/e2e/(ui|claude_code|load)/' \
+            | sort -u | tr '\n' ' ' | sed 's/ $//')" || true
+          if [ -z "${tests}" ] && printf '%s\n' "${files}" | grep -v '^tests/e2e/ui/' \
+            | grep -qE '^(tests/e2e/|\.github/e2e-stack/|\.github/workflows/test-e2e-changed\.yml$)'; then
+            tests="${SMOKE_TESTS}"
+            echo "harness or stack changed without a test file; running the smoke suite"
+          fi
+          echo "tests=${tests}" >> "${GITHUB_OUTPUT}"
+          if [ -n "${tests}" ]; then
+            echo "any=true" >> "${GITHUB_OUTPUT}"
+            echo "selected e2e tests: ${tests}"
+          else
+            echo "any=false" >> "${GITHUB_OUTPUT}"
+            echo "no e2e changes; nothing to run"
+          fi
+
+  run:
+    name: Run changed e2e tests against the stage-mirror stack
+    needs: detect
+    if: needs.detect.outputs.any == 'true' && github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    environment: e2e-changed
+    permissions:
+      contents: read
+      id-token: write
+    services:
+      postgres:
+        image: postgres:16.6
+        env:
+          POSTGRES_USER: litellm
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U litellm"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+      jaeger:
+        image: jaegertracing/jaeger:2.10.0
+        ports:
+          - 4318:4318
+          - 16686:16686
+    steps:
+      - name: Validate configuration
+        env:
+          ROLE: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }}
+        run: test -n "${ROLE}" || { echo "::error::Set repo variable E2E_AWS_ROLE_TO_ASSUME to an OIDC role with read access to the e2e secrets"; exit 1; }
+
+      - name: Checkout
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.13"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
+      - name: Install dependencies
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen \
+            --extra proxy --extra proxy-runtime --extra extra_proxy \
+            --extra semantic-router --extra bedrock-realtime \
+            --group ci --group proxy-dev --group e2e-dev
+          uv pip install "pipecat-ai[openai]==1.4.0"
+
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
+      - name: Generate Prisma client
+        run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Install Playwright chromium
+        run: uv run --no-sync playwright install --with-deps chromium
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0
+        with:
+          role-to-assume: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }}
+          aws-region: us-east-1
+          role-session-name: litellm-e2e-changed-${{ github.run_id }}
+
+      - name: Fetch provider credentials from AWS Secrets Manager
+        run: |
+          aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-provider-keys \
+            --query SecretString --output text \
+            | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env
+          aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-license \
+            --query SecretString --output text \
+            | jq -R -s '{"LITELLM_LICENSE": rtrimstr("\n")}' \
+            | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env
+
+      - name: Boot the stage-mirror stack
+        run: bash .github/e2e-stack/up.sh
+
+      - name: Export stack environment
+        run: |
+          master_key="$(grep '^LITELLM_MASTER_KEY=' "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" | cut -d= -f2-)"
+          echo "::add-mask::${master_key}"
+          cat "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" >> "${GITHUB_ENV}"
+
+      - name: Run the selected tests three times with retries off
+        env:
+          TESTS: ${{ needs.detect.outputs.tests }}
+        run: |
+          read -r -a test_files <<< "${TESTS}"
+          for pass in 1 2 3; do
+            echo "::group::pass ${pass} of 3"
+            set +e
+            uv run --no-sync pytest "${test_files[@]}" --reruns 0 -v -rA --tb=short -p no:cacheprovider
+            status=$?
+            set -e
+            echo "::endgroup::"
+            if [ "${status}" = "5" ]; then
+              echo "selected files collected no runnable tests"
+              exit 0
+            fi
+            if [ "${status}" != "0" ]; then
+              echo "::error::pass ${pass} of 3 failed with exit code ${status}"
+              exit "${status}"
+            fi
+          done
+
+      - name: Show stack logs on failure
+        if: failure()
+        run: tail -n 200 "${RUNNER_TEMP}/litellm-e2e-stack/logs"/*.log
diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md
index 871d6b3904c..e04781ab205 100644
--- a/tests/e2e/CONTRIBUTING.md
+++ b/tests/e2e/CONTRIBUTING.md
@@ -52,6 +52,10 @@ The suites run against a live proxy, so bring one up first by running the litell
 
 Some suites need extra services the bare proxy does not start. The `logging/` OTEL trace-completeness tests read spans back from a jaeger query API at `http://localhost:16686` (override with `E2E_OTEL_QUERY_URL`); run a `jaegertracing/all-in-one` and point `PHOENIX_COLLECTOR_HTTP_ENDPOINT` at its OTLP ingest. The `mcp/` suite needs the deterministic upstream MCP server in `mcp_tests/mcp_e2e_upstream_server.py` reachable by the proxy
 
+### The pull request check
+
+Every PR that adds or modifies a `tests/e2e/**/test_*.py` file (outside `ui/`, `claude_code/`, and `load/`, which have their own lanes) runs exactly those files three times, with retries off, against a stage-mirror stack booted on a GitHub Actions runner: migrations, a control-plane backend, two gateway processes behind an nginx load balancer, Postgres, Jaeger, and a TLS cluster-mode Valkey, wired the way stage is deployed. A PR that only touches the harness or the stack itself runs the `access_control/` suite as a smoke instead. Three green passes are the bar because the check exists to catch a flaky test before it reaches the release gate, so a red pass is a failure to fix, not a retry candidate. The same stack boots on a laptop with `bash .github/e2e-stack/up.sh`: it reads provider keys from `tests/e2e/.env`, writes the pytest environment to `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}/stack.env`, every port is overridable through `E2E_*_PORT` variables, and `down.sh` tears it all down
+
 ### Record and replay
 
 Record/replay scopes to the proxy's provider-bound traffic only. In `E2E_FIXTURE_MODE=record` the harness boots a local provider-edge server, edge-wired tests register their deployments with an `api_base` pointing at it, and every provider call the proxy makes is forwarded verbatim and written to a fixture bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`). `E2E_FIXTURE_MODE=replay` runs the same tests against the same live proxy and database, but the edge answers the proxy's provider calls from the bundle instead of the provider, so the run makes zero provider calls and spends nothing while key auth, routing, cost calculation, and spend-log writes all still execute for real. Unset (or `live`) behaves exactly as before the knob existed. Both record and replay need the proxy up; only the provider is taken out of the loop
diff --git a/tests/e2e/gateway/stage_mirror_ci_config.yml b/tests/e2e/gateway/stage_mirror_ci_config.yml
new file mode 100644
index 00000000000..57a92fd47fb
--- /dev/null
+++ b/tests/e2e/gateway/stage_mirror_ci_config.yml
@@ -0,0 +1,63 @@
+general_settings:
+  store_prompts_in_spend_logs: true
+  database_connection_pool_limit: 10
+  forward_client_headers_to_llm_api: false
+  maximum_spend_logs_retention_period: "60d"
+  maximum_spend_logs_cleanup_cron: "0 1 * * *"
+  proxy_budget_rescheduler_min_time: 15
+  proxy_budget_rescheduler_max_time: 20
+
+litellm_settings:
+  drop_params: true
+  default_redis_ttl: 20
+  request_timeout: 600
+  num_retries: 3
+  json_logs: true
+  store_audit_logs: true
+  cache: true
+  cache_params:
+    type: redis
+    host: 127.0.0.1
+    port: 6379
+    redis_startup_nodes:
+      - host: 127.0.0.1
+        port: 6379
+    ssl: true
+  callbacks: ["arize_phoenix", "datadog", "smtp_email", "prometheus", "otel"]
+  require_auth_for_metrics_endpoint: false
+
+router_settings:
+  routing_strategy: simple-shuffle
+  num_retries: 3
+  allowed_fails: 5
+  cooldown_time: 30
+
+model_list:
+  - model_name: gpt-5.5
+    litellm_params:
+      model: openai/gpt-5.5
+      api_key: os.environ/OPENAI_API_KEY
+  - model_name: claude-haiku-4-5
+    litellm_params:
+      model: anthropic/claude-haiku-4-5
+      api_key: os.environ/ANTHROPIC_API_KEY
+  - model_name: gemini-2.5-flash-vertex
+    litellm_params:
+      model: vertex_ai/gemini-2.5-flash
+      vertex_project: os.environ/VERTEXAI_PROJECT
+      vertex_location: us-central1
+      vertex_credentials: os.environ/VERTEXAI_CREDENTIALS
+  - model_name: gemini-2.5-flash
+    litellm_params:
+      model: gemini/gemini-2.5-flash
+      api_key: os.environ/GEMINI_API_KEY
+  - model_name: gemini-2.5-flash
+    litellm_params:
+      model: gemini/gemini-2.5-flash
+      api_key: os.environ/GEMINI_API_KEY
+
+mcp_servers:
+  devin:
+    url: "https://mcp.devin.ai/mcp"
+    auth_type: api_key
+    auth_value: os.environ/DEVIN_API_KEY

From b348ed7f09709647d3f9bce38be3ba49741be307 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Wed, 2 Sep 2026 14:56:52 -0700
Subject: [PATCH 02/16] ci(e2e): only tail stack logs when the stack actually
 booted

---
 .github/workflows/test-e2e-changed.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml
index 6ad252e7246..a4466bba3fd 100644
--- a/.github/workflows/test-e2e-changed.yml
+++ b/.github/workflows/test-e2e-changed.yml
@@ -136,6 +136,7 @@ jobs:
             | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env
 
       - name: Boot the stage-mirror stack
+        id: boot
         run: bash .github/e2e-stack/up.sh
 
       - name: Export stack environment
@@ -167,5 +168,5 @@ jobs:
           done
 
       - name: Show stack logs on failure
-        if: failure()
+        if: failure() && steps.boot.conclusion != 'skipped'
         run: tail -n 200 "${RUNNER_TEMP}/litellm-e2e-stack/logs"/*.log

From 743f94f82abf9d4963c087809fab782d47761b09 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Wed, 2 Sep 2026 14:59:15 -0700
Subject: [PATCH 03/16] ci(e2e): fail when nothing collects and ignore lanes
 with their own checks in the smoke trigger

---
 .github/workflows/test-e2e-changed.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml
index a4466bba3fd..7a638801161 100644
--- a/.github/workflows/test-e2e-changed.yml
+++ b/.github/workflows/test-e2e-changed.yml
@@ -34,7 +34,7 @@ jobs:
             | grep -E '^tests/e2e/([A-Za-z0-9_.-]+/)*test_[A-Za-z0-9_.-]+\.py$' \
             | grep -vE '^tests/e2e/(ui|claude_code|load)/' \
             | sort -u | tr '\n' ' ' | sed 's/ $//')" || true
-          if [ -z "${tests}" ] && printf '%s\n' "${files}" | grep -v '^tests/e2e/ui/' \
+          if [ -z "${tests}" ] && printf '%s\n' "${files}" | grep -vE '^tests/e2e/(ui|claude_code|load)/' \
             | grep -qE '^(tests/e2e/|\.github/e2e-stack/|\.github/workflows/test-e2e-changed\.yml$)'; then
             tests="${SMOKE_TESTS}"
             echo "harness or stack changed without a test file; running the smoke suite"
@@ -158,8 +158,8 @@ jobs:
             set -e
             echo "::endgroup::"
             if [ "${status}" = "5" ]; then
-              echo "selected files collected no runnable tests"
-              exit 0
+              echo "::error::the selected files collected no runnable tests, so nothing was verified"
+              exit 1
             fi
             if [ "${status}" != "0" ]; then
               echo "::error::pass ${pass} of 3 failed with exit code ${status}"

From c62643f0cf72c2c02f08c2073a621e81d906b056 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Wed, 2 Sep 2026 15:39:21 -0700
Subject: [PATCH 04/16] ci(e2e): drop trailing newlines from fetched secret
 values before writing the env

---
 .github/e2e-stack/secrets_to_env.py    | 2 +-
 .github/workflows/test-e2e-changed.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/e2e-stack/secrets_to_env.py b/.github/e2e-stack/secrets_to_env.py
index 25b22555c29..65022931d81 100644
--- a/.github/e2e-stack/secrets_to_env.py
+++ b/.github/e2e-stack/secrets_to_env.py
@@ -8,7 +8,7 @@ secrets_adapter: TypeAdapter[dict[str, str]] = TypeAdapter(dict[str, str])
 
 def main() -> int:
     env_path = Path(sys.argv[1])
-    secrets = secrets_adapter.validate_json(sys.stdin.read())
+    secrets = {key: value.rstrip("\r\n") for key, value in secrets_adapter.validate_json(sys.stdin.read()).items()}
     unwritable = tuple(
         key for key, value in secrets.items() if "'" in value or "\n" in value or "\r" in value
     )
diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml
index 7a638801161..b20c84af02d 100644
--- a/.github/workflows/test-e2e-changed.yml
+++ b/.github/workflows/test-e2e-changed.yml
@@ -132,7 +132,7 @@ jobs:
             | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env
           aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-license \
             --query SecretString --output text \
-            | jq -R -s '{"LITELLM_LICENSE": rtrimstr("\n")}' \
+            | jq -R -s '{"LITELLM_LICENSE": .}' \
             | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env
 
       - name: Boot the stage-mirror stack

From 2bce27cfa6a0277c7237649c1c322883887e781a Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Wed, 2 Sep 2026 15:52:42 -0700
Subject: [PATCH 05/16] docs(e2e): describe the dedicated, least-privilege,
 capped credentials behind the pull request check

---
 tests/e2e/CONTRIBUTING.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md
index e04781ab205..b984791c738 100644
--- a/tests/e2e/CONTRIBUTING.md
+++ b/tests/e2e/CONTRIBUTING.md
@@ -56,6 +56,8 @@ Some suites need extra services the bare proxy does not start. The `logging/` OT
 
 Every PR that adds or modifies a `tests/e2e/**/test_*.py` file (outside `ui/`, `claude_code/`, and `load/`, which have their own lanes) runs exactly those files three times, with retries off, against a stage-mirror stack booted on a GitHub Actions runner: migrations, a control-plane backend, two gateway processes behind an nginx load balancer, Postgres, Jaeger, and a TLS cluster-mode Valkey, wired the way stage is deployed. A PR that only touches the harness or the stack itself runs the `access_control/` suite as a smoke instead. Three green passes are the bar because the check exists to catch a flaky test before it reaches the release gate, so a red pass is a failure to fix, not a retry candidate. The same stack boots on a laptop with `bash .github/e2e-stack/up.sh`: it reads provider keys from `tests/e2e/.env`, writes the pytest environment to `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}/stack.env`, every port is overridable through `E2E_*_PORT` variables, and `down.sh` tears it all down
 
+Credentials: the lane never sees the stage keys. Everything the runner writes into `tests/e2e/.env` comes from two AWS Secrets Manager secrets in us-east-1, `litellm-e2e-changed-provider-keys` and `litellm-e2e-changed-license`, and the first holds only the names that the tests and `tests/e2e/gateway/stage_mirror_ci_config.yml` read. Each credential in it is dedicated to this lane and can do only what the tests do: the AWS pair belongs to the `litellm-e2e-changed` IAM user, whose inline policy allows Bedrock inference, the one guardrail the tests use, the batch job APIs, the batch S3 bucket, and assuming the batch test's role; the Vertex key belongs to a service account holding only `roles/aiplatform.user` on the Vertex project; the OpenAI, Anthropic, Gemini, Cohere, and Devin keys are per-lane keys with hard monthly spend caps; and the Datadog application key is scoped to log search. A leaked key can therefore spend at most one month of a small capped budget or call a handful of Bedrock APIs, and rotating one is a single `aws secretsmanager put-secret-value` on the secret, with no workflow change
+
 ### Record and replay
 
 Record/replay scopes to the proxy's provider-bound traffic only. In `E2E_FIXTURE_MODE=record` the harness boots a local provider-edge server, edge-wired tests register their deployments with an `api_base` pointing at it, and every provider call the proxy makes is forwarded verbatim and written to a fixture bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`). `E2E_FIXTURE_MODE=replay` runs the same tests against the same live proxy and database, but the edge answers the proxy's provider calls from the bundle instead of the provider, so the run makes zero provider calls and spends nothing while key auth, routing, cost calculation, and spend-log writes all still execute for real. Unset (or `live`) behaves exactly as before the knob existed. Both record and replay need the proxy up; only the provider is taken out of the loop

From 1c4e46f17e813d5eb8e5fa0c2577f7bef3a32b9a Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Wed, 2 Sep 2026 17:15:36 -0700
Subject: [PATCH 06/16] ci(e2e): reload the stack config every 7s so the
 harness propagation budget holds, and only mask credential-named values

---
 .github/e2e-stack/secrets_to_env.py          | 7 +++++--
 tests/e2e/gateway/stage_mirror_ci_config.yml | 1 +
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/e2e-stack/secrets_to_env.py b/.github/e2e-stack/secrets_to_env.py
index 65022931d81..d2d6675d690 100644
--- a/.github/e2e-stack/secrets_to_env.py
+++ b/.github/e2e-stack/secrets_to_env.py
@@ -1,9 +1,12 @@
+import re
 import sys
 from pathlib import Path
+from typing import Final
 
 from pydantic import TypeAdapter
 
 secrets_adapter: TypeAdapter[dict[str, str]] = TypeAdapter(dict[str, str])
+SECRET_NAME: Final = re.compile(r"KEY|SECRET|TOKEN|PASS|CREDENTIAL|LICENSE|AUTH")
 
 
 def main() -> int:
@@ -18,8 +21,8 @@ def main() -> int:
     lines = tuple(f"{key}='{value}'" for key, value in secrets.items() if value)
     with env_path.open("a") as handle:
         _ = handle.write("\n".join(lines) + "\n")
-    for value in secrets.values():
-        if value:
+    for key, value in secrets.items():
+        if value and SECRET_NAME.search(key):
             _ = sys.stdout.write(f"::add-mask::{value}\n")
     return 0
 
diff --git a/tests/e2e/gateway/stage_mirror_ci_config.yml b/tests/e2e/gateway/stage_mirror_ci_config.yml
index 57a92fd47fb..21664c2d0a1 100644
--- a/tests/e2e/gateway/stage_mirror_ci_config.yml
+++ b/tests/e2e/gateway/stage_mirror_ci_config.yml
@@ -1,4 +1,5 @@
 general_settings:
+  proxy_config_reload_interval_seconds: 7
   store_prompts_in_spend_logs: true
   database_connection_pool_limit: 10
   forward_client_headers_to_llm_api: false

From 1af9c229fa452c1a226bcc22f34f74cfffd54c54 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Wed, 2 Sep 2026 17:29:56 -0700
Subject: [PATCH 07/16] ci(e2e): leave the managed-files opt-in file to its own
 lane instead of failing on an empty collection

---
 .github/workflows/test-e2e-changed.yml | 5 +++--
 tests/e2e/CONTRIBUTING.md              | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml
index b20c84af02d..c1a74f007ce 100644
--- a/.github/workflows/test-e2e-changed.yml
+++ b/.github/workflows/test-e2e-changed.yml
@@ -27,14 +27,15 @@ jobs:
           REPO: ${{ github.repository }}
           PR_NUMBER: ${{ github.event.pull_request.number }}
           SMOKE_TESTS: tests/e2e/access_control
+          OWN_LANE: '^tests/e2e/(ui|claude_code|load)/|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$'
         run: |
           files="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" --paginate \
             --jq '.[] | select(.status != "removed") | .filename')"
           tests="$(printf '%s\n' "${files}" \
             | grep -E '^tests/e2e/([A-Za-z0-9_.-]+/)*test_[A-Za-z0-9_.-]+\.py$' \
-            | grep -vE '^tests/e2e/(ui|claude_code|load)/' \
+            | grep -vE "${OWN_LANE}" \
             | sort -u | tr '\n' ' ' | sed 's/ $//')" || true
-          if [ -z "${tests}" ] && printf '%s\n' "${files}" | grep -vE '^tests/e2e/(ui|claude_code|load)/' \
+          if [ -z "${tests}" ] && printf '%s\n' "${files}" | grep -vE "${OWN_LANE}" \
             | grep -qE '^(tests/e2e/|\.github/e2e-stack/|\.github/workflows/test-e2e-changed\.yml$)'; then
             tests="${SMOKE_TESTS}"
             echo "harness or stack changed without a test file; running the smoke suite"
diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md
index b984791c738..81fcdf84e74 100644
--- a/tests/e2e/CONTRIBUTING.md
+++ b/tests/e2e/CONTRIBUTING.md
@@ -54,7 +54,7 @@ Some suites need extra services the bare proxy does not start. The `logging/` OT
 
 ### The pull request check
 
-Every PR that adds or modifies a `tests/e2e/**/test_*.py` file (outside `ui/`, `claude_code/`, and `load/`, which have their own lanes) runs exactly those files three times, with retries off, against a stage-mirror stack booted on a GitHub Actions runner: migrations, a control-plane backend, two gateway processes behind an nginx load balancer, Postgres, Jaeger, and a TLS cluster-mode Valkey, wired the way stage is deployed. A PR that only touches the harness or the stack itself runs the `access_control/` suite as a smoke instead. Three green passes are the bar because the check exists to catch a flaky test before it reaches the release gate, so a red pass is a failure to fix, not a retry candidate. The same stack boots on a laptop with `bash .github/e2e-stack/up.sh`: it reads provider keys from `tests/e2e/.env`, writes the pytest environment to `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}/stack.env`, every port is overridable through `E2E_*_PORT` variables, and `down.sh` tears it all down
+Every PR that adds or modifies a `tests/e2e/**/test_*.py` file (outside `ui/`, `claude_code/`, `load/`, and `batches/test_managed_files_enforcement_e2e.py`, which have their own lanes or need a differently configured proxy) runs exactly those files three times, with retries off, against a stage-mirror stack booted on a GitHub Actions runner: migrations, a control-plane backend, two gateway processes behind an nginx load balancer, Postgres, Jaeger, and a TLS cluster-mode Valkey, wired the way stage is deployed. A PR that only touches the harness or the stack itself runs the `access_control/` suite as a smoke instead. Three green passes are the bar because the check exists to catch a flaky test before it reaches the release gate, so a red pass is a failure to fix, not a retry candidate. The same stack boots on a laptop with `bash .github/e2e-stack/up.sh`: it reads provider keys from `tests/e2e/.env`, writes the pytest environment to `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}/stack.env`, every port is overridable through `E2E_*_PORT` variables, and `down.sh` tears it all down
 
 Credentials: the lane never sees the stage keys. Everything the runner writes into `tests/e2e/.env` comes from two AWS Secrets Manager secrets in us-east-1, `litellm-e2e-changed-provider-keys` and `litellm-e2e-changed-license`, and the first holds only the names that the tests and `tests/e2e/gateway/stage_mirror_ci_config.yml` read. Each credential in it is dedicated to this lane and can do only what the tests do: the AWS pair belongs to the `litellm-e2e-changed` IAM user, whose inline policy allows Bedrock inference, the one guardrail the tests use, the batch job APIs, the batch S3 bucket, and assuming the batch test's role; the Vertex key belongs to a service account holding only `roles/aiplatform.user` on the Vertex project; the OpenAI, Anthropic, Gemini, Cohere, and Devin keys are per-lane keys with hard monthly spend caps; and the Datadog application key is scoped to log search. A leaked key can therefore spend at most one month of a small capped budget or call a handful of Bedrock APIs, and rotating one is a single `aws secretsmanager put-secret-value` on the secret, with no workflow change
 

From a3ee6b25667d7640214465b58364d04554c02f20 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Wed, 2 Sep 2026 17:50:35 -0700
Subject: [PATCH 08/16] ci(e2e): fail a pass whose every collected test was
 skipped

---
 .github/e2e-stack/assert_tests_ran.py  | 21 +++++++++++++++++++++
 .github/workflows/test-e2e-changed.yml |  4 +++-
 tests/e2e/CONTRIBUTING.md              |  2 +-
 3 files changed, 25 insertions(+), 2 deletions(-)
 create mode 100644 .github/e2e-stack/assert_tests_ran.py

diff --git a/.github/e2e-stack/assert_tests_ran.py b/.github/e2e-stack/assert_tests_ran.py
new file mode 100644
index 00000000000..092a9db7256
--- /dev/null
+++ b/.github/e2e-stack/assert_tests_ran.py
@@ -0,0 +1,21 @@
+import sys
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from typing import Final
+
+
+def main() -> int:
+    report: Final = ET.parse(Path(sys.argv[1])).getroot()
+    suites: Final = tuple(report.iter("testsuite"))
+    collected: Final = sum(int(suite.get("tests", "0")) for suite in suites)
+    skipped: Final = sum(int(suite.get("skipped", "0")) for suite in suites)
+    executed: Final = collected - skipped
+    _ = sys.stdout.write(f"executed {executed} of {collected} collected tests ({skipped} skipped)\n")
+    if executed > 0:
+        return 0
+    _ = sys.stdout.write("::error::every selected test was skipped, so nothing was verified\n")
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml
index c1a74f007ce..e08660d0412 100644
--- a/.github/workflows/test-e2e-changed.yml
+++ b/.github/workflows/test-e2e-changed.yml
@@ -152,9 +152,10 @@ jobs:
         run: |
           read -r -a test_files <<< "${TESTS}"
           for pass in 1 2 3; do
+            report="${RUNNER_TEMP}/e2e-pass-${pass}.xml"
             echo "::group::pass ${pass} of 3"
             set +e
-            uv run --no-sync pytest "${test_files[@]}" --reruns 0 -v -rA --tb=short -p no:cacheprovider
+            uv run --no-sync pytest "${test_files[@]}" --reruns 0 -v -rA --tb=short -p no:cacheprovider --junitxml="${report}"
             status=$?
             set -e
             echo "::endgroup::"
@@ -166,6 +167,7 @@ jobs:
               echo "::error::pass ${pass} of 3 failed with exit code ${status}"
               exit "${status}"
             fi
+            uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}"
           done
 
       - name: Show stack logs on failure
diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md
index 81fcdf84e74..ef6fbb5405d 100644
--- a/tests/e2e/CONTRIBUTING.md
+++ b/tests/e2e/CONTRIBUTING.md
@@ -54,7 +54,7 @@ Some suites need extra services the bare proxy does not start. The `logging/` OT
 
 ### The pull request check
 
-Every PR that adds or modifies a `tests/e2e/**/test_*.py` file (outside `ui/`, `claude_code/`, `load/`, and `batches/test_managed_files_enforcement_e2e.py`, which have their own lanes or need a differently configured proxy) runs exactly those files three times, with retries off, against a stage-mirror stack booted on a GitHub Actions runner: migrations, a control-plane backend, two gateway processes behind an nginx load balancer, Postgres, Jaeger, and a TLS cluster-mode Valkey, wired the way stage is deployed. A PR that only touches the harness or the stack itself runs the `access_control/` suite as a smoke instead. Three green passes are the bar because the check exists to catch a flaky test before it reaches the release gate, so a red pass is a failure to fix, not a retry candidate. The same stack boots on a laptop with `bash .github/e2e-stack/up.sh`: it reads provider keys from `tests/e2e/.env`, writes the pytest environment to `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}/stack.env`, every port is overridable through `E2E_*_PORT` variables, and `down.sh` tears it all down
+Every PR that adds or modifies a `tests/e2e/**/test_*.py` file (outside `ui/`, `claude_code/`, `load/`, and `batches/test_managed_files_enforcement_e2e.py`, which have their own lanes or need a differently configured proxy) runs exactly those files three times, with retries off, against a stage-mirror stack booted on a GitHub Actions runner: migrations, a control-plane backend, two gateway processes behind an nginx load balancer, Postgres, Jaeger, and a TLS cluster-mode Valkey, wired the way stage is deployed. A PR that only touches the harness or the stack itself runs the `access_control/` suite as a smoke instead. Three green passes are the bar because the check exists to catch a flaky test before it reaches the release gate, so a red pass is a failure to fix, not a retry candidate. A pass that executes nothing is also red: each pass writes a JUnit report and fails when every collected test was skipped, so a skipped-out file cannot pass on paper. The same stack boots on a laptop with `bash .github/e2e-stack/up.sh`: it reads provider keys from `tests/e2e/.env`, writes the pytest environment to `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}/stack.env`, every port is overridable through `E2E_*_PORT` variables, and `down.sh` tears it all down
 
 Credentials: the lane never sees the stage keys. Everything the runner writes into `tests/e2e/.env` comes from two AWS Secrets Manager secrets in us-east-1, `litellm-e2e-changed-provider-keys` and `litellm-e2e-changed-license`, and the first holds only the names that the tests and `tests/e2e/gateway/stage_mirror_ci_config.yml` read. Each credential in it is dedicated to this lane and can do only what the tests do: the AWS pair belongs to the `litellm-e2e-changed` IAM user, whose inline policy allows Bedrock inference, the one guardrail the tests use, the batch job APIs, the batch S3 bucket, and assuming the batch test's role; the Vertex key belongs to a service account holding only `roles/aiplatform.user` on the Vertex project; the OpenAI, Anthropic, Gemini, Cohere, and Devin keys are per-lane keys with hard monthly spend caps; and the Datadog application key is scoped to log search. A leaked key can therefore spend at most one month of a small capped budget or call a handful of Bedrock APIs, and rotating one is a single `aws secretsmanager put-secret-value` on the secret, with no workflow change
 

From a9bef8d370ebd553071729b32a9f27f729f1d17e Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Wed, 2 Sep 2026 18:01:19 -0700
Subject: [PATCH 09/16] docs(e2e): drop the spend-cap claim from the
 credentials paragraph

---
 tests/e2e/CONTRIBUTING.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md
index ef6fbb5405d..1e88f6505cf 100644
--- a/tests/e2e/CONTRIBUTING.md
+++ b/tests/e2e/CONTRIBUTING.md
@@ -56,7 +56,7 @@ Some suites need extra services the bare proxy does not start. The `logging/` OT
 
 Every PR that adds or modifies a `tests/e2e/**/test_*.py` file (outside `ui/`, `claude_code/`, `load/`, and `batches/test_managed_files_enforcement_e2e.py`, which have their own lanes or need a differently configured proxy) runs exactly those files three times, with retries off, against a stage-mirror stack booted on a GitHub Actions runner: migrations, a control-plane backend, two gateway processes behind an nginx load balancer, Postgres, Jaeger, and a TLS cluster-mode Valkey, wired the way stage is deployed. A PR that only touches the harness or the stack itself runs the `access_control/` suite as a smoke instead. Three green passes are the bar because the check exists to catch a flaky test before it reaches the release gate, so a red pass is a failure to fix, not a retry candidate. A pass that executes nothing is also red: each pass writes a JUnit report and fails when every collected test was skipped, so a skipped-out file cannot pass on paper. The same stack boots on a laptop with `bash .github/e2e-stack/up.sh`: it reads provider keys from `tests/e2e/.env`, writes the pytest environment to `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}/stack.env`, every port is overridable through `E2E_*_PORT` variables, and `down.sh` tears it all down
 
-Credentials: the lane never sees the stage keys. Everything the runner writes into `tests/e2e/.env` comes from two AWS Secrets Manager secrets in us-east-1, `litellm-e2e-changed-provider-keys` and `litellm-e2e-changed-license`, and the first holds only the names that the tests and `tests/e2e/gateway/stage_mirror_ci_config.yml` read. Each credential in it is dedicated to this lane and can do only what the tests do: the AWS pair belongs to the `litellm-e2e-changed` IAM user, whose inline policy allows Bedrock inference, the one guardrail the tests use, the batch job APIs, the batch S3 bucket, and assuming the batch test's role; the Vertex key belongs to a service account holding only `roles/aiplatform.user` on the Vertex project; the OpenAI, Anthropic, Gemini, Cohere, and Devin keys are per-lane keys with hard monthly spend caps; and the Datadog application key is scoped to log search. A leaked key can therefore spend at most one month of a small capped budget or call a handful of Bedrock APIs, and rotating one is a single `aws secretsmanager put-secret-value` on the secret, with no workflow change
+Credentials: everything the runner writes into `tests/e2e/.env` comes from two AWS Secrets Manager secrets in us-east-1, `litellm-e2e-changed-provider-keys` and `litellm-e2e-changed-license`, and the first holds only the names that the tests and `tests/e2e/gateway/stage_mirror_ci_config.yml` read. The cloud credentials in it are dedicated to this lane and can do only what the tests do: the AWS pair belongs to the `litellm-e2e-changed` IAM user, whose inline policy allows Bedrock inference, the one guardrail the tests use, the batch job APIs, the batch S3 bucket, and assuming the batch test's role, and the Vertex key belongs to a service account holding only `roles/aiplatform.user` on the Vertex project. The provider API keys carry no lane-specific spend cap, since a cap that trips mid-month would fail every run until it resets, so the reviewer's approval of the `e2e-changed` environment is the control on how a PR's tests use them. Rotating any value is a single `aws secretsmanager put-secret-value` on the secret, with no workflow change
 
 ### Record and replay
 

From 0f59b6fb7a996debcb350f6bf10a18e5ba276a62 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Sat, 5 Sep 2026 12:03:42 -0700
Subject: [PATCH 10/16] ci(e2e): refine changed-test selection and runner
 lifecycle

---
 .github/e2e-stack/assert_tests_ran.py         | 29 +++++--
 .github/e2e-stack/secrets_to_env.py           | 44 ++++++----
 .github/e2e-stack/up.sh                       |  8 +-
 .github/workflows/test-e2e-changed.yml        | 80 ++++++++++++++++---
 .../test_e2e_changed_gate.py                  | 55 +++++++++++++
 tests/e2e/CONTRIBUTING.md                     | 12 ++-
 tests/e2e/gateway/stage_mirror_ci_config.yml  |  4 -
 7 files changed, 186 insertions(+), 46 deletions(-)
 create mode 100644 tests/code_coverage_tests/test_e2e_changed_gate.py

diff --git a/.github/e2e-stack/assert_tests_ran.py b/.github/e2e-stack/assert_tests_ran.py
index 092a9db7256..7fd3f7c7c33 100644
--- a/.github/e2e-stack/assert_tests_ran.py
+++ b/.github/e2e-stack/assert_tests_ran.py
@@ -5,15 +5,28 @@ from typing import Final
 
 
 def main() -> int:
-    report: Final = ET.parse(Path(sys.argv[1])).getroot()
-    suites: Final = tuple(report.iter("testsuite"))
-    collected: Final = sum(int(suite.get("tests", "0")) for suite in suites)
-    skipped: Final = sum(int(suite.get("skipped", "0")) for suite in suites)
-    executed: Final = collected - skipped
-    _ = sys.stdout.write(f"executed {executed} of {collected} collected tests ({skipped} skipped)\n")
-    if executed > 0:
+    selected: Final = tuple(sys.argv[2:])
+    try:
+        report: Final = ET.parse(Path(sys.argv[1])).getroot()
+    except (ET.ParseError, OSError):
+        _ = sys.stdout.write("::error::could not read the test execution report\n")
+        return 1
+    cases: Final = tuple(report.iter("testcase"))
+    passed: Final = frozenset(
+        case.get("file") for case in cases if all(case.find(tag) is None for tag in ("skipped", "failure", "error"))
+    )
+    missing: Final = tuple(path for path in selected if path not in passed)
+    for path in selected:
+        collected: Final = sum(case.get("file") == path for case in cases)
+        skipped: Final = sum(case.get("file") == path and case.find("skipped") is not None for case in cases)
+        _ = sys.stdout.write(f"{path}: {collected} collected, {skipped} skipped\n")
+    if (
+        selected
+        and not missing
+        and not any(case.find(tag) is not None for case in cases for tag in ("failure", "error"))
+    ):
         return 0
-    _ = sys.stdout.write("::error::every selected test was skipped, so nothing was verified\n")
+    _ = sys.stdout.write("::error::every selected file must execute a passing test, with no failures or errors\n")
     return 1
 
 
diff --git a/.github/e2e-stack/secrets_to_env.py b/.github/e2e-stack/secrets_to_env.py
index d2d6675d690..99e717b271d 100644
--- a/.github/e2e-stack/secrets_to_env.py
+++ b/.github/e2e-stack/secrets_to_env.py
@@ -1,29 +1,41 @@
+import os
 import re
 import sys
 from pathlib import Path
 from typing import Final
 
-from pydantic import TypeAdapter
+from pydantic import TypeAdapter, ValidationError
 
-secrets_adapter: TypeAdapter[dict[str, str]] = TypeAdapter(dict[str, str])
-SECRET_NAME: Final = re.compile(r"KEY|SECRET|TOKEN|PASS|CREDENTIAL|LICENSE|AUTH")
+secrets_adapter: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str])
+ENV_NAME: Final = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
 
 
 def main() -> int:
-    env_path = Path(sys.argv[1])
-    secrets = {key: value.rstrip("\r\n") for key, value in secrets_adapter.validate_json(sys.stdin.read()).items()}
-    unwritable = tuple(
-        key for key, value in secrets.items() if "'" in value or "\n" in value or "\r" in value
-    )
-    if unwritable:
-        _ = sys.stderr.write(f"values contain characters unsafe for both bash and dotenv: {', '.join(unwritable)}\n")
+    env_path: Final = Path(sys.argv[1])
+    try:
+        secrets: Final = {
+            key: value.rstrip("\r\n") for key, value in secrets_adapter.validate_json(sys.stdin.read()).items()
+        }
+    except (ValidationError, UnicodeError):
+        _ = sys.stderr.write("expected a JSON object containing string environment values\n")
+        return 1
+    if any(
+        ENV_NAME.fullmatch(key) is None or any(char in value for char in "'\n\r\0") for key, value in secrets.items()
+    ):
+        _ = sys.stderr.write("environment names or values cannot be represented in both bash and dotenv\n")
+        return 1
+    for value in secrets.values():
+        if value:
+            _ = sys.stdout.write(f"::add-mask::{value.replace('%', '%25')}\n")
+    sys.stdout.flush()
+    lines: Final = tuple(f"{key}='{value}'" for key, value in secrets.items() if value)
+    try:
+        with os.fdopen(os.open(env_path, os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW, 0o600), "w") as handle:
+            os.fchmod(handle.fileno(), 0o600)
+            _ = handle.write("\n".join(lines) + "\n")
+    except OSError:
+        _ = sys.stderr.write("could not write the environment file\n")
         return 1
-    lines = tuple(f"{key}='{value}'" for key, value in secrets.items() if value)
-    with env_path.open("a") as handle:
-        _ = handle.write("\n".join(lines) + "\n")
-    for key, value in secrets.items():
-        if value and SECRET_NAME.search(key):
-            _ = sys.stdout.write(f"::add-mask::{value}\n")
     return 0
 
 
diff --git a/.github/e2e-stack/up.sh b/.github/e2e-stack/up.sh
index 06fce35a896..e891add341d 100755
--- a/.github/e2e-stack/up.sh
+++ b/.github/e2e-stack/up.sh
@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 set -euo pipefail
+umask 077
 
 REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
 STACK_DIR="${E2E_STACK_DIR:-${RUNNER_TEMP:-/tmp}/litellm-e2e-stack}"
@@ -8,9 +9,9 @@ LOGS_DIR="${STACK_DIR}/logs"
 PIDS_DIR="${STACK_DIR}/pids"
 
 POSTGRES_IMAGE="${E2E_POSTGRES_IMAGE:-postgres:16.6}"
-VALKEY_IMAGE="${E2E_VALKEY_IMAGE:-valkey/valkey:8.1}"
+VALKEY_IMAGE="${E2E_VALKEY_IMAGE:-valkey/valkey:8.1.4@sha256:81db6d39e1bba3b3ff32bd3a1b19a6d69690f94a3954ec131277b9a26b95b3aa}"
 JAEGER_IMAGE="${E2E_JAEGER_IMAGE:-jaegertracing/jaeger:2.10.0}"
-NGINX_IMAGE="${E2E_NGINX_IMAGE:-nginx:1.29-alpine}"
+NGINX_IMAGE="${E2E_NGINX_IMAGE:-nginx:1.29.1-alpine@sha256:42a516af16b852e33b7682d5ef8acbd5d13fe08fecadc7ed98605ba5e3b26ab8}"
 
 LB_PORT="${E2E_LB_PORT:-4000}"
 GATEWAY_PORT_1="${E2E_GATEWAY_PORT_1:-4010}"
@@ -28,6 +29,8 @@ JAEGER_QUERY_PORT="${E2E_JAEGER_QUERY_PORT:-16686}"
 MASTER_KEY="${LITELLM_MASTER_KEY:-sk-e2e-$(openssl rand -hex 16)}"
 
 mkdir -p "${CERTS_DIR}" "${LOGS_DIR}" "${PIDS_DIR}"
+chmod 700 "${STACK_DIR}" "${LOGS_DIR}" "${PIDS_DIR}"
+chmod 755 "${CERTS_DIR}"
 
 log() { printf 'e2e-stack: %s\n' "$*"; }
 
@@ -38,7 +41,6 @@ wait_for() {
   until eval "${check}"; do
     if ((SECONDS >= deadline)); then
       log "timed out waiting for ${label}"
-      tail -n 60 "${LOGS_DIR}"/*.log 2>/dev/null || true
       exit 1
     fi
     sleep 2
diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml
index e08660d0412..b802e9173f5 100644
--- a/.github/workflows/test-e2e-changed.yml
+++ b/.github/workflows/test-e2e-changed.yml
@@ -26,27 +26,26 @@ jobs:
           GH_TOKEN: ${{ github.token }}
           REPO: ${{ github.repository }}
           PR_NUMBER: ${{ github.event.pull_request.number }}
-          SMOKE_TESTS: tests/e2e/access_control
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
           OWN_LANE: '^tests/e2e/(ui|claude_code|load)/|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$'
         run: |
+          gh api "repos/${REPO}/pulls/${PR_NUMBER}" \
+            --jq 'select(.head.sha == env.HEAD_SHA and .changed_files < 3000) | .head.sha' \
+            | grep -Fxq "${HEAD_SHA}"
           files="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" --paginate \
             --jq '.[] | select(.status != "removed") | .filename')"
+          gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.head.sha' | grep -Fxq "${HEAD_SHA}"
           tests="$(printf '%s\n' "${files}" \
             | grep -E '^tests/e2e/([A-Za-z0-9_.-]+/)*test_[A-Za-z0-9_.-]+\.py$' \
             | grep -vE "${OWN_LANE}" \
             | sort -u | tr '\n' ' ' | sed 's/ $//')" || true
-          if [ -z "${tests}" ] && printf '%s\n' "${files}" | grep -vE "${OWN_LANE}" \
-            | grep -qE '^(tests/e2e/|\.github/e2e-stack/|\.github/workflows/test-e2e-changed\.yml$)'; then
-            tests="${SMOKE_TESTS}"
-            echo "harness or stack changed without a test file; running the smoke suite"
-          fi
           echo "tests=${tests}" >> "${GITHUB_OUTPUT}"
           if [ -n "${tests}" ]; then
             echo "any=true" >> "${GITHUB_OUTPUT}"
             echo "selected e2e tests: ${tests}"
           else
             echo "any=false" >> "${GITHUB_OUTPUT}"
-            echo "no e2e changes; nothing to run"
+            echo "no changed e2e test files supported by this stack; nothing to run"
           fi
 
   run:
@@ -88,6 +87,7 @@ jobs:
         uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
           persist-credentials: false
+          ref: ${{ github.sha }}
 
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
@@ -120,14 +120,24 @@ jobs:
         run: uv run --no-sync playwright install --with-deps chromium
 
       - name: Configure AWS credentials
+        id: aws
         uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0
         with:
           role-to-assume: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }}
           aws-region: us-east-1
           role-session-name: litellm-e2e-changed-${{ github.run_id }}
+          role-duration-seconds: 900
+          output-env-credentials: false
+          output-credentials: true
 
       - name: Fetch provider credentials from AWS Secrets Manager
+        env:
+          AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }}
+          AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }}
+          AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }}
+          AWS_DEFAULT_REGION: us-east-1
         run: |
+          umask 077
           aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-provider-keys \
             --query SecretString --output text \
             | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env
@@ -138,7 +148,12 @@ jobs:
 
       - name: Boot the stage-mirror stack
         id: boot
-        run: bash .github/e2e-stack/up.sh
+        run: |
+          umask 077
+          if ! bash .github/e2e-stack/up.sh > "${RUNNER_TEMP}/e2e-boot.log" 2>&1; then
+            echo "::error::stage-mirror stack failed to boot; raw logs are not published"
+            exit 1
+          fi
 
       - name: Export stack environment
         run: |
@@ -149,13 +164,16 @@ jobs:
       - name: Run the selected tests three times with retries off
         env:
           TESTS: ${{ needs.detect.outputs.tests }}
+          E2E_FIXTURE_MODE: live
         run: |
+          umask 077
           read -r -a test_files <<< "${TESTS}"
           for pass in 1 2 3; do
             report="${RUNNER_TEMP}/e2e-pass-${pass}.xml"
             echo "::group::pass ${pass} of 3"
             set +e
-            uv run --no-sync pytest "${test_files[@]}" --reruns 0 -v -rA --tb=short -p no:cacheprovider --junitxml="${report}"
+            uv run --no-sync pytest "${test_files[@]}" --rootdir=. --reruns 0 -v -p no:cacheprovider \
+              -o junit_family=xunit1 --junitxml="${report}" > "${RUNNER_TEMP}/e2e-pass-${pass}.log" 2>&1
             status=$?
             set -e
             echo "::endgroup::"
@@ -163,13 +181,49 @@ jobs:
               echo "::error::the selected files collected no runnable tests, so nothing was verified"
               exit 1
             fi
+            if ! uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}" "${test_files[@]}"; then
+              echo "::error::pass ${pass} of 3 did not verify every selected file"
+              exit 1
+            fi
             if [ "${status}" != "0" ]; then
               echo "::error::pass ${pass} of 3 failed with exit code ${status}"
               exit "${status}"
             fi
-            uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}"
+            echo "pass ${pass} of 3 passed"
           done
 
-      - name: Show stack logs on failure
-        if: failure() && steps.boot.conclusion != 'skipped'
-        run: tail -n 200 "${RUNNER_TEMP}/litellm-e2e-stack/logs"/*.log
+      - name: Stop the stack
+        if: always() && steps.boot.outcome != 'skipped'
+        run: bash .github/e2e-stack/down.sh
+
+      - name: Remove credentials and raw output
+        if: always()
+        run: |
+          rm -f tests/e2e/.env "${RUNNER_TEMP}/e2e-boot.log" "${RUNNER_TEMP}"/e2e-pass-*.log "${RUNNER_TEMP}"/e2e-pass-*.xml
+          rm -rf "${RUNNER_TEMP}/litellm-e2e-stack"
+
+  gate:
+    name: e2e-changed-tests
+    needs: [detect, run]
+    if: always()
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Require three successful passes when tests changed
+        env:
+          DETECT_RESULT: ${{ needs.detect.result }}
+          ANY_TESTS: ${{ needs.detect.outputs.any }}
+          RUN_RESULT: ${{ needs.run.result }}
+        run: |
+          if [ "${DETECT_RESULT}" != "success" ]; then
+            echo "::error::changed-test detection did not succeed"
+            exit 1
+          fi
+          if [ "${ANY_TESTS}" = "false" ]; then
+            echo "no changed e2e test files supported by this stack; nothing to run"
+            exit 0
+          fi
+          if [ "${ANY_TESTS}" != "true" ] || [ "${RUN_RESULT}" != "success" ]; then
+            echo "::error::selected e2e tests require an approved, successful run; fork PRs must run from a reviewed same-repository branch"
+            exit 1
+          fi
diff --git a/tests/code_coverage_tests/test_e2e_changed_gate.py b/tests/code_coverage_tests/test_e2e_changed_gate.py
new file mode 100644
index 00000000000..7a82a8ebb60
--- /dev/null
+++ b/tests/code_coverage_tests/test_e2e_changed_gate.py
@@ -0,0 +1,55 @@
+import subprocess
+import sys
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from typing import Final
+
+import pytest
+
+GATE: Final = Path(__file__).resolve().parents[2] / ".github/e2e-stack/assert_tests_ran.py"
+SELECTED: Final = ("tests/e2e/access_control/test_a.py", "tests/e2e/access_control/test_b.py")
+
+
+@pytest.mark.parametrize(
+    ("second_outcome", "expected_status"),
+    (("passed", 0), ("skipped", 1), ("failure", 1), ("error", 1), ("deselected", 1)),
+)
+def test_each_changed_file_must_run(tmp_path: Path, second_outcome: str, expected_status: int) -> None:
+    suite: Final = ET.Element("testsuite")
+    _ = ET.SubElement(suite, "testcase", file=SELECTED[0])
+    if second_outcome != "deselected":
+        second: Final = ET.SubElement(suite, "testcase", file=SELECTED[1])
+        if second_outcome != "passed":
+            _ = ET.SubElement(second, second_outcome)
+    report: Final = tmp_path / "report.xml"
+    ET.ElementTree(suite).write(report)
+
+    result: Final = subprocess.run([sys.executable, str(GATE), str(report), *SELECTED], capture_output=True, text=True)
+
+    assert result.returncode == expected_status, result.stdout
+
+
+@pytest.mark.parametrize("outcome", ("failure", "error"))
+def test_passing_case_does_not_hide_a_failure_in_the_same_file(tmp_path: Path, outcome: str) -> None:
+    suite: Final = ET.Element("testsuite")
+    _ = ET.SubElement(suite, "testcase", file=SELECTED[0])
+    failed: Final = ET.SubElement(suite, "testcase", file=SELECTED[0])
+    _ = ET.SubElement(failed, outcome)
+    report: Final = tmp_path / "report.xml"
+    ET.ElementTree(suite).write(report)
+
+    result: Final = subprocess.run(
+        [sys.executable, str(GATE), str(report), SELECTED[0]], capture_output=True, text=True
+    )
+
+    assert result.returncode == 1
+
+
+@pytest.mark.parametrize("contents", ("<testsuite/>", "<testsuite", '<testsuite><testcase name="a"/></testsuite>'))
+def test_missing_execution_evidence_fails(tmp_path: Path, contents: str) -> None:
+    report: Final = tmp_path / "report.xml"
+    _ = report.write_text(contents)
+
+    result: Final = subprocess.run([sys.executable, str(GATE), str(report), *SELECTED], capture_output=True, text=True)
+
+    assert result.returncode == 1
diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md
index 1e88f6505cf..d51e729fd1c 100644
--- a/tests/e2e/CONTRIBUTING.md
+++ b/tests/e2e/CONTRIBUTING.md
@@ -54,9 +54,17 @@ Some suites need extra services the bare proxy does not start. The `logging/` OT
 
 ### The pull request check
 
-Every PR that adds or modifies a `tests/e2e/**/test_*.py` file (outside `ui/`, `claude_code/`, `load/`, and `batches/test_managed_files_enforcement_e2e.py`, which have their own lanes or need a differently configured proxy) runs exactly those files three times, with retries off, against a stage-mirror stack booted on a GitHub Actions runner: migrations, a control-plane backend, two gateway processes behind an nginx load balancer, Postgres, Jaeger, and a TLS cluster-mode Valkey, wired the way stage is deployed. A PR that only touches the harness or the stack itself runs the `access_control/` suite as a smoke instead. Three green passes are the bar because the check exists to catch a flaky test before it reaches the release gate, so a red pass is a failure to fix, not a retry candidate. A pass that executes nothing is also red: each pass writes a JUnit report and fails when every collected test was skipped, so a skipped-out file cannot pass on paper. The same stack boots on a laptop with `bash .github/e2e-stack/up.sh`: it reads provider keys from `tests/e2e/.env`, writes the pytest environment to `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}/stack.env`, every port is overridable through `E2E_*_PORT` variables, and `down.sh` tears it all down
+Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times with pytest retries off. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. Documentation, harness, configuration, deleted-file, and workflow-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories and `batches/test_managed_files_enforcement_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack
 
-Credentials: everything the runner writes into `tests/e2e/.env` comes from two AWS Secrets Manager secrets in us-east-1, `litellm-e2e-changed-provider-keys` and `litellm-e2e-changed-license`, and the first holds only the names that the tests and `tests/e2e/gateway/stage_mirror_ci_config.yml` read. The cloud credentials in it are dedicated to this lane and can do only what the tests do: the AWS pair belongs to the `litellm-e2e-changed` IAM user, whose inline policy allows Bedrock inference, the one guardrail the tests use, the batch job APIs, the batch S3 bucket, and assuming the batch test's role, and the Vertex key belongs to a service account holding only `roles/aiplatform.user` on the Vertex project. The provider API keys carry no lane-specific spend cap, since a cap that trips mid-month would fail every run until it resets, so the reviewer's approval of the `e2e-changed` environment is the control on how a PR's tests use them. Rotating any value is a single `aws secretsmanager put-secret-value` on the secret, with no workflow change
+Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A failed pass stops the run without retrying. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch
+
+Repository admins must require the `e2e-changed-tests` status check for merging and configure the `e2e-changed` environment with required reviewers, self-review disabled, and admin bypass disabled. Each push cancels the previous run; a new run that selects tests needs a fresh approval. Reviewers must inspect the entire executable PR diff, including application code, dependencies, tests, and workflow helpers, before approving the exact revision. Approved code executes with provider credentials, so environment approval is a trust decision about that code
+
+Credentials come from the existing AWS Secrets Manager secrets in us-east-1, `litellm-e2e-changed-provider-keys` and `litellm-e2e-changed-license`. The OIDC role must trust only `repo:BerriAI/litellm:environment:e2e-changed` with audience `sts.amazonaws.com` and have read access only to these secrets. The short-lived reader credentials are scoped to the fetch step. Provider credentials must cover the selected suites, including Datadog credentials when logging or MCP tests need them; missing credentials fail the run. Keep provider credentials dedicated to this lane with only the permissions those tests need
+
+Fetched values are masked before use, and credential files and raw output are private to the runner. Public logs contain selected file names, counts, and pass status; raw pytest output, reports, and stack logs are not uploaded or printed. The workflow removes them and the credential files during cleanup. To diagnose a failed pass, reproduce the selected files locally with the appropriate credentials and inspect the local logs
+
+To reproduce the CI topology on a dedicated machine, `bash .github/e2e-stack/up.sh` reads `tests/e2e/.env`, writes `stack.env` under `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}`, and `bash .github/e2e-stack/down.sh` stops it. Keep this directory private and remove its credential files and logs after use
 
 ### Record and replay
 
diff --git a/tests/e2e/gateway/stage_mirror_ci_config.yml b/tests/e2e/gateway/stage_mirror_ci_config.yml
index 21664c2d0a1..6b3756b5f9f 100644
--- a/tests/e2e/gateway/stage_mirror_ci_config.yml
+++ b/tests/e2e/gateway/stage_mirror_ci_config.yml
@@ -52,10 +52,6 @@ model_list:
     litellm_params:
       model: gemini/gemini-2.5-flash
       api_key: os.environ/GEMINI_API_KEY
-  - model_name: gemini-2.5-flash
-    litellm_params:
-      model: gemini/gemini-2.5-flash
-      api_key: os.environ/GEMINI_API_KEY
 
 mcp_servers:
   devin:

From 5051e6d44acb627d912cba11b4afdc91ae5ff8c6 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Sat, 5 Sep 2026 12:08:11 -0700
Subject: [PATCH 11/16] ci(e2e): include execution gate checks in code quality

---
 .github/workflows/test-code-quality.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/test-code-quality.yml b/.github/workflows/test-code-quality.yml
index c112bf2bb22..a0e0ce6665a 100644
--- a/.github/workflows/test-code-quality.yml
+++ b/.github/workflows/test-code-quality.yml
@@ -74,6 +74,9 @@ jobs:
       - name: check_workflow_startup_safety
         run: uv run --no-sync python ./tests/code_coverage_tests/check_workflow_startup_safety.py
 
+      - name: test_e2e_changed_gate
+        run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py
+
       - name: router_code_coverage
         run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
 

From 65d8bbb8ac4610c9722ecae03ffa16b352c47256 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 5 Sep 2026 16:10:40 -0700
Subject: [PATCH 12/16] fix(e2e): wait for every gateway before using a new
 model and keep the network rerun

The changed-tests workflow overrode the suite's `--reruns 1` with `--reruns 0`, so a
transport blip failed a pass that pytest.ini already scopes to network errors and
5xx responses. Pass 2 of run 33692484803 also went red 15s after a model write with
"no healthy deployments": the barrier only polled /v1/models through nginx, which
proves one gateway converged, and the next request rolled the other. The stack now
exports LITELLM_PROXY_REPLICA_URLS, the barrier polls every replica with the full
budget before settling, and up.sh refuses to boot without DD_API_KEY, since the
gateway config enables the datadog callback on every run
---
 .github/e2e-stack/up.sh                |   7 ++
 .github/workflows/test-e2e-changed.yml |   4 +-
 tests/e2e/CONTRIBUTING.md              |   6 +-
 tests/e2e/claude_code/_env.py          |   1 +
 tests/e2e/claude_code/conftest.py      |   1 +
 tests/e2e/e2e_config.py                |  18 +++-
 tests/e2e/proxy_client.py              | 117 +++++++++++++++++++------
 tests/e2e/test_proxy_client.py         |  87 ++++++++++++++++++
 8 files changed, 206 insertions(+), 35 deletions(-)
 create mode 100644 tests/e2e/test_proxy_client.py

diff --git a/.github/e2e-stack/up.sh b/.github/e2e-stack/up.sh
index e891add341d..2f2e6c6f9a8 100755
--- a/.github/e2e-stack/up.sh
+++ b/.github/e2e-stack/up.sh
@@ -54,6 +54,12 @@ if [[ -f "${REPO_ROOT}/tests/e2e/.env" ]]; then
   set +a
 fi
 
+if [[ -z "${DD_API_KEY:-}" ]]; then
+  log "DD_API_KEY is empty; the gateway config enables the datadog callback, so put a Datadog API key in tests/e2e/.env"
+  exit 1
+fi
+export DD_SITE="${DD_SITE:-datadoghq.com}"
+
 if ! port_open "${DATABASE_PORT}"; then
   docker run -d --name e2e-postgres -p "${DATABASE_PORT}:5432" \
     -e "POSTGRES_USER=${DATABASE_USER}" -e "POSTGRES_PASSWORD=${DATABASE_PASSWORD}" -e "POSTGRES_DB=${DATABASE_NAME}" \
@@ -189,6 +195,7 @@ wait_for "load balancer" "curl -fs http://127.0.0.1:${LB_PORT}/health/liveliness
 cat > "${STACK_DIR}/stack.env" <<EOF
 LITELLM_PROXY_URL=http://127.0.0.1:${LB_PORT}
 LITELLM_CONTROL_PLANE_URL=http://127.0.0.1:${BACKEND_PORT}
+LITELLM_PROXY_REPLICA_URLS=http://127.0.0.1:${GATEWAY_PORT_1},http://127.0.0.1:${GATEWAY_PORT_2}
 LITELLM_MASTER_KEY=${MASTER_KEY}
 REDIS_HOST=127.0.0.1
 REDIS_PORT=${REDIS_PORT}
diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml
index b802e9173f5..16d84beab2d 100644
--- a/.github/workflows/test-e2e-changed.yml
+++ b/.github/workflows/test-e2e-changed.yml
@@ -161,7 +161,7 @@ jobs:
           echo "::add-mask::${master_key}"
           cat "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" >> "${GITHUB_ENV}"
 
-      - name: Run the selected tests three times with retries off
+      - name: Run the selected tests three times
         env:
           TESTS: ${{ needs.detect.outputs.tests }}
           E2E_FIXTURE_MODE: live
@@ -172,7 +172,7 @@ jobs:
             report="${RUNNER_TEMP}/e2e-pass-${pass}.xml"
             echo "::group::pass ${pass} of 3"
             set +e
-            uv run --no-sync pytest "${test_files[@]}" --rootdir=. --reruns 0 -v -p no:cacheprovider \
+            uv run --no-sync pytest "${test_files[@]}" --rootdir=. -v -p no:cacheprovider \
               -o junit_family=xunit1 --junitxml="${report}" > "${RUNNER_TEMP}/e2e-pass-${pass}.log" 2>&1
             status=$?
             set -e
diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md
index d51e729fd1c..7bfd02023dd 100644
--- a/tests/e2e/CONTRIBUTING.md
+++ b/tests/e2e/CONTRIBUTING.md
@@ -54,13 +54,13 @@ Some suites need extra services the bare proxy does not start. The `logging/` OT
 
 ### The pull request check
 
-Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times with pytest retries off. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. Documentation, harness, configuration, deleted-file, and workflow-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories and `batches/test_managed_files_enforcement_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack
+Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, harness, configuration, deleted-file, and workflow-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories and `batches/test_managed_files_enforcement_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack
 
-Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A failed pass stops the run without retrying. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch
+Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A failed pass stops the run. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch
 
 Repository admins must require the `e2e-changed-tests` status check for merging and configure the `e2e-changed` environment with required reviewers, self-review disabled, and admin bypass disabled. Each push cancels the previous run; a new run that selects tests needs a fresh approval. Reviewers must inspect the entire executable PR diff, including application code, dependencies, tests, and workflow helpers, before approving the exact revision. Approved code executes with provider credentials, so environment approval is a trust decision about that code
 
-Credentials come from the existing AWS Secrets Manager secrets in us-east-1, `litellm-e2e-changed-provider-keys` and `litellm-e2e-changed-license`. The OIDC role must trust only `repo:BerriAI/litellm:environment:e2e-changed` with audience `sts.amazonaws.com` and have read access only to these secrets. The short-lived reader credentials are scoped to the fetch step. Provider credentials must cover the selected suites, including Datadog credentials when logging or MCP tests need them; missing credentials fail the run. Keep provider credentials dedicated to this lane with only the permissions those tests need
+Credentials come from the existing AWS Secrets Manager secrets in us-east-1, `litellm-e2e-changed-provider-keys` and `litellm-e2e-changed-license`. The OIDC role must trust only `repo:BerriAI/litellm:environment:e2e-changed` with audience `sts.amazonaws.com` and have read access only to these secrets. The short-lived reader credentials are scoped to the fetch step. Provider credentials must cover the selected suites, including Datadog credentials when logging or MCP tests need them; missing credentials fail the run. `up.sh` refuses to start without `DD_API_KEY`, because the stack's gateway config enables the Datadog callback for every run and a gateway booted without the key fails readiness. Keep provider credentials dedicated to this lane with only the permissions those tests need
 
 Fetched values are masked before use, and credential files and raw output are private to the runner. Public logs contain selected file names, counts, and pass status; raw pytest output, reports, and stack logs are not uploaded or printed. The workflow removes them and the credential files during cleanup. To diagnose a failed pass, reproduce the selected files locally with the appropriate credentials and inspect the local logs
 
diff --git a/tests/e2e/claude_code/_env.py b/tests/e2e/claude_code/_env.py
index ed2da1fe1e2..889d8f848dd 100644
--- a/tests/e2e/claude_code/_env.py
+++ b/tests/e2e/claude_code/_env.py
@@ -99,5 +99,6 @@ def require_proxy_client(
         base_url=cfg.base_url,
         master_key=cfg.api_key,
         control_plane_base_url=cfg.base_url,
+        replica_urls=(cfg.base_url,),
     )
     return ProxyClientConfig(client=client, api_key=cfg.api_key)
diff --git a/tests/e2e/claude_code/conftest.py b/tests/e2e/claude_code/conftest.py
index aaad7667936..6e3dce0377e 100644
--- a/tests/e2e/claude_code/conftest.py
+++ b/tests/e2e/claude_code/conftest.py
@@ -594,6 +594,7 @@ def _build_control_plane_client(proxy_config: ProxyConfig):
         base_url=proxy_config.base_url,
         master_key=proxy_config.api_key,
         control_plane_base_url=proxy_config.base_url,
+        replica_urls=(proxy_config.base_url,),
     )
 
 
diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py
index 21c5a338dc3..70d1180cb38 100644
--- a/tests/e2e/e2e_config.py
+++ b/tests/e2e/e2e_config.py
@@ -10,6 +10,7 @@ import os
 import time
 import uuid
 from pathlib import Path
+from typing import Final
 
 from dotenv import load_dotenv
 
@@ -32,6 +33,14 @@ CONTROL_PLANE_BASE_URL = os.environ.get(
     "LITELLM_CONTROL_PLANE_URL", PROXY_BASE_URL
 ).rstrip("/")
 
+
+def parse_replica_urls(raw: str, fallback: str) -> tuple[str, ...]:
+    urls: Final = tuple(url.strip().rstrip("/") for url in raw.split(",") if url.strip())
+    return urls or (fallback,)
+
+
+PROXY_REPLICA_URLS: Final = parse_replica_urls(os.environ.get("LITELLM_PROXY_REPLICA_URLS", ""), PROXY_BASE_URL)
+
 UI_USERNAME = os.environ.get("E2E_UI_USERNAME", "admin")
 UI_PASSWORD = os.environ.get("E2E_UI_PASSWORD", MASTER_KEY)
 
@@ -86,10 +95,11 @@ SLOW_PROVIDER_TIMEOUT_SECONDS = float(os.environ.get("E2E_SLOW_PROVIDER_TIMEOUT"
 # (`proxy_config_reload_interval_seconds`, 30s by default and 7s on the e2e stack)
 # plus margin.
 #
-# The barriers below wait this out instead of returning on first sight, because a
-# single successful read only proves ONE replica converged: every request opens a
-# fresh connection, so a load-balanced Service routes each one independently and
-# the next call re-rolls. See ProxyClient._await_model_servable.
+# The barriers below wait this out on top of polling /v1/models on every replica in
+# PROXY_REPLICA_URLS: that poll proves each addressed gateway converged, but not the
+# workers behind it, and behind a load balancer (PROXY_REPLICA_URLS unset) a
+# successful read only proves ONE replica converged, because every request opens a
+# fresh connection and the next call re-rolls. See ProxyClient._await_model_servable.
 PROPAGATION_TIMEOUT = float(os.environ.get("E2E_PROPAGATION_TIMEOUT", "15"))
 
 EXPECT_RUST = os.environ.get("E2E_EXPECT_RUST", "").strip().lower() in ("1", "true", "yes")
diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py
index 2d382a610e1..c5b34a534b7 100644
--- a/tests/e2e/proxy_client.py
+++ b/tests/e2e/proxy_client.py
@@ -10,12 +10,15 @@ from __future__ import annotations
 
 import time
 import warnings
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
 from dataclasses import dataclass
 from datetime import datetime
+from types import MappingProxyType
+from typing import Final
 
 from e2e_http import (
     AnthropicHeaders,
+    AuthHeaders,
     NoBody,
     ProbeResult,
     Result,
@@ -71,6 +74,7 @@ from e2e_config import (
     POLL_INTERVAL,
     POLL_TIMEOUT,
     PROXY_BASE_URL,
+    PROXY_REPLICA_URLS,
     REQUEST_TIMEOUT,
     SLOW_PROVIDER_TIMEOUT_SECONDS,
     settle_propagation,
@@ -79,10 +83,11 @@ from transport import HttpTransport, SplitTransport, Transport
 
 RowsPredicate = Callable[[list[SpendLogRow]], bool]
 
-# After /model/new, poll data-plane /v1/models until the model is listed (or fail).
-# Bound by MODEL_SERVABLE_TIMEOUT so a stuck reload does not burn the spend
-# poll_timeout (120s). Return on first listing; settle_propagation owns the separate
-# wait that lets every worker and replica reload before the caller uses the model.
+# After /model/new, poll /v1/models on every replica in PROXY_REPLICA_URLS until each
+# lists the model (or fail). Bound by MODEL_SERVABLE_TIMEOUT per replica so a stuck
+# reload does not burn the spend poll_timeout (120s). Return on first listing;
+# settle_propagation owns the separate wait that lets the workers behind each replica
+# reload before the caller uses the model.
 MODEL_SERVABLE_TIMEOUT = 40.0
 MODEL_SERVABLE_DB_SYNC_SECONDS = 0.0
 MODEL_SERVABLE_INTERVAL = 2.0
@@ -108,6 +113,16 @@ class NotServable:
 
 ServableOutcome = Servable | NotServable
 
+type ModelsPoller = Callable[[float], Result[ModelsListResponse]]
+
+
+@dataclass(frozen=True, slots=True)
+class NotServableOn:
+    """`NotServable` labeled with the replica whose /v1/models never listed the model."""
+
+    replica: str
+    last_result: Result[ModelsListResponse] | None
+
 
 def await_servable(
     list_models: Callable[[float], Result[ModelsListResponse]],
@@ -171,9 +186,41 @@ def await_servable(
             sleep(wait)
 
 
+def await_servable_everywhere(
+    pollers: Mapping[str, ModelsPoller],
+    *,
+    model_name: str,
+    timeout: float,
+    interval: float,
+    request_timeout: float,
+    db_sync_seconds: float,
+    now: Callable[[], float],
+    sleep: Callable[[float], None],
+) -> Servable | NotServableOn:
+    """`await_servable` against every replica in turn, each with the full budget, so
+    the model is only servable once every replica has listed it."""
+    for replica, list_models in pollers.items():
+        match await_servable(
+            list_models,
+            model_name=model_name,
+            timeout=timeout,
+            interval=interval,
+            request_timeout=request_timeout,
+            db_sync_seconds=db_sync_seconds,
+            now=now,
+            sleep=sleep,
+        ):
+            case NotServable(last_result=last_result):
+                return NotServableOn(replica=replica, last_result=last_result)
+            case Servable():
+                continue
+    return Servable()
+
+
 def servable_timeout_message(
     *,
     model_name: str,
+    replica: str,
     timeout: float,
     db_sync_seconds: float,
     last_result: Result[ModelsListResponse] | None,
@@ -184,8 +231,8 @@ def servable_timeout_message(
         else ""
     )
     return (
-        f"model {model_name!r} was created but never became servable on the data "
-        f"plane within {timeout}s of first listing (plus {db_sync_seconds}s continuous "
+        f"model {model_name!r} was created but never became servable on {replica} "
+        f"within {timeout}s of first listing (plus {db_sync_seconds}s continuous "
         f"DB sync) after /model/new (control/data-plane propagation or "
         f"STORE_MODEL_IN_DB reload issue){last_error}"
     )
@@ -194,6 +241,7 @@ def servable_timeout_message(
 @dataclass(frozen=True, slots=True)
 class ProxyClient:
     transport: Transport
+    replicas: Mapping[str, Transport]
     poll_timeout: float = 120.0
     poll_interval: float = 5.0
     model_servable_timeout: float = MODEL_SERVABLE_TIMEOUT
@@ -310,12 +358,13 @@ class ProxyClient:
         model name passed". We poll the data-plane /v1/models until the model
         appears, then settle for the remainder of the propagation budget.
 
-        Both steps are needed, and the second is the one that matters at >1 replica.
-        The poll proves *a* replica is serving the model; it cannot prove they all
-        are, because every request opens a fresh connection and a load-balanced
-        Service routes each one independently -- so the caller's next request
-        re-rolls and can land on a replica that has not reloaded yet. Waiting out
-        PROPAGATION_TIMEOUT is what makes the model safe to use anywhere."""
+        Both steps are needed. The poll asks every replica in PROXY_REPLICA_URLS
+        directly, so behind the stack's load balancer it proves each gateway serves
+        the model rather than whichever one the balancer routed the poll to. It still
+        cannot see the workers behind a gateway, nor any replica when only the
+        balancer address is configured (every request opens a fresh connection, so
+        the caller's next request re-rolls), so waiting out PROPAGATION_TIMEOUT is
+        what makes the model safe to use anywhere."""
         model_id = unwrap(
             self.transport.post(
                 "/model/new",
@@ -330,16 +379,10 @@ class ProxyClient:
         return model_id
 
     def _await_model_servable(self, model_name: str, listed_for: str | None = None) -> None:
-        """Block until the data plane lists `model_name`, or fail at model_servable_timeout."""
-        headers = self.transport.master if listed_for is None else self.transport.bearer(listed_for)
-        outcome = await_servable(
-            lambda poll_timeout: self.transport.get(
-                "/v1/models",
-                headers=headers,
-                params=NoBody(),
-                response_type=ModelsListResponse,
-                timeout=poll_timeout,
-            ),
+        """Block until every replica lists `model_name`, or fail at model_servable_timeout."""
+        headers: Final = self.transport.master if listed_for is None else self.transport.bearer(listed_for)
+        outcome: Final = await_servable_everywhere(
+            {url: self._models_poller(transport, headers) for url, transport in self.replicas.items()},
             model_name=model_name,
             timeout=self.model_servable_timeout,
             interval=self.model_servable_interval,
@@ -351,16 +394,27 @@ class ProxyClient:
         match outcome:
             case Servable():
                 return
-            case NotServable(last_result=last_result):
+            case NotServableOn(replica=replica, last_result=last_result):
                 raise AssertionError(
                     servable_timeout_message(
                         model_name=model_name,
+                        replica=replica,
                         timeout=self.model_servable_timeout,
                         db_sync_seconds=self.model_servable_db_sync_seconds,
                         last_result=last_result,
                     )
                 )
 
+    @staticmethod
+    def _models_poller(transport: Transport, headers: AuthHeaders) -> ModelsPoller:
+        return lambda poll_timeout: transport.get(
+            "/v1/models",
+            headers=headers,
+            params=NoBody(),
+            response_type=ModelsListResponse,
+            timeout=poll_timeout,
+        )
+
     def update_model(self, model_id: str, litellm_params: LiteLLMParamsBody) -> None:
         """Merge `litellm_params` over the deployment `model_id`'s stored params via
         POST /model/update. The proxy overlays only the non-null fields and clears
@@ -547,16 +601,20 @@ def build_proxy_client(
     base_url: str = PROXY_BASE_URL,
     master_key: str = MASTER_KEY,
     control_plane_base_url: str = CONTROL_PLANE_BASE_URL,
+    replica_urls: tuple[str, ...] = PROXY_REPLICA_URLS,
 ) -> ProxyClient:
     """The ProxyClient every suite's client is built from: a SplitTransport that routes
     LLM calls to the data plane (PROXY_BASE_URL) and management/admin calls to the
     control plane (CONTROL_PLANE_BASE_URL), with the shared poll budget. The two
     base URLs are the same for a monolithic proxy, so routing is then a no-op.
+    ``replica_urls`` (PROXY_REPLICA_URLS) names every data-plane replica the model
+    barrier polls directly; it is the data-plane URL itself unless the stack
+    exports each gateway's own address.
 
     The endpoints are injectable for callers that resolve the proxy some other
     way than ``e2e_config``'s env names (see ``claude_code/_env.py``); they must
-    pass all three together, since a caller that overrides only the data plane
-    would leave management calls pointed at the env default.
+    pass all four together, since a caller that overrides only the data plane
+    would leave management calls and the replica poll pointed at the env defaults.
 
     Test-to-proxy traffic always goes over the wire, in every E2E_FIXTURE_MODE:
     record and replay scope to the proxy's provider-bound calls via the
@@ -573,8 +631,15 @@ def build_proxy_client(
             request_timeout=REQUEST_TIMEOUT,
         ),
     )
+    replicas: Final = MappingProxyType(
+        {
+            url: HttpTransport(base_url=url, master_key=master_key, request_timeout=REQUEST_TIMEOUT)
+            for url in replica_urls
+        }
+    )
     return ProxyClient(
         transport=split,
+        replicas=replicas,
         poll_timeout=POLL_TIMEOUT,
         poll_interval=POLL_INTERVAL,
     )
diff --git a/tests/e2e/test_proxy_client.py b/tests/e2e/test_proxy_client.py
new file mode 100644
index 00000000000..a508c97b9fb
--- /dev/null
+++ b/tests/e2e/test_proxy_client.py
@@ -0,0 +1,87 @@
+"""Harness coverage for the model barrier that gates on every replica.
+
+No proxy needed and no ``e2e`` marker: this pins that a model registered through
+the control plane only counts as servable once every configured replica lists it
+on /v1/models, which is what keeps a two-gateway stack from handing a test a
+model that one gateway has not reloaded yet. The fakes are plain pollers and an
+injected clock, so nothing here monkeypatches anything.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
+from itertools import chain, repeat
+from typing import Final
+
+import pytest
+
+from e2e_config import parse_replica_urls
+from e2e_http import Success
+from models import ModelListEntry, ModelsListResponse
+from proxy_client import ModelsPoller, NotServableOn, Servable, await_servable_everywhere
+
+MODEL: Final = "gpt-under-test"
+TIMEOUT: Final = 10.0
+INTERVAL: Final = 2.0
+
+
+@dataclass
+class FakeClock:
+    elapsed: float = 0.0
+
+    def now(self) -> float:
+        return self.elapsed
+
+    def sleep(self, seconds: float) -> None:
+        self.elapsed += seconds
+
+
+def _listing(*model_ids: str) -> Success[ModelsListResponse]:
+    entries: Final = tuple(ModelListEntry(id=model_id) for model_id in model_ids)
+    return Success(status_code=200, data=ModelsListResponse(data=entries))
+
+
+def _poller(results: Iterable[Success[ModelsListResponse]]) -> ModelsPoller:
+    it: Final = iter(results)
+    return lambda _timeout: next(it)
+
+
+def _await(pollers: Mapping[str, ModelsPoller]) -> Servable | NotServableOn:
+    clock: Final = FakeClock()
+    return await_servable_everywhere(
+        pollers,
+        model_name=MODEL,
+        timeout=TIMEOUT,
+        interval=INTERVAL,
+        request_timeout=5.0,
+        db_sync_seconds=0.0,
+        now=clock.now,
+        sleep=clock.sleep,
+    )
+
+
+class TestAwaitServableEverywhere:
+    @pytest.mark.parametrize("missing", ["gateway-1", "gateway-2"])
+    def test_fails_on_the_replica_that_never_lists_the_model(self, missing: str) -> None:
+        pollers: Final = {
+            "gateway-1": _poller(repeat(_listing(MODEL))),
+            "gateway-2": _poller(repeat(_listing(MODEL))),
+        } | {missing: _poller(repeat(_listing()))}
+        assert _await(pollers) == NotServableOn(replica=missing, last_result=_listing())
+
+    def test_passes_once_every_replica_lists_the_model(self) -> None:
+        pollers: Final = {
+            "gateway-1": _poller(repeat(_listing(MODEL))),
+            "gateway-2": _poller(chain(repeat(_listing(), 2), repeat(_listing(MODEL)))),
+        }
+        assert _await(pollers) == Servable()
+
+
+class TestParseReplicaUrls:
+    def test_splits_and_trims_the_gateway_addresses(self) -> None:
+        raw: Final = " http://127.0.0.1:4010/, http://127.0.0.1:4011 "
+        assert parse_replica_urls(raw, "http://lb") == ("http://127.0.0.1:4010", "http://127.0.0.1:4011")
+
+    def test_falls_back_to_the_data_plane_address_when_unset(self) -> None:
+        assert parse_replica_urls("", "http://lb") == ("http://lb",)

From 5a06845db18a2e1709bcd5e696cafdd4c190bd7e Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 5 Sep 2026 16:33:15 -0700
Subject: [PATCH 13/16] fix(ci): mask only credential-length values in the
 e2e-changed log

A one-character value in the provider secret bundle was masked too, which
turned every 1 in the run log into ***, including the pass numbers and the
gateway addresses, so the only public diagnostics were unreadable
---
 .github/e2e-stack/secrets_to_env.py              |  3 ++-
 .../code_coverage_tests/test_e2e_changed_gate.py | 16 ++++++++++++++++
 tests/e2e/CONTRIBUTING.md                        |  2 +-
 3 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/.github/e2e-stack/secrets_to_env.py b/.github/e2e-stack/secrets_to_env.py
index 99e717b271d..e7870a32e42 100644
--- a/.github/e2e-stack/secrets_to_env.py
+++ b/.github/e2e-stack/secrets_to_env.py
@@ -8,6 +8,7 @@ from pydantic import TypeAdapter, ValidationError
 
 secrets_adapter: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str])
 ENV_NAME: Final = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
+MIN_MASKED_LENGTH: Final = 8
 
 
 def main() -> int:
@@ -25,7 +26,7 @@ def main() -> int:
         _ = sys.stderr.write("environment names or values cannot be represented in both bash and dotenv\n")
         return 1
     for value in secrets.values():
-        if value:
+        if len(value) >= MIN_MASKED_LENGTH:
             _ = sys.stdout.write(f"::add-mask::{value.replace('%', '%25')}\n")
     sys.stdout.flush()
     lines: Final = tuple(f"{key}='{value}'" for key, value in secrets.items() if value)
diff --git a/tests/code_coverage_tests/test_e2e_changed_gate.py b/tests/code_coverage_tests/test_e2e_changed_gate.py
index 7a82a8ebb60..2e95c33cf20 100644
--- a/tests/code_coverage_tests/test_e2e_changed_gate.py
+++ b/tests/code_coverage_tests/test_e2e_changed_gate.py
@@ -7,6 +7,7 @@ from typing import Final
 import pytest
 
 GATE: Final = Path(__file__).resolve().parents[2] / ".github/e2e-stack/assert_tests_ran.py"
+SECRETS_TO_ENV: Final = GATE.with_name("secrets_to_env.py")
 SELECTED: Final = ("tests/e2e/access_control/test_a.py", "tests/e2e/access_control/test_b.py")
 
 
@@ -53,3 +54,18 @@ def test_missing_execution_evidence_fails(tmp_path: Path, contents: str) -> None
     result: Final = subprocess.run([sys.executable, str(GATE), str(report), *SELECTED], capture_output=True, text=True)
 
     assert result.returncode == 1
+
+
+def test_short_values_are_written_without_masking_every_digit_in_the_log(tmp_path: Path) -> None:
+    env_path: Final = tmp_path / ".env"
+
+    result: Final = subprocess.run(
+        [sys.executable, str(SECRETS_TO_ENV), str(env_path)],
+        input='{"FLAG": "1", "API_KEY": "sk-0123456789abcdef"}',
+        capture_output=True,
+        text=True,
+    )
+
+    assert result.returncode == 0, result.stderr
+    assert result.stdout == "::add-mask::sk-0123456789abcdef\n"
+    assert env_path.read_text() == "FLAG='1'\nAPI_KEY='sk-0123456789abcdef'\n"
diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md
index 7b90b9de5c8..0b8859d9260 100644
--- a/tests/e2e/CONTRIBUTING.md
+++ b/tests/e2e/CONTRIBUTING.md
@@ -64,7 +64,7 @@ Repository admins must require the `e2e-changed-tests` status check for merging
 
 Credentials come from the existing AWS Secrets Manager secrets in us-east-1, `litellm-e2e-changed-provider-keys` and `litellm-e2e-changed-license`. The OIDC role must trust only `repo:BerriAI/litellm:environment:e2e-changed` with audience `sts.amazonaws.com` and have read access only to these secrets. The short-lived reader credentials are scoped to the fetch step. Provider credentials must cover the selected suites, including Datadog credentials when logging or MCP tests need them; missing credentials fail the run. `up.sh` refuses to start without `DD_API_KEY`, because the stack's gateway config enables the Datadog callback for every run and a gateway booted without the key fails readiness. Keep provider credentials dedicated to this lane with only the permissions those tests need
 
-Fetched values are masked before use, and credential files and raw output are private to the runner. Public logs contain selected file names, counts, and pass status; raw pytest output, reports, and stack logs are not uploaded or printed. The workflow removes them and the credential files during cleanup. To diagnose a failed pass, reproduce the selected files locally with the appropriate credentials and inspect the local logs
+Fetched values of eight characters or more are masked before use, while shorter values such as flags stay unmasked because masking a one-character value would blank every matching digit in the log, and credential files and raw output are private to the runner. Public logs contain selected file names, counts, and pass status; raw pytest output, reports, and stack logs are not uploaded or printed. The workflow removes them and the credential files during cleanup. To diagnose a failed pass, reproduce the selected files locally with the appropriate credentials and inspect the local logs
 
 To reproduce the CI topology on a dedicated machine, `bash .github/e2e-stack/up.sh` reads `tests/e2e/.env`, writes `stack.env` under `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}`, and `bash .github/e2e-stack/down.sh` stops it. Keep this directory private and remove its credential files and logs after use
 

From a9e918577bb9cc5c35bd4fcf90948aa4522ea059 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 5 Sep 2026 18:46:51 -0700
Subject: [PATCH 14/16] ci(e2e): run the access_control canary on harness
 changes and name failed tests

A harness-only change (proxy_client.py, conftest.py, pytest.ini, the gateway
config, .github/e2e-stack, or the workflow) selected nothing, so the stack was
never exercised by the change that touched it. select_tests.py keeps the
changed-file rule and adds the access_control suite whenever a harness file
changes. The run step now reports the pytest exit code before the evidence
check, prints pytest's summary line per pass so the rerun count is visible,
and assert_tests_ran.py names each failed or errored test as classname::name
---
 .github/e2e-stack/assert_tests_ran.py         |  4 +
 .github/e2e-stack/select_tests.py             | 31 +++++++
 .github/workflows/test-e2e-changed.yml        | 29 +++++--
 .../test_e2e_changed_gate.py                  | 85 +++++++++++++++++++
 tests/e2e/CONTRIBUTING.md                     |  6 +-
 5 files changed, 143 insertions(+), 12 deletions(-)
 create mode 100644 .github/e2e-stack/select_tests.py

diff --git a/.github/e2e-stack/assert_tests_ran.py b/.github/e2e-stack/assert_tests_ran.py
index 7fd3f7c7c33..c4348c20873 100644
--- a/.github/e2e-stack/assert_tests_ran.py
+++ b/.github/e2e-stack/assert_tests_ran.py
@@ -20,6 +20,10 @@ def main() -> int:
         collected: Final = sum(case.get("file") == path for case in cases)
         skipped: Final = sum(case.get("file") == path and case.find("skipped") is not None for case in cases)
         _ = sys.stdout.write(f"{path}: {collected} collected, {skipped} skipped\n")
+        for case in cases:
+            if case.get("file") != path or all(case.find(tag) is None for tag in ("failure", "error")):
+                continue
+            _ = sys.stdout.write(f"  failed: {case.get('classname', '')}::{case.get('name', '')}\n")
     if (
         selected
         and not missing
diff --git a/.github/e2e-stack/select_tests.py b/.github/e2e-stack/select_tests.py
new file mode 100644
index 00000000000..cbe582b5b42
--- /dev/null
+++ b/.github/e2e-stack/select_tests.py
@@ -0,0 +1,31 @@
+import re
+import sys
+from typing import Final
+
+SELECTABLE: Final = re.compile(r"^tests/e2e/([A-Za-z0-9_.-]+/)*test_[A-Za-z0-9_.-]+\.py$")
+OWN_LANE: Final = re.compile(
+    r"^tests/e2e/(ui|claude_code|load)/|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$"
+)
+HARNESS: Final = re.compile(
+    r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$"
+    r"|^tests/e2e/gateway/"
+    r"|^\.github/e2e-stack/"
+    r"|^\.github/workflows/test-e2e-changed\.yml$"
+)
+
+
+def select(changed: tuple[str, ...], canary: tuple[str, ...]) -> tuple[str, ...]:
+    direct: Final = frozenset(path for path in changed if SELECTABLE.match(path) and not OWN_LANE.match(path))
+    harness_changed: Final = any(HARNESS.match(path) for path in changed)
+    canary_tests: Final = frozenset(path for path in canary if harness_changed and SELECTABLE.match(path))
+    return tuple(sorted(direct | canary_tests))
+
+
+def main() -> int:
+    changed: Final = tuple(line.strip() for line in sys.stdin if line.strip())
+    _ = sys.stdout.write(" ".join(select(changed, tuple(sys.argv[1:]))) + "\n")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml
index 16d84beab2d..23ab6dfcfe4 100644
--- a/.github/workflows/test-e2e-changed.yml
+++ b/.github/workflows/test-e2e-changed.yml
@@ -15,11 +15,21 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 5
     permissions:
+      contents: read
       pull-requests: read
     outputs:
       tests: ${{ steps.changed.outputs.tests }}
       any: ${{ steps.changed.outputs.any }}
     steps:
+      - name: Checkout the selector and the canary suite
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          sparse-checkout: |
+            .github/e2e-stack
+            tests/e2e/access_control
+          persist-credentials: false
+          ref: ${{ github.sha }}
+
       - name: List the e2e test files this PR added or modified
         id: changed
         env:
@@ -27,7 +37,6 @@ jobs:
           REPO: ${{ github.repository }}
           PR_NUMBER: ${{ github.event.pull_request.number }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
-          OWN_LANE: '^tests/e2e/(ui|claude_code|load)/|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$'
         run: |
           gh api "repos/${REPO}/pulls/${PR_NUMBER}" \
             --jq 'select(.head.sha == env.HEAD_SHA and .changed_files < 3000) | .head.sha' \
@@ -36,9 +45,7 @@ jobs:
             --jq '.[] | select(.status != "removed") | .filename')"
           gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.head.sha' | grep -Fxq "${HEAD_SHA}"
           tests="$(printf '%s\n' "${files}" \
-            | grep -E '^tests/e2e/([A-Za-z0-9_.-]+/)*test_[A-Za-z0-9_.-]+\.py$' \
-            | grep -vE "${OWN_LANE}" \
-            | sort -u | tr '\n' ' ' | sed 's/ $//')" || true
+            | python3 .github/e2e-stack/select_tests.py tests/e2e/access_control/test_*.py)"
           echo "tests=${tests}" >> "${GITHUB_OUTPUT}"
           if [ -n "${tests}" ]; then
             echo "any=true" >> "${GITHUB_OUTPUT}"
@@ -170,25 +177,29 @@ jobs:
           read -r -a test_files <<< "${TESTS}"
           for pass in 1 2 3; do
             report="${RUNNER_TEMP}/e2e-pass-${pass}.xml"
+            log="${RUNNER_TEMP}/e2e-pass-${pass}.log"
             echo "::group::pass ${pass} of 3"
             set +e
             uv run --no-sync pytest "${test_files[@]}" --rootdir=. -v -p no:cacheprovider \
-              -o junit_family=xunit1 --junitxml="${report}" > "${RUNNER_TEMP}/e2e-pass-${pass}.log" 2>&1
+              -o junit_family=xunit1 --junitxml="${report}" > "${log}" 2>&1
             status=$?
+            uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}" "${test_files[@]}"
+            verified=$?
             set -e
+            grep -E '^=+ .* in [0-9.]+s( \([0-9:]+\))? =+$' "${log}" | tail -n 1
             echo "::endgroup::"
             if [ "${status}" = "5" ]; then
               echo "::error::the selected files collected no runnable tests, so nothing was verified"
               exit 1
             fi
-            if ! uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}" "${test_files[@]}"; then
-              echo "::error::pass ${pass} of 3 did not verify every selected file"
-              exit 1
-            fi
             if [ "${status}" != "0" ]; then
               echo "::error::pass ${pass} of 3 failed with exit code ${status}"
               exit "${status}"
             fi
+            if [ "${verified}" != "0" ]; then
+              echo "::error::pass ${pass} of 3 did not verify every selected file"
+              exit 1
+            fi
             echo "pass ${pass} of 3 passed"
           done
 
diff --git a/tests/code_coverage_tests/test_e2e_changed_gate.py b/tests/code_coverage_tests/test_e2e_changed_gate.py
index 2e95c33cf20..f7c4a5e2527 100644
--- a/tests/code_coverage_tests/test_e2e_changed_gate.py
+++ b/tests/code_coverage_tests/test_e2e_changed_gate.py
@@ -8,6 +8,8 @@ import pytest
 
 GATE: Final = Path(__file__).resolve().parents[2] / ".github/e2e-stack/assert_tests_ran.py"
 SECRETS_TO_ENV: Final = GATE.with_name("secrets_to_env.py")
+SELECT_TESTS: Final = GATE.with_name("select_tests.py")
+CANARY: Final = ("tests/e2e/access_control/test_a.py", "tests/e2e/access_control/test_b.py")
 SELECTED: Final = ("tests/e2e/access_control/test_a.py", "tests/e2e/access_control/test_b.py")
 
 
@@ -46,6 +48,29 @@ def test_passing_case_does_not_hide_a_failure_in_the_same_file(tmp_path: Path, o
     assert result.returncode == 1
 
 
+def test_failed_cases_are_named_per_selected_file(tmp_path: Path) -> None:
+    suite: Final = ET.Element("testsuite")
+    _ = ET.SubElement(suite, "testcase", file=SELECTED[0], classname="tests.e2e.access_control.test_a", name="test_ok")
+    failed: Final = ET.SubElement(
+        suite, "testcase", file=SELECTED[0], classname="tests.e2e.access_control.test_a", name="test_boom"
+    )
+    _ = ET.SubElement(failed, "failure", message="secret-bearing message")
+    errored: Final = ET.SubElement(
+        suite, "testcase", file=SELECTED[1], classname="tests.e2e.access_control.test_b", name="test_setup"
+    )
+    _ = ET.SubElement(errored, "error")
+    report: Final = tmp_path / "report.xml"
+    ET.ElementTree(suite).write(report)
+
+    result: Final = subprocess.run([sys.executable, str(GATE), str(report), *SELECTED], capture_output=True, text=True)
+
+    assert result.returncode == 1
+    assert "  failed: tests.e2e.access_control.test_a::test_boom\n" in result.stdout
+    assert "  failed: tests.e2e.access_control.test_b::test_setup\n" in result.stdout
+    assert "test_ok" not in result.stdout
+    assert "secret-bearing message" not in result.stdout
+
+
 @pytest.mark.parametrize("contents", ("<testsuite/>", "<testsuite", '<testsuite><testcase name="a"/></testsuite>'))
 def test_missing_execution_evidence_fails(tmp_path: Path, contents: str) -> None:
     report: Final = tmp_path / "report.xml"
@@ -69,3 +94,63 @@ def test_short_values_are_written_without_masking_every_digit_in_the_log(tmp_pat
     assert result.returncode == 0, result.stderr
     assert result.stdout == "::add-mask::sk-0123456789abcdef\n"
     assert env_path.read_text() == "FLAG='1'\nAPI_KEY='sk-0123456789abcdef'\n"
+
+
+def select_tests(changed: tuple[str, ...]) -> tuple[str, ...]:
+    result: Final = subprocess.run(
+        [sys.executable, str(SELECT_TESTS), *CANARY],
+        input="".join(f"{path}\n" for path in changed),
+        capture_output=True,
+        text=True,
+    )
+    assert result.returncode == 0, result.stderr
+    return tuple(result.stdout.split())
+
+
+@pytest.mark.parametrize(
+    ("changed", "expected"),
+    (
+        (("tests/e2e/logging/test_datadog_e2e.py", "litellm/router.py"), ("tests/e2e/logging/test_datadog_e2e.py",)),
+        (("tests/e2e/ui/test_keys.py", "tests/e2e/claude_code/test_cli.py", "tests/e2e/load/test_burst.py"), ()),
+        (("tests/e2e/batches/test_managed_files_enforcement_e2e.py",), ()),
+        (("tests/e2e/logging/helpers.py", "docs/my-website/docs/index.md", "tests/e2e/CLAUDE.md"), ()),
+        (
+            ("tests/e2e/logging/test_datadog_e2e.py", "tests/e2e/logging/test_datadog_e2e.py"),
+            ("tests/e2e/logging/test_datadog_e2e.py",),
+        ),
+    ),
+)
+def test_changed_suite_files_are_selected_outside_the_own_lane(
+    changed: tuple[str, ...], expected: tuple[str, ...]
+) -> None:
+    assert select_tests(changed) == expected
+
+
+@pytest.mark.parametrize(
+    "harness_file",
+    (
+        "tests/e2e/proxy_client.py",
+        "tests/e2e/conftest.py",
+        "tests/e2e/pytest.ini",
+        "tests/e2e/gateway/stage_mirror_ci_config.yml",
+        ".github/e2e-stack/up.sh",
+        ".github/workflows/test-e2e-changed.yml",
+    ),
+)
+def test_harness_changes_run_the_canary_suite(harness_file: str) -> None:
+    assert select_tests((harness_file, "litellm/router.py")) == CANARY
+
+
+def test_a_changed_canary_file_is_selected_once_alongside_a_harness_change() -> None:
+    assert select_tests((CANARY[1], "tests/e2e/proxy_client.py")) == CANARY
+
+
+def test_the_canary_joins_directly_selected_files_in_sorted_order() -> None:
+    assert select_tests(("tests/e2e/logging/test_datadog_e2e.py", ".github/e2e-stack/up.sh")) == (
+        *CANARY,
+        "tests/e2e/logging/test_datadog_e2e.py",
+    )
+
+
+def test_a_harness_unit_test_change_runs_itself_and_the_canary() -> None:
+    assert select_tests(("tests/e2e/test_proxy_client.py",)) == (*CANARY, "tests/e2e/test_proxy_client.py")
diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md
index 0b8859d9260..be282e2a77e 100644
--- a/tests/e2e/CONTRIBUTING.md
+++ b/tests/e2e/CONTRIBUTING.md
@@ -56,15 +56,15 @@ A couple of logging destinations are configured on the proxy rather than by the
 
 ### The pull request check
 
-Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, harness, configuration, deleted-file, and workflow-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories and `batches/test_managed_files_enforcement_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack
+Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories and `batches/test_managed_files_enforcement_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack
 
-Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A failed pass stops the run. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch
+Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch
 
 Repository admins must require the `e2e-changed-tests` status check for merging and configure the `e2e-changed` environment with required reviewers, self-review disabled, and admin bypass disabled. Each push cancels the previous run; a new run that selects tests needs a fresh approval. Reviewers must inspect the entire executable PR diff, including application code, dependencies, tests, and workflow helpers, before approving the exact revision. Approved code executes with provider credentials, so environment approval is a trust decision about that code
 
 Credentials come from the existing AWS Secrets Manager secrets in us-east-1, `litellm-e2e-changed-provider-keys` and `litellm-e2e-changed-license`. The OIDC role must trust only `repo:BerriAI/litellm:environment:e2e-changed` with audience `sts.amazonaws.com` and have read access only to these secrets. The short-lived reader credentials are scoped to the fetch step. Provider credentials must cover the selected suites, including Datadog credentials when logging or MCP tests need them; missing credentials fail the run. `up.sh` refuses to start without `DD_API_KEY`, because the stack's gateway config enables the Datadog callback for every run and a gateway booted without the key fails readiness. Keep provider credentials dedicated to this lane with only the permissions those tests need
 
-Fetched values of eight characters or more are masked before use, while shorter values such as flags stay unmasked because masking a one-character value would blank every matching digit in the log, and credential files and raw output are private to the runner. Public logs contain selected file names, counts, and pass status; raw pytest output, reports, and stack logs are not uploaded or printed. The workflow removes them and the credential files during cleanup. To diagnose a failed pass, reproduce the selected files locally with the appropriate credentials and inspect the local logs
+Fetched values of eight characters or more are masked before use, while shorter values such as flags stay unmasked because masking a one-character value would blank every matching digit in the log, and credential files and raw output are private to the runner. Public logs contain selected file names, counts, pytest's summary line, failed test ids, and pass status; raw pytest output, reports, and stack logs are not uploaded or printed. The workflow removes them and the credential files during cleanup. To diagnose a failed pass, reproduce the selected files locally with the appropriate credentials and inspect the local logs
 
 To reproduce the CI topology on a dedicated machine, `bash .github/e2e-stack/up.sh` reads `tests/e2e/.env`, writes `stack.env` under `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}`, and `bash .github/e2e-stack/down.sh` stops it. Keep this directory private and remove its credential files and logs after use
 

From b2e93ba99f3e8ba8e87809d433d2bb9db5cd791a Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 5 Sep 2026 18:58:07 -0700
Subject: [PATCH 15/16] ci(e2e): declare the embedding model the access_control
 canary calls

The first canary run failed pass 1 because the stage-mirror config had no
openai-text-embedding-3-small while test_llm_api_routes_group_grants_every_llm_endpoint
calls /embeddings with it; the public log named the test, which is the
behavior the previous commit added
---
 tests/e2e/CONTRIBUTING.md                    | 2 +-
 tests/e2e/gateway/stage_mirror_ci_config.yml | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md
index be282e2a77e..106cfa31f0e 100644
--- a/tests/e2e/CONTRIBUTING.md
+++ b/tests/e2e/CONTRIBUTING.md
@@ -56,7 +56,7 @@ A couple of logging destinations are configured on the proxy rather than by the
 
 ### The pull request check
 
-Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories and `batches/test_managed_files_enforcement_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack
+Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories and `batches/test_managed_files_enforcement_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack
 
 Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch
 
diff --git a/tests/e2e/gateway/stage_mirror_ci_config.yml b/tests/e2e/gateway/stage_mirror_ci_config.yml
index 6b3756b5f9f..229e8514dee 100644
--- a/tests/e2e/gateway/stage_mirror_ci_config.yml
+++ b/tests/e2e/gateway/stage_mirror_ci_config.yml
@@ -52,6 +52,10 @@ model_list:
     litellm_params:
       model: gemini/gemini-2.5-flash
       api_key: os.environ/GEMINI_API_KEY
+  - model_name: openai-text-embedding-3-small
+    litellm_params:
+      model: openai/text-embedding-3-small
+      api_key: os.environ/OPENAI_API_KEY
 
 mcp_servers:
   devin:

From 116f88b023996a52a46f02b2dd8bffc8c15c9246 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 5 Sep 2026 21:03:50 -0700
Subject: [PATCH 16/16] fix(e2e-changed): keep the gate off suites the stack
 cannot run

The selector picked up two suites that can never pass in this stack, so
editing either one turned the check permanently red: the presidio masking
suite calls pytest.fail without an analyzer and anonymizer that up.sh
never starts, and the pipecat audio suite skips itself at import time
unless the NLTK punkt_tab data is present, which nothing installs.

tests/e2e/coverage_registry/test_collector.py had the same problem for a
different reason. Its nested pytest.main autoloads pytest-retry from the
ci group the workflow installs and dies with "INTERNALERROR: no option
named 'filtered_exceptions'", so the collect-only pass now disables that
plugin. The plugin's entry point is pytest-retry, not retry, so the same
one-word fix lands on mutmut's pytest_add_cli_args, where "-p no:retry"
was disabling nothing.

Two smaller holes in the harness: a canary argument the shell never
expanded used to select nothing and let the gate pass green, and a secret
that cannot be represented in both bash and dotenv was rejected without
naming the key.
---
 .github/e2e-stack/secrets_to_env.py           | 13 +++--
 .github/e2e-stack/select_tests.py             | 23 +++++++--
 pyproject.toml                                |  2 +-
 .../test_e2e_changed_gate.py                  | 49 ++++++++++++++++++-
 tests/e2e/CONTRIBUTING.md                     |  4 +-
 tests/e2e/coverage_registry/collector.py      |  2 +
 6 files changed, 80 insertions(+), 13 deletions(-)

diff --git a/.github/e2e-stack/secrets_to_env.py b/.github/e2e-stack/secrets_to_env.py
index e7870a32e42..7913b25918b 100644
--- a/.github/e2e-stack/secrets_to_env.py
+++ b/.github/e2e-stack/secrets_to_env.py
@@ -20,10 +20,15 @@ def main() -> int:
     except (ValidationError, UnicodeError):
         _ = sys.stderr.write("expected a JSON object containing string environment values\n")
         return 1
-    if any(
-        ENV_NAME.fullmatch(key) is None or any(char in value for char in "'\n\r\0") for key, value in secrets.items()
-    ):
-        _ = sys.stderr.write("environment names or values cannot be represented in both bash and dotenv\n")
+    unusable: Final = tuple(
+        key
+        for key, value in secrets.items()
+        if ENV_NAME.fullmatch(key) is None or any(char in value for char in "'\n\r\0")
+    )
+    if unusable:
+        _ = sys.stderr.write(
+            f"these names or values cannot be represented in both bash and dotenv: {' '.join(sorted(unusable))}\n"
+        )
         return 1
     for value in secrets.values():
         if len(value) >= MIN_MASKED_LENGTH:
diff --git a/.github/e2e-stack/select_tests.py b/.github/e2e-stack/select_tests.py
index cbe582b5b42..a62358f81ff 100644
--- a/.github/e2e-stack/select_tests.py
+++ b/.github/e2e-stack/select_tests.py
@@ -3,8 +3,11 @@ import sys
 from typing import Final
 
 SELECTABLE: Final = re.compile(r"^tests/e2e/([A-Za-z0-9_.-]+/)*test_[A-Za-z0-9_.-]+\.py$")
-OWN_LANE: Final = re.compile(
-    r"^tests/e2e/(ui|claude_code|load)/|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$"
+UNSUPPORTED: Final = re.compile(
+    r"^tests/e2e/(ui|claude_code|load)/"
+    r"|^tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e\.py$"
+    r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$"
+    r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$"
 )
 HARNESS: Final = re.compile(
     r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$"
@@ -12,18 +15,28 @@ HARNESS: Final = re.compile(
     r"|^\.github/e2e-stack/"
     r"|^\.github/workflows/test-e2e-changed\.yml$"
 )
+UNEXPANDED: Final = re.compile(r"[*?\[]")
+
+
+def is_selectable(path: str) -> bool:
+    return SELECTABLE.match(path) is not None and UNSUPPORTED.match(path) is None
 
 
 def select(changed: tuple[str, ...], canary: tuple[str, ...]) -> tuple[str, ...]:
-    direct: Final = frozenset(path for path in changed if SELECTABLE.match(path) and not OWN_LANE.match(path))
+    direct: Final = frozenset(path for path in changed if is_selectable(path))
     harness_changed: Final = any(HARNESS.match(path) for path in changed)
-    canary_tests: Final = frozenset(path for path in canary if harness_changed and SELECTABLE.match(path))
+    canary_tests: Final = frozenset(path for path in canary if harness_changed and is_selectable(path))
     return tuple(sorted(direct | canary_tests))
 
 
 def main() -> int:
+    canary: Final = tuple(sys.argv[1:])
+    unexpanded: Final = tuple(path for path in canary if UNEXPANDED.search(path))
+    if unexpanded:
+        _ = sys.stderr.write(f"the canary paths reached the selector unexpanded: {' '.join(unexpanded)}\n")
+        return 1
     changed: Final = tuple(line.strip() for line in sys.stdin if line.strip())
-    _ = sys.stdout.write(" ".join(select(changed, tuple(sys.argv[1:]))) + "\n")
+    _ = sys.stdout.write(" ".join(select(changed, canary)) + "\n")
     return 0
 
 
diff --git a/pyproject.toml b/pyproject.toml
index b889a3a0e60..f4f238dd4b9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -388,7 +388,7 @@ mutate_only_covered_lines = true
 # registered hash algorithm". Nothing to do with mutation coverage, and one
 # erroring test is enough to end the stats phase before any mutant runs.
 pytest_add_cli_args = [
-    "-p", "no:retry",
+    "-p", "no:pytest-retry",
     "-p", "no:rerunfailures",
     "-p", "no:xdist",
     "--ignore=tests/test_litellm/proxy/management_endpoints/test_saml_sso.py",
diff --git a/tests/code_coverage_tests/test_e2e_changed_gate.py b/tests/code_coverage_tests/test_e2e_changed_gate.py
index f7c4a5e2527..d2b842364a6 100644
--- a/tests/code_coverage_tests/test_e2e_changed_gate.py
+++ b/tests/code_coverage_tests/test_e2e_changed_gate.py
@@ -113,6 +113,16 @@ def select_tests(changed: tuple[str, ...]) -> tuple[str, ...]:
         (("tests/e2e/logging/test_datadog_e2e.py", "litellm/router.py"), ("tests/e2e/logging/test_datadog_e2e.py",)),
         (("tests/e2e/ui/test_keys.py", "tests/e2e/claude_code/test_cli.py", "tests/e2e/load/test_burst.py"), ()),
         (("tests/e2e/batches/test_managed_files_enforcement_e2e.py",), ()),
+        (("tests/e2e/guardrails/test_presidio_masking_e2e.py",), ()),
+        (("tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py",), ()),
+        (
+            ("tests/e2e/llm_translation/realtime/test_realtime_e2e.py",),
+            ("tests/e2e/llm_translation/realtime/test_realtime_e2e.py",),
+        ),
+        (
+            ("tests/e2e/guardrails/test_bedrock_guardrail_e2e.py",),
+            ("tests/e2e/guardrails/test_bedrock_guardrail_e2e.py",),
+        ),
         (("tests/e2e/logging/helpers.py", "docs/my-website/docs/index.md", "tests/e2e/CLAUDE.md"), ()),
         (
             ("tests/e2e/logging/test_datadog_e2e.py", "tests/e2e/logging/test_datadog_e2e.py"),
@@ -120,7 +130,7 @@ def select_tests(changed: tuple[str, ...]) -> tuple[str, ...]:
         ),
     ),
 )
-def test_changed_suite_files_are_selected_outside_the_own_lane(
+def test_changed_suite_files_are_selected_unless_the_stack_cannot_run_them(
     changed: tuple[str, ...], expected: tuple[str, ...]
 ) -> None:
     assert select_tests(changed) == expected
@@ -154,3 +164,40 @@ def test_the_canary_joins_directly_selected_files_in_sorted_order() -> None:
 
 def test_a_harness_unit_test_change_runs_itself_and_the_canary() -> None:
     assert select_tests(("tests/e2e/test_proxy_client.py",)) == (*CANARY, "tests/e2e/test_proxy_client.py")
+
+
+def test_a_canary_argument_the_shell_never_expanded_fails_the_selector() -> None:
+    result: Final = subprocess.run(
+        [sys.executable, str(SELECT_TESTS), "tests/e2e/access_control/test_*.py"],
+        input="tests/e2e/proxy_client.py\n",
+        capture_output=True,
+        text=True,
+    )
+
+    assert result.returncode == 1
+    assert "tests/e2e/access_control/test_*.py" in result.stderr
+    assert result.stdout == ""
+
+
+@pytest.mark.parametrize(
+    ("secrets", "offender", "unprintable"),
+    (
+        ('{"AWS_ACCESS_KEY_ID": "AKIAEXAMPLE", "BAD-NAME": "shibboleth"}', "BAD-NAME", "shibboleth"),
+        ("""{"AWS_SECRET_ACCESS_KEY": "quote'shibboleth"}""", "AWS_SECRET_ACCESS_KEY", "shibboleth"),
+        ('{"DD_API_KEY": "line\\nshibboleth"}', "DD_API_KEY", "shibboleth"),
+    ),
+)
+def test_an_unusable_secret_is_named_without_printing_its_value(
+    tmp_path: Path, secrets: str, offender: str, unprintable: str
+) -> None:
+    env_path: Final = tmp_path / ".env"
+
+    result: Final = subprocess.run(
+        [sys.executable, str(SECRETS_TO_ENV), str(env_path)], input=secrets, capture_output=True, text=True
+    )
+
+    assert result.returncode == 1
+    assert offender in result.stderr
+    assert unprintable not in result.stderr
+    assert result.stdout == ""
+    assert not env_path.exists()
diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md
index 106cfa31f0e..b83300cc8d5 100644
--- a/tests/e2e/CONTRIBUTING.md
+++ b/tests/e2e/CONTRIBUTING.md
@@ -56,9 +56,9 @@ A couple of logging destinations are configured on the proxy rather than by the
 
 ### The pull request check
 
-Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories and `batches/test_managed_files_enforcement_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack
+Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start
 
-Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch
+Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch
 
 Repository admins must require the `e2e-changed-tests` status check for merging and configure the `e2e-changed` environment with required reviewers, self-review disabled, and admin bypass disabled. Each push cancels the previous run; a new run that selects tests needs a fresh approval. Reviewers must inspect the entire executable PR diff, including application code, dependencies, tests, and workflow helpers, before approving the exact revision. Approved code executes with provider credentials, so environment approval is a trust decision about that code
 
diff --git a/tests/e2e/coverage_registry/collector.py b/tests/e2e/coverage_registry/collector.py
index e20f7884f55..c5d09820fe7 100644
--- a/tests/e2e/coverage_registry/collector.py
+++ b/tests/e2e/coverage_registry/collector.py
@@ -105,6 +105,8 @@ def collect_markers(e2e_dir: Path = E2E_DIR) -> CollectedMarkers:
                 "--continue-on-collection-errors",
                 "-p",
                 "no:cacheprovider",
+                "-p",
+                "no:pytest-retry",
                 str(e2e_dir),
             ],
             plugins=[sink],