ci(e2e): fail a pass whose every collected test was skipped

.github/e2e-stack/assert_tests_ran.py

@@ -0,0 +1,21 @@
+import sys
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from typing import Final
+
+
+def main() -> int:
+    report: Final = ET.parse(Path(sys.argv[1])).getroot()
+    suites: Final = tuple(report.iter("testsuite"))
+    collected: Final = sum(int(suite.get("tests", "0")) for suite in suites)
+    skipped: Final = sum(int(suite.get("skipped", "0")) for suite in suites)
+    executed: Final = collected - skipped
+    _ = sys.stdout.write(f"executed {executed} of {collected} collected tests ({skipped} skipped)\n")
+    if executed > 0:
+        return 0
+    _ = sys.stdout.write("::error::every selected test was skipped, so nothing was verified\n")
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/workflows/test-e2e-changed.yml

@@ -152,9 +152,10 @@ jobs:
         run: |
           read -r -a test_files <<< "${TESTS}"
           for pass in 1 2 3; do
+            report="${RUNNER_TEMP}/e2e-pass-${pass}.xml"
             echo "::group::pass ${pass} of 3"
             set +e
-            uv run --no-sync pytest "${test_files[@]}" --reruns 0 -v -rA --tb=short -p no:cacheprovider
+            uv run --no-sync pytest "${test_files[@]}" --reruns 0 -v -rA --tb=short -p no:cacheprovider --junitxml="${report}"
             status=$?
             set -e
             echo "::endgroup::"
@@ -166,6 +167,7 @@ jobs:
               echo "::error::pass ${pass} of 3 failed with exit code ${status}"
               exit "${status}"
             fi
+            uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}"
           done
 
       - name: Show stack logs on failure

tests/e2e/CONTRIBUTING.md

@@ -54,7 +54,7 @@ Some suites need extra services the bare proxy does not start. The `logging/` OT
 
 ### The pull request check
 
-Every PR that adds or modifies a `tests/e2e/**/test_*.py` file (outside `ui/`, `claude_code/`, `load/`, and `batches/test_managed_files_enforcement_e2e.py`, which have their own lanes or need a differently configured proxy) runs exactly those files three times, with retries off, against a stage-mirror stack booted on a GitHub Actions runner: migrations, a control-plane backend, two gateway processes behind an nginx load balancer, Postgres, Jaeger, and a TLS cluster-mode Valkey, wired the way stage is deployed. A PR that only touches the harness or the stack itself runs the `access_control/` suite as a smoke instead. Three green passes are the bar because the check exists to catch a flaky test before it reaches the release gate, so a red pass is a failure to fix, not a retry candidate. The same stack boots on a laptop with `bash .github/e2e-stack/up.sh`: it reads provider keys from `tests/e2e/.env`, writes the pytest environment to `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}/stack.env`, every port is overridable through `E2E_*_PORT` variables, and `down.sh` tears it all down
+Every PR that adds or modifies a `tests/e2e/**/test_*.py` file (outside `ui/`, `claude_code/`, `load/`, and `batches/test_managed_files_enforcement_e2e.py`, which have their own lanes or need a differently configured proxy) runs exactly those files three times, with retries off, against a stage-mirror stack booted on a GitHub Actions runner: migrations, a control-plane backend, two gateway processes behind an nginx load balancer, Postgres, Jaeger, and a TLS cluster-mode Valkey, wired the way stage is deployed. A PR that only touches the harness or the stack itself runs the `access_control/` suite as a smoke instead. Three green passes are the bar because the check exists to catch a flaky test before it reaches the release gate, so a red pass is a failure to fix, not a retry candidate. A pass that executes nothing is also red: each pass writes a JUnit report and fails when every collected test was skipped, so a skipped-out file cannot pass on paper. The same stack boots on a laptop with `bash .github/e2e-stack/up.sh`: it reads provider keys from `tests/e2e/.env`, writes the pytest environment to `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}/stack.env`, every port is overridable through `E2E_*_PORT` variables, and `down.sh` tears it all down
 
 Credentials: the lane never sees the stage keys. Everything the runner writes into `tests/e2e/.env` comes from two AWS Secrets Manager secrets in us-east-1, `litellm-e2e-changed-provider-keys` and `litellm-e2e-changed-license`, and the first holds only the names that the tests and `tests/e2e/gateway/stage_mirror_ci_config.yml` read. Each credential in it is dedicated to this lane and can do only what the tests do: the AWS pair belongs to the `litellm-e2e-changed` IAM user, whose inline policy allows Bedrock inference, the one guardrail the tests use, the batch job APIs, the batch S3 bucket, and assuming the batch test's role; the Vertex key belongs to a service account holding only `roles/aiplatform.user` on the Vertex project; the OpenAI, Anthropic, Gemini, Cohere, and Devin keys are per-lane keys with hard monthly spend caps; and the Datadog application key is scoped to log search. A leaked key can therefore spend at most one month of a small capped budget or call a handful of Bedrock APIs, and rotating one is a single `aws secretsmanager put-secret-value` on the secret, with no workflow change