diff --git a/.github/e2e-stack/assert_tests_ran.py b/.github/e2e-stack/assert_tests_ran.py new file mode 100644 index 00000000000..092a9db7256 --- /dev/null +++ b/.github/e2e-stack/assert_tests_ran.py @@ -0,0 +1,21 @@ +import sys +import xml.etree.ElementTree as ET +from pathlib import Path +from typing import Final + + +def main() -> int: + report: Final = ET.parse(Path(sys.argv[1])).getroot() + suites: Final = tuple(report.iter("testsuite")) + collected: Final = sum(int(suite.get("tests", "0")) for suite in suites) + skipped: Final = sum(int(suite.get("skipped", "0")) for suite in suites) + executed: Final = collected - skipped + _ = sys.stdout.write(f"executed {executed} of {collected} collected tests ({skipped} skipped)\n") + if executed > 0: + return 0 + _ = sys.stdout.write("::error::every selected test was skipped, so nothing was verified\n") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml index c1a74f007ce..e08660d0412 100644 --- a/.github/workflows/test-e2e-changed.yml +++ b/.github/workflows/test-e2e-changed.yml @@ -152,9 +152,10 @@ jobs: run: | read -r -a test_files <<< "${TESTS}" for pass in 1 2 3; do + report="${RUNNER_TEMP}/e2e-pass-${pass}.xml" echo "::group::pass ${pass} of 3" set +e - uv run --no-sync pytest "${test_files[@]}" --reruns 0 -v -rA --tb=short -p no:cacheprovider + uv run --no-sync pytest "${test_files[@]}" --reruns 0 -v -rA --tb=short -p no:cacheprovider --junitxml="${report}" status=$? set -e echo "::endgroup::" @@ -166,6 +167,7 @@ jobs: echo "::error::pass ${pass} of 3 failed with exit code ${status}" exit "${status}" fi + uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}" done - name: Show stack logs on failure diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 81fcdf84e74..ef6fbb5405d 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -54,7 +54,7 @@ Some suites need extra services the bare proxy does not start. The `logging/` OT ### The pull request check -Every PR that adds or modifies a `tests/e2e/**/test_*.py` file (outside `ui/`, `claude_code/`, `load/`, and `batches/test_managed_files_enforcement_e2e.py`, which have their own lanes or need a differently configured proxy) runs exactly those files three times, with retries off, against a stage-mirror stack booted on a GitHub Actions runner: migrations, a control-plane backend, two gateway processes behind an nginx load balancer, Postgres, Jaeger, and a TLS cluster-mode Valkey, wired the way stage is deployed. A PR that only touches the harness or the stack itself runs the `access_control/` suite as a smoke instead. Three green passes are the bar because the check exists to catch a flaky test before it reaches the release gate, so a red pass is a failure to fix, not a retry candidate. The same stack boots on a laptop with `bash .github/e2e-stack/up.sh`: it reads provider keys from `tests/e2e/.env`, writes the pytest environment to `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}/stack.env`, every port is overridable through `E2E_*_PORT` variables, and `down.sh` tears it all down +Every PR that adds or modifies a `tests/e2e/**/test_*.py` file (outside `ui/`, `claude_code/`, `load/`, and `batches/test_managed_files_enforcement_e2e.py`, which have their own lanes or need a differently configured proxy) runs exactly those files three times, with retries off, against a stage-mirror stack booted on a GitHub Actions runner: migrations, a control-plane backend, two gateway processes behind an nginx load balancer, Postgres, Jaeger, and a TLS cluster-mode Valkey, wired the way stage is deployed. A PR that only touches the harness or the stack itself runs the `access_control/` suite as a smoke instead. Three green passes are the bar because the check exists to catch a flaky test before it reaches the release gate, so a red pass is a failure to fix, not a retry candidate. A pass that executes nothing is also red: each pass writes a JUnit report and fails when every collected test was skipped, so a skipped-out file cannot pass on paper. The same stack boots on a laptop with `bash .github/e2e-stack/up.sh`: it reads provider keys from `tests/e2e/.env`, writes the pytest environment to `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}/stack.env`, every port is overridable through `E2E_*_PORT` variables, and `down.sh` tears it all down Credentials: the lane never sees the stage keys. Everything the runner writes into `tests/e2e/.env` comes from two AWS Secrets Manager secrets in us-east-1, `litellm-e2e-changed-provider-keys` and `litellm-e2e-changed-license`, and the first holds only the names that the tests and `tests/e2e/gateway/stage_mirror_ci_config.yml` read. Each credential in it is dedicated to this lane and can do only what the tests do: the AWS pair belongs to the `litellm-e2e-changed` IAM user, whose inline policy allows Bedrock inference, the one guardrail the tests use, the batch job APIs, the batch S3 bucket, and assuming the batch test's role; the Vertex key belongs to a service account holding only `roles/aiplatform.user` on the Vertex project; the OpenAI, Anthropic, Gemini, Cohere, and Devin keys are per-lane keys with hard monthly spend caps; and the Datadog application key is scoped to log search. A leaked key can therefore spend at most one month of a small capped budget or call a handful of Bedrock APIs, and rotating one is a single `aws secretsmanager put-secret-value` on the secret, with no workflow change