diff --git a/.github/e2e-stack/up.sh b/.github/e2e-stack/up.sh index 38519869346..06fce35a896 100755 --- a/.github/e2e-stack/up.sh +++ b/.github/e2e-stack/up.sh @@ -67,12 +67,14 @@ wait_for "jaeger" "curl -fs http://127.0.0.1:${JAEGER_QUERY_PORT}/api/services > openssl genrsa -out "${CERTS_DIR}/ca.key" 2048 2>/dev/null openssl req -x509 -new -nodes -key "${CERTS_DIR}/ca.key" -sha256 -days 7 \ - -subj "/CN=litellm-e2e-ca" -out "${CERTS_DIR}/ca.crt" 2>/dev/null + -subj "/CN=litellm-e2e-ca" \ + -addext "basicConstraints=critical,CA:TRUE" -addext "keyUsage=critical,keyCertSign,cRLSign" \ + -out "${CERTS_DIR}/ca.crt" 2>/dev/null openssl genrsa -out "${CERTS_DIR}/server.key" 2048 2>/dev/null openssl req -new -key "${CERTS_DIR}/server.key" -subj "/CN=localhost" -out "${CERTS_DIR}/server.csr" 2>/dev/null openssl x509 -req -in "${CERTS_DIR}/server.csr" -CA "${CERTS_DIR}/ca.crt" -CAkey "${CERTS_DIR}/ca.key" \ -CAcreateserial -days 7 -sha256 \ - -extfile <(printf 'subjectAltName=DNS:localhost,IP:127.0.0.1') \ + -extfile <(printf 'basicConstraints=CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\nsubjectAltName=DNS:localhost,IP:127.0.0.1\n') \ -out "${CERTS_DIR}/server.crt" 2>/dev/null chmod 644 "${CERTS_DIR}"/*.key "${CERTS_DIR}"/*.crt @@ -90,10 +92,11 @@ wait_for "valkey" "${VALKEY_CLI} ping 2>/dev/null | grep -q PONG" ${VALKEY_CLI} cluster addslotsrange 0 16383 >/dev/null wait_for "valkey cluster" "${VALKEY_CLI} cluster info 2>/dev/null | grep -q cluster_state:ok" -CONFIG_PATH="${REPO_ROOT}/tests/e2e/gateway/litellm-config.yml" +CONFIG_SOURCE="${REPO_ROOT}/tests/e2e/gateway/stage_mirror_ci_config.yml" +CONFIG_PATH="${CONFIG_SOURCE}" if [[ "${REDIS_PORT}" != "6379" ]]; then CONFIG_PATH="${STACK_DIR}/litellm-config.yml" - sed "s/port: 6379/port: ${REDIS_PORT}/" "${REPO_ROOT}/tests/e2e/gateway/litellm-config.yml" > "${CONFIG_PATH}" + sed "s/port: 6379/port: ${REDIS_PORT}/" "${CONFIG_SOURCE}" > "${CONFIG_PATH}" fi SERVER_ENV=( diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml index ea45dee6161..e19293d9d07 100644 --- a/.github/workflows/test-e2e-changed.yml +++ b/.github/workflows/test-e2e-changed.yml @@ -2,43 +2,50 @@ name: e2e-changed-tests on: pull_request: - paths: - - "tests/e2e/**" concurrency: group: e2e-changed-${{ github.event.pull_request.number }} cancel-in-progress: true -permissions: - contents: read +permissions: {} jobs: detect: name: Detect changed e2e tests runs-on: ubuntu-latest timeout-minutes: 5 + permissions: + pull-requests: read outputs: tests: ${{ steps.changed.outputs.tests }} any: ${{ steps.changed.outputs.any }} steps: - - name: List added or modified test files + - name: List the e2e test files this PR added or modified id: changed env: GH_TOKEN: ${{ github.token }} REPO: ${{ github.repository }} PR_NUMBER: ${{ github.event.pull_request.number }} + SMOKE_TESTS: tests/e2e/access_control run: | - tests="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" --paginate \ - --jq '.[] | select(.status != "removed") | .filename' \ + files="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" --paginate \ + --jq '.[] | select(.status != "removed") | .filename')" + tests="$(printf '%s\n' "${files}" \ | grep -E '^tests/e2e/([A-Za-z0-9_.-]+/)*test_[A-Za-z0-9_.-]+\.py$' \ - | grep -v '^tests/e2e/ui/' | sort -u | tr '\n' ' ' | sed 's/ $//')" || true + | grep -vE '^tests/e2e/(ui|claude_code|load)/' \ + | sort -u | tr '\n' ' ' | sed 's/ $//')" || true + if [ -z "${tests}" ] && printf '%s\n' "${files}" | grep -v '^tests/e2e/ui/' \ + | grep -qE '^(tests/e2e/|\.github/e2e-stack/|\.github/workflows/test-e2e-changed\.yml$)'; then + tests="${SMOKE_TESTS}" + echo "harness or stack changed without a test file; running the smoke suite" + fi echo "tests=${tests}" >> "${GITHUB_OUTPUT}" if [ -n "${tests}" ]; then echo "any=true" >> "${GITHUB_OUTPUT}" - echo "changed e2e tests: ${tests}" + echo "selected e2e tests: ${tests}" else echo "any=false" >> "${GITHUB_OUTPUT}" - echo "no changed e2e test files" + echo "no e2e changes; nothing to run" fi run: @@ -46,7 +53,8 @@ jobs: needs: detect if: needs.detect.outputs.any == 'true' && github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 90 + environment: e2e-changed permissions: contents: read id-token: write @@ -90,8 +98,8 @@ jobs: .github/scripts/uv_sync_with_retries.sh --frozen \ --extra proxy --extra proxy-runtime --extra extra_proxy \ --extra semantic-router --extra bedrock-realtime \ - --group ci --group e2e-dev - uv pip install redis==7.4.1 prometheus-client "pipecat-ai[openai]==1.4.0" + --group ci --group proxy-dev --group e2e-dev + uv pip install "pipecat-ai[openai]==1.4.0" - name: Generate Prisma client env: @@ -115,10 +123,10 @@ jobs: - name: Fetch provider credentials from AWS Secrets Manager run: | - aws secretsmanager get-secret-value --secret-id berrie-litellm-stage-provider-keys \ + aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-provider-keys \ --query SecretString --output text \ | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env - aws secretsmanager get-secret-value --secret-id litellm-license \ + aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-license \ --query SecretString --output text \ | jq -R -s '{"LITELLM_LICENSE": rtrimstr("\n")}' \ | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env @@ -132,29 +140,28 @@ jobs: echo "::add-mask::${master_key}" cat "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" >> "${GITHUB_ENV}" - - name: Run the changed tests + - name: Run the selected tests three times with retries off env: TESTS: ${{ needs.detect.outputs.tests }} run: | read -r -a test_files <<< "${TESTS}" - set +e - uv run --no-sync pytest "${test_files[@]}" -v -rA --tb=short -p no:cacheprovider - status=$? - set -e - if [ "${status}" = "5" ]; then - echo "changed files collected no runnable tests" - exit 0 - fi - exit "${status}" + for pass in 1 2 3; do + echo "::group::pass ${pass} of 3" + set +e + uv run --no-sync pytest "${test_files[@]}" --reruns 0 -v -rA --tb=short -p no:cacheprovider + status=$? + set -e + echo "::endgroup::" + if [ "${status}" = "5" ]; then + echo "selected files collected no runnable tests" + exit 0 + fi + if [ "${status}" != "0" ]; then + echo "::error::pass ${pass} of 3 failed with exit code ${status}" + exit "${status}" + fi + done - name: Show stack logs on failure if: failure() run: tail -n 200 "${RUNNER_TEMP}/litellm-e2e-stack/logs"/*.log - - - name: Upload stack logs - if: failure() - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 - with: - name: e2e-stack-logs - path: ${{ runner.temp }}/litellm-e2e-stack/logs/ - retention-days: 7 diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 871d6b3904c..e04781ab205 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -52,6 +52,10 @@ The suites run against a live proxy, so bring one up first by running the litell Some suites need extra services the bare proxy does not start. The `logging/` OTEL trace-completeness tests read spans back from a jaeger query API at `http://localhost:16686` (override with `E2E_OTEL_QUERY_URL`); run a `jaegertracing/all-in-one` and point `PHOENIX_COLLECTOR_HTTP_ENDPOINT` at its OTLP ingest. The `mcp/` suite needs the deterministic upstream MCP server in `mcp_tests/mcp_e2e_upstream_server.py` reachable by the proxy +### The pull request check + +Every PR that adds or modifies a `tests/e2e/**/test_*.py` file (outside `ui/`, `claude_code/`, and `load/`, which have their own lanes) runs exactly those files three times, with retries off, against a stage-mirror stack booted on a GitHub Actions runner: migrations, a control-plane backend, two gateway processes behind an nginx load balancer, Postgres, Jaeger, and a TLS cluster-mode Valkey, wired the way stage is deployed. A PR that only touches the harness or the stack itself runs the `access_control/` suite as a smoke instead. Three green passes are the bar because the check exists to catch a flaky test before it reaches the release gate, so a red pass is a failure to fix, not a retry candidate. The same stack boots on a laptop with `bash .github/e2e-stack/up.sh`: it reads provider keys from `tests/e2e/.env`, writes the pytest environment to `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}/stack.env`, every port is overridable through `E2E_*_PORT` variables, and `down.sh` tears it all down + ### Record and replay Record/replay scopes to the proxy's provider-bound traffic only. In `E2E_FIXTURE_MODE=record` the harness boots a local provider-edge server, edge-wired tests register their deployments with an `api_base` pointing at it, and every provider call the proxy makes is forwarded verbatim and written to a fixture bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`). `E2E_FIXTURE_MODE=replay` runs the same tests against the same live proxy and database, but the edge answers the proxy's provider calls from the bundle instead of the provider, so the run makes zero provider calls and spends nothing while key auth, routing, cost calculation, and spend-log writes all still execute for real. Unset (or `live`) behaves exactly as before the knob existed. Both record and replay need the proxy up; only the provider is taken out of the loop diff --git a/tests/e2e/gateway/litellm-config.yml b/tests/e2e/gateway/stage_mirror_ci_config.yml similarity index 100% rename from tests/e2e/gateway/litellm-config.yml rename to tests/e2e/gateway/stage_mirror_ci_config.yml