From 4a646dd9a0d7acd2ea7c3fbd57de1e17ead7cec8 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 2 Sep 2026 14:53:40 -0700 Subject: [PATCH] ci(e2e): run a PR's changed e2e tests three times behind a human-approved environment Adds a required-check candidate that selects the tests/e2e test files a PR added or modified, boots a stage-mirror stack on the runner (migrations, backend, two gateway processes behind nginx, Postgres, Jaeger, TLS cluster Valkey), and runs those files three times with retries off. The run job sits behind the e2e-changed GitHub environment, so a reviewer approves each run before the OIDC token that reads the provider keys from AWS Secrets Manager exists. Supersedes #34981 --- .github/e2e-stack/down.sh | 17 ++ .github/e2e-stack/secrets_to_env.py | 28 +++ .github/e2e-stack/up.sh | 198 +++++++++++++++++++ .github/workflows/test-e2e-changed.yml | 171 ++++++++++++++++ tests/e2e/CONTRIBUTING.md | 4 + tests/e2e/gateway/stage_mirror_ci_config.yml | 63 ++++++ 6 files changed, 481 insertions(+) create mode 100755 .github/e2e-stack/down.sh create mode 100644 .github/e2e-stack/secrets_to_env.py create mode 100755 .github/e2e-stack/up.sh create mode 100644 .github/workflows/test-e2e-changed.yml create mode 100644 tests/e2e/gateway/stage_mirror_ci_config.yml diff --git a/.github/e2e-stack/down.sh b/.github/e2e-stack/down.sh new file mode 100755 index 00000000000..9f72f2d6e64 --- /dev/null +++ b/.github/e2e-stack/down.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -uo pipefail + +STACK_DIR="${E2E_STACK_DIR:-${RUNNER_TEMP:-/tmp}/litellm-e2e-stack}" + +for pid_file in "${STACK_DIR}"/pids/*.pid; do + [[ -f "${pid_file}" ]] || continue + pkill -TERM -P "$(cat "${pid_file}")" 2>/dev/null + kill -TERM "$(cat "${pid_file}")" 2>/dev/null + rm -f "${pid_file}" +done + +for container in e2e-nginx e2e-valkey e2e-jaeger e2e-postgres; do + docker rm -f "${container}" >/dev/null 2>&1 +done + +exit 0 diff --git a/.github/e2e-stack/secrets_to_env.py b/.github/e2e-stack/secrets_to_env.py new file mode 100644 index 00000000000..25b22555c29 --- /dev/null +++ b/.github/e2e-stack/secrets_to_env.py @@ -0,0 +1,28 @@ +import sys +from pathlib import Path + +from pydantic import TypeAdapter + +secrets_adapter: TypeAdapter[dict[str, str]] = TypeAdapter(dict[str, str]) + + +def main() -> int: + env_path = Path(sys.argv[1]) + secrets = secrets_adapter.validate_json(sys.stdin.read()) + unwritable = tuple( + key for key, value in secrets.items() if "'" in value or "\n" in value or "\r" in value + ) + if unwritable: + _ = sys.stderr.write(f"values contain characters unsafe for both bash and dotenv: {', '.join(unwritable)}\n") + return 1 + lines = tuple(f"{key}='{value}'" for key, value in secrets.items() if value) + with env_path.open("a") as handle: + _ = handle.write("\n".join(lines) + "\n") + for value in secrets.values(): + if value: + _ = sys.stdout.write(f"::add-mask::{value}\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/e2e-stack/up.sh b/.github/e2e-stack/up.sh new file mode 100755 index 00000000000..06fce35a896 --- /dev/null +++ b/.github/e2e-stack/up.sh @@ -0,0 +1,198 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +STACK_DIR="${E2E_STACK_DIR:-${RUNNER_TEMP:-/tmp}/litellm-e2e-stack}" +CERTS_DIR="${STACK_DIR}/certs" +LOGS_DIR="${STACK_DIR}/logs" +PIDS_DIR="${STACK_DIR}/pids" + +POSTGRES_IMAGE="${E2E_POSTGRES_IMAGE:-postgres:16.6}" +VALKEY_IMAGE="${E2E_VALKEY_IMAGE:-valkey/valkey:8.1}" +JAEGER_IMAGE="${E2E_JAEGER_IMAGE:-jaegertracing/jaeger:2.10.0}" +NGINX_IMAGE="${E2E_NGINX_IMAGE:-nginx:1.29-alpine}" + +LB_PORT="${E2E_LB_PORT:-4000}" +GATEWAY_PORT_1="${E2E_GATEWAY_PORT_1:-4010}" +GATEWAY_PORT_2="${E2E_GATEWAY_PORT_2:-4011}" +BACKEND_PORT="${E2E_BACKEND_PORT:-4001}" +REDIS_PORT="${E2E_REDIS_PORT:-6379}" +DATABASE_HOST="${E2E_DATABASE_HOST:-127.0.0.1}" +DATABASE_PORT="${E2E_DATABASE_PORT:-5432}" +DATABASE_USER="${E2E_DATABASE_USER:-litellm}" +DATABASE_PASSWORD="${E2E_DATABASE_PASSWORD:-dbpassword9090}" +DATABASE_NAME="${E2E_DATABASE_NAME:-litellm}" +JAEGER_OTLP_PORT="${E2E_JAEGER_OTLP_PORT:-4318}" +JAEGER_QUERY_PORT="${E2E_JAEGER_QUERY_PORT:-16686}" + +MASTER_KEY="${LITELLM_MASTER_KEY:-sk-e2e-$(openssl rand -hex 16)}" + +mkdir -p "${CERTS_DIR}" "${LOGS_DIR}" "${PIDS_DIR}" + +log() { printf 'e2e-stack: %s\n' "$*"; } + +port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null; } + +wait_for() { + local label="$1" check="$2" deadline=$((SECONDS + ${3:-120})) + until eval "${check}"; do + if ((SECONDS >= deadline)); then + log "timed out waiting for ${label}" + tail -n 60 "${LOGS_DIR}"/*.log 2>/dev/null || true + exit 1 + fi + sleep 2 + done + log "${label} is up" +} + +if [[ -f "${REPO_ROOT}/tests/e2e/.env" ]]; then + set -a + source "${REPO_ROOT}/tests/e2e/.env" + set +a +fi + +if ! port_open "${DATABASE_PORT}"; then + docker run -d --name e2e-postgres -p "${DATABASE_PORT}:5432" \ + -e "POSTGRES_USER=${DATABASE_USER}" -e "POSTGRES_PASSWORD=${DATABASE_PASSWORD}" -e "POSTGRES_DB=${DATABASE_NAME}" \ + "${POSTGRES_IMAGE}" >/dev/null +fi +wait_for "postgres" "port_open ${DATABASE_PORT}" + +if ! port_open "${JAEGER_QUERY_PORT}"; then + docker run -d --name e2e-jaeger -p "${JAEGER_OTLP_PORT}:4318" -p "${JAEGER_QUERY_PORT}:16686" \ + "${JAEGER_IMAGE}" >/dev/null +fi +wait_for "jaeger" "curl -fs http://127.0.0.1:${JAEGER_QUERY_PORT}/api/services >/dev/null" + +openssl genrsa -out "${CERTS_DIR}/ca.key" 2048 2>/dev/null +openssl req -x509 -new -nodes -key "${CERTS_DIR}/ca.key" -sha256 -days 7 \ + -subj "/CN=litellm-e2e-ca" \ + -addext "basicConstraints=critical,CA:TRUE" -addext "keyUsage=critical,keyCertSign,cRLSign" \ + -out "${CERTS_DIR}/ca.crt" 2>/dev/null +openssl genrsa -out "${CERTS_DIR}/server.key" 2048 2>/dev/null +openssl req -new -key "${CERTS_DIR}/server.key" -subj "/CN=localhost" -out "${CERTS_DIR}/server.csr" 2>/dev/null +openssl x509 -req -in "${CERTS_DIR}/server.csr" -CA "${CERTS_DIR}/ca.crt" -CAkey "${CERTS_DIR}/ca.key" \ + -CAcreateserial -days 7 -sha256 \ + -extfile <(printf 'basicConstraints=CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\nsubjectAltName=DNS:localhost,IP:127.0.0.1\n') \ + -out "${CERTS_DIR}/server.crt" 2>/dev/null +chmod 644 "${CERTS_DIR}"/*.key "${CERTS_DIR}"/*.crt + +CERTIFI_BUNDLE="$(cd "${REPO_ROOT}" && uv run --no-sync python -c 'import certifi; print(certifi.where())')" +cat "${CERTIFI_BUNDLE}" "${CERTS_DIR}/ca.crt" > "${CERTS_DIR}/ca-bundle.pem" + +docker rm -f e2e-valkey >/dev/null 2>&1 || true +docker run -d --name e2e-valkey -p "${REDIS_PORT}:${REDIS_PORT}" -v "${CERTS_DIR}:/certs:ro" \ + "${VALKEY_IMAGE}" valkey-server \ + --cluster-enabled yes --port 0 --tls-port "${REDIS_PORT}" \ + --tls-cert-file /certs/server.crt --tls-key-file /certs/server.key --tls-ca-cert-file /certs/ca.crt \ + --tls-auth-clients no --cluster-announce-ip 127.0.0.1 >/dev/null +VALKEY_CLI="docker exec e2e-valkey valkey-cli --tls --cacert /certs/ca.crt -h 127.0.0.1 -p ${REDIS_PORT}" +wait_for "valkey" "${VALKEY_CLI} ping 2>/dev/null | grep -q PONG" +${VALKEY_CLI} cluster addslotsrange 0 16383 >/dev/null +wait_for "valkey cluster" "${VALKEY_CLI} cluster info 2>/dev/null | grep -q cluster_state:ok" + +CONFIG_SOURCE="${REPO_ROOT}/tests/e2e/gateway/stage_mirror_ci_config.yml" +CONFIG_PATH="${CONFIG_SOURCE}" +if [[ "${REDIS_PORT}" != "6379" ]]; then + CONFIG_PATH="${STACK_DIR}/litellm-config.yml" + sed "s/port: 6379/port: ${REDIS_PORT}/" "${CONFIG_SOURCE}" > "${CONFIG_PATH}" +fi + +SERVER_ENV=( + "LITELLM_MASTER_KEY=${MASTER_KEY}" + "DATABASE_HOST=${DATABASE_HOST}" + "DATABASE_PORT=${DATABASE_PORT}" + "DATABASE_USER=${DATABASE_USER}" + "DATABASE_PASSWORD=${DATABASE_PASSWORD}" + "DATABASE_NAME=${DATABASE_NAME}" + "DISABLE_SCHEMA_UPDATE=true" + "REDIS_HOST=127.0.0.1" + "REDIS_PORT=${REDIS_PORT}" + "REDIS_CLUSTER_NODES=[{\"host\":\"127.0.0.1\",\"port\":${REDIS_PORT}}]" + "CONFIG_FILE_PATH=${CONFIG_PATH}" + "STORE_MODEL_IN_DB=True" + "OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf" + "OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:${JAEGER_OTLP_PORT}" + "SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem" + "PYTHONPATH=${REPO_ROOT}" +) +if [[ -n "${VERTEXAI_CREDENTIALS:-}" ]]; then + printf '%s' "${VERTEXAI_CREDENTIALS}" > "${STACK_DIR}/vertex-adc.json" + SERVER_ENV+=("GOOGLE_APPLICATION_CREDENTIALS=${STACK_DIR}/vertex-adc.json") +fi + +cd "${REPO_ROOT}" + +log "running migrations" +env "${SERVER_ENV[@]}" uv run --no-sync python migrations/run.py >"${LOGS_DIR}/migrations.log" 2>&1 + +start_server() { + local name="$1"; shift + env "${SERVER_ENV[@]}" "$@" >"${LOGS_DIR}/${name}.log" 2>&1 & + echo $! > "${PIDS_DIR}/${name}.pid" +} + +start_server backend uv run --no-sync uvicorn backend.main:app --host 0.0.0.0 --port "${BACKEND_PORT}" +start_server gateway-1 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_1}" +start_server gateway-2 uv run --no-sync uvicorn gateway.main:app --workers 1 --host 0.0.0.0 --port "${GATEWAY_PORT_2}" + +if [[ "$(uname)" == "Linux" ]]; then + NGINX_UPSTREAM_HOST=127.0.0.1 + NGINX_DOCKER_ARGS=(--network host) +else + NGINX_UPSTREAM_HOST=host.docker.internal + NGINX_DOCKER_ARGS=(-p "${LB_PORT}:${LB_PORT}") +fi + +cat > "${STACK_DIR}/nginx.conf" </dev/null 2>&1 || true +docker run -d --name e2e-nginx "${NGINX_DOCKER_ARGS[@]}" \ + -v "${STACK_DIR}/nginx.conf:/etc/nginx/nginx.conf:ro" "${NGINX_IMAGE}" >/dev/null + +wait_for "backend" "curl -fs http://127.0.0.1:${BACKEND_PORT}/health/liveliness >/dev/null" 300 +wait_for "gateway-1" "curl -fs http://127.0.0.1:${GATEWAY_PORT_1}/health/liveliness >/dev/null" 300 +wait_for "gateway-2" "curl -fs http://127.0.0.1:${GATEWAY_PORT_2}/health/liveliness >/dev/null" 300 +wait_for "load balancer" "curl -fs http://127.0.0.1:${LB_PORT}/health/liveliness >/dev/null" 60 + +cat > "${STACK_DIR}/stack.env" <> "${GITHUB_OUTPUT}" + if [ -n "${tests}" ]; then + echo "any=true" >> "${GITHUB_OUTPUT}" + echo "selected e2e tests: ${tests}" + else + echo "any=false" >> "${GITHUB_OUTPUT}" + echo "no e2e changes; nothing to run" + fi + + run: + name: Run changed e2e tests against the stage-mirror stack + needs: detect + if: needs.detect.outputs.any == 'true' && github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + timeout-minutes: 90 + environment: e2e-changed + permissions: + contents: read + id-token: write + services: + postgres: + image: postgres:16.6 + env: + POSTGRES_USER: litellm + POSTGRES_PASSWORD: dbpassword9090 + POSTGRES_DB: litellm + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U litellm" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + jaeger: + image: jaegertracing/jaeger:2.10.0 + ports: + - 4318:4318 + - 16686:16686 + steps: + - name: Validate configuration + env: + ROLE: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }} + run: test -n "${ROLE}" || { echo "::error::Set repo variable E2E_AWS_ROLE_TO_ASSUME to an OIDC role with read access to the e2e secrets"; exit 1; } + + - name: Checkout + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.13" + + - name: Set up uv + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - name: Cache the Rust build + uses: ./.github/actions/cache-cargo-build + + - name: Install dependencies + run: | + .github/scripts/uv_sync_with_retries.sh --frozen \ + --extra proxy --extra proxy-runtime --extra extra_proxy \ + --extra semantic-router --extra bedrock-realtime \ + --group ci --group proxy-dev --group e2e-dev + uv pip install "pipecat-ai[openai]==1.4.0" + + - name: Cache Prisma binaries + uses: ./.github/actions/cache-prisma-binaries + + - name: Generate Prisma client + run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma + + - name: Install Playwright chromium + run: uv run --no-sync playwright install --with-deps chromium + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0 + with: + role-to-assume: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }} + aws-region: us-east-1 + role-session-name: litellm-e2e-changed-${{ github.run_id }} + + - name: Fetch provider credentials from AWS Secrets Manager + run: | + aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-provider-keys \ + --query SecretString --output text \ + | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env + aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-license \ + --query SecretString --output text \ + | jq -R -s '{"LITELLM_LICENSE": rtrimstr("\n")}' \ + | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env + + - name: Boot the stage-mirror stack + run: bash .github/e2e-stack/up.sh + + - name: Export stack environment + run: | + master_key="$(grep '^LITELLM_MASTER_KEY=' "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" | cut -d= -f2-)" + echo "::add-mask::${master_key}" + cat "${RUNNER_TEMP}/litellm-e2e-stack/stack.env" >> "${GITHUB_ENV}" + + - name: Run the selected tests three times with retries off + env: + TESTS: ${{ needs.detect.outputs.tests }} + run: | + read -r -a test_files <<< "${TESTS}" + for pass in 1 2 3; do + echo "::group::pass ${pass} of 3" + set +e + uv run --no-sync pytest "${test_files[@]}" --reruns 0 -v -rA --tb=short -p no:cacheprovider + status=$? + set -e + echo "::endgroup::" + if [ "${status}" = "5" ]; then + echo "selected files collected no runnable tests" + exit 0 + fi + if [ "${status}" != "0" ]; then + echo "::error::pass ${pass} of 3 failed with exit code ${status}" + exit "${status}" + fi + done + + - name: Show stack logs on failure + if: failure() + run: tail -n 200 "${RUNNER_TEMP}/litellm-e2e-stack/logs"/*.log diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 871d6b3904c..e04781ab205 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -52,6 +52,10 @@ The suites run against a live proxy, so bring one up first by running the litell Some suites need extra services the bare proxy does not start. The `logging/` OTEL trace-completeness tests read spans back from a jaeger query API at `http://localhost:16686` (override with `E2E_OTEL_QUERY_URL`); run a `jaegertracing/all-in-one` and point `PHOENIX_COLLECTOR_HTTP_ENDPOINT` at its OTLP ingest. The `mcp/` suite needs the deterministic upstream MCP server in `mcp_tests/mcp_e2e_upstream_server.py` reachable by the proxy +### The pull request check + +Every PR that adds or modifies a `tests/e2e/**/test_*.py` file (outside `ui/`, `claude_code/`, and `load/`, which have their own lanes) runs exactly those files three times, with retries off, against a stage-mirror stack booted on a GitHub Actions runner: migrations, a control-plane backend, two gateway processes behind an nginx load balancer, Postgres, Jaeger, and a TLS cluster-mode Valkey, wired the way stage is deployed. A PR that only touches the harness or the stack itself runs the `access_control/` suite as a smoke instead. Three green passes are the bar because the check exists to catch a flaky test before it reaches the release gate, so a red pass is a failure to fix, not a retry candidate. The same stack boots on a laptop with `bash .github/e2e-stack/up.sh`: it reads provider keys from `tests/e2e/.env`, writes the pytest environment to `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}/stack.env`, every port is overridable through `E2E_*_PORT` variables, and `down.sh` tears it all down + ### Record and replay Record/replay scopes to the proxy's provider-bound traffic only. In `E2E_FIXTURE_MODE=record` the harness boots a local provider-edge server, edge-wired tests register their deployments with an `api_base` pointing at it, and every provider call the proxy makes is forwarded verbatim and written to a fixture bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`). `E2E_FIXTURE_MODE=replay` runs the same tests against the same live proxy and database, but the edge answers the proxy's provider calls from the bundle instead of the provider, so the run makes zero provider calls and spends nothing while key auth, routing, cost calculation, and spend-log writes all still execute for real. Unset (or `live`) behaves exactly as before the knob existed. Both record and replay need the proxy up; only the provider is taken out of the loop diff --git a/tests/e2e/gateway/stage_mirror_ci_config.yml b/tests/e2e/gateway/stage_mirror_ci_config.yml new file mode 100644 index 00000000000..57a92fd47fb --- /dev/null +++ b/tests/e2e/gateway/stage_mirror_ci_config.yml @@ -0,0 +1,63 @@ +general_settings: + store_prompts_in_spend_logs: true + database_connection_pool_limit: 10 + forward_client_headers_to_llm_api: false + maximum_spend_logs_retention_period: "60d" + maximum_spend_logs_cleanup_cron: "0 1 * * *" + proxy_budget_rescheduler_min_time: 15 + proxy_budget_rescheduler_max_time: 20 + +litellm_settings: + drop_params: true + default_redis_ttl: 20 + request_timeout: 600 + num_retries: 3 + json_logs: true + store_audit_logs: true + cache: true + cache_params: + type: redis + host: 127.0.0.1 + port: 6379 + redis_startup_nodes: + - host: 127.0.0.1 + port: 6379 + ssl: true + callbacks: ["arize_phoenix", "datadog", "smtp_email", "prometheus", "otel"] + require_auth_for_metrics_endpoint: false + +router_settings: + routing_strategy: simple-shuffle + num_retries: 3 + allowed_fails: 5 + cooldown_time: 30 + +model_list: + - model_name: gpt-5.5 + litellm_params: + model: openai/gpt-5.5 + api_key: os.environ/OPENAI_API_KEY + - model_name: claude-haiku-4-5 + litellm_params: + model: anthropic/claude-haiku-4-5 + api_key: os.environ/ANTHROPIC_API_KEY + - model_name: gemini-2.5-flash-vertex + litellm_params: + model: vertex_ai/gemini-2.5-flash + vertex_project: os.environ/VERTEXAI_PROJECT + vertex_location: us-central1 + vertex_credentials: os.environ/VERTEXAI_CREDENTIALS + - model_name: gemini-2.5-flash + litellm_params: + model: gemini/gemini-2.5-flash + api_key: os.environ/GEMINI_API_KEY + - model_name: gemini-2.5-flash + litellm_params: + model: gemini/gemini-2.5-flash + api_key: os.environ/GEMINI_API_KEY + +mcp_servers: + devin: + url: "https://mcp.devin.ai/mcp" + auth_type: api_key + auth_value: os.environ/DEVIN_API_KEY