From 77f406dc002190cf1717a865fe656c20a62175cf Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 11 Sep 2026 17:09:35 -0700 Subject: [PATCH] test(e2e): start persistent Keycloak in the changed-test runner --- .github/e2e-stack/down.sh | 2 +- .github/e2e-stack/select_tests.py | 1 + .github/e2e-stack/start-idp.sh | 45 ++++++++ .github/e2e-stack/up.sh | 9 ++ .github/workflows/test-e2e-changed.yml | 5 +- .../test_e2e_changed_gate.py | 2 + .../code_coverage_tests/test_e2e_idp_stack.py | 106 ++++++++++++++++++ tests/e2e/CONTRIBUTING.md | 4 +- tests/e2e/gateway/stage_mirror_ci_config.yml | 7 ++ 9 files changed, 177 insertions(+), 4 deletions(-) create mode 100644 .github/e2e-stack/start-idp.sh create mode 100644 tests/code_coverage_tests/test_e2e_idp_stack.py diff --git a/.github/e2e-stack/down.sh b/.github/e2e-stack/down.sh index 9f72f2d6e64..740d626beea 100755 --- a/.github/e2e-stack/down.sh +++ b/.github/e2e-stack/down.sh @@ -10,7 +10,7 @@ for pid_file in "${STACK_DIR}"/pids/*.pid; do rm -f "${pid_file}" done -for container in e2e-nginx e2e-valkey e2e-jaeger e2e-postgres; do +for container in e2e-nginx e2e-keycloak e2e-valkey e2e-jaeger e2e-postgres; do docker rm -f "${container}" >/dev/null 2>&1 done diff --git a/.github/e2e-stack/select_tests.py b/.github/e2e-stack/select_tests.py index a62358f81ff..238818a0d36 100644 --- a/.github/e2e-stack/select_tests.py +++ b/.github/e2e-stack/select_tests.py @@ -11,6 +11,7 @@ UNSUPPORTED: Final = re.compile( ) HARNESS: Final = re.compile( r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$" + r"|^tests/e2e/idp_realm\.json$" r"|^tests/e2e/gateway/" r"|^\.github/e2e-stack/" r"|^\.github/workflows/test-e2e-changed\.yml$" diff --git a/.github/e2e-stack/start-idp.sh b/.github/e2e-stack/start-idp.sh new file mode 100644 index 00000000000..e59a7ade34c --- /dev/null +++ b/.github/e2e-stack/start-idp.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +KEYCLOAK_IMAGE="${E2E_KEYCLOAK_IMAGE:-quay.io/keycloak/keycloak@sha256:ff4257d0d64efbe99ed1ddfaf07765cc3c36dc7518bf8324d41961327f441c54}" +KEYCLOAK_PORT="${E2E_KEYCLOAK_PORT:-8081}" +POSTGRES_IMAGE="${E2E_POSTGRES_IMAGE:-postgres:16.6}" +: "${DATABASE_HOST:?}" "${DATABASE_PORT:?}" "${DATABASE_USER:?}" "${DATABASE_PASSWORD:?}" "${DATABASE_NAME:?}" + +DB_HOST="${DATABASE_HOST}" +DB_NETWORK_ARGS=(--network bridge) +IDP_NETWORK_ARGS=(-p "127.0.0.1:${KEYCLOAK_PORT}:${KEYCLOAK_PORT}") +if [[ "$(uname)" == "Linux" ]]; then + DB_NETWORK_ARGS=(--network host) + IDP_NETWORK_ARGS=(--network host) +elif [[ "${DB_HOST}" == "127.0.0.1" || "${DB_HOST}" == "localhost" ]]; then + DB_HOST=host.docker.internal +fi + +docker run --rm "${DB_NETWORK_ARGS[@]}" -e "PGPASSWORD=${DATABASE_PASSWORD}" \ + "${POSTGRES_IMAGE}" psql -h "${DB_HOST}" -p "${DATABASE_PORT}" \ + -U "${DATABASE_USER}" -d "${DATABASE_NAME}" -v ON_ERROR_STOP=1 \ + -c 'CREATE SCHEMA IF NOT EXISTS keycloak' >/dev/null + +docker rm -f e2e-keycloak >/dev/null 2>&1 || true +docker run -d --name e2e-keycloak "${IDP_NETWORK_ARGS[@]}" --memory 1536m \ + -v "${REPO_ROOT}/tests/e2e/idp_realm.json:/opt/keycloak/data/import/realm.json:ro" \ + -e KC_DB=postgres -e "KC_DB_URL_HOST=${DB_HOST}" -e "KC_DB_URL_PORT=${DATABASE_PORT}" \ + -e "KC_DB_URL_DATABASE=${DATABASE_NAME}" -e KC_DB_SCHEMA=keycloak \ + -e "KC_DB_USERNAME=${DATABASE_USER}" -e "KC_DB_PASSWORD=${DATABASE_PASSWORD}" \ + -e KC_DB_POOL_INITIAL_SIZE=2 -e KC_DB_POOL_MIN_SIZE=2 -e KC_DB_POOL_MAX_SIZE=10 \ + -e "KC_HTTP_PORT=${KEYCLOAK_PORT}" -e KC_BOOTSTRAP_ADMIN_USERNAME=admin \ + -e KC_BOOTSTRAP_ADMIN_PASSWORD=e2e-ephemeral-idp-not-a-secret \ + "${KEYCLOAK_IMAGE}" start-dev --import-realm >/dev/null + +deadline=$((SECONDS + ${E2E_KEYCLOAK_STARTUP_TIMEOUT:-300})) +until curl -fsS --connect-timeout 2 --max-time 3 \ + "http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e/.well-known/openid-configuration" >/dev/null 2>&1; do + if ((SECONDS >= deadline)); then + echo 'e2e-stack: timed out waiting for the Keycloak realm' >&2 + exit 1 + fi + sleep 2 +done +echo 'e2e-stack: Keycloak realm is up' diff --git a/.github/e2e-stack/up.sh b/.github/e2e-stack/up.sh index 2f2e6c6f9a8..a789a570483 100755 --- a/.github/e2e-stack/up.sh +++ b/.github/e2e-stack/up.sh @@ -25,6 +25,7 @@ DATABASE_PASSWORD="${E2E_DATABASE_PASSWORD:-dbpassword9090}" DATABASE_NAME="${E2E_DATABASE_NAME:-litellm}" JAEGER_OTLP_PORT="${E2E_JAEGER_OTLP_PORT:-4318}" JAEGER_QUERY_PORT="${E2E_JAEGER_QUERY_PORT:-16686}" +KEYCLOAK_PORT="${E2E_KEYCLOAK_PORT:-8081}" MASTER_KEY="${LITELLM_MASTER_KEY:-sk-e2e-$(openssl rand -hex 16)}" @@ -124,6 +125,9 @@ SERVER_ENV=( "OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:${JAEGER_OTLP_PORT}" "SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem" "PYTHONPATH=${REPO_ROOT}" + "JWT_PUBLIC_KEY_URL=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e/protocol/openid-connect/certs" + "JWT_ISSUER=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e" + "JWT_AUDIENCE=litellm-e2e" ) if [[ -n "${VERTEXAI_CREDENTIALS:-}" ]]; then printf '%s' "${VERTEXAI_CREDENTIALS}" > "${STACK_DIR}/vertex-adc.json" @@ -132,6 +136,8 @@ fi cd "${REPO_ROOT}" +env "${SERVER_ENV[@]}" "E2E_KEYCLOAK_PORT=${KEYCLOAK_PORT}" bash .github/e2e-stack/start-idp.sh + log "running migrations" env "${SERVER_ENV[@]}" uv run --no-sync python migrations/run.py >"${LOGS_DIR}/migrations.log" 2>&1 @@ -200,6 +206,9 @@ LITELLM_MASTER_KEY=${MASTER_KEY} REDIS_HOST=127.0.0.1 REDIS_PORT=${REDIS_PORT} E2E_OTEL_QUERY_URL=http://127.0.0.1:${JAEGER_QUERY_PORT} +E2E_KEYCLOAK_URL=http://127.0.0.1:${KEYCLOAK_PORT} +E2E_KEYCLOAK_ADMIN_USER=admin +E2E_KEYCLOAK_ADMIN_PASSWORD=e2e-ephemeral-idp-not-a-secret SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem DATABASE_URL=postgresql://${DATABASE_USER}:${DATABASE_PASSWORD}@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_NAME} EOF diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml index 23ab6dfcfe4..1db597ff673 100644 --- a/.github/workflows/test-e2e-changed.yml +++ b/.github/workflows/test-e2e-changed.yml @@ -27,6 +27,8 @@ jobs: sparse-checkout: | .github/e2e-stack tests/e2e/access_control + tests/e2e/management/test_jwt_management_e2e.py + tests/e2e/other/test_jwt_auth_e2e.py persist-credentials: false ref: ${{ github.sha }} @@ -45,7 +47,8 @@ jobs: --jq '.[] | select(.status != "removed") | .filename')" gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.head.sha' | grep -Fxq "${HEAD_SHA}" tests="$(printf '%s\n' "${files}" \ - | python3 .github/e2e-stack/select_tests.py tests/e2e/access_control/test_*.py)" + | python3 .github/e2e-stack/select_tests.py tests/e2e/access_control/test_*.py \ + tests/e2e/management/test_jwt_management_e2e.py tests/e2e/other/test_jwt_auth_e2e.py)" echo "tests=${tests}" >> "${GITHUB_OUTPUT}" if [ -n "${tests}" ]; then echo "any=true" >> "${GITHUB_OUTPUT}" diff --git a/tests/code_coverage_tests/test_e2e_changed_gate.py b/tests/code_coverage_tests/test_e2e_changed_gate.py index d2b842364a6..588402e3996 100644 --- a/tests/code_coverage_tests/test_e2e_changed_gate.py +++ b/tests/code_coverage_tests/test_e2e_changed_gate.py @@ -144,6 +144,8 @@ def test_changed_suite_files_are_selected_unless_the_stack_cannot_run_them( "tests/e2e/pytest.ini", "tests/e2e/gateway/stage_mirror_ci_config.yml", ".github/e2e-stack/up.sh", + ".github/e2e-stack/start-idp.sh", + "tests/e2e/idp_realm.json", ".github/workflows/test-e2e-changed.yml", ), ) diff --git a/tests/code_coverage_tests/test_e2e_idp_stack.py b/tests/code_coverage_tests/test_e2e_idp_stack.py new file mode 100644 index 00000000000..3b994fe521f --- /dev/null +++ b/tests/code_coverage_tests/test_e2e_idp_stack.py @@ -0,0 +1,106 @@ +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[2] +START_IDP = ROOT / ".github/e2e-stack/start-idp.sh" + + +def run_start(tmp_path: Path, *, platform: str = "Linux", failure: str = ""): + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + calls = tmp_path / "docker.jsonl" + programs = { + "docker": """import json, os, sys +with open(os.environ['DOCKER_LOG'], 'a') as out: + out.write(json.dumps(sys.argv[1:]) + '\\n') +if os.environ['FAILURE'] == 'schema' and 'psql' in sys.argv: + sys.exit(17) +if os.environ['FAILURE'] == 'launch' and '--name' in sys.argv: + sys.exit(18) +""", + "curl": "import os, sys; sys.exit(1 if os.environ['FAILURE'] == 'readiness' else 0)\n", + "uname": "import os; print(os.environ['PLATFORM'])\n", + } + for name, source in programs.items(): + program = bin_dir / name + program.write_text(f"#!{sys.executable}\n{source}") + program.chmod(0o755) + result = subprocess.run( + ["bash", str(START_IDP)], + env={ + **os.environ, + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "DOCKER_LOG": str(calls), + "PLATFORM": platform, + "FAILURE": failure, + "DATABASE_HOST": "127.0.0.1", + "DATABASE_PORT": "5544", + "DATABASE_USER": "fixture_user", + "DATABASE_PASSWORD": "fixture_password", + "DATABASE_NAME": "fixture_db", + "E2E_KEYCLOAK_PORT": "8181", + "E2E_KEYCLOAK_STARTUP_TIMEOUT": "0", + }, + capture_output=True, + text=True, + timeout=10, + ) + return result, [json.loads(line) for line in calls.read_text().splitlines()] + + +@pytest.mark.parametrize("platform", ("Linux", "Darwin")) +def test_idp_uses_existing_database_and_imports_runner_realm(tmp_path: Path, platform: str) -> None: + result, calls = run_start(tmp_path, platform=platform) + + assert result.returncode == 0, result.stderr + schema, _, launch = calls + host = "127.0.0.1" if platform == "Linux" else "host.docker.internal" + assert schema[schema.index("-h") + 1] == host + assert schema[schema.index("-p") + 1] == "5544" + assert "ON_ERROR_STOP=1" in schema + assert "CREATE SCHEMA IF NOT EXISTS keycloak" in schema + assert f"KC_DB_URL_HOST={host}" in launch + assert "KC_DB_URL_PORT=5544" in launch + assert "KC_DB_SCHEMA=keycloak" in launch + assert "KC_DB_POOL_MAX_SIZE=10" in launch + assert f"{ROOT}/tests/e2e/idp_realm.json:/opt/keycloak/data/import/realm.json:ro" in launch + assert "KC_HTTP_PORT=8181" in launch + if platform == "Linux": + assert launch[launch.index("--network") + 1] == "host" + else: + assert launch[launch.index("-p") + 1] == "127.0.0.1:8181:8181" + assert "Keycloak realm is up" in result.stdout + + +@pytest.mark.parametrize(("failure", "code"), (("schema", 17), ("launch", 18), ("readiness", 1))) +def test_idp_failure_stops_stack_startup(tmp_path: Path, failure: str, code: int) -> None: + result, calls = run_start(tmp_path, failure=failure) + + assert result.returncode == code + assert "Keycloak realm is up" not in result.stdout + if failure == "schema": + assert len(calls) == 1, "do not replace an IdP when its database is unavailable" + + +def test_proxy_and_test_runner_share_the_same_jwt_configuration() -> None: + config = yaml.safe_load((ROOT / "tests/e2e/gateway/stage_mirror_ci_config.yml").read_text()) + general = config["general_settings"] + assert general["enable_jwt_auth"] is True + assert general["litellm_jwtauth"] == { + "user_id_jwt_field": "sub", + "user_email_jwt_field": "email", + "team_ids_jwt_field": "groups", + "user_id_upsert": True, + } + up = (START_IDP.parent / "up.sh").read_text() + assert '"JWT_ISSUER=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e"' in up + assert '"JWT_AUDIENCE=litellm-e2e"' in up + assert "E2E_KEYCLOAK_URL=http://127.0.0.1:${KEYCLOAK_PORT}" in up + assert up.index("bash .github/e2e-stack/start-idp.sh") < up.index("start_server backend") + assert "e2e-keycloak" in (START_IDP.parent / "down.sh").read_text() diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 53fcdbfe381..948ad819a8c 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -69,7 +69,7 @@ The suites run against a live proxy, so bring one up first by running the litell uv run pytest tests/e2e/other/test_jwt_auth_e2e.py tests/e2e/management/test_jwt_management_e2e.py --reruns 0 -v ``` - CI runs this suite against a Keycloak deployed beside the ephemeral stack, and that deployment (the JWT config block, `JWT_PUBLIC_KEY_URL`, and the admin credential handed to the run pod) lives in the project-releaser repo, not here. CI fetches the realm from the test-runner revision even when it reuses a gateway image from another commit. Keycloak stores its realm, keys and users in a separate schema in the build's PostgreSQL, so replacing the IdP pod preserves token validity. Its startup probe waits for the imported realm. Losing the whole ephemeral database invalidates the stack. Keycloak skips imports into an existing realm, so changes to the realm export require a fresh stack (or deliberately replacing the local data volume). A stack without it fails the JWT tests rather than skipping them + Buildkite runs this suite against a Keycloak deployed beside the ephemeral stack by project-releaser. It fetches the realm from the test-runner revision even when it reuses a gateway image from another commit. The GitHub Actions changed-test stack starts the same digest-pinned Keycloak through `.github/e2e-stack/start-idp.sh`, imports the checked-out realm, and exports the IdP URL and credentials in `stack.env`. Both runners configure issuer/audience validation and store the realm, keys and users in a separate schema in the stack's PostgreSQL, so replacing Keycloak preserves token validity. Both wait for realm discovery before running tests. Losing the whole ephemeral database invalidates the stack. Keycloak skips imports into an existing realm, so changes to the realm export require a fresh stack (or deliberately replacing the local data volume). A stack without it fails the JWT tests rather than skipping them 4. Run a suite against it; the harness reads `LITELLM_PROXY_URL` (default `http://localhost:4000`): @@ -101,7 +101,7 @@ A couple of logging destinations are configured on the proxy rather than by the ### The pull request check -Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start +Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch diff --git a/tests/e2e/gateway/stage_mirror_ci_config.yml b/tests/e2e/gateway/stage_mirror_ci_config.yml index 229e8514dee..8c8e64443cb 100644 --- a/tests/e2e/gateway/stage_mirror_ci_config.yml +++ b/tests/e2e/gateway/stage_mirror_ci_config.yml @@ -1,4 +1,11 @@ general_settings: + proxy_batch_write_at: 5 + enable_jwt_auth: true + litellm_jwtauth: + user_id_jwt_field: sub + user_email_jwt_field: email + team_ids_jwt_field: groups + user_id_upsert: true proxy_config_reload_interval_seconds: 7 store_prompts_in_spend_logs: true database_connection_pool_limit: 10