test(e2e): start persistent Keycloak in the changed-test runner

.github/e2e-stack/down.sh

@@ -10,7 +10,7 @@ for pid_file in "${STACK_DIR}"/pids/*.pid; do
   rm -f "${pid_file}"
 done
 
-for container in e2e-nginx e2e-valkey e2e-jaeger e2e-postgres; do
+for container in e2e-nginx e2e-keycloak e2e-valkey e2e-jaeger e2e-postgres; do
   docker rm -f "${container}" >/dev/null 2>&1
 done
 

.github/e2e-stack/select_tests.py

@@ -11,6 +11,7 @@ UNSUPPORTED: Final = re.compile(
 )
 HARNESS: Final = re.compile(
     r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$"
+    r"|^tests/e2e/idp_realm\.json$"
     r"|^tests/e2e/gateway/"
     r"|^\.github/e2e-stack/"
     r"|^\.github/workflows/test-e2e-changed\.yml$"

.github/e2e-stack/start-idp.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+KEYCLOAK_IMAGE="${E2E_KEYCLOAK_IMAGE:-quay.io/keycloak/keycloak@sha256:ff4257d0d64efbe99ed1ddfaf07765cc3c36dc7518bf8324d41961327f441c54}"
+KEYCLOAK_PORT="${E2E_KEYCLOAK_PORT:-8081}"
+POSTGRES_IMAGE="${E2E_POSTGRES_IMAGE:-postgres:16.6}"
+: "${DATABASE_HOST:?}" "${DATABASE_PORT:?}" "${DATABASE_USER:?}" "${DATABASE_PASSWORD:?}" "${DATABASE_NAME:?}"
+
+DB_HOST="${DATABASE_HOST}"
+DB_NETWORK_ARGS=(--network bridge)
+IDP_NETWORK_ARGS=(-p "127.0.0.1:${KEYCLOAK_PORT}:${KEYCLOAK_PORT}")
+if [[ "$(uname)" == "Linux" ]]; then
+  DB_NETWORK_ARGS=(--network host)
+  IDP_NETWORK_ARGS=(--network host)
+elif [[ "${DB_HOST}" == "127.0.0.1" || "${DB_HOST}" == "localhost" ]]; then
+  DB_HOST=host.docker.internal
+fi
+
+docker run --rm "${DB_NETWORK_ARGS[@]}" -e "PGPASSWORD=${DATABASE_PASSWORD}" \
+  "${POSTGRES_IMAGE}" psql -h "${DB_HOST}" -p "${DATABASE_PORT}" \
+  -U "${DATABASE_USER}" -d "${DATABASE_NAME}" -v ON_ERROR_STOP=1 \
+  -c 'CREATE SCHEMA IF NOT EXISTS keycloak' >/dev/null
+
+docker rm -f e2e-keycloak >/dev/null 2>&1 || true
+docker run -d --name e2e-keycloak "${IDP_NETWORK_ARGS[@]}" --memory 1536m \
+  -v "${REPO_ROOT}/tests/e2e/idp_realm.json:/opt/keycloak/data/import/realm.json:ro" \
+  -e KC_DB=postgres -e "KC_DB_URL_HOST=${DB_HOST}" -e "KC_DB_URL_PORT=${DATABASE_PORT}" \
+  -e "KC_DB_URL_DATABASE=${DATABASE_NAME}" -e KC_DB_SCHEMA=keycloak \
+  -e "KC_DB_USERNAME=${DATABASE_USER}" -e "KC_DB_PASSWORD=${DATABASE_PASSWORD}" \
+  -e KC_DB_POOL_INITIAL_SIZE=2 -e KC_DB_POOL_MIN_SIZE=2 -e KC_DB_POOL_MAX_SIZE=10 \
+  -e "KC_HTTP_PORT=${KEYCLOAK_PORT}" -e KC_BOOTSTRAP_ADMIN_USERNAME=admin \
+  -e KC_BOOTSTRAP_ADMIN_PASSWORD=e2e-ephemeral-idp-not-a-secret \
+  "${KEYCLOAK_IMAGE}" start-dev --import-realm >/dev/null
+
+deadline=$((SECONDS + ${E2E_KEYCLOAK_STARTUP_TIMEOUT:-300}))
+until curl -fsS --connect-timeout 2 --max-time 3 \
+  "http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e/.well-known/openid-configuration" >/dev/null 2>&1; do
+  if ((SECONDS >= deadline)); then
+    echo 'e2e-stack: timed out waiting for the Keycloak realm' >&2
+    exit 1
+  fi
+  sleep 2
+done
+echo 'e2e-stack: Keycloak realm is up'

.github/e2e-stack/up.sh

@@ -25,6 +25,7 @@ DATABASE_PASSWORD="${E2E_DATABASE_PASSWORD:-dbpassword9090}"
 DATABASE_NAME="${E2E_DATABASE_NAME:-litellm}"
 JAEGER_OTLP_PORT="${E2E_JAEGER_OTLP_PORT:-4318}"
 JAEGER_QUERY_PORT="${E2E_JAEGER_QUERY_PORT:-16686}"
+KEYCLOAK_PORT="${E2E_KEYCLOAK_PORT:-8081}"
 
 MASTER_KEY="${LITELLM_MASTER_KEY:-sk-e2e-$(openssl rand -hex 16)}"
 
@@ -124,6 +125,9 @@ SERVER_ENV=(
   "OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:${JAEGER_OTLP_PORT}"
   "SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem"
   "PYTHONPATH=${REPO_ROOT}"
+  "JWT_PUBLIC_KEY_URL=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e/protocol/openid-connect/certs"
+  "JWT_ISSUER=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e"
+  "JWT_AUDIENCE=litellm-e2e"
 )
 if [[ -n "${VERTEXAI_CREDENTIALS:-}" ]]; then
   printf '%s' "${VERTEXAI_CREDENTIALS}" > "${STACK_DIR}/vertex-adc.json"
@@ -132,6 +136,8 @@ fi
 
 cd "${REPO_ROOT}"
 
+env "${SERVER_ENV[@]}" "E2E_KEYCLOAK_PORT=${KEYCLOAK_PORT}" bash .github/e2e-stack/start-idp.sh
+
 log "running migrations"
 env "${SERVER_ENV[@]}" uv run --no-sync python migrations/run.py >"${LOGS_DIR}/migrations.log" 2>&1
 
@@ -200,6 +206,9 @@ LITELLM_MASTER_KEY=${MASTER_KEY}
 REDIS_HOST=127.0.0.1
 REDIS_PORT=${REDIS_PORT}
 E2E_OTEL_QUERY_URL=http://127.0.0.1:${JAEGER_QUERY_PORT}
+E2E_KEYCLOAK_URL=http://127.0.0.1:${KEYCLOAK_PORT}
+E2E_KEYCLOAK_ADMIN_USER=admin
+E2E_KEYCLOAK_ADMIN_PASSWORD=e2e-ephemeral-idp-not-a-secret
 SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem
 DATABASE_URL=postgresql://${DATABASE_USER}:${DATABASE_PASSWORD}@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_NAME}
 EOF

.github/workflows/test-e2e-changed.yml

@@ -27,6 +27,8 @@ jobs:
           sparse-checkout: |
             .github/e2e-stack
             tests/e2e/access_control
+            tests/e2e/management/test_jwt_management_e2e.py
+            tests/e2e/other/test_jwt_auth_e2e.py
           persist-credentials: false
           ref: ${{ github.sha }}
 
@@ -45,7 +47,8 @@ jobs:
             --jq '.[] | select(.status != "removed") | .filename')"
           gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.head.sha' | grep -Fxq "${HEAD_SHA}"
           tests="$(printf '%s\n' "${files}" \
-            | python3 .github/e2e-stack/select_tests.py tests/e2e/access_control/test_*.py)"
+            | python3 .github/e2e-stack/select_tests.py tests/e2e/access_control/test_*.py \
+                tests/e2e/management/test_jwt_management_e2e.py tests/e2e/other/test_jwt_auth_e2e.py)"
           echo "tests=${tests}" >> "${GITHUB_OUTPUT}"
           if [ -n "${tests}" ]; then
             echo "any=true" >> "${GITHUB_OUTPUT}"

tests/code_coverage_tests/test_e2e_changed_gate.py

@@ -144,6 +144,8 @@ def test_changed_suite_files_are_selected_unless_the_stack_cannot_run_them(
         "tests/e2e/pytest.ini",
         "tests/e2e/gateway/stage_mirror_ci_config.yml",
         ".github/e2e-stack/up.sh",
+        ".github/e2e-stack/start-idp.sh",
+        "tests/e2e/idp_realm.json",
         ".github/workflows/test-e2e-changed.yml",
     ),
 )

tests/code_coverage_tests/test_e2e_idp_stack.py

@@ -0,0 +1,106 @@
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+import yaml
+
+ROOT = Path(__file__).resolve().parents[2]
+START_IDP = ROOT / ".github/e2e-stack/start-idp.sh"
+
+
+def run_start(tmp_path: Path, *, platform: str = "Linux", failure: str = ""):
+    bin_dir = tmp_path / "bin"
+    bin_dir.mkdir()
+    calls = tmp_path / "docker.jsonl"
+    programs = {
+        "docker": """import json, os, sys
+with open(os.environ['DOCKER_LOG'], 'a') as out:
+    out.write(json.dumps(sys.argv[1:]) + '\\n')
+if os.environ['FAILURE'] == 'schema' and 'psql' in sys.argv:
+    sys.exit(17)
+if os.environ['FAILURE'] == 'launch' and '--name' in sys.argv:
+    sys.exit(18)
+""",
+        "curl": "import os, sys; sys.exit(1 if os.environ['FAILURE'] == 'readiness' else 0)\n",
+        "uname": "import os; print(os.environ['PLATFORM'])\n",
+    }
+    for name, source in programs.items():
+        program = bin_dir / name
+        program.write_text(f"#!{sys.executable}\n{source}")
+        program.chmod(0o755)
+    result = subprocess.run(
+        ["bash", str(START_IDP)],
+        env={
+            **os.environ,
+            "PATH": f"{bin_dir}:{os.environ['PATH']}",
+            "DOCKER_LOG": str(calls),
+            "PLATFORM": platform,
+            "FAILURE": failure,
+            "DATABASE_HOST": "127.0.0.1",
+            "DATABASE_PORT": "5544",
+            "DATABASE_USER": "fixture_user",
+            "DATABASE_PASSWORD": "fixture_password",
+            "DATABASE_NAME": "fixture_db",
+            "E2E_KEYCLOAK_PORT": "8181",
+            "E2E_KEYCLOAK_STARTUP_TIMEOUT": "0",
+        },
+        capture_output=True,
+        text=True,
+        timeout=10,
+    )
+    return result, [json.loads(line) for line in calls.read_text().splitlines()]
+
+
+@pytest.mark.parametrize("platform", ("Linux", "Darwin"))
+def test_idp_uses_existing_database_and_imports_runner_realm(tmp_path: Path, platform: str) -> None:
+    result, calls = run_start(tmp_path, platform=platform)
+
+    assert result.returncode == 0, result.stderr
+    schema, _, launch = calls
+    host = "127.0.0.1" if platform == "Linux" else "host.docker.internal"
+    assert schema[schema.index("-h") + 1] == host
+    assert schema[schema.index("-p") + 1] == "5544"
+    assert "ON_ERROR_STOP=1" in schema
+    assert "CREATE SCHEMA IF NOT EXISTS keycloak" in schema
+    assert f"KC_DB_URL_HOST={host}" in launch
+    assert "KC_DB_URL_PORT=5544" in launch
+    assert "KC_DB_SCHEMA=keycloak" in launch
+    assert "KC_DB_POOL_MAX_SIZE=10" in launch
+    assert f"{ROOT}/tests/e2e/idp_realm.json:/opt/keycloak/data/import/realm.json:ro" in launch
+    assert "KC_HTTP_PORT=8181" in launch
+    if platform == "Linux":
+        assert launch[launch.index("--network") + 1] == "host"
+    else:
+        assert launch[launch.index("-p") + 1] == "127.0.0.1:8181:8181"
+    assert "Keycloak realm is up" in result.stdout
+
+
+@pytest.mark.parametrize(("failure", "code"), (("schema", 17), ("launch", 18), ("readiness", 1)))
+def test_idp_failure_stops_stack_startup(tmp_path: Path, failure: str, code: int) -> None:
+    result, calls = run_start(tmp_path, failure=failure)
+
+    assert result.returncode == code
+    assert "Keycloak realm is up" not in result.stdout
+    if failure == "schema":
+        assert len(calls) == 1, "do not replace an IdP when its database is unavailable"
+
+
+def test_proxy_and_test_runner_share_the_same_jwt_configuration() -> None:
+    config = yaml.safe_load((ROOT / "tests/e2e/gateway/stage_mirror_ci_config.yml").read_text())
+    general = config["general_settings"]
+    assert general["enable_jwt_auth"] is True
+    assert general["litellm_jwtauth"] == {
+        "user_id_jwt_field": "sub",
+        "user_email_jwt_field": "email",
+        "team_ids_jwt_field": "groups",
+        "user_id_upsert": True,
+    }
+    up = (START_IDP.parent / "up.sh").read_text()
+    assert '"JWT_ISSUER=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e"' in up
+    assert '"JWT_AUDIENCE=litellm-e2e"' in up
+    assert "E2E_KEYCLOAK_URL=http://127.0.0.1:${KEYCLOAK_PORT}" in up
+    assert up.index("bash .github/e2e-stack/start-idp.sh") < up.index("start_server backend")
+    assert "e2e-keycloak" in (START_IDP.parent / "down.sh").read_text()

tests/e2e/CONTRIBUTING.md

@@ -69,7 +69,7 @@ The suites run against a live proxy, so bring one up first by running the litell
      uv run pytest tests/e2e/other/test_jwt_auth_e2e.py tests/e2e/management/test_jwt_management_e2e.py --reruns 0 -v
    ```
 
-   CI runs this suite against a Keycloak deployed beside the ephemeral stack, and that deployment (the JWT config block, `JWT_PUBLIC_KEY_URL`, and the admin credential handed to the run pod) lives in the project-releaser repo, not here. CI fetches the realm from the test-runner revision even when it reuses a gateway image from another commit. Keycloak stores its realm, keys and users in a separate schema in the build's PostgreSQL, so replacing the IdP pod preserves token validity. Its startup probe waits for the imported realm. Losing the whole ephemeral database invalidates the stack. Keycloak skips imports into an existing realm, so changes to the realm export require a fresh stack (or deliberately replacing the local data volume). A stack without it fails the JWT tests rather than skipping them
+   Buildkite runs this suite against a Keycloak deployed beside the ephemeral stack by project-releaser. It fetches the realm from the test-runner revision even when it reuses a gateway image from another commit. The GitHub Actions changed-test stack starts the same digest-pinned Keycloak through `.github/e2e-stack/start-idp.sh`, imports the checked-out realm, and exports the IdP URL and credentials in `stack.env`. Both runners configure issuer/audience validation and store the realm, keys and users in a separate schema in the stack's PostgreSQL, so replacing Keycloak preserves token validity. Both wait for realm discovery before running tests. Losing the whole ephemeral database invalidates the stack. Keycloak skips imports into an existing realm, so changes to the realm export require a fresh stack (or deliberately replacing the local data volume). A stack without it fails the JWT tests rather than skipping them
 
 4. Run a suite against it; the harness reads `LITELLM_PROXY_URL` (default `http://localhost:4000`):
 
@@ -101,7 +101,7 @@ A couple of logging destinations are configured on the proxy rather than by the
 
 ### The pull request check
 
-Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start
+Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start
 
 Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch
 

tests/e2e/gateway/stage_mirror_ci_config.yml

@@ -1,4 +1,11 @@
 general_settings:
+  proxy_batch_write_at: 5
+  enable_jwt_auth: true
+  litellm_jwtauth:
+    user_id_jwt_field: sub
+    user_email_jwt_field: email
+    team_ids_jwt_field: groups
+    user_id_upsert: true
   proxy_config_reload_interval_seconds: 7
   store_prompts_in_spend_logs: true
   database_connection_pool_limit: 10