From 90ac77e58e32eeabb12b8df28144bf2ebae28dee Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 5 Sep 2026 17:25:02 -0700 Subject: [PATCH 1/7] test(e2e): add JWT issuer harness and first live JWT auth tests Add tests/e2e/jwt_issuer.py, a test-only RS256 issuer that serves a JWKS document and signs arbitrary claims over an open loopback-only /token endpoint, so e2e tests can mint tokens without ever holding a signing key. One key per process keeps the proxy's cached JWKS valid for the whole run. Add the first five live JWT auth tests in tests/e2e/other: a valid token for an existing team is accepted and its spend row carries the claimed team and user, a tampered signature and an expired token are refused with 401, a token naming a team that does not exist is refused with 403, and a plain sk- virtual key keeps working with enable_jwt_auth on. Harness unit tests cover the issuer itself. Team and user create/delete land on the shared ProxyClient (warn-only teardown, /user/delete typed as the int it returns) instead of a fourth per-suite copy. Document the issuer command, the E2E_JWT_ISSUER_PORT convention (default 4190), JWT_PUBLIC_KEY_URL and the litellm_jwtauth config block in tests/e2e/CONTRIBUTING.md, and register the new cells in other.yaml. --- tests/e2e/CLAUDE.md | 2 +- tests/e2e/CONTRIBUTING.md | 21 ++- tests/e2e/coverage_registry/other.yaml | 9 +- tests/e2e/e2e_config.py | 4 + tests/e2e/e2e_http.py | 21 +++ tests/e2e/jwt_issuer.py | 230 +++++++++++++++++++++++++ tests/e2e/models.py | 12 ++ tests/e2e/other/other_client.py | 34 +++- tests/e2e/other/test_jwt_auth_e2e.py | 129 ++++++++++++++ tests/e2e/proxy_client.py | 41 +++++ tests/e2e/test_jwt_issuer.py | 141 +++++++++++++++ 11 files changed, 633 insertions(+), 11 deletions(-) create mode 100644 tests/e2e/jwt_issuer.py create mode 100644 tests/e2e/other/test_jwt_auth_e2e.py create mode 100644 tests/e2e/test_jwt_issuer.py diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 89c04208d65..4985da9e95d 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -19,7 +19,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `security/` - secret handling and log-leak protection - `router/` - routing and reliability behavior (fallbacks, cooldowns) - `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What remains here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`) and markerless harness unit tests for the Locust/session-anomaly aggregation logic -- `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite +- `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (RS256 tokens minted by the test-only issuer in `jwt_issuer.py`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at; see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke - `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream; see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport; the Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json` diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index b270feb820e..ad414bb6647 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -27,14 +27,31 @@ The suites run against a live proxy, so bring one up first by running the litell 2. Bring up a Postgres and a Redis for the proxy to use. The repo-root `docker-compose.yml` already defines a Postgres on `5432`; a `docker run -p 6379:6379 redis:7` covers Redis. Point `DATABASE_URL` / `REDIS_HOST` / `REDIS_PORT` at whatever you run. Tests that read Redis directly default to the deployed shape (TLS + cluster mode) whenever `REDIS_HOST` is set, so for a local standalone Redis also set `REDIS_CLUSTER=false` and `REDIS_SSL=false` (plus `REDIS_PASSWORD` when your Redis requires auth) -3. Start the litellm proxy locally against your config and confirm it is live: +3. Start the test-only JWT issuer the `other/` suite mints tokens from, then the litellm proxy against your config, and confirm both are live. The issuer is a fake identity provider (`jwt_issuer.py`) with an open mint endpoint, so it binds loopback only; it generates one RSA key per process and serves it as a JWKS, and the proxy caches that JWKS for `public_key_ttl` (600s) without refetching on an unknown `kid`, so restart the proxy whenever you restart the issuer: ```bash set -a && source .env && set +a - litellm --config .yml --port 4000 + uv run python tests/e2e/jwt_issuer.py & + curl -fs http://127.0.0.1:4190/.well-known/jwks.json + JWT_PUBLIC_KEY_URL=http://127.0.0.1:4190/.well-known/jwks.json litellm --config .yml --port 4000 curl -fs http://localhost:4000/health/liveliness ``` + The issuer's port is `E2E_JWT_ISSUER_PORT` (default `4190`, the port in the URLs above), read by both the issuer process and the tests, so set it in one place if you change it. JWT auth is an enterprise feature, so the proxy also needs `LITELLM_LICENSE` in its environment, and its config needs the JWT block below. `enable_jwt_auth` only routes bearer tokens with three dot-separated segments into the JWT path, so `sk-` virtual keys and the master key keep working for every other suite. `proxy_batch_write_at` is lowered so the JWT spend-attribution test sees its row well inside the poll deadline: + + ```yaml + general_settings: + proxy_batch_write_at: 5 + enable_jwt_auth: true + litellm_jwtauth: + user_id_jwt_field: sub + user_email_jwt_field: email + team_ids_jwt_field: groups + user_id_upsert: true + ``` + + Leave `JWT_AUDIENCE` and `JWT_ISSUER` unset unless you also put matching `aud` / `iss` claims in the tokens the tests mint; the issuer sets `iss` to its own base URL + 4. Run a suite against it; the harness reads `LITELLM_PROXY_URL` (default `http://localhost:4000`): ```bash diff --git a/tests/e2e/coverage_registry/other.yaml b/tests/e2e/coverage_registry/other.yaml index 814ebae2e0b..8b9cb4d2adf 100644 --- a/tests/e2e/coverage_registry/other.yaml +++ b/tests/e2e/coverage_registry/other.yaml @@ -9,9 +9,12 @@ - {id: other.auth.llm_chat.not_bearer_scheme_denied, module: other, tier: P0, area: auth, assertions: [not_bearer_scheme_denied], source: "vendor testing strategy §11.1 / LIT-4778", rationale: "NotBearer scheme on chat is 401/403"} - {id: other.auth.realtime.missing_header_denied, module: other, tier: P1, area: auth, assertions: [missing_header_denied], source: "vendor testing strategy §9.19 / LIT-4778", rationale: "Realtime client-secret and calls routes reject requests without Authorization"} - {id: other.config.responses.metadata_redis_ttl_bounded, module: other, tier: P0, area: config, assertions: [ttl_bounded], source: "responses + redis cache", rationale: "Responses store+metadata must not leave TTL-unbounded Redis entries (LIT-1201)"} -- {id: other.auth.jwt.valid_token_allows, module: other, tier: P0, area: auth, assertions: [valid_token_allows], source: "handle_jwt.py:77-150", rationale: "Valid JWT with correct issuer + claims grants access"} -- {id: other.auth.jwt.expired_denied, module: other, tier: P0, area: auth, assertions: [expired_denied], source: "handle_jwt.py:125-135", rationale: "Expired JWT rejected even with valid signature"} -- {id: other.auth.jwt.invalid_signature_denied, module: other, tier: P0, area: auth, assertions: [invalid_signature_denied], source: "handle_jwt.py:145-150", rationale: "Bad/missing signature fails verification"} +- {id: other.auth.jwt.valid_token_allows, module: other, tier: P0, area: auth, assertions: [valid_token_allows], source: "handle_jwt.py:1217-1256 auth_jwt / user_api_key_auth.py:1365-1377", rationale: "An RS256 JWT signed by the configured JWKS whose groups claim names an existing team is accepted on /chat/completions"} +- {id: other.auth.jwt.spend_attributed_to_claims, module: other, tier: P0, area: auth, assertions: [spend_attributed_to_claims], source: "handle_jwt.py:2224 auth_builder / user_api_key_auth.py:1438-1474", rationale: "The spend log row for a JWT-authenticated call carries the team_id from the groups claim and the user_id from sub, not a virtual key's identity. Single-group claim only: the proxy picks the team from a set, so attribution over several groups is unordered"} +- {id: other.auth.jwt.expired_denied, module: other, tier: P0, area: auth, assertions: [expired_denied], source: "handle_jwt.py:1244-1250", rationale: "Expired JWT rejected 401 (Token Expired) even with valid signature; leeway is 0"} +- {id: other.auth.jwt.invalid_signature_denied, module: other, tier: P0, area: auth, assertions: [invalid_signature_denied], source: "handle_jwt.py:1158-1166 _decode_jwt_with_public_key", rationale: "A genuine token whose signature bytes were altered fails verification with 401"} +- {id: other.auth.jwt.unknown_team_denied, module: other, tier: P0, area: auth, assertions: [unknown_team_denied], source: "handle_jwt.py:1549-1632 find_team_with_model_access", rationale: "A verified JWT whose groups claim resolves to no existing team is denied with 403 naming the unresolved team, never silently admitted without a team. The proxy words it as a model-access denial, the same body an existing team without model access gets"} +- {id: other.auth.jwt.virtual_key_unaffected, module: other, tier: P0, area: auth, assertions: [virtual_key_unaffected], source: "handle_jwt.py:213 is_jwt / user_api_key_auth.py:1332-1333", rationale: "enable_jwt_auth only routes three-segment bearer tokens into the JWT branch, so sk- virtual keys keep working on the same proxy"} - {id: other.auth.model_access_group.wildcard_bare_name_allowed, module: other, tier: P0, area: auth, assertions: [wildcard_bare_name_allowed], source: "auth_checks.py:3232 / LIT-5813", fail_before_fix: proven, rationale: "A grant of a group holding a wildcard deployment covers the bare model names callers actually send, not only the provider-prefixed spelling"} - {id: other.auth.model_access_group.member_allowed, module: other, tier: P0, area: auth, assertions: [member_allowed], source: "auth_checks.py:3232", rationale: "A key whose allow-list is a model access group can call the deployments in that group"} - {id: other.auth.model_access_group.non_member_denied, module: other, tier: P0, area: auth, assertions: [non_member_denied], source: "auth_checks.py:3232", rationale: "That same grant reaches nothing outside the group, including provider models the group's wildcard does not cover"} diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 09d17b9a0db..0e7f4c0ad2b 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -15,6 +15,7 @@ from typing import Final from dotenv import load_dotenv from fixture_mode import deterministic_marker, parse_fixture_mode +from jwt_issuer import jwt_issuer_url from provider_edge import provider_edge_api_base # Local runs keep provider / DataDog keys in tests/e2e/.env (see CONTRIBUTING.md). @@ -43,6 +44,9 @@ UI_BASE_URL = os.environ.get("E2E_UI_BASE_URL", PROXY_BASE_URL).rstrip("/") CHEAP_ANTHROPIC_MODEL = os.environ.get("E2E_CHEAP_ANTHROPIC_MODEL", "claude-haiku-4-5") CHEAP_OPENAI_MODEL = os.environ.get("E2E_CHEAP_OPENAI_MODEL", "gpt-5.5") +# Test-only JWT issuer (jwt_issuer.py); the port is E2E_JWT_ISSUER_PORT (see CONTRIBUTING.md). +JWT_ISSUER_URL = jwt_issuer_url() + LINEAR_MCP_URL = os.environ.get("E2E_LINEAR_MCP_URL", "https://mcp.linear.app/mcp") LINEAR_STORAGE_STATE = os.environ.get("E2E_LINEAR_STORAGE_STATE", "") diff --git a/tests/e2e/e2e_http.py b/tests/e2e/e2e_http.py index 9d5f1658e91..4c65c53b653 100644 --- a/tests/e2e/e2e_http.py +++ b/tests/e2e/e2e_http.py @@ -386,6 +386,27 @@ def get_external[R: BaseModel]( return _classify(resp, response_type) +def post_external[R: BaseModel]( + url: str, + *, + json: BaseModel, + response_type: type[R], + timeout: float = 30.0, +) -> Result[R]: + """POST an absolute URL outside the proxy (e.g. the e2e JWT issuer's mint + endpoint). Like get_external: no proxy base url, no proxy auth, and the same + tagged-union classification as every other call.""" + try: + resp = requests.post( + url, + json=json.model_dump(by_alias=True, exclude_none=True), + timeout=timeout, + ) + except requests.RequestException as exc: + return NetworkError(message=str(exc)) + return _classify(resp, response_type) + + def delete[R: BaseModel]( url: URL, *, diff --git a/tests/e2e/jwt_issuer.py b/tests/e2e/jwt_issuer.py new file mode 100644 index 00000000000..4006fcaa314 --- /dev/null +++ b/tests/e2e/jwt_issuer.py @@ -0,0 +1,230 @@ +"""Test-only fake identity provider for the e2e JWT suite. Never deploy it. + +Run it next to the proxy (`uv run python tests/e2e/jwt_issuer.py`). On start it +generates one RSA signing key and keeps it for the life of the process, serving +the public half at `GET /.well-known/jwks.json` and signing whatever claims are +POSTed to `/token`. The proxy's `JWT_PUBLIC_KEY_URL` points at the JWKS URL, and +tests mint RS256 tokens by POSTing claims, so the private key never leaves this +process and no test holds it. + +One key per process, rather than per pytest run, is what survives the proxy's +JWKS cache: the proxy caches the JWKS for `litellm_jwtauth.public_key_ttl` +(600s by default) and does not refetch on an unknown `kid`, so a key rotated +every run would be rejected until the cache expired. Restart the proxy whenever +you restart the issuer. + +The mint endpoint takes no credential: anyone who can reach it gets a token the +proxy trusts. It therefore binds 127.0.0.1 only, must never be exposed beyond +loopback, and must only ever be trusted by a proxy under test. +""" + +from __future__ import annotations + +import logging +import os +import threading +import time +import uuid +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Final, Literal + +import jwt +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from jwt.utils import to_base64url_uint +from pydantic import BaseModel, RootModel, ValidationError + +JWT_ISSUER_PORT_ENV: Final = "E2E_JWT_ISSUER_PORT" +DEFAULT_JWT_ISSUER_PORT: Final = 4190 +LOOPBACK_HOST: Final = "127.0.0.1" +JWKS_PATH: Final = "/.well-known/jwks.json" +TOKEN_PATH: Final = "/token" +DEFAULT_TOKEN_LIFETIME_SECONDS: Final = 300 + +ClaimValue = str | int | float | bool | None | list[str] + + +def jwt_issuer_port() -> int: + raw: Final = os.environ.get(JWT_ISSUER_PORT_ENV, "").strip() + return int(raw) if raw else DEFAULT_JWT_ISSUER_PORT + + +def jwt_issuer_url() -> str: + return f"http://{LOOPBACK_HOST}:{jwt_issuer_port()}" + + +class RsaJwk(BaseModel): + kty: Literal["RSA"] = "RSA" + alg: Literal["RS256"] = "RS256" + use: Literal["sig"] = "sig" + kid: str + n: str + e: str + + +class JwksDocument(BaseModel): + keys: tuple[RsaJwk, ...] + + +class TokenRequest(RootModel[Mapping[str, ClaimValue]]): + """The JSON body of POST /token: the claims to sign, verbatim. Nested objects + are not supported; every value is a scalar or a list of strings.""" + + +class MintedToken(BaseModel): + token: str + + +class IssuerError(BaseModel): + error: str + + +@dataclass(frozen=True, slots=True) +class SigningKey: + kid: str + private_pem: str + jwk: RsaJwk + + +def generate_signing_key(kid: str | None = None) -> SigningKey: + private_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + numbers: Final = private_key.public_key().public_numbers() + resolved_kid: Final = kid if kid is not None else uuid.uuid4().hex + return SigningKey( + kid=resolved_kid, + private_pem=private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ).decode(), + jwk=RsaJwk( + kid=resolved_kid, + n=to_base64url_uint(numbers.n).decode(), + e=to_base64url_uint(numbers.e).decode(), + ), + ) + + +def mint( + key: SigningKey, + claims: Mapping[str, ClaimValue], + *, + issuer: str, + now: int, + lifetime_seconds: int = DEFAULT_TOKEN_LIFETIME_SECONDS, +) -> str: + """Sign `claims` as a compact RS256 JWT carrying `key.kid` in its header. + `iss`, `iat`, and `exp` are filled in when absent and left alone when the + caller sets them, so a test can mint an already-expired token.""" + payload: Final[dict[str, ClaimValue]] = { + "iss": issuer, + "iat": now, + "exp": now + lifetime_seconds, + **claims, + } + return jwt.encode(payload, key.private_pem, algorithm="RS256", headers={"kid": key.kid}) + + +class _IssuerServer(ThreadingHTTPServer): + daemon_threads = True + + def __init__(self, bind: tuple[str, int], *, key: SigningKey, clock: Callable[[], int]) -> None: + super().__init__(bind, _IssuerHandler) + self.key: Final = key + self.clock: Final = clock + + @property + def url(self) -> str: + host, port = self.server_address[0], self.server_address[1] + return f"http://{host}:{port}" + + +class _IssuerHandler(BaseHTTPRequestHandler): + def _issuer(self) -> _IssuerServer: + issuer: Final = self.server + assert isinstance(issuer, _IssuerServer) + return issuer + + def do_GET(self) -> None: + if self.path != JWKS_PATH: + self._send(404, IssuerError(error=f"unknown path {self.path}; the JWKS is at {JWKS_PATH}")) + return + self._send(200, JwksDocument(keys=(self._issuer().key.jwk,))) + + def do_POST(self) -> None: + if self.path != TOKEN_PATH: + self._send(404, IssuerError(error=f"unknown path {self.path}; mint tokens at {TOKEN_PATH}")) + return + length: Final = int(self.headers.get("Content-Length", "0")) + try: + request: Final = TokenRequest.model_validate_json(self.rfile.read(length)) + except ValidationError as exc: + self._send(400, IssuerError(error=f"claims must be a JSON object of scalar or string-list values: {exc}")) + return + issuer: Final = self._issuer() + token: Final = mint(issuer.key, request.root, issuer=issuer.url, now=issuer.clock()) + self._send(200, MintedToken(token=token)) + + def _send(self, status: int, body: BaseModel) -> None: + payload: Final = body.model_dump_json().encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + +@dataclass(frozen=True, slots=True) +class RunningIssuer: + url: str + key: SigningKey + server: _IssuerServer + + @property + def jwks_url(self) -> str: + return f"{self.url}{JWKS_PATH}" + + def shutdown(self) -> None: + self.server.shutdown() + self.server.server_close() + + +def _wall_clock() -> int: + return int(time.time()) + + +def start_jwt_issuer( + *, + port: int = 0, + key: SigningKey | None = None, + clock: Callable[[], int] = _wall_clock, +) -> RunningIssuer: + """Serve the issuer on loopback in a daemon thread. `port=0` takes an + OS-assigned port for in-process tests; the CLI passes the documented one.""" + server: Final = _IssuerServer((LOOPBACK_HOST, port), key=key or generate_signing_key(), clock=clock) + thread: Final = threading.Thread(target=server.serve_forever, name="e2e-jwt-issuer", daemon=True) + thread.start() + return RunningIssuer(url=server.url, key=server.key, server=server) + + +def main() -> None: + logging.basicConfig(level=logging.INFO, format="%(message)s") + running: Final = start_jwt_issuer(port=jwt_issuer_port()) + logging.getLogger(__name__).info( + "e2e jwt issuer listening on %s jwks=%s mint=POST %s%s kid=%s (test-only, loopback, ctrl-c to stop)", + running.url, + running.jwks_url, + running.url, + TOKEN_PATH, + running.key.kid, + ) + try: + threading.Event().wait() + except KeyboardInterrupt: + running.shutdown() + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 2c6c0e9bbd4..17f5418bd73 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -1078,6 +1078,18 @@ class UserListResponse(BaseModel): total: int +class JwtClaimsBody(BaseModel): + """Claims POSTed to the e2e JWT issuer's /token: exactly what the + `litellm_jwtauth` block in CONTRIBUTING.md reads (sub -> user_id, email -> + user_email, groups -> team ids), plus an explicit `exp` for the expired case; + the issuer fills in iss/iat/exp when they are left unset.""" + + sub: str + email: str + groups: Sequence[str] + exp: int | None = None + + class OrgNewBody(BaseModel): organization_alias: str models: list[str] = [] diff --git a/tests/e2e/other/other_client.py b/tests/e2e/other/other_client.py index 1aa83ac42c7..b51f3338f52 100644 --- a/tests/e2e/other/other_client.py +++ b/tests/e2e/other/other_client.py @@ -1,19 +1,27 @@ """Client for the `other` holding-pen suite: the auth gate (master key vs an -invalid key on an admin route) and the process-lifecycle health probes -(liveness, public readiness, authenticated readiness diagnostics). +invalid key on an admin route), JWT auth against the test-only issuer +(jwt_issuer.py), and the process-lifecycle health probes (liveness, public +readiness, authenticated readiness diagnostics). Holds the shared ProxyClient so `resources` / `scoped_key` still clean up, and adds only the routes these behaviors need. The health probes deliberately send no auth header (public routes), so they go through the transport with an empty -headers model rather than a bearer. +headers model rather than a bearer. Tokens are minted by POSTing claims to the +issuer, so no test ever holds a signing key. """ from __future__ import annotations from dataclasses import dataclass +from typing import Final -from e2e_http import NoBody, ProbeResult, Result +import pytest + +from e2e_config import JWT_ISSUER_URL +from e2e_http import NetworkError, NoBody, ProbeResult, Result, Success, post_external +from jwt_issuer import TOKEN_PATH, MintedToken from models import ( + JwtClaimsBody, ReadinessDetailsResponse, ReadinessResponse, UserListParams, @@ -25,6 +33,7 @@ from proxy_client import ProxyClient @dataclass(frozen=True, slots=True) class OtherClient: proxy: ProxyClient + jwt_issuer_url: str def liveness(self) -> ProbeResult: """GET /health/liveliness. Unauthenticated; the probe returns status + @@ -68,6 +77,21 @@ class OtherClient: response_type=UserListResponse, ) + def mint_jwt(self, claims: JwtClaimsBody) -> str: + """Have the test-only issuer sign `claims` into a compact RS256 JWT. A + missing issuer is a hard failure naming the start command, not a skip.""" + result: Final = post_external(f"{self.jwt_issuer_url}{TOKEN_PATH}", json=claims, response_type=MintedToken) + match result: + case Success(data=minted): + return minted.token + case NetworkError(message=message): + pytest.fail( + f"No live JWT issuer at {self.jwt_issuer_url}: {message}. Start it next to the proxy with " + "`uv run python tests/e2e/jwt_issuer.py` (see CONTRIBUTING.md)" + ) + case _: + raise AssertionError(result) + def build_client(proxy: ProxyClient) -> OtherClient: - return OtherClient(proxy=proxy) + return OtherClient(proxy=proxy, jwt_issuer_url=JWT_ISSUER_URL) diff --git a/tests/e2e/other/test_jwt_auth_e2e.py b/tests/e2e/other/test_jwt_auth_e2e.py new file mode 100644 index 00000000000..49a9350a3a2 --- /dev/null +++ b/tests/e2e/other/test_jwt_auth_e2e.py @@ -0,0 +1,129 @@ +"""Live e2e: RS256 JWTs minted by the test-only issuer (jwt_issuer.py) against a +proxy running with `enable_jwt_auth: true` and the `litellm_jwtauth` block from +CONTRIBUTING.md (sub -> user_id, email -> user_email, groups -> team ids, +user_id_upsert). + +Every case mints through the issuer, so the tests never hold a signing key: the +bad-signature case corrupts a genuine signature, the expired case asks the +issuer for a token whose `exp` is already in the past. Those identities get +their own freshly created team so a rejection can only be blamed on the token, +while the unknown-team case names a team that was never created. An acceptance +is proven twice, at the boundary (200 from a real provider) and in the spend log +the proxy attributes to the claims. The last case keeps a plain `sk-` virtual +key working on the same proxy, guarding against the flag turning JWT on for +everyone. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Final + +import pytest + +from e2e_config import CHEAP_OPENAI_MODEL, unique_marker +from e2e_http import UnauthorizedError, UnknownApiError, unwrap +from lifecycle import ResourceManager +from models import ChatBody, ChatMessage, JwtClaimsBody, TeamNewBody +from other_client import OtherClient + +pytestmark = pytest.mark.e2e + + +@dataclass(frozen=True, slots=True) +class JwtIdentity: + user_id: str + team_id: str + + def claims(self, *, exp: int | None = None) -> JwtClaimsBody: + return JwtClaimsBody(sub=self.user_id, email=f"{self.user_id}@example.com", groups=(self.team_id,), exp=exp) + + +@pytest.fixture +def identity(client: OtherClient, resources: ResourceManager) -> JwtIdentity: + marker: Final = unique_marker() + team_id: Final = client.proxy.create_team( + TeamNewBody(team_alias=f"e2e-jwt-{marker}", team_id=f"e2e-jwt-team-{marker}") + ) + resources.defer(lambda: client.proxy.delete_team(team_id)) + user_id: Final = f"e2e-jwt-user-{marker}" + resources.defer(lambda: client.proxy.delete_user(user_id)) + return JwtIdentity(user_id=user_id, team_id=team_id) + + +def _ping() -> ChatBody: + return ChatBody( + model=CHEAP_OPENAI_MODEL, + messages=[ChatMessage(role="user", content=f"Reply with the single word pong. {unique_marker()}")], + max_tokens=16, + ) + + +def _corrupt_signature(token: str) -> str: + header, payload, signature = token.split(".") + flipped: Final = "A" if signature[10] != "A" else "B" + return f"{header}.{payload}.{signature[:10]}{flipped}{signature[11:]}" + + +class TestJwtAuth: + @pytest.mark.covers("other.auth.jwt.valid_token_allows", "other.auth.jwt.spend_attributed_to_claims") + def test_valid_token_for_an_existing_team_is_accepted_and_attributed( + self, client: OtherClient, identity: JwtIdentity + ) -> None: + token: Final = client.mint_jwt(identity.claims()) + + response: Final = unwrap(client.proxy.chat(token, _ping())) + assert response.id is not None and response.choices, ( + f"chat under a valid JWT returned no completion: {response}" + ) + + rows: Final = client.proxy.poll_logs_for_request_id(response.id) + assert rows, f"no spend log row for request {response.id} within the poll deadline" + row: Final = rows[0] + assert row.team_id == identity.team_id, ( + f"spend row must carry the team from the JWT groups claim {identity.team_id!r}, got {row.team_id!r}" + ) + assert row.user == identity.user_id, ( + f"spend row must carry the user from the JWT sub claim {identity.user_id!r}, got {row.user!r}" + ) + + @pytest.mark.covers("other.auth.jwt.invalid_signature_denied") + def test_tampered_signature_is_rejected(self, client: OtherClient, identity: JwtIdentity) -> None: + tampered: Final = _corrupt_signature(client.mint_jwt(identity.claims())) + + result: Final = client.proxy.chat(tampered, _ping()) + assert isinstance(result, UnauthorizedError), ( + f"a JWT whose signature does not verify must be rejected with 401, got {result}" + ) + assert "signature verification failed" in result.body.lower(), ( + f"the 401 must come from signature verification, not another auth failure, got {result.body[:300]}" + ) + + @pytest.mark.covers("other.auth.jwt.expired_denied") + def test_expired_token_is_rejected(self, client: OtherClient, identity: JwtIdentity) -> None: + expired: Final = client.mint_jwt(identity.claims(exp=1)) + + result: Final = client.proxy.chat(expired, _ping()) + assert isinstance(result, UnauthorizedError), ( + f"an expired JWT must be rejected with 401 even though its signature verifies, got {result}" + ) + assert "expired" in result.body.lower(), f"the 401 must say the token expired, got {result.body[:300]}" + + @pytest.mark.covers("other.auth.jwt.unknown_team_denied") + def test_token_naming_a_team_that_does_not_exist_is_rejected(self, client: OtherClient) -> None: + marker: Final = unique_marker() + never_created: Final = JwtIdentity(user_id=f"e2e-jwt-user-{marker}", team_id=f"e2e-jwt-missing-team-{marker}") + token: Final = client.mint_jwt(never_created.claims()) + + result: Final = client.proxy.chat(token, _ping()) + assert isinstance(result, UnknownApiError) and result.status_code == 403, ( + f"a valid JWT whose groups name no existing team must be rejected with 403, got {result}" + ) + assert never_created.team_id in result.body, ( + f"the 403 must name the team it could not resolve ({never_created.team_id}), got {result.body[:300]}" + ) + + @pytest.mark.covers("other.auth.jwt.virtual_key_unaffected") + def test_plain_virtual_key_still_works_with_jwt_auth_enabled(self, client: OtherClient, scoped_key: str) -> None: + response: Final = unwrap(client.proxy.chat(scoped_key, _ping())) + assert response.choices, f"an sk- key must keep working on a proxy with enable_jwt_auth, got {response}" diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index cdc20e5299a..5292f56b324 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -21,6 +21,7 @@ from e2e_http import ( Result, StreamingResponse, Success, + UnknownApiError, is_ok, unwrap, ) @@ -65,6 +66,11 @@ from models import ( SpendLogsPage, SpendLogsPageParams, SpendLogsParams, + TeamDeleteBody, + TeamNewBody, + TeamNewResponse, + UserDeleteBody, + UserDeleteResponse, ) from e2e_config import ( CONTROL_PLANE_BASE_URL, @@ -409,6 +415,41 @@ class ProxyClient: if not is_ok(result): warnings.warn(f"delete_credential({credential_name!r}) failed: {result}", stacklevel=2) + def create_team(self, body: TeamNewBody) -> str: + return unwrap( + self.transport.post( + "/team/new", + headers=self.transport.master, + json=body, + response_type=TeamNewResponse, + ) + ).team_id + + def delete_team(self, team_id: str) -> None: + result = self.transport.post( + "/team/delete", + headers=self.transport.master, + json=TeamDeleteBody(team_ids=[team_id]), + response_type=NoBody, + ) + if not is_ok(result): + warnings.warn(f"delete_team({team_id!r}) failed: {result}", stacklevel=2) + + def delete_user(self, user_id: str) -> None: + """Best-effort teardown; a 404 is not a leak, since JWT tests defer this for + a user the proxy only upserts after a successful auth.""" + result = self.transport.post( + "/user/delete", + headers=self.transport.master, + json=UserDeleteBody(user_ids=[user_id]), + response_type=UserDeleteResponse, + ) + match result: + case Success() | UnknownApiError(status_code=404): + return + case _: + warnings.warn(f"delete_user({user_id!r}) failed: {result}", stacklevel=2) + # ---- LLM calls ------------------------------------------------------ def chat(self, key: str, body: ChatBody) -> Result[ChatResponse]: diff --git a/tests/e2e/test_jwt_issuer.py b/tests/e2e/test_jwt_issuer.py new file mode 100644 index 00000000000..7acaaf13949 --- /dev/null +++ b/tests/e2e/test_jwt_issuer.py @@ -0,0 +1,141 @@ +"""Harness coverage for the test-only JWT issuer (jwt_issuer.py). + +No proxy and no ``e2e`` marker. The issuer is booted in-process on an +OS-assigned port with a fixed clock and driven over HTTP through +``e2e_http.post_external`` / ``get_external``, the same transport the live +suite uses, so what is pinned here is the contract the live JWT tests lean on: +a token minted at ``/token`` verifies against the key served at +``/.well-known/jwks.json`` under the ``kid`` in its header, ``iss``/``iat``/``exp`` +are filled in only when the caller left them out, and malformed claim bodies or +unknown paths are refused instead of signed. +""" + +from __future__ import annotations + +import time +from collections.abc import Iterator +from typing import Final + +import jwt +import pytest +from pydantic import BaseModel, RootModel + +from e2e_http import UnknownApiError, get_external, post_external, unwrap +from jwt_issuer import ( + DEFAULT_TOKEN_LIFETIME_SECONDS, + JWKS_PATH, + JWT_ISSUER_PORT_ENV, + TOKEN_PATH, + JwksDocument, + MintedToken, + RunningIssuer, + jwt_issuer_port, + start_jwt_issuer, +) + +FROZEN_NOW: Final = int(time.time()) + + +class _Claims(BaseModel): + sub: str + groups: tuple[str, ...] = () + exp: int | None = None + + +class _DecodedClaims(BaseModel): + sub: str + iss: str + iat: int + exp: int + groups: tuple[str, ...] = () + + +class _NotAnObject(RootModel[tuple[str, ...]]): + pass + + +@pytest.fixture(scope="module") +def issuer() -> Iterator[RunningIssuer]: + running: Final = start_jwt_issuer(clock=lambda: FROZEN_NOW) + yield running + running.shutdown() + + +def _mint(issuer: RunningIssuer, claims: _Claims) -> str: + return unwrap(post_external(f"{issuer.url}{TOKEN_PATH}", json=claims, response_type=MintedToken)).token + + +def _served_jwks(issuer: RunningIssuer) -> JwksDocument: + return unwrap(get_external(issuer.jwks_url, response_type=JwksDocument)) + + +def _decode(token: str, jwks: JwksDocument, *, verify_exp: bool = True) -> _DecodedClaims: + key: Final = jwt.PyJWK.from_json(jwks.keys[0].model_dump_json()) + decoded: Final = jwt.decode(token, key, algorithms=["RS256"], options={"verify_exp": verify_exp}) + return _DecodedClaims.model_validate(decoded) + + +class TestJwtIssuer: + def test_minted_token_verifies_against_the_served_jwks(self, issuer: RunningIssuer) -> None: + token: Final = _mint(issuer, _Claims(sub="alice", groups=("team-a",))) + jwks: Final = _served_jwks(issuer) + + assert len(jwks.keys) == 1 + assert jwt.get_unverified_header(token)["kid"] == jwks.keys[0].kid + claims: Final = _decode(token, jwks) + assert claims.sub == "alice" + assert claims.groups == ("team-a",) + assert claims.iss == issuer.url + assert claims.iat == FROZEN_NOW + assert claims.exp == FROZEN_NOW + DEFAULT_TOKEN_LIFETIME_SECONDS + + def test_a_token_signed_by_another_key_does_not_verify(self, issuer: RunningIssuer) -> None: + other: Final = start_jwt_issuer(clock=lambda: FROZEN_NOW) + try: + foreign_token: Final = _mint(other, _Claims(sub="alice")) + finally: + other.shutdown() + + with pytest.raises(jwt.InvalidSignatureError): + _decode(foreign_token, _served_jwks(issuer)) + + def test_an_explicit_exp_is_signed_as_given(self, issuer: RunningIssuer) -> None: + expired_at: Final = FROZEN_NOW - 60 + token: Final = _mint(issuer, _Claims(sub="alice", exp=expired_at)) + jwks: Final = _served_jwks(issuer) + + assert _decode(token, jwks, verify_exp=False).exp == expired_at + with pytest.raises(jwt.ExpiredSignatureError): + _ = _decode(token, jwks) + + def test_non_object_claims_are_refused(self, issuer: RunningIssuer) -> None: + result: Final = post_external( + f"{issuer.url}{TOKEN_PATH}", json=_NotAnObject(("not", "claims")), response_type=MintedToken + ) + assert isinstance(result, UnknownApiError) + assert result.status_code == 400 + + @pytest.mark.parametrize( + ("method", "path"), + [("GET", TOKEN_PATH), ("POST", JWKS_PATH), ("GET", "/token/anything")], + ids=["get-token", "post-jwks", "get-other"], + ) + def test_unknown_routes_are_404(self, issuer: RunningIssuer, method: str, path: str) -> None: + url: Final = f"{issuer.url}{path}" + result: Final = ( + get_external(url, response_type=MintedToken) + if method == "GET" + else post_external(url, json=_Claims(sub="alice"), response_type=MintedToken) + ) + assert isinstance(result, UnknownApiError) + assert result.status_code == 404 + + +class TestIssuerPort: + def test_defaults_to_the_documented_port(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv(JWT_ISSUER_PORT_ENV, raising=False) + assert jwt_issuer_port() == 4190, "CONTRIBUTING.md hardcodes 4190 in JWT_PUBLIC_KEY_URL" + + def test_env_override_wins(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(JWT_ISSUER_PORT_ENV, " 4321 ") + assert jwt_issuer_port() == 4321 From 2085a37b82f100edc9652e6859713b97e0fb7cdd Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 9 Sep 2026 16:56:07 -0700 Subject: [PATCH 2/7] test(e2e): issue the JWT suite's tokens from a real Keycloak realm The suite used to mint its own RS256 tokens from a stand-in issuer, which could only ever prove the proxy agreed with the tests: the claims were whatever the tests chose to sign. Every JWT bug worth catching lives in the shape of what an identity provider really emits, so the suite now runs against Keycloak (realm in idp_realm.json), provisions a group and a user per test through its admin API, and signs in through the direct-access grant. That changes what the tokens look like: sub is Keycloak's opaque user uuid rather than a friendly name, groups arrives from a protocol mapper, aud is the IdP's own audience, and the JWKS carries an encryption key beside the signing key so the proxy has to select on kid. The expiry case now takes a one-second token from a second client in the realm and waits for it to lapse instead of forging a stale exp. The proxy config the suite needs is unchanged. CI runs it against a Keycloak deployed beside the ephemeral stack, which lives in the releaser repo. Claude-Session: https://claude.ai/code/session_01EX13mWex6RaBo9PYnkAtFW --- tests/e2e/CLAUDE.md | 2 +- tests/e2e/CONTRIBUTING.md | 17 +- tests/e2e/coverage_registry/other.yaml | 6 +- tests/e2e/e2e_config.py | 4 - tests/e2e/e2e_http.py | 72 ++++++-- tests/e2e/idp.py | 226 ++++++++++++++++++++++++ tests/e2e/idp_realm.json | 56 ++++++ tests/e2e/jwt_issuer.py | 230 ------------------------- tests/e2e/models.py | 12 -- tests/e2e/other/other_client.py | 43 ++--- tests/e2e/other/test_jwt_auth_e2e.py | 93 +++++----- tests/e2e/test_idp.py | 79 +++++++++ tests/e2e/test_jwt_issuer.py | 141 --------------- 13 files changed, 502 insertions(+), 479 deletions(-) create mode 100644 tests/e2e/idp.py create mode 100644 tests/e2e/idp_realm.json delete mode 100644 tests/e2e/jwt_issuer.py create mode 100644 tests/e2e/test_idp.py delete mode 100644 tests/e2e/test_jwt_issuer.py diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 4985da9e95d..c954c03dfb7 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -19,7 +19,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `security/` - secret handling and log-leak protection - `router/` - routing and reliability behavior (fallbacks, cooldowns) - `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What remains here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`) and markerless harness unit tests for the Locust/session-anomaly aggregation logic -- `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (RS256 tokens minted by the test-only issuer in `jwt_issuer.py`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at; see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite +- `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at; see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke - `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream; see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport; the Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json` diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index ad414bb6647..4802feb10c4 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -27,17 +27,20 @@ The suites run against a live proxy, so bring one up first by running the litell 2. Bring up a Postgres and a Redis for the proxy to use. The repo-root `docker-compose.yml` already defines a Postgres on `5432`; a `docker run -p 6379:6379 redis:7` covers Redis. Point `DATABASE_URL` / `REDIS_HOST` / `REDIS_PORT` at whatever you run. Tests that read Redis directly default to the deployed shape (TLS + cluster mode) whenever `REDIS_HOST` is set, so for a local standalone Redis also set `REDIS_CLUSTER=false` and `REDIS_SSL=false` (plus `REDIS_PASSWORD` when your Redis requires auth) -3. Start the test-only JWT issuer the `other/` suite mints tokens from, then the litellm proxy against your config, and confirm both are live. The issuer is a fake identity provider (`jwt_issuer.py`) with an open mint endpoint, so it binds loopback only; it generates one RSA key per process and serves it as a JWKS, and the proxy caches that JWKS for `public_key_ttl` (600s) without refetching on an unknown `kid`, so restart the proxy whenever you restart the issuer: +3. Start the identity provider the `other/` suite authenticates against, then the litellm proxy against your config, and confirm both are live. It is a real Keycloak, running the realm in `tests/e2e/idp_realm.json`, and the proxy trusts it because `JWT_PUBLIC_KEY_URL` points at that realm's JWKS. The proxy caches the JWKS for `public_key_ttl` (600s) and does not refetch on an unknown `kid`, so restart the proxy whenever you recreate the container: ```bash set -a && source .env && set +a - uv run python tests/e2e/jwt_issuer.py & - curl -fs http://127.0.0.1:4190/.well-known/jwks.json - JWT_PUBLIC_KEY_URL=http://127.0.0.1:4190/.well-known/jwks.json litellm --config .yml --port 4000 + docker run -d --name litellm-e2e-idp -p 8480:8080 \ + -e KC_BOOTSTRAP_ADMIN_USERNAME=admin -e KC_BOOTSTRAP_ADMIN_PASSWORD=admin \ + -v "$PWD/tests/e2e/idp_realm.json:/opt/keycloak/data/import/realm.json:ro" \ + quay.io/keycloak/keycloak:26.7.3 start-dev --import-realm + curl -fs --retry 30 --retry-delay 2 --retry-all-errors http://127.0.0.1:8480/realms/litellm-e2e/.well-known/openid-configuration + JWT_PUBLIC_KEY_URL=http://127.0.0.1:8480/realms/litellm-e2e/protocol/openid-connect/certs litellm --config .yml --port 4000 curl -fs http://localhost:4000/health/liveliness ``` - The issuer's port is `E2E_JWT_ISSUER_PORT` (default `4190`, the port in the URLs above), read by both the issuer process and the tests, so set it in one place if you change it. JWT auth is an enterprise feature, so the proxy also needs `LITELLM_LICENSE` in its environment, and its config needs the JWT block below. `enable_jwt_auth` only routes bearer tokens with three dot-separated segments into the JWT path, so `sk-` virtual keys and the master key keep working for every other suite. `proxy_batch_write_at` is lowered so the JWT spend-attribution test sees its row well inside the poll deadline: + The tests reach Keycloak at `E2E_KEYCLOAK_URL` (default `http://127.0.0.1:8480`) and provision their identities through its admin API, so they also need `E2E_KEYCLOAK_ADMIN_USER` and `E2E_KEYCLOAK_ADMIN_PASSWORD` (`admin` / `admin` for the throwaway container above; the deployed stacks take theirs from a secret). JWT auth is an enterprise feature, so the proxy needs `LITELLM_LICENSE` in its environment, and its config needs the JWT block below. `enable_jwt_auth` only routes bearer tokens with three dot-separated segments into the JWT path, so `sk-` virtual keys and the master key keep working for every other suite. `proxy_batch_write_at` is lowered so the JWT spend-attribution test sees its row well inside the poll deadline: ```yaml general_settings: @@ -50,7 +53,9 @@ The suites run against a live proxy, so bring one up first by running the litell user_id_upsert: true ``` - Leave `JWT_AUDIENCE` and `JWT_ISSUER` unset unless you also put matching `aud` / `iss` claims in the tokens the tests mint; the issuer sets `iss` to its own base URL + Leave `JWT_AUDIENCE` and `JWT_ISSUER` unset. Keycloak's access tokens carry `aud: account`, its own audience rather than the proxy's, and their `iss` is whatever base URL the token was requested through, so pinning either one only makes sense once the deployment fixes Keycloak's hostname + + CI runs this suite against a Keycloak deployed beside the ephemeral stack, and that deployment (the JWT config block, `JWT_PUBLIC_KEY_URL`, and the admin credential handed to the run pod) lives in the project-releaser repo, not here. A stack without it fails the JWT tests rather than skipping them 4. Run a suite against it; the harness reads `LITELLM_PROXY_URL` (default `http://localhost:4000`): diff --git a/tests/e2e/coverage_registry/other.yaml b/tests/e2e/coverage_registry/other.yaml index 8b9cb4d2adf..ff25155924b 100644 --- a/tests/e2e/coverage_registry/other.yaml +++ b/tests/e2e/coverage_registry/other.yaml @@ -9,9 +9,9 @@ - {id: other.auth.llm_chat.not_bearer_scheme_denied, module: other, tier: P0, area: auth, assertions: [not_bearer_scheme_denied], source: "vendor testing strategy §11.1 / LIT-4778", rationale: "NotBearer scheme on chat is 401/403"} - {id: other.auth.realtime.missing_header_denied, module: other, tier: P1, area: auth, assertions: [missing_header_denied], source: "vendor testing strategy §9.19 / LIT-4778", rationale: "Realtime client-secret and calls routes reject requests without Authorization"} - {id: other.config.responses.metadata_redis_ttl_bounded, module: other, tier: P0, area: config, assertions: [ttl_bounded], source: "responses + redis cache", rationale: "Responses store+metadata must not leave TTL-unbounded Redis entries (LIT-1201)"} -- {id: other.auth.jwt.valid_token_allows, module: other, tier: P0, area: auth, assertions: [valid_token_allows], source: "handle_jwt.py:1217-1256 auth_jwt / user_api_key_auth.py:1365-1377", rationale: "An RS256 JWT signed by the configured JWKS whose groups claim names an existing team is accepted on /chat/completions"} -- {id: other.auth.jwt.spend_attributed_to_claims, module: other, tier: P0, area: auth, assertions: [spend_attributed_to_claims], source: "handle_jwt.py:2224 auth_builder / user_api_key_auth.py:1438-1474", rationale: "The spend log row for a JWT-authenticated call carries the team_id from the groups claim and the user_id from sub, not a virtual key's identity. Single-group claim only: the proxy picks the team from a set, so attribution over several groups is unordered"} -- {id: other.auth.jwt.expired_denied, module: other, tier: P0, area: auth, assertions: [expired_denied], source: "handle_jwt.py:1244-1250", rationale: "Expired JWT rejected 401 (Token Expired) even with valid signature; leeway is 0"} +- {id: other.auth.jwt.valid_token_allows, module: other, tier: P0, area: auth, assertions: [valid_token_allows], source: "handle_jwt.py:1217-1256 auth_jwt / user_api_key_auth.py:1365-1377", rationale: "An access token issued by the configured IdP whose groups claim names an existing team is accepted on /chat/completions"} +- {id: other.auth.jwt.spend_attributed_to_claims, module: other, tier: P0, area: auth, assertions: [spend_attributed_to_claims], source: "handle_jwt.py:2224 auth_builder / user_api_key_auth.py:1438-1474", rationale: "The spend log row for a JWT-authenticated call carries the team_id from the groups claim and the user_id from sub, which for a real IdP is an opaque uuid, not a virtual key's identity. Single-group claim only: the proxy picks the team from a set, so attribution over several groups is unordered"} +- {id: other.auth.jwt.expired_denied, module: other, tier: P0, area: auth, assertions: [expired_denied], source: "handle_jwt.py:1244-1250", rationale: "A token the IdP issued with a one-second lifespan is rejected 401 (Token Expired) once it lapses, even though its signature still verifies; leeway is 0"} - {id: other.auth.jwt.invalid_signature_denied, module: other, tier: P0, area: auth, assertions: [invalid_signature_denied], source: "handle_jwt.py:1158-1166 _decode_jwt_with_public_key", rationale: "A genuine token whose signature bytes were altered fails verification with 401"} - {id: other.auth.jwt.unknown_team_denied, module: other, tier: P0, area: auth, assertions: [unknown_team_denied], source: "handle_jwt.py:1549-1632 find_team_with_model_access", rationale: "A verified JWT whose groups claim resolves to no existing team is denied with 403 naming the unresolved team, never silently admitted without a team. The proxy words it as a model-access denial, the same body an existing team without model access gets"} - {id: other.auth.jwt.virtual_key_unaffected, module: other, tier: P0, area: auth, assertions: [virtual_key_unaffected], source: "handle_jwt.py:213 is_jwt / user_api_key_auth.py:1332-1333", rationale: "enable_jwt_auth only routes three-segment bearer tokens into the JWT branch, so sk- virtual keys keep working on the same proxy"} diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 0e7f4c0ad2b..09d17b9a0db 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -15,7 +15,6 @@ from typing import Final from dotenv import load_dotenv from fixture_mode import deterministic_marker, parse_fixture_mode -from jwt_issuer import jwt_issuer_url from provider_edge import provider_edge_api_base # Local runs keep provider / DataDog keys in tests/e2e/.env (see CONTRIBUTING.md). @@ -44,9 +43,6 @@ UI_BASE_URL = os.environ.get("E2E_UI_BASE_URL", PROXY_BASE_URL).rstrip("/") CHEAP_ANTHROPIC_MODEL = os.environ.get("E2E_CHEAP_ANTHROPIC_MODEL", "claude-haiku-4-5") CHEAP_OPENAI_MODEL = os.environ.get("E2E_CHEAP_OPENAI_MODEL", "gpt-5.5") -# Test-only JWT issuer (jwt_issuer.py); the port is E2E_JWT_ISSUER_PORT (see CONTRIBUTING.md). -JWT_ISSUER_URL = jwt_issuer_url() - LINEAR_MCP_URL = os.environ.get("E2E_LINEAR_MCP_URL", "https://mcp.linear.app/mcp") LINEAR_STORAGE_STATE = os.environ.get("E2E_LINEAR_STORAGE_STATE", "") diff --git a/tests/e2e/e2e_http.py b/tests/e2e/e2e_http.py index 4c65c53b653..d6924aff5c0 100644 --- a/tests/e2e/e2e_http.py +++ b/tests/e2e/e2e_http.py @@ -125,6 +125,20 @@ class ProbeResult(BaseModel): return 200 <= self.status_code < 500 and self.status_code != 404 +class ExternalWrite(BaseModel): + """Outcome of a write to a non-proxy API (an identity provider's admin API) + that answers with a status and, on create, a Location header naming the new + resource rather than a JSON body.""" + + status_code: int + location: str = "" + body: str = "" + + @property + def ok(self) -> bool: + return 200 <= self.status_code < 300 + + class StreamingResponse(BaseModel): """Raw outcome for calls whose body is provider-native or streamed: status, the x-litellm-call-id header, the x-litellm-response-cost header (StandardLogging @@ -252,16 +266,17 @@ def assert_auth_denied(result: StreamingResponse, context: str) -> None: f"{context}: expected 401/403, got {result.status_code}: {result.body[:300]}" ) -def _headers(headers: BaseModel) -> dict[str, str]: - dumped: dict[str, object] = headers.model_dump(by_alias=True, exclude_none=True) +def _flat(model: BaseModel) -> dict[str, str]: + dumped: dict[str, object] = model.model_dump(by_alias=True, exclude_none=True) return {key: str(value) for key, value in dumped.items()} +def _headers(headers: BaseModel) -> dict[str, str]: + return _flat(headers) + + def _params(params: BaseModel | None) -> dict[str, str]: - if params is None: - return {} - dumped: dict[str, object] = params.model_dump(by_alias=True, exclude_none=True) - return {key: str(value) for key, value in dumped.items()} + return _flat(params) if params is not None else {} TRANSIENT_STATUSES: frozenset[int] = frozenset({529}) @@ -386,20 +401,20 @@ def get_external[R: BaseModel]( return _classify(resp, response_type) -def post_external[R: BaseModel]( +def post_form_external[R: BaseModel]( url: str, *, - json: BaseModel, + form: BaseModel, response_type: type[R], timeout: float = 30.0, ) -> Result[R]: - """POST an absolute URL outside the proxy (e.g. the e2e JWT issuer's mint - endpoint). Like get_external: no proxy base url, no proxy auth, and the same - tagged-union classification as every other call.""" + """POST an absolute URL outside the proxy as `application/x-www-form-urlencoded`, + the encoding OAuth 2 token endpoints take. Like get_external: no proxy base url, + no proxy auth, and the same tagged-union classification as every other call.""" try: resp = requests.post( url, - json=json.model_dump(by_alias=True, exclude_none=True), + data=_flat(form), timeout=timeout, ) except requests.RequestException as exc: @@ -407,6 +422,39 @@ def post_external[R: BaseModel]( return _classify(resp, response_type) +def post_json_external( + url: str, + *, + headers: BaseModel, + json: BaseModel, + timeout: float = 30.0, +) -> ExternalWrite: + """POST an absolute URL outside the proxy under its own bearer, for an API that + answers a create with a status and a Location header rather than a JSON body.""" + try: + resp = requests.post( + url, + headers=_headers(headers), + json=json.model_dump(by_alias=True, exclude_none=True), + timeout=timeout, + ) + except requests.RequestException as exc: + return ExternalWrite(status_code=-1, body=str(exc)) + return ExternalWrite( + status_code=resp.status_code, + location=resp.headers.get("Location", ""), + body=resp.text, + ) + + +def delete_external(url: str, *, headers: BaseModel, timeout: float = 30.0) -> ExternalWrite: + try: + resp = requests.delete(url, headers=_headers(headers), timeout=timeout) + except requests.RequestException as exc: + return ExternalWrite(status_code=-1, body=str(exc)) + return ExternalWrite(status_code=resp.status_code, body=resp.text) + + def delete[R: BaseModel]( url: URL, *, diff --git a/tests/e2e/idp.py b/tests/e2e/idp.py new file mode 100644 index 00000000000..50cf44b4159 --- /dev/null +++ b/tests/e2e/idp.py @@ -0,0 +1,226 @@ +"""The identity provider the JWT suite authenticates against: a real Keycloak +realm, imported from `idp_realm.json`. + +A real IdP rather than a hand-rolled signer because every JWT bug this suite +exists to catch lives in the shape of what an IdP actually emits: `sub` is an +opaque uuid and not a friendly name, group membership arrives as a claim built +by a protocol mapper, the JWKS carries a signing key next to an encryption key +so the proxy has to select on `kid`, and `aud` is the IdP's own audience rather +than the proxy's. A stand-in issuer that mints exactly the claims the tests +assert on can only prove the proxy agrees with the tests. + +Tests never hold a signing key. They provision an identity through Keycloak's +admin API (a group named after the litellm team, a user in it with a password +generated for that test alone), then ask Keycloak for an access token through +the direct-access grant, the same way a CLI or service account signs in. The +proxy's `JWT_PUBLIC_KEY_URL` points at this realm's JWKS, so the token the +tests carry is trusted for exactly one reason: Keycloak signed it. + +The realm declares two clients. `litellm-e2e-tests` mints ordinary tokens; the +`litellm-e2e-shortlived` client sets `access.token.lifespan` to one second, so +the expiry test lets a genuine token expire instead of forging a stale `exp`. + +Connection details come from the environment (`E2E_KEYCLOAK_URL` and the admin +credential). A missing or unreachable IdP is a hard failure naming the start +command, never a skip, so a stack deployed without it turns the run red. +""" + +from __future__ import annotations + +import os +import secrets +from dataclasses import dataclass +from typing import Final, Literal + +import pytest +from pydantic import BaseModel, Field + +from e2e_http import ( + AuthHeaders, + ExternalWrite, + NetworkError, + Result, + Success, + delete_external, + post_form_external, + post_json_external, +) + +KEYCLOAK_URL_ENV: Final = "E2E_KEYCLOAK_URL" +KEYCLOAK_REALM_ENV: Final = "E2E_KEYCLOAK_REALM" +KEYCLOAK_ADMIN_USER_ENV: Final = "E2E_KEYCLOAK_ADMIN_USER" +KEYCLOAK_ADMIN_PASSWORD_ENV: Final = "E2E_KEYCLOAK_ADMIN_PASSWORD" + +DEFAULT_KEYCLOAK_URL: Final = "http://127.0.0.1:8480" +DEFAULT_REALM: Final = "litellm-e2e" +TESTS_CLIENT_ID: Final = "litellm-e2e-tests" +SHORT_LIVED_CLIENT_ID: Final = "litellm-e2e-shortlived" +SHORT_LIVED_TOKEN_SECONDS: Final = 1 + +_START_HINT: Final = ( + "Start it with the `docker run ... quay.io/keycloak/keycloak` command in tests/e2e/CONTRIBUTING.md, " + f"and point {KEYCLOAK_URL_ENV} / {KEYCLOAK_ADMIN_USER_ENV} / {KEYCLOAK_ADMIN_PASSWORD_ENV} at it" +) + + +class TokenGrantForm(BaseModel): + """The direct-access (password) grant an OAuth 2 token endpoint takes, form encoded.""" + + grant_type: Literal["password"] = "password" + client_id: str + username: str + password: str + + +class TokenResponse(BaseModel): + access_token: str + + +class GroupCreateBody(BaseModel): + name: str + + +class PasswordCredential(BaseModel): + type: Literal["password"] = "password" + value: str + temporary: bool = False + + +class UserCreateBody(BaseModel): + """Keycloak's admin representation of a new user. `firstName` / `lastName` and + an empty `requiredActions` matter: a realm's default VERIFY_PROFILE action + otherwise leaves the account "not fully set up" and every grant fails.""" + + username: str + email: str + email_verified: bool = Field(default=True, alias="emailVerified") + first_name: str = Field(default="E2E", alias="firstName") + last_name: str = Field(default="Tester", alias="lastName") + enabled: bool = True + groups: tuple[str, ...] + credentials: tuple[PasswordCredential, ...] + required_actions: tuple[str, ...] = Field(default=(), alias="requiredActions") + + +def created_id(write: ExternalWrite, context: str) -> str: + """The new resource's id, which Keycloak returns only as the last segment of + the Location header on a 201.""" + if not write.ok: + pytest.fail(f"Keycloak refused to create {context}: HTTP {write.status_code} {write.body[:300]}") + return write.location.rsplit("/", 1)[-1] + + +@dataclass(frozen=True, slots=True) +class Identity: + """One provisioned IdP user: the `sub` the proxy will see, the credential the + test signs in with, and the group whose name the litellm team carries.""" + + user_id: str + username: str + password: str + group: str + group_id: str + + +@dataclass(frozen=True, slots=True) +class Keycloak: + base_url: str + realm: str + admin_username: str + admin_password: str + + @property + def issuer(self) -> str: + return f"{self.base_url}/realms/{self.realm}" + + @property + def jwks_url(self) -> str: + return f"{self.issuer}/protocol/openid-connect/certs" + + def token_url(self, realm: str) -> str: + return f"{self.base_url}/realms/{realm}/protocol/openid-connect/token" + + def _admin_url(self, path: str) -> str: + return f"{self.base_url}/admin/realms/{self.realm}{path}" + + def _admin_headers(self) -> AuthHeaders: + """A fresh admin token per call: the master realm's tokens are short lived, + and a cached one would expire in the middle of a slow test.""" + form: Final = TokenGrantForm(client_id="admin-cli", username=self.admin_username, password=self.admin_password) + result: Final = post_form_external(self.token_url("master"), form=form, response_type=TokenResponse) + return AuthHeaders(authorization=f"Bearer {self._token(result, 'the Keycloak admin credential')}") + + def _token(self, result: Result[TokenResponse], context: str) -> str: + match result: + case Success(data=granted): + return granted.access_token + case NetworkError(message=message): + return pytest.fail(f"No live Keycloak at {self.base_url} for {context}: {message}. {_START_HINT}") + case _: + return pytest.fail(f"Keycloak refused {context}: {result}") + + def create_group(self, name: str) -> str: + return created_id( + post_json_external( + self._admin_url("/groups"), headers=self._admin_headers(), json=GroupCreateBody(name=name) + ), + f"group {name}", + ) + + def create_user(self, *, username: str, email: str, password: str, group: str) -> str: + return created_id( + post_json_external( + self._admin_url("/users"), + headers=self._admin_headers(), + json=UserCreateBody( + username=username, + email=email, + groups=(group,), + credentials=(PasswordCredential(value=password),), + ), + ), + f"user {username}", + ) + + def delete_user(self, user_id: str) -> None: + delete_external(self._admin_url(f"/users/{user_id}"), headers=self._admin_headers()) + + def delete_group(self, group_id: str) -> None: + delete_external(self._admin_url(f"/groups/{group_id}"), headers=self._admin_headers()) + + def provision(self, *, marker: str, group: str) -> Identity: + """Create `group` and a user in it, credentialed with a password generated + for this test alone, and hand back the identity a token can be minted for.""" + group_id: Final = self.create_group(group) + username: Final = f"e2e-jwt-user-{marker}" + password: Final = secrets.token_urlsafe(24) + user_id: Final = self.create_user( + username=username, email=f"{username}@example.com", password=password, group=group + ) + return Identity(user_id=user_id, username=username, password=password, group=group, group_id=group_id) + + def access_token(self, identity: Identity, *, client_id: str = TESTS_CLIENT_ID) -> str: + """Sign `identity` in through the direct-access grant and hand back the + access token Keycloak signed, exactly as it came off the wire.""" + result: Final = post_form_external( + self.token_url(self.realm), + form=TokenGrantForm(client_id=client_id, username=identity.username, password=identity.password), + response_type=TokenResponse, + ) + return self._token(result, f"a token for {identity.username}") + + +def keycloak_from_env() -> Keycloak: + admin_username: Final = os.environ.get(KEYCLOAK_ADMIN_USER_ENV, "").strip() + admin_password: Final = os.environ.get(KEYCLOAK_ADMIN_PASSWORD_ENV, "").strip() + if not admin_username or not admin_password: + pytest.fail( + f"The JWT suite needs {KEYCLOAK_ADMIN_USER_ENV} and {KEYCLOAK_ADMIN_PASSWORD_ENV} to provision " + f"identities in its Keycloak realm, and neither may be empty. {_START_HINT}" + ) + return Keycloak( + base_url=os.environ.get(KEYCLOAK_URL_ENV, DEFAULT_KEYCLOAK_URL).rstrip("/"), + realm=os.environ.get(KEYCLOAK_REALM_ENV, "").strip() or DEFAULT_REALM, + admin_username=admin_username, + admin_password=admin_password, + ) diff --git a/tests/e2e/idp_realm.json b/tests/e2e/idp_realm.json new file mode 100644 index 00000000000..2a8bd89b428 --- /dev/null +++ b/tests/e2e/idp_realm.json @@ -0,0 +1,56 @@ +{ + "realm": "litellm-e2e", + "enabled": true, + "sslRequired": "none", + "registrationAllowed": false, + "accessTokenLifespan": 300, + "clients": [ + { + "clientId": "litellm-e2e-tests", + "enabled": true, + "publicClient": true, + "standardFlowEnabled": false, + "directAccessGrantsEnabled": true, + "protocolMappers": [ + { + "name": "groups", + "protocol": "openid-connect", + "protocolMapper": "oidc-group-membership-mapper", + "consentRequired": false, + "config": { + "claim.name": "groups", + "full.path": "false", + "access.token.claim": "true", + "id.token.claim": "true", + "userinfo.token.claim": "true" + } + } + ] + }, + { + "clientId": "litellm-e2e-shortlived", + "enabled": true, + "publicClient": true, + "standardFlowEnabled": false, + "directAccessGrantsEnabled": true, + "attributes": { + "access.token.lifespan": "1" + }, + "protocolMappers": [ + { + "name": "groups", + "protocol": "openid-connect", + "protocolMapper": "oidc-group-membership-mapper", + "consentRequired": false, + "config": { + "claim.name": "groups", + "full.path": "false", + "access.token.claim": "true", + "id.token.claim": "true", + "userinfo.token.claim": "true" + } + } + ] + } + ] +} diff --git a/tests/e2e/jwt_issuer.py b/tests/e2e/jwt_issuer.py deleted file mode 100644 index 4006fcaa314..00000000000 --- a/tests/e2e/jwt_issuer.py +++ /dev/null @@ -1,230 +0,0 @@ -"""Test-only fake identity provider for the e2e JWT suite. Never deploy it. - -Run it next to the proxy (`uv run python tests/e2e/jwt_issuer.py`). On start it -generates one RSA signing key and keeps it for the life of the process, serving -the public half at `GET /.well-known/jwks.json` and signing whatever claims are -POSTed to `/token`. The proxy's `JWT_PUBLIC_KEY_URL` points at the JWKS URL, and -tests mint RS256 tokens by POSTing claims, so the private key never leaves this -process and no test holds it. - -One key per process, rather than per pytest run, is what survives the proxy's -JWKS cache: the proxy caches the JWKS for `litellm_jwtauth.public_key_ttl` -(600s by default) and does not refetch on an unknown `kid`, so a key rotated -every run would be rejected until the cache expired. Restart the proxy whenever -you restart the issuer. - -The mint endpoint takes no credential: anyone who can reach it gets a token the -proxy trusts. It therefore binds 127.0.0.1 only, must never be exposed beyond -loopback, and must only ever be trusted by a proxy under test. -""" - -from __future__ import annotations - -import logging -import os -import threading -import time -import uuid -from collections.abc import Callable, Mapping -from dataclasses import dataclass -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from typing import Final, Literal - -import jwt -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric import rsa -from jwt.utils import to_base64url_uint -from pydantic import BaseModel, RootModel, ValidationError - -JWT_ISSUER_PORT_ENV: Final = "E2E_JWT_ISSUER_PORT" -DEFAULT_JWT_ISSUER_PORT: Final = 4190 -LOOPBACK_HOST: Final = "127.0.0.1" -JWKS_PATH: Final = "/.well-known/jwks.json" -TOKEN_PATH: Final = "/token" -DEFAULT_TOKEN_LIFETIME_SECONDS: Final = 300 - -ClaimValue = str | int | float | bool | None | list[str] - - -def jwt_issuer_port() -> int: - raw: Final = os.environ.get(JWT_ISSUER_PORT_ENV, "").strip() - return int(raw) if raw else DEFAULT_JWT_ISSUER_PORT - - -def jwt_issuer_url() -> str: - return f"http://{LOOPBACK_HOST}:{jwt_issuer_port()}" - - -class RsaJwk(BaseModel): - kty: Literal["RSA"] = "RSA" - alg: Literal["RS256"] = "RS256" - use: Literal["sig"] = "sig" - kid: str - n: str - e: str - - -class JwksDocument(BaseModel): - keys: tuple[RsaJwk, ...] - - -class TokenRequest(RootModel[Mapping[str, ClaimValue]]): - """The JSON body of POST /token: the claims to sign, verbatim. Nested objects - are not supported; every value is a scalar or a list of strings.""" - - -class MintedToken(BaseModel): - token: str - - -class IssuerError(BaseModel): - error: str - - -@dataclass(frozen=True, slots=True) -class SigningKey: - kid: str - private_pem: str - jwk: RsaJwk - - -def generate_signing_key(kid: str | None = None) -> SigningKey: - private_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) - numbers: Final = private_key.public_key().public_numbers() - resolved_kid: Final = kid if kid is not None else uuid.uuid4().hex - return SigningKey( - kid=resolved_kid, - private_pem=private_key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.PKCS8, - encryption_algorithm=serialization.NoEncryption(), - ).decode(), - jwk=RsaJwk( - kid=resolved_kid, - n=to_base64url_uint(numbers.n).decode(), - e=to_base64url_uint(numbers.e).decode(), - ), - ) - - -def mint( - key: SigningKey, - claims: Mapping[str, ClaimValue], - *, - issuer: str, - now: int, - lifetime_seconds: int = DEFAULT_TOKEN_LIFETIME_SECONDS, -) -> str: - """Sign `claims` as a compact RS256 JWT carrying `key.kid` in its header. - `iss`, `iat`, and `exp` are filled in when absent and left alone when the - caller sets them, so a test can mint an already-expired token.""" - payload: Final[dict[str, ClaimValue]] = { - "iss": issuer, - "iat": now, - "exp": now + lifetime_seconds, - **claims, - } - return jwt.encode(payload, key.private_pem, algorithm="RS256", headers={"kid": key.kid}) - - -class _IssuerServer(ThreadingHTTPServer): - daemon_threads = True - - def __init__(self, bind: tuple[str, int], *, key: SigningKey, clock: Callable[[], int]) -> None: - super().__init__(bind, _IssuerHandler) - self.key: Final = key - self.clock: Final = clock - - @property - def url(self) -> str: - host, port = self.server_address[0], self.server_address[1] - return f"http://{host}:{port}" - - -class _IssuerHandler(BaseHTTPRequestHandler): - def _issuer(self) -> _IssuerServer: - issuer: Final = self.server - assert isinstance(issuer, _IssuerServer) - return issuer - - def do_GET(self) -> None: - if self.path != JWKS_PATH: - self._send(404, IssuerError(error=f"unknown path {self.path}; the JWKS is at {JWKS_PATH}")) - return - self._send(200, JwksDocument(keys=(self._issuer().key.jwk,))) - - def do_POST(self) -> None: - if self.path != TOKEN_PATH: - self._send(404, IssuerError(error=f"unknown path {self.path}; mint tokens at {TOKEN_PATH}")) - return - length: Final = int(self.headers.get("Content-Length", "0")) - try: - request: Final = TokenRequest.model_validate_json(self.rfile.read(length)) - except ValidationError as exc: - self._send(400, IssuerError(error=f"claims must be a JSON object of scalar or string-list values: {exc}")) - return - issuer: Final = self._issuer() - token: Final = mint(issuer.key, request.root, issuer=issuer.url, now=issuer.clock()) - self._send(200, MintedToken(token=token)) - - def _send(self, status: int, body: BaseModel) -> None: - payload: Final = body.model_dump_json().encode() - self.send_response(status) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(payload))) - self.end_headers() - self.wfile.write(payload) - - -@dataclass(frozen=True, slots=True) -class RunningIssuer: - url: str - key: SigningKey - server: _IssuerServer - - @property - def jwks_url(self) -> str: - return f"{self.url}{JWKS_PATH}" - - def shutdown(self) -> None: - self.server.shutdown() - self.server.server_close() - - -def _wall_clock() -> int: - return int(time.time()) - - -def start_jwt_issuer( - *, - port: int = 0, - key: SigningKey | None = None, - clock: Callable[[], int] = _wall_clock, -) -> RunningIssuer: - """Serve the issuer on loopback in a daemon thread. `port=0` takes an - OS-assigned port for in-process tests; the CLI passes the documented one.""" - server: Final = _IssuerServer((LOOPBACK_HOST, port), key=key or generate_signing_key(), clock=clock) - thread: Final = threading.Thread(target=server.serve_forever, name="e2e-jwt-issuer", daemon=True) - thread.start() - return RunningIssuer(url=server.url, key=server.key, server=server) - - -def main() -> None: - logging.basicConfig(level=logging.INFO, format="%(message)s") - running: Final = start_jwt_issuer(port=jwt_issuer_port()) - logging.getLogger(__name__).info( - "e2e jwt issuer listening on %s jwks=%s mint=POST %s%s kid=%s (test-only, loopback, ctrl-c to stop)", - running.url, - running.jwks_url, - running.url, - TOKEN_PATH, - running.key.kid, - ) - try: - threading.Event().wait() - except KeyboardInterrupt: - running.shutdown() - - -if __name__ == "__main__": - main() diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 17f5418bd73..2c6c0e9bbd4 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -1078,18 +1078,6 @@ class UserListResponse(BaseModel): total: int -class JwtClaimsBody(BaseModel): - """Claims POSTed to the e2e JWT issuer's /token: exactly what the - `litellm_jwtauth` block in CONTRIBUTING.md reads (sub -> user_id, email -> - user_email, groups -> team ids), plus an explicit `exp` for the expired case; - the issuer fills in iss/iat/exp when they are left unset.""" - - sub: str - email: str - groups: Sequence[str] - exp: int | None = None - - class OrgNewBody(BaseModel): organization_alias: str models: list[str] = [] diff --git a/tests/e2e/other/other_client.py b/tests/e2e/other/other_client.py index b51f3338f52..4313bbe4068 100644 --- a/tests/e2e/other/other_client.py +++ b/tests/e2e/other/other_client.py @@ -1,27 +1,23 @@ """Client for the `other` holding-pen suite: the auth gate (master key vs an -invalid key on an admin route), JWT auth against the test-only issuer -(jwt_issuer.py), and the process-lifecycle health probes (liveness, public -readiness, authenticated readiness diagnostics). +invalid key on an admin route), JWT auth against the suite's Keycloak realm +(idp.py), and the process-lifecycle health probes (liveness, public readiness, +authenticated readiness diagnostics). Holds the shared ProxyClient so `resources` / `scoped_key` still clean up, and adds only the routes these behaviors need. The health probes deliberately send no auth header (public routes), so they go through the transport with an empty -headers model rather than a bearer. Tokens are minted by POSTing claims to the -issuer, so no test ever holds a signing key. +headers model rather than a bearer. JWT tests reach the identity provider +through `idp`, which provisions identities and mints tokens through Keycloak's +own endpoints, so no test ever holds a signing key. """ from __future__ import annotations from dataclasses import dataclass -from typing import Final -import pytest - -from e2e_config import JWT_ISSUER_URL -from e2e_http import NetworkError, NoBody, ProbeResult, Result, Success, post_external -from jwt_issuer import TOKEN_PATH, MintedToken +from e2e_http import NoBody, ProbeResult, Result +from idp import Keycloak, keycloak_from_env from models import ( - JwtClaimsBody, ReadinessDetailsResponse, ReadinessResponse, UserListParams, @@ -33,7 +29,11 @@ from proxy_client import ProxyClient @dataclass(frozen=True, slots=True) class OtherClient: proxy: ProxyClient - jwt_issuer_url: str + + @property + def idp(self) -> Keycloak: + """Resolved per use, so the suite's non-JWT tests never need the IdP env.""" + return keycloak_from_env() def liveness(self) -> ProbeResult: """GET /health/liveliness. Unauthenticated; the probe returns status + @@ -77,21 +77,6 @@ class OtherClient: response_type=UserListResponse, ) - def mint_jwt(self, claims: JwtClaimsBody) -> str: - """Have the test-only issuer sign `claims` into a compact RS256 JWT. A - missing issuer is a hard failure naming the start command, not a skip.""" - result: Final = post_external(f"{self.jwt_issuer_url}{TOKEN_PATH}", json=claims, response_type=MintedToken) - match result: - case Success(data=minted): - return minted.token - case NetworkError(message=message): - pytest.fail( - f"No live JWT issuer at {self.jwt_issuer_url}: {message}. Start it next to the proxy with " - "`uv run python tests/e2e/jwt_issuer.py` (see CONTRIBUTING.md)" - ) - case _: - raise AssertionError(result) - def build_client(proxy: ProxyClient) -> OtherClient: - return OtherClient(proxy=proxy, jwt_issuer_url=JWT_ISSUER_URL) + return OtherClient(proxy=proxy) diff --git a/tests/e2e/other/test_jwt_auth_e2e.py b/tests/e2e/other/test_jwt_auth_e2e.py index 49a9350a3a2..25a5e7d4b9a 100644 --- a/tests/e2e/other/test_jwt_auth_e2e.py +++ b/tests/e2e/other/test_jwt_auth_e2e.py @@ -1,54 +1,63 @@ -"""Live e2e: RS256 JWTs minted by the test-only issuer (jwt_issuer.py) against a +"""Live e2e: access tokens issued by a real Keycloak realm (idp.py) against a proxy running with `enable_jwt_auth: true` and the `litellm_jwtauth` block from CONTRIBUTING.md (sub -> user_id, email -> user_email, groups -> team ids, user_id_upsert). -Every case mints through the issuer, so the tests never hold a signing key: the -bad-signature case corrupts a genuine signature, the expired case asks the -issuer for a token whose `exp` is already in the past. Those identities get -their own freshly created team so a rejection can only be blamed on the token, -while the unknown-team case names a team that was never created. An acceptance -is proven twice, at the boundary (200 from a real provider) and in the spend log -the proxy attributes to the claims. The last case keeps a plain `sk-` virtual -key working on the same proxy, guarding against the flag turning JWT on for -everyone. +Every identity is provisioned in Keycloak for the test that uses it: a group +named after the litellm team, and a user in that group whose password exists +only for the length of the test. Tokens then come from Keycloak's direct-access +grant, so no test ever holds a signing key and the claims the proxy reads are +the ones an IdP really emits (`sub` is Keycloak's user uuid, `groups` comes off +a protocol mapper, `aud` is Keycloak's own audience). + +The rejection cases stay honest about where the rejection has to come from: the +bad-signature case corrupts a genuine signature, and the expiry case takes its +token from the realm's one-second client and waits for it to lapse rather than +forging a stale `exp`. Those identities get their own freshly created team, so a +rejection can only be blamed on the token, while the unknown-team case names a +group no litellm team was ever created for. An acceptance is proven twice, at +the boundary (200 from a real provider) and in the spend log the proxy +attributes to the claims. The last case keeps a plain `sk-` virtual key working +on the same proxy, guarding against the flag turning JWT on for everyone. """ from __future__ import annotations -from dataclasses import dataclass +import time from typing import Final import pytest from e2e_config import CHEAP_OPENAI_MODEL, unique_marker from e2e_http import UnauthorizedError, UnknownApiError, unwrap +from idp import SHORT_LIVED_CLIENT_ID, SHORT_LIVED_TOKEN_SECONDS, Identity from lifecycle import ResourceManager -from models import ChatBody, ChatMessage, JwtClaimsBody, TeamNewBody +from models import ChatBody, ChatMessage, TeamNewBody from other_client import OtherClient pytestmark = pytest.mark.e2e -@dataclass(frozen=True, slots=True) -class JwtIdentity: - user_id: str - team_id: str - - def claims(self, *, exp: int | None = None) -> JwtClaimsBody: - return JwtClaimsBody(sub=self.user_id, email=f"{self.user_id}@example.com", groups=(self.team_id,), exp=exp) +def _provision(client: OtherClient, resources: ResourceManager, *, marker: str) -> Identity: + """A Keycloak group and a user in it, torn down with the test. The group name + is what the token's `groups` claim carries, which is what the proxy resolves + as a litellm team id.""" + identity: Final = client.idp.provision(marker=marker, group=f"e2e-jwt-team-{marker}") + resources.defer(lambda: client.idp.delete_user(identity.user_id)) + resources.defer(lambda: client.idp.delete_group(identity.group_id)) + resources.defer(lambda: client.proxy.delete_user(identity.user_id)) + return identity @pytest.fixture -def identity(client: OtherClient, resources: ResourceManager) -> JwtIdentity: +def identity(client: OtherClient, resources: ResourceManager) -> Identity: + """An IdP identity whose group is also a real litellm team, so anything the + proxy rejects is about the token and never about an unresolvable team.""" marker: Final = unique_marker() - team_id: Final = client.proxy.create_team( - TeamNewBody(team_alias=f"e2e-jwt-{marker}", team_id=f"e2e-jwt-team-{marker}") - ) + provisioned: Final = _provision(client, resources, marker=marker) + team_id: Final = client.proxy.create_team(TeamNewBody(team_alias=f"e2e-jwt-{marker}", team_id=provisioned.group)) resources.defer(lambda: client.proxy.delete_team(team_id)) - user_id: Final = f"e2e-jwt-user-{marker}" - resources.defer(lambda: client.proxy.delete_user(user_id)) - return JwtIdentity(user_id=user_id, team_id=team_id) + return provisioned def _ping() -> ChatBody: @@ -68,9 +77,9 @@ def _corrupt_signature(token: str) -> str: class TestJwtAuth: @pytest.mark.covers("other.auth.jwt.valid_token_allows", "other.auth.jwt.spend_attributed_to_claims") def test_valid_token_for_an_existing_team_is_accepted_and_attributed( - self, client: OtherClient, identity: JwtIdentity + self, client: OtherClient, identity: Identity ) -> None: - token: Final = client.mint_jwt(identity.claims()) + token: Final = client.idp.access_token(identity) response: Final = unwrap(client.proxy.chat(token, _ping())) assert response.id is not None and response.choices, ( @@ -80,16 +89,16 @@ class TestJwtAuth: rows: Final = client.proxy.poll_logs_for_request_id(response.id) assert rows, f"no spend log row for request {response.id} within the poll deadline" row: Final = rows[0] - assert row.team_id == identity.team_id, ( - f"spend row must carry the team from the JWT groups claim {identity.team_id!r}, got {row.team_id!r}" + assert row.team_id == identity.group, ( + f"spend row must carry the team from the JWT groups claim {identity.group!r}, got {row.team_id!r}" ) assert row.user == identity.user_id, ( f"spend row must carry the user from the JWT sub claim {identity.user_id!r}, got {row.user!r}" ) @pytest.mark.covers("other.auth.jwt.invalid_signature_denied") - def test_tampered_signature_is_rejected(self, client: OtherClient, identity: JwtIdentity) -> None: - tampered: Final = _corrupt_signature(client.mint_jwt(identity.claims())) + def test_tampered_signature_is_rejected(self, client: OtherClient, identity: Identity) -> None: + tampered: Final = _corrupt_signature(client.idp.access_token(identity)) result: Final = client.proxy.chat(tampered, _ping()) assert isinstance(result, UnauthorizedError), ( @@ -100,27 +109,29 @@ class TestJwtAuth: ) @pytest.mark.covers("other.auth.jwt.expired_denied") - def test_expired_token_is_rejected(self, client: OtherClient, identity: JwtIdentity) -> None: - expired: Final = client.mint_jwt(identity.claims(exp=1)) + def test_expired_token_is_rejected(self, client: OtherClient, identity: Identity) -> None: + expiring: Final = client.idp.access_token(identity, client_id=SHORT_LIVED_CLIENT_ID) + time.sleep(SHORT_LIVED_TOKEN_SECONDS + 1) - result: Final = client.proxy.chat(expired, _ping()) + result: Final = client.proxy.chat(expiring, _ping()) assert isinstance(result, UnauthorizedError), ( f"an expired JWT must be rejected with 401 even though its signature verifies, got {result}" ) assert "expired" in result.body.lower(), f"the 401 must say the token expired, got {result.body[:300]}" @pytest.mark.covers("other.auth.jwt.unknown_team_denied") - def test_token_naming_a_team_that_does_not_exist_is_rejected(self, client: OtherClient) -> None: - marker: Final = unique_marker() - never_created: Final = JwtIdentity(user_id=f"e2e-jwt-user-{marker}", team_id=f"e2e-jwt-missing-team-{marker}") - token: Final = client.mint_jwt(never_created.claims()) + def test_token_naming_a_team_that_does_not_exist_is_rejected( + self, client: OtherClient, resources: ResourceManager + ) -> None: + stranger: Final = _provision(client, resources, marker=unique_marker()) + token: Final = client.idp.access_token(stranger) result: Final = client.proxy.chat(token, _ping()) assert isinstance(result, UnknownApiError) and result.status_code == 403, ( f"a valid JWT whose groups name no existing team must be rejected with 403, got {result}" ) - assert never_created.team_id in result.body, ( - f"the 403 must name the team it could not resolve ({never_created.team_id}), got {result.body[:300]}" + assert stranger.group in result.body, ( + f"the 403 must name the team it could not resolve ({stranger.group}), got {result.body[:300]}" ) @pytest.mark.covers("other.auth.jwt.virtual_key_unaffected") diff --git a/tests/e2e/test_idp.py b/tests/e2e/test_idp.py new file mode 100644 index 00000000000..93be2806eb5 --- /dev/null +++ b/tests/e2e/test_idp.py @@ -0,0 +1,79 @@ +"""Harness coverage for idp.py: the pure parts of the Keycloak client, which are +the ones a wrong value in silently mistargets. No proxy and no IdP needed, so +these carry no `e2e` marker and run everywhere.""" + +from __future__ import annotations + +from typing import Final + +import pytest + +from e2e_http import ExternalWrite +from idp import ( + KEYCLOAK_ADMIN_PASSWORD_ENV, + KEYCLOAK_ADMIN_USER_ENV, + KEYCLOAK_REALM_ENV, + KEYCLOAK_URL_ENV, + Keycloak, + PasswordCredential, + created_id, + UserCreateBody, + keycloak_from_env, +) + +_REALM: Final = Keycloak( + base_url="http://keycloak:8080", realm="litellm-e2e", admin_username="admin", admin_password="pw" +) + + +def test_realm_urls_match_keycloaks_own_layout() -> None: + assert _REALM.issuer == "http://keycloak:8080/realms/litellm-e2e" + assert _REALM.jwks_url == "http://keycloak:8080/realms/litellm-e2e/protocol/openid-connect/certs" + assert _REALM.token_url("master") == "http://keycloak:8080/realms/master/protocol/openid-connect/token" + + +def test_created_id_is_the_last_segment_of_the_location_header() -> None: + created: Final = ExternalWrite( + status_code=201, location="http://keycloak:8080/admin/realms/litellm-e2e/groups/abc-123" + ) + assert created_id(created, "a group") == "abc-123" + + +def test_a_refused_create_fails_the_test_with_the_idps_own_words() -> None: + with pytest.raises(BaseException, match=r"409.*already exists"): + created_id(ExternalWrite(status_code=409, body="Group already exists"), "a group") + + +def test_new_users_are_born_fully_set_up() -> None: + """A user without a profile or with a pending required action authenticates + nowhere: Keycloak answers every grant with "Account is not fully set up".""" + body: Final = UserCreateBody( + username="e2e", email="e2e@example.com", groups=("team",), credentials=(PasswordCredential(value="pw"),) + ).model_dump(by_alias=True) + + assert body["requiredActions"] == () + assert body["firstName"] and body["lastName"] and body["emailVerified"] is True + assert body["credentials"][0]["temporary"] is False + + +def test_connection_details_come_from_the_environment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(KEYCLOAK_URL_ENV, "http://keycloak.litellm.svc.cluster.local:8080/") + monkeypatch.setenv(KEYCLOAK_REALM_ENV, "other-realm") + monkeypatch.setenv(KEYCLOAK_ADMIN_USER_ENV, "admin") + monkeypatch.setenv(KEYCLOAK_ADMIN_PASSWORD_ENV, "pw") + + resolved: Final = keycloak_from_env() + + assert resolved.issuer == "http://keycloak.litellm.svc.cluster.local:8080/realms/other-realm" + assert resolved.admin_username == "admin" and resolved.admin_password == "pw" + + +@pytest.mark.parametrize("blank", ["", " "]) +def test_a_missing_admin_credential_fails_loudly_instead_of_skipping( + monkeypatch: pytest.MonkeyPatch, blank: str +) -> None: + monkeypatch.setenv(KEYCLOAK_ADMIN_USER_ENV, "admin") + monkeypatch.setenv(KEYCLOAK_ADMIN_PASSWORD_ENV, blank) + + with pytest.raises(BaseException, match=KEYCLOAK_ADMIN_PASSWORD_ENV): + keycloak_from_env() diff --git a/tests/e2e/test_jwt_issuer.py b/tests/e2e/test_jwt_issuer.py deleted file mode 100644 index 7acaaf13949..00000000000 --- a/tests/e2e/test_jwt_issuer.py +++ /dev/null @@ -1,141 +0,0 @@ -"""Harness coverage for the test-only JWT issuer (jwt_issuer.py). - -No proxy and no ``e2e`` marker. The issuer is booted in-process on an -OS-assigned port with a fixed clock and driven over HTTP through -``e2e_http.post_external`` / ``get_external``, the same transport the live -suite uses, so what is pinned here is the contract the live JWT tests lean on: -a token minted at ``/token`` verifies against the key served at -``/.well-known/jwks.json`` under the ``kid`` in its header, ``iss``/``iat``/``exp`` -are filled in only when the caller left them out, and malformed claim bodies or -unknown paths are refused instead of signed. -""" - -from __future__ import annotations - -import time -from collections.abc import Iterator -from typing import Final - -import jwt -import pytest -from pydantic import BaseModel, RootModel - -from e2e_http import UnknownApiError, get_external, post_external, unwrap -from jwt_issuer import ( - DEFAULT_TOKEN_LIFETIME_SECONDS, - JWKS_PATH, - JWT_ISSUER_PORT_ENV, - TOKEN_PATH, - JwksDocument, - MintedToken, - RunningIssuer, - jwt_issuer_port, - start_jwt_issuer, -) - -FROZEN_NOW: Final = int(time.time()) - - -class _Claims(BaseModel): - sub: str - groups: tuple[str, ...] = () - exp: int | None = None - - -class _DecodedClaims(BaseModel): - sub: str - iss: str - iat: int - exp: int - groups: tuple[str, ...] = () - - -class _NotAnObject(RootModel[tuple[str, ...]]): - pass - - -@pytest.fixture(scope="module") -def issuer() -> Iterator[RunningIssuer]: - running: Final = start_jwt_issuer(clock=lambda: FROZEN_NOW) - yield running - running.shutdown() - - -def _mint(issuer: RunningIssuer, claims: _Claims) -> str: - return unwrap(post_external(f"{issuer.url}{TOKEN_PATH}", json=claims, response_type=MintedToken)).token - - -def _served_jwks(issuer: RunningIssuer) -> JwksDocument: - return unwrap(get_external(issuer.jwks_url, response_type=JwksDocument)) - - -def _decode(token: str, jwks: JwksDocument, *, verify_exp: bool = True) -> _DecodedClaims: - key: Final = jwt.PyJWK.from_json(jwks.keys[0].model_dump_json()) - decoded: Final = jwt.decode(token, key, algorithms=["RS256"], options={"verify_exp": verify_exp}) - return _DecodedClaims.model_validate(decoded) - - -class TestJwtIssuer: - def test_minted_token_verifies_against_the_served_jwks(self, issuer: RunningIssuer) -> None: - token: Final = _mint(issuer, _Claims(sub="alice", groups=("team-a",))) - jwks: Final = _served_jwks(issuer) - - assert len(jwks.keys) == 1 - assert jwt.get_unverified_header(token)["kid"] == jwks.keys[0].kid - claims: Final = _decode(token, jwks) - assert claims.sub == "alice" - assert claims.groups == ("team-a",) - assert claims.iss == issuer.url - assert claims.iat == FROZEN_NOW - assert claims.exp == FROZEN_NOW + DEFAULT_TOKEN_LIFETIME_SECONDS - - def test_a_token_signed_by_another_key_does_not_verify(self, issuer: RunningIssuer) -> None: - other: Final = start_jwt_issuer(clock=lambda: FROZEN_NOW) - try: - foreign_token: Final = _mint(other, _Claims(sub="alice")) - finally: - other.shutdown() - - with pytest.raises(jwt.InvalidSignatureError): - _decode(foreign_token, _served_jwks(issuer)) - - def test_an_explicit_exp_is_signed_as_given(self, issuer: RunningIssuer) -> None: - expired_at: Final = FROZEN_NOW - 60 - token: Final = _mint(issuer, _Claims(sub="alice", exp=expired_at)) - jwks: Final = _served_jwks(issuer) - - assert _decode(token, jwks, verify_exp=False).exp == expired_at - with pytest.raises(jwt.ExpiredSignatureError): - _ = _decode(token, jwks) - - def test_non_object_claims_are_refused(self, issuer: RunningIssuer) -> None: - result: Final = post_external( - f"{issuer.url}{TOKEN_PATH}", json=_NotAnObject(("not", "claims")), response_type=MintedToken - ) - assert isinstance(result, UnknownApiError) - assert result.status_code == 400 - - @pytest.mark.parametrize( - ("method", "path"), - [("GET", TOKEN_PATH), ("POST", JWKS_PATH), ("GET", "/token/anything")], - ids=["get-token", "post-jwks", "get-other"], - ) - def test_unknown_routes_are_404(self, issuer: RunningIssuer, method: str, path: str) -> None: - url: Final = f"{issuer.url}{path}" - result: Final = ( - get_external(url, response_type=MintedToken) - if method == "GET" - else post_external(url, json=_Claims(sub="alice"), response_type=MintedToken) - ) - assert isinstance(result, UnknownApiError) - assert result.status_code == 404 - - -class TestIssuerPort: - def test_defaults_to_the_documented_port(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv(JWT_ISSUER_PORT_ENV, raising=False) - assert jwt_issuer_port() == 4190, "CONTRIBUTING.md hardcodes 4190 in JWT_PUBLIC_KEY_URL" - - def test_env_override_wins(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv(JWT_ISSUER_PORT_ENV, " 4321 ") - assert jwt_issuer_port() == 4321 From 9a80bf2ad4e6254ef20d2ffe16562c9e9554db66 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 11 Sep 2026 16:36:12 -0700 Subject: [PATCH 3/7] test(e2e): harden JWT fixtures and cover management lifecycles --- tests/e2e/CONTRIBUTING.md | 24 ++- tests/e2e/conftest.py | 37 ++++- tests/e2e/coverage_registry/mgmt.yaml | 4 + tests/e2e/coverage_registry/other.yaml | 3 + tests/e2e/e2e_http.py | 2 + tests/e2e/idp.py | 78 ++++----- tests/e2e/idp_realm.json | 154 ++++++++++++++++++ tests/e2e/management/management_client.py | 17 +- .../e2e/management/test_jwt_management_e2e.py | 91 +++++++++++ tests/e2e/other/test_jwt_auth_e2e.py | 73 +++++---- tests/e2e/test_idp.py | 102 +++++++++++- 11 files changed, 504 insertions(+), 81 deletions(-) create mode 100644 tests/e2e/management/test_jwt_management_e2e.py diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 4802feb10c4..3383ce79cf8 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -27,16 +27,19 @@ The suites run against a live proxy, so bring one up first by running the litell 2. Bring up a Postgres and a Redis for the proxy to use. The repo-root `docker-compose.yml` already defines a Postgres on `5432`; a `docker run -p 6379:6379 redis:7` covers Redis. Point `DATABASE_URL` / `REDIS_HOST` / `REDIS_PORT` at whatever you run. Tests that read Redis directly default to the deployed shape (TLS + cluster mode) whenever `REDIS_HOST` is set, so for a local standalone Redis also set `REDIS_CLUSTER=false` and `REDIS_SSL=false` (plus `REDIS_PASSWORD` when your Redis requires auth) -3. Start the identity provider the `other/` suite authenticates against, then the litellm proxy against your config, and confirm both are live. It is a real Keycloak, running the realm in `tests/e2e/idp_realm.json`, and the proxy trusts it because `JWT_PUBLIC_KEY_URL` points at that realm's JWKS. The proxy caches the JWKS for `public_key_ttl` (600s) and does not refetch on an unknown `kid`, so restart the proxy whenever you recreate the container: +3. Start the identity provider the JWT API tests authenticate against, then the litellm proxy against your config, and confirm both are live. It is a real Keycloak, running the realm in `tests/e2e/idp_realm.json`, and the proxy trusts it because `JWT_PUBLIC_KEY_URL` points at that realm's JWKS. The proxy caches the JWKS for `public_key_ttl` (600s) and does not refetch on an unknown `kid`, so keep its data volume across restarts; restart the proxy if you deliberately replace that volume: ```bash - set -a && source .env && set +a docker run -d --name litellm-e2e-idp -p 8480:8080 \ -e KC_BOOTSTRAP_ADMIN_USERNAME=admin -e KC_BOOTSTRAP_ADMIN_PASSWORD=admin \ -v "$PWD/tests/e2e/idp_realm.json:/opt/keycloak/data/import/realm.json:ro" \ + -v litellm-e2e-idp-data:/opt/keycloak/data \ quay.io/keycloak/keycloak:26.7.3 start-dev --import-realm curl -fs --retry 30 --retry-delay 2 --retry-all-errors http://127.0.0.1:8480/realms/litellm-e2e/.well-known/openid-configuration - JWT_PUBLIC_KEY_URL=http://127.0.0.1:8480/realms/litellm-e2e/protocol/openid-connect/certs litellm --config .yml --port 4000 + export JWT_ISSUER=http://127.0.0.1:8480/realms/litellm-e2e + export JWT_AUDIENCE=litellm-e2e + export JWT_PUBLIC_KEY_URL="$JWT_ISSUER/protocol/openid-connect/certs" + litellm --config .yml --port 4000 curl -fs http://localhost:4000/health/liveliness ``` @@ -53,9 +56,20 @@ The suites run against a live proxy, so bring one up first by running the litell user_id_upsert: true ``` - Leave `JWT_AUDIENCE` and `JWT_ISSUER` unset. Keycloak's access tokens carry `aud: account`, its own audience rather than the proxy's, and their `iss` is whatever base URL the token was requested through, so pinning either one only makes sense once the deployment fixes Keycloak's hostname + Set `JWT_ISSUER` to the exact realm URL used by the test runner and `JWT_AUDIENCE=litellm-e2e`. The realm explicitly maps this audience, `sub`, `email`, and `groups`; the proxy fetches real signing keys from its JWKS endpoint. The rejection tests obtain signed tokens with a different audience or issuer and verify the corresponding rejection reason. The issuer test uses a different HTTP Host when requesting a token from the isolated, dynamically named test IdP. - CI runs this suite against a Keycloak deployed beside the ephemeral stack, and that deployment (the JWT config block, `JWT_PUBLIC_KEY_URL`, and the admin credential handed to the run pod) lives in the project-releaser repo, not here. A stack without it fails the JWT tests rather than skipping them + Keycloak's password grant is a test-only provisioning shortcut, not a production login recommendation. The `litellm-e2e-admin` client adds the proxy's admin scope; the normal client does not. Never reuse this permissive realm outside an isolated test stack. + + Management tests can use the shared `idp` and `jwt_identity` fixtures. Each test gets a unique Keycloak group/user and a matching proxy user/team. Setup and fallback cleanup use the master key; the operations and read-backs being tested must explicitly use `caller_key=idp.access_token(jwt_identity, client_id=ADMIN_CLIENT_ID)` (or a member token). See `management/test_jwt_management_e2e.py` for create/read/update/clear/delete and tenant-denial examples. A group claim alone is not database team membership: permission tests explicitly add the member and prove an allowed read before asserting the denied write. + + Every successful IdP create immediately registers cleanup, including partial setup failures. Cleanup failures emit warnings. Tokens are minted on demand, and the expiration test waits relative to the token's actual `exp` with a bounded clock-drift check. To check first-attempt behavior locally, run both files with `--reruns 0`: + + ```bash + E2E_KEYCLOAK_ADMIN_USER=admin E2E_KEYCLOAK_ADMIN_PASSWORD=admin \ + uv run pytest tests/e2e/other/test_jwt_auth_e2e.py tests/e2e/management/test_jwt_management_e2e.py --reruns 0 -v + ``` + + CI runs this suite against a Keycloak deployed beside the ephemeral stack, and that deployment (the JWT config block, `JWT_PUBLIC_KEY_URL`, and the admin credential handed to the run pod) lives in the project-releaser repo, not here. CI fetches the realm from the test-runner revision even when it reuses a gateway image from another commit. Keycloak stores its realm, keys and users in a separate schema in the build's PostgreSQL, so replacing the IdP pod preserves token validity. Its startup probe waits for the imported realm. Losing the whole ephemeral database invalidates the stack. Keycloak skips imports into an existing realm, so changes to the realm export require a fresh stack (or deliberately replacing the local data volume). A stack without it fails the JWT tests rather than skipping them 4. Run a suite against it; the harness reads `LITELLM_PROXY_URL` (default `http://localhost:4000`): diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index e1b987cbfd9..8f82c7a12bd 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -17,23 +17,52 @@ import functools import os from collections.abc import Generator, Iterator from datetime import datetime, timezone +from typing import Final import pytest import requests - -from e2e_config import CONTROL_PLANE_BASE_URL, FIXTURE_DIR, FIXTURE_MODE_RAW, PROXY_BASE_URL +from e2e_config import CONTROL_PLANE_BASE_URL, FIXTURE_DIR, FIXTURE_MODE_RAW, PROXY_BASE_URL, unique_marker from e2e_db import RESET_OPT_IN_ENV, reset_spend_logs, run_spend_log_cleanup +from e2e_http import unwrap from fixture_mode import fixture_mode_collection_error, fixture_report_lines -from provider_edge import replay_leftover_error +from idp import Identity, Keycloak, keycloak_from_env from junit_properties import attach_result_properties from lifecycle import ProxyClientProvider, ResourceManager +from models import TeamNewBody, UserNewBody, UserNewResponse +from provider_edge import replay_leftover_error from proxy_client import ProxyClient, build_proxy_client - _E2E_TEST_RAN = pytest.StashKey[bool]() _CALL_PASSED = pytest.StashKey[bool]() +@pytest.fixture(scope="session") +def idp() -> Keycloak: + return keycloak_from_env() + + +@pytest.fixture +def jwt_identity(idp: Keycloak, resources: ResourceManager, proxy: ProxyClient) -> Identity: + marker: Final = unique_marker() + identity: Final = idp.provision(marker=marker, group=f"e2e-jwt-team-{marker}", defer=resources.defer) + resources.defer(lambda: proxy.delete_user(identity.user_id)) + # Seed the canonical user before any JWT call populates the auth cache. + # Group claims grant team access; management membership is added by the test. + unwrap( + proxy.transport.post( + "/user/new", + headers=proxy.transport.master, + json=UserNewBody( + user_id=identity.user_id, user_email=f"{identity.username}@example.com", user_role="internal_user" + ), + response_type=UserNewResponse, + ) + ) + team_id: Final = proxy.create_team(TeamNewBody(team_alias=f"e2e-jwt-{marker}", team_id=identity.group)) + resources.defer(lambda: proxy.delete_team(team_id)) + return identity + + def pytest_configure(config: pytest.Config) -> None: config.addinivalue_line( "markers", diff --git a/tests/e2e/coverage_registry/mgmt.yaml b/tests/e2e/coverage_registry/mgmt.yaml index d571fb36546..c3d7892aa79 100644 --- a/tests/e2e/coverage_registry/mgmt.yaml +++ b/tests/e2e/coverage_registry/mgmt.yaml @@ -74,3 +74,7 @@ - {id: mgmt.credential_migration.check.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "key_management_endpoints.py:4252", rationale: "Encryption migration (smoke)"} - {id: mgmt.credential.new.serves_request, module: mgmt, tier: P1, surface: api, assertions: [serves_request], source: "credential_endpoints/endpoints.py:42", rationale: "Stored credential resolves into a deployment and serves a live /messages request"} - {id: mgmt.model.test_connection.happy_path, module: mgmt, tier: P0, surface: api, assertions: [happy_path], source: "_health_endpoints.py:1785", rationale: "Test Connection for a responses-mode Bedrock Mantle deployment reaches the live provider and reports success; this exact shape 500ed on an acompletion partial before v1.91.0", fail_before_fix: proven} + +- {id: mgmt.key.jwt.lifecycle, module: mgmt, tier: P0, surface: api, assertions: [lifecycle], source: "management_endpoints/key_management_endpoints.py", rationale: "An IdP-issued admin JWT creates, reads, updates, clears and deletes a key; omitted fields survive updates"} +- {id: mgmt.key.jwt.member_denied, module: mgmt, tier: P0, surface: api, assertions: [member_denied], source: "auth/handle_jwt.py", rationale: "A valid member JWT cannot update an admin-managed key and denial leaves it unchanged"} +- {id: mgmt.key.jwt.other_team_denied, module: mgmt, tier: P0, surface: api, assertions: [other_team_denied], source: "auth/handle_jwt.py", rationale: "A valid JWT for another existing team cannot read the key"} diff --git a/tests/e2e/coverage_registry/other.yaml b/tests/e2e/coverage_registry/other.yaml index ff25155924b..9292e5f07db 100644 --- a/tests/e2e/coverage_registry/other.yaml +++ b/tests/e2e/coverage_registry/other.yaml @@ -51,3 +51,6 @@ - {id: other.a2a.message_send.bridge_invokes, module: other, tier: P1, area: a2a, assertions: [bridge_invokes], source: "a2a_protocol/litellm_completion_bridge/handler.py", rationale: "A2A message/send routes through the completion bridge to a real provider and logs an asend_message spend row"} - {id: other.a2a.version.serves_pinned_0_3, module: other, tier: P1, area: a2a, assertions: [serves_pinned_0_3], source: "agent_endpoints/a2a_endpoints.py _served_version", rationale: "An agent pinning 0.3 returns the flat 0.3 message shape (parts on the result)"} - {id: other.a2a.version.serves_pinned_1_0, module: other, tier: P1, area: a2a, assertions: [serves_pinned_1_0], source: "agent_endpoints/a2a_endpoints.py _served_version", rationale: "An agent pinning 1.0 returns the nested 1.0 message shape (result.message with ROLE_AGENT)"} + +- {id: other.auth.jwt.wrong_issuer_denied, module: other, tier: P0, area: auth, assertions: [wrong_issuer_denied], source: "auth/handle_jwt.py", rationale: "A signed token with the correct audience and an unexpected issuer is rejected"} +- {id: other.auth.jwt.wrong_audience_denied, module: other, tier: P0, area: auth, assertions: [wrong_audience_denied], source: "auth/handle_jwt.py", rationale: "A signed token from the trusted issuer intended for another app is rejected"} diff --git a/tests/e2e/e2e_http.py b/tests/e2e/e2e_http.py index d6924aff5c0..061fbc23fde 100644 --- a/tests/e2e/e2e_http.py +++ b/tests/e2e/e2e_http.py @@ -406,6 +406,7 @@ def post_form_external[R: BaseModel]( *, form: BaseModel, response_type: type[R], + headers: BaseModel | None = None, timeout: float = 30.0, ) -> Result[R]: """POST an absolute URL outside the proxy as `application/x-www-form-urlencoded`, @@ -415,6 +416,7 @@ def post_form_external[R: BaseModel]( resp = requests.post( url, data=_flat(form), + headers=_headers(headers) if headers is not None else None, timeout=timeout, ) except requests.RequestException as exc: diff --git a/tests/e2e/idp.py b/tests/e2e/idp.py index 50cf44b4159..6d2fc84eb27 100644 --- a/tests/e2e/idp.py +++ b/tests/e2e/idp.py @@ -1,40 +1,15 @@ -"""The identity provider the JWT suite authenticates against: a real Keycloak -realm, imported from `idp_realm.json`. - -A real IdP rather than a hand-rolled signer because every JWT bug this suite -exists to catch lives in the shape of what an IdP actually emits: `sub` is an -opaque uuid and not a friendly name, group membership arrives as a claim built -by a protocol mapper, the JWKS carries a signing key next to an encryption key -so the proxy has to select on `kid`, and `aud` is the IdP's own audience rather -than the proxy's. A stand-in issuer that mints exactly the claims the tests -assert on can only prove the proxy agrees with the tests. - -Tests never hold a signing key. They provision an identity through Keycloak's -admin API (a group named after the litellm team, a user in it with a password -generated for that test alone), then ask Keycloak for an access token through -the direct-access grant, the same way a CLI or service account signs in. The -proxy's `JWT_PUBLIC_KEY_URL` points at this realm's JWKS, so the token the -tests carry is trusted for exactly one reason: Keycloak signed it. - -The realm declares two clients. `litellm-e2e-tests` mints ordinary tokens; the -`litellm-e2e-shortlived` client sets `access.token.lifespan` to one second, so -the expiry test lets a genuine token expire instead of forging a stale `exp`. - -Connection details come from the environment (`E2E_KEYCLOAK_URL` and the admin -credential). A missing or unreachable IdP is a hard failure naming the start -command, never a skip, so a stack deployed without it turns the run red. -""" +"""Provision isolated identities and obtain signed tokens from the test Keycloak realm.""" from __future__ import annotations import os import secrets -from dataclasses import dataclass +import warnings +from collections.abc import Callable +from dataclasses import dataclass, field from typing import Final, Literal import pytest -from pydantic import BaseModel, Field - from e2e_http import ( AuthHeaders, ExternalWrite, @@ -45,6 +20,7 @@ from e2e_http import ( post_form_external, post_json_external, ) +from pydantic import BaseModel, Field KEYCLOAK_URL_ENV: Final = "E2E_KEYCLOAK_URL" KEYCLOAK_REALM_ENV: Final = "E2E_KEYCLOAK_REALM" @@ -55,7 +31,8 @@ DEFAULT_KEYCLOAK_URL: Final = "http://127.0.0.1:8480" DEFAULT_REALM: Final = "litellm-e2e" TESTS_CLIENT_ID: Final = "litellm-e2e-tests" SHORT_LIVED_CLIENT_ID: Final = "litellm-e2e-shortlived" -SHORT_LIVED_TOKEN_SECONDS: Final = 1 +ADMIN_CLIENT_ID: Final = "litellm-e2e-admin" +WRONG_AUDIENCE_CLIENT_ID: Final = "litellm-e2e-other-app" _START_HINT: Final = ( "Start it with the `docker run ... quay.io/keycloak/keycloak` command in tests/e2e/CONTRIBUTING.md, " @@ -73,7 +50,11 @@ class TokenGrantForm(BaseModel): class TokenResponse(BaseModel): - access_token: str + access_token: str = Field(repr=False) + + +class TokenRequestHeaders(BaseModel): + host: str | None = None class GroupCreateBody(BaseModel): @@ -105,8 +86,10 @@ class UserCreateBody(BaseModel): def created_id(write: ExternalWrite, context: str) -> str: """The new resource's id, which Keycloak returns only as the last segment of the Location header on a 201.""" - if not write.ok: + if write.status_code != 201: pytest.fail(f"Keycloak refused to create {context}: HTTP {write.status_code} {write.body[:300]}") + if not write.location or write.location.endswith("/"): + pytest.fail(f"Keycloak created {context} without a resource id in its Location header") return write.location.rsplit("/", 1)[-1] @@ -117,7 +100,7 @@ class Identity: user_id: str username: str - password: str + password: str = field(repr=False) group: str group_id: str @@ -127,7 +110,7 @@ class Keycloak: base_url: str realm: str admin_username: str - admin_password: str + admin_password: str = field(repr=False) @property def issuer(self) -> str: @@ -183,29 +166,48 @@ class Keycloak: ) def delete_user(self, user_id: str) -> None: - delete_external(self._admin_url(f"/users/{user_id}"), headers=self._admin_headers()) + self._delete(f"/users/{user_id}") def delete_group(self, group_id: str) -> None: - delete_external(self._admin_url(f"/groups/{group_id}"), headers=self._admin_headers()) + self._delete(f"/groups/{group_id}") - def provision(self, *, marker: str, group: str) -> Identity: + def _delete(self, path: str) -> None: + try: + headers: Final = self._admin_headers() + except pytest.fail.Exception as exc: + warnings.warn(f"Keycloak cleanup could not authenticate for {path}: {exc}", RuntimeWarning, stacklevel=2) + return + result: Final = delete_external(self._admin_url(path), headers=headers) + if result.status_code not in (204, 404): + warnings.warn( + f"Keycloak cleanup failed for {path}: HTTP {result.status_code} {result.body[:300]}", + RuntimeWarning, + stacklevel=2, + ) + + def provision(self, *, marker: str, group: str, defer: Callable[[Callable[[], object]], None]) -> Identity: """Create `group` and a user in it, credentialed with a password generated for this test alone, and hand back the identity a token can be minted for.""" group_id: Final = self.create_group(group) + defer(lambda: self.delete_group(group_id)) username: Final = f"e2e-jwt-user-{marker}" password: Final = secrets.token_urlsafe(24) user_id: Final = self.create_user( username=username, email=f"{username}@example.com", password=password, group=group ) + defer(lambda: self.delete_user(user_id)) return Identity(user_id=user_id, username=username, password=password, group=group, group_id=group_id) - def access_token(self, identity: Identity, *, client_id: str = TESTS_CLIENT_ID) -> str: + def access_token( + self, identity: Identity, *, client_id: str = TESTS_CLIENT_ID, issuer_host: str | None = None + ) -> str: """Sign `identity` in through the direct-access grant and hand back the access token Keycloak signed, exactly as it came off the wire.""" result: Final = post_form_external( self.token_url(self.realm), form=TokenGrantForm(client_id=client_id, username=identity.username, password=identity.password), response_type=TokenResponse, + headers=TokenRequestHeaders(host=issuer_host), ) return self._token(result, f"a token for {identity.username}") diff --git a/tests/e2e/idp_realm.json b/tests/e2e/idp_realm.json index 2a8bd89b428..3b747e7a5dd 100644 --- a/tests/e2e/idp_realm.json +++ b/tests/e2e/idp_realm.json @@ -24,7 +24,22 @@ "id.token.claim": "true", "userinfo.token.claim": "true" } + }, + { + "name": "litellm-audience", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "consentRequired": false, + "config": { + "included.custom.audience": "litellm-e2e", + "access.token.claim": "true", + "id.token.claim": "false" + } } + ], + "defaultClientScopes": [ + "email", + "basic" ] }, { @@ -49,6 +64,145 @@ "id.token.claim": "true", "userinfo.token.claim": "true" } + }, + { + "name": "litellm-audience", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "consentRequired": false, + "config": { + "included.custom.audience": "litellm-e2e", + "access.token.claim": "true", + "id.token.claim": "false" + } + } + ], + "defaultClientScopes": [ + "email", + "basic" + ] + }, + { + "clientId": "litellm-e2e-admin", + "enabled": true, + "publicClient": true, + "standardFlowEnabled": false, + "directAccessGrantsEnabled": true, + "defaultClientScopes": [ + "email", + "litellm_proxy_admin", + "basic" + ], + "protocolMappers": [ + { + "name": "groups", + "protocol": "openid-connect", + "protocolMapper": "oidc-group-membership-mapper", + "consentRequired": false, + "config": { + "claim.name": "groups", + "full.path": "false", + "access.token.claim": "true", + "id.token.claim": "true", + "userinfo.token.claim": "true" + } + }, + { + "name": "litellm-audience", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "consentRequired": false, + "config": { + "included.custom.audience": "litellm-e2e", + "access.token.claim": "true", + "id.token.claim": "false" + } + } + ] + }, + { + "clientId": "litellm-e2e-other-app", + "enabled": true, + "publicClient": true, + "standardFlowEnabled": false, + "directAccessGrantsEnabled": true, + "defaultClientScopes": [ + "email", + "basic" + ], + "protocolMappers": [ + { + "name": "groups", + "protocol": "openid-connect", + "protocolMapper": "oidc-group-membership-mapper", + "consentRequired": false, + "config": { + "claim.name": "groups", + "full.path": "false", + "access.token.claim": "true", + "id.token.claim": "true", + "userinfo.token.claim": "true" + } + }, + { + "name": "litellm-audience", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "consentRequired": false, + "config": { + "included.custom.audience": "litellm-e2e-other-app", + "access.token.claim": "true", + "id.token.claim": "false" + } + } + ] + } + ], + "clientScopes": [ + { + "name": "litellm_proxy_admin", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "false" + } + }, + { + "name": "email", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true" + }, + "protocolMappers": [ + { + "name": "email", + "protocol": "openid-connect", + "protocolMapper": "oidc-usermodel-property-mapper", + "config": { + "user.attribute": "email", + "claim.name": "email", + "jsonType.label": "String", + "access.token.claim": "true", + "id.token.claim": "true", + "userinfo.token.claim": "true" + } + } + ] + }, + { + "name": "basic", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "false" + }, + "protocolMappers": [ + { + "name": "sub", + "protocol": "openid-connect", + "protocolMapper": "oidc-sub-mapper", + "config": { + "access.token.claim": "true" + } } ] } diff --git a/tests/e2e/management/management_client.py b/tests/e2e/management/management_client.py index 2b897f5f07f..e588d19d5e8 100644 --- a/tests/e2e/management/management_client.py +++ b/tests/e2e/management/management_client.py @@ -10,9 +10,7 @@ import time from dataclasses import dataclass import jwt - from e2e_config import MASTER_KEY -from proxy_client import ProxyClient from e2e_http import ( AuthHeaders, NetworkError, @@ -37,6 +35,8 @@ from models import ( KeyDeleteBody, KeyGenerateBody, KeyGenerateResponse, + KeyInfoParams, + KeyInfoResponse, KeyListParams, KeyListResponse, KeyRegenerateBody, @@ -78,6 +78,7 @@ from models import ( UserNewResponse, UserUpdateBody, ) +from proxy_client import ProxyClient MODEL_ACCESS_DENIED_MARKER = "key_model_access_denied" ROUTE_NOT_ALLOWED_MARKER = "not allowed to call this route" @@ -149,13 +150,21 @@ class ManagementClient: def update_key_models(self, key: str, models: list[str]) -> None: _ = unwrap(self.update_key(KeyUpdateBody(key=key, models=models))) - def delete_key_strict(self, key: str) -> None: + def key_info_as(self, key: str, *, caller_key: str) -> Result[KeyInfoResponse]: + return self.proxy.transport.get( + "/key/info", + headers=self.proxy.transport.bearer(caller_key), + params=KeyInfoParams(key=key), + response_type=KeyInfoResponse, + ) + + def delete_key_strict(self, key: str, *, caller_key: str | None = None) -> None: """Strict delete for the act phase of a test: a failed delete is a hard failure, unlike the warn-only ProxyClient.delete_key used at teardown.""" _ = unwrap( self.proxy.transport.post( "/key/delete", - headers=self.proxy.transport.master, + headers=self.proxy.transport.master if caller_key is None else self.proxy.transport.bearer(caller_key), json=KeyDeleteBody(keys=[key]), response_type=NoBody, ) diff --git a/tests/e2e/management/test_jwt_management_e2e.py b/tests/e2e/management/test_jwt_management_e2e.py new file mode 100644 index 00000000000..22306da8eb8 --- /dev/null +++ b/tests/e2e/management/test_jwt_management_e2e.py @@ -0,0 +1,91 @@ +"""Management writes and tenant isolation under credentials issued by Keycloak.""" + +from __future__ import annotations + +from typing import Final + +import pytest +from e2e_config import CHEAP_OPENAI_MODEL, unique_marker +from e2e_http import UnauthorizedError, UnknownApiError, unwrap +from idp import ADMIN_CLIENT_ID, Identity, Keycloak +from lifecycle import ResourceManager +from management_client import ManagementClient +from models import KeyGenerateBody, KeyUpdateBody, TeamNewBody, UserNewBody + +pytestmark = pytest.mark.e2e + + +class TestJwtManagement: + @pytest.mark.covers("mgmt.key.jwt.lifecycle") + def test_admin_creates_reads_updates_clears_and_deletes_a_key( + self, client: ManagementClient, idp: Keycloak, jwt_identity: Identity, resources: ResourceManager + ) -> None: + admin: Final = idp.access_token(jwt_identity, client_id=ADMIN_CLIENT_ID) + alias: Final = f"e2e-jwt-key-{unique_marker()}" + created: Final = unwrap( + client.generate_key( + KeyGenerateBody(key_alias=alias, team_id=jwt_identity.group, models=[CHEAP_OPENAI_MODEL]), + caller_key=admin, + ) + ) + resources.defer(lambda: client.proxy.delete_key(created.key)) + + original: Final = unwrap(client.key_info_as(created.key, caller_key=admin)).info + assert original.key_alias == alias and original.team_id == jwt_identity.group + assert original.models == [CHEAP_OPENAI_MODEL] + + updated_alias: Final = f"{alias}-updated" + unwrap( + client.update_key(KeyUpdateBody(key=created.key, key_alias=updated_alias, rpm_limit=120), caller_key=admin) + ) + updated: Final = unwrap(client.key_info_as(created.key, caller_key=admin)).info + assert updated.key_alias == updated_alias and updated.rpm_limit == 120 + assert updated.models == [CHEAP_OPENAI_MODEL], "omitted models must preserve the restriction" + + unwrap(client.update_key(KeyUpdateBody(key=created.key, models=[]), caller_key=admin)) + cleared: Final = unwrap(client.key_info_as(created.key, caller_key=admin)).info + assert cleared.models == [] and cleared.rpm_limit == 120 + + assert unwrap(client.key_list(updated_alias, caller_key=admin)).total_count == 1 + client.delete_key_strict(created.key, caller_key=admin) + assert unwrap(client.key_list(updated_alias, caller_key=admin)).total_count == 0 + + @pytest.mark.covers("mgmt.key.jwt.member_denied", "mgmt.key.jwt.other_team_denied") + def test_member_cannot_write_and_another_team_cannot_read_the_key( + self, client: ManagementClient, idp: Keycloak, jwt_identity: Identity, resources: ResourceManager + ) -> None: + admin: Final = idp.access_token(jwt_identity, client_id=ADMIN_CLIENT_ID) + member: Final = idp.access_token(jwt_identity) + alias: Final = f"e2e-jwt-owned-{unique_marker()}" + created: Final = unwrap( + client.generate_key(KeyGenerateBody(key_alias=alias, team_id=jwt_identity.group), caller_key=admin) + ) + resources.defer(lambda: client.proxy.delete_key(created.key)) + + client.add_team_member(jwt_identity.group, jwt_identity.user_id) + assert unwrap(client.key_info_as(created.key, caller_key=member)).info.key_alias == alias + + refused: Final = client.update_key(KeyUpdateBody(key=created.key, key_alias="forbidden"), caller_key=member) + assert isinstance(refused, UnauthorizedError), f"member write was accepted: {refused}" + assert "does not have permissions for endpoint" in refused.body.lower(), ( + f"expected a permission denial: {refused}" + ) + assert unwrap(client.key_info_as(created.key, caller_key=admin)).info.key_alias == alias + + marker: Final = unique_marker() + outsider: Final = idp.provision(marker=marker, group=f"e2e-jwt-team-{marker}", defer=resources.defer) + resources.defer(lambda: client.proxy.delete_user(outsider.user_id)) + client.create_user( + UserNewBody( + user_id=outsider.user_id, user_email=f"{outsider.username}@example.com", user_role="internal_user" + ) + ) + team_id: Final = client.proxy.create_team(TeamNewBody(team_alias=marker, team_id=outsider.group)) + resources.defer(lambda: client.proxy.delete_team(team_id)) + client.add_team_member(outsider.group, outsider.user_id) + outsider_token: Final = idp.access_token(outsider) + hidden: Final = client.key_info_as(created.key, caller_key=outsider_token) + assert isinstance(hidden, UnknownApiError) and hidden.status_code == 403, ( + f"another team must not read this key: {hidden}" + ) + assert unwrap(client.key_info_as(created.key, caller_key=admin)).info.team_id == jwt_identity.group diff --git a/tests/e2e/other/test_jwt_auth_e2e.py b/tests/e2e/other/test_jwt_auth_e2e.py index 25a5e7d4b9a..2bed40f4d69 100644 --- a/tests/e2e/other/test_jwt_auth_e2e.py +++ b/tests/e2e/other/test_jwt_auth_e2e.py @@ -1,50 +1,42 @@ -"""Live e2e: access tokens issued by a real Keycloak realm (idp.py) against a -proxy running with `enable_jwt_auth: true` and the `litellm_jwtauth` block from -CONTRIBUTING.md (sub -> user_id, email -> user_email, groups -> team ids, -user_id_upsert). - -Every identity is provisioned in Keycloak for the test that uses it: a group -named after the litellm team, and a user in that group whose password exists -only for the length of the test. Tokens then come from Keycloak's direct-access -grant, so no test ever holds a signing key and the claims the proxy reads are -the ones an IdP really emits (`sub` is Keycloak's user uuid, `groups` comes off -a protocol mapper, `aud` is Keycloak's own audience). - -The rejection cases stay honest about where the rejection has to come from: the -bad-signature case corrupts a genuine signature, and the expiry case takes its -token from the realm's one-second client and waits for it to lapse rather than -forging a stale `exp`. Those identities get their own freshly created team, so a -rejection can only be blamed on the token, while the unknown-team case names a -group no litellm team was ever created for. An acceptance is proven twice, at -the boundary (200 from a real provider) and in the spend log the proxy -attributes to the claims. The last case keeps a plain `sk-` virtual key working -on the same proxy, guarding against the flag turning JWT on for everyone. -""" +"""Real Keycloak tokens exercise verification, attribution and virtual-key coexistence.""" from __future__ import annotations +import base64 import time from typing import Final import pytest - from e2e_config import CHEAP_OPENAI_MODEL, unique_marker from e2e_http import UnauthorizedError, UnknownApiError, unwrap -from idp import SHORT_LIVED_CLIENT_ID, SHORT_LIVED_TOKEN_SECONDS, Identity +from idp import SHORT_LIVED_CLIENT_ID, WRONG_AUDIENCE_CLIENT_ID, Identity from lifecycle import ResourceManager from models import ChatBody, ChatMessage, TeamNewBody from other_client import OtherClient +from pydantic import BaseModel pytestmark = pytest.mark.e2e +class IssuedClaims(BaseModel): + """Read the IdP's signed payload only to check the test precondition.""" + + exp: int + sub: str + iss: str + aud: str | list[str] + + +def _claims(token: str) -> IssuedClaims: + payload: Final = token.split(".")[1] + return IssuedClaims.model_validate_json(base64.urlsafe_b64decode(payload + "=" * (-len(payload) % 4))) + + def _provision(client: OtherClient, resources: ResourceManager, *, marker: str) -> Identity: """A Keycloak group and a user in it, torn down with the test. The group name is what the token's `groups` claim carries, which is what the proxy resolves as a litellm team id.""" - identity: Final = client.idp.provision(marker=marker, group=f"e2e-jwt-team-{marker}") - resources.defer(lambda: client.idp.delete_user(identity.user_id)) - resources.defer(lambda: client.idp.delete_group(identity.group_id)) + identity: Final = client.idp.provision(marker=marker, group=f"e2e-jwt-team-{marker}", defer=resources.defer) resources.defer(lambda: client.proxy.delete_user(identity.user_id)) return identity @@ -81,6 +73,7 @@ class TestJwtAuth: ) -> None: token: Final = client.idp.access_token(identity) + assert _claims(token).sub == identity.user_id, "IdP must emit the provisioned user as sub" response: Final = unwrap(client.proxy.chat(token, _ping())) assert response.id is not None and response.choices, ( f"chat under a valid JWT returned no completion: {response}" @@ -111,7 +104,9 @@ class TestJwtAuth: @pytest.mark.covers("other.auth.jwt.expired_denied") def test_expired_token_is_rejected(self, client: OtherClient, identity: Identity) -> None: expiring: Final = client.idp.access_token(identity, client_id=SHORT_LIVED_CLIENT_ID) - time.sleep(SHORT_LIVED_TOKEN_SECONDS + 1) + delay: Final = _claims(expiring).exp - time.time() + 1 + assert delay <= 5, f"short-lived client expiry or IdP clock drifted: wait would be {delay}s" + time.sleep(max(0, delay)) result: Final = client.proxy.chat(expiring, _ping()) assert isinstance(result, UnauthorizedError), ( @@ -119,6 +114,28 @@ class TestJwtAuth: ) assert "expired" in result.body.lower(), f"the 401 must say the token expired, got {result.body[:300]}" + @pytest.mark.covers("other.auth.jwt.wrong_issuer_denied") + def test_signed_token_from_the_wrong_issuer_is_rejected(self, client: OtherClient, identity: Identity) -> None: + token: Final = client.idp.access_token(identity, issuer_host="unexpected-issuer.invalid") + claims: Final = _claims(token) + assert claims.iss != client.idp.issuer and "litellm-e2e" in claims.aud + + result: Final = client.proxy.chat(token, _ping()) + assert isinstance(result, UnauthorizedError), f"wrong issuer must be rejected: {result}" + assert "issuer" in result.body.lower(), f"expected issuer validation to reject the token: {result}" + + @pytest.mark.covers("other.auth.jwt.wrong_audience_denied") + def test_signed_token_for_another_application_is_rejected(self, client: OtherClient, identity: Identity) -> None: + token: Final = client.idp.access_token(identity, client_id=WRONG_AUDIENCE_CLIENT_ID) + claims: Final = _claims(token) + assert claims.iss == client.idp.issuer and "litellm-e2e" not in ( + [claims.aud] if isinstance(claims.aud, str) else claims.aud + ) + + result: Final = client.proxy.chat(token, _ping()) + assert isinstance(result, UnauthorizedError), f"wrong audience must be rejected: {result}" + assert "audience" in result.body.lower(), f"expected audience validation to reject the token: {result}" + @pytest.mark.covers("other.auth.jwt.unknown_team_denied") def test_token_naming_a_team_that_does_not_exist_is_rejected( self, client: OtherClient, resources: ResourceManager diff --git a/tests/e2e/test_idp.py b/tests/e2e/test_idp.py index 93be2806eb5..90a85de8765 100644 --- a/tests/e2e/test_idp.py +++ b/tests/e2e/test_idp.py @@ -4,10 +4,14 @@ these carry no `e2e` marker and run everywhere.""" from __future__ import annotations +from collections.abc import Callable, Generator +from contextlib import ExitStack, contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from queue import SimpleQueue +from threading import Thread from typing import Final import pytest - from e2e_http import ExternalWrite from idp import ( KEYCLOAK_ADMIN_PASSWORD_ENV, @@ -16,8 +20,8 @@ from idp import ( KEYCLOAK_URL_ENV, Keycloak, PasswordCredential, - created_id, UserCreateBody, + created_id, keycloak_from_env, ) @@ -44,6 +48,100 @@ def test_a_refused_create_fails_the_test_with_the_idps_own_words() -> None: created_id(ExternalWrite(status_code=409, body="Group already exists"), "a group") +@pytest.mark.parametrize("location", ["", "http://keycloak/groups/"]) +def test_create_without_a_resource_id_fails(location: str) -> None: + with pytest.raises(pytest.fail.Exception, match="resource id"): + created_id(ExternalWrite(status_code=201, location=location), "a group") + + +@contextmanager +def _idp_server( + *, user_status: int = 201, delete_status: int = 204, admin_status: int = 200 +) -> Generator[tuple[Keycloak, SimpleQueue[str]]]: + """Exercise provisioning failures through the same HTTP transport as live tests.""" + deletions: SimpleQueue[str] = SimpleQueue() + + class Handler(BaseHTTPRequestHandler): + def log_message(self, format: str, *args: object) -> None: + pass + + def do_POST(self) -> None: + self.rfile.read(int(self.headers.get("Content-Length", "0"))) + if self.path.endswith("/token"): + self.send_response(admin_status) + self.end_headers() + self.wfile.write(b'{"access_token":"synthetic-harness-token"}') + else: + self.send_response(user_status if self.path.endswith("/users") else 201) + self.send_header("Location", f"{self.path}/resource-1") + self.end_headers() + if user_status != 201 and self.path.endswith("/users"): + self.wfile.write(b"injected create failure") + + def do_DELETE(self) -> None: + deletions.put(self.path) + self.send_response(delete_status) + self.end_headers() + + server: Final = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread: Final = Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield ( + Keycloak( + base_url=f"http://127.0.0.1:{server.server_port}", + realm="test", + admin_username="admin", + admin_password="pw", + ), + deletions, + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +def test_partial_provisioning_removes_the_group_when_user_creation_fails() -> None: + with _idp_server(user_status=500) as (idp, deletions): + with ExitStack() as cleanup: + + def defer(callback: Callable[[], object]) -> None: + cleanup.callback(callback) + + with pytest.raises(pytest.fail.Exception, match="injected create failure"): + idp.provision(marker="partial", group="team", defer=defer) + assert deletions.get_nowait() == "/admin/realms/test/groups/resource-1" + assert deletions.empty() + + +def test_successful_provisioning_cleans_up_user_before_group() -> None: + with _idp_server() as (idp, deletions): + with ExitStack() as cleanup: + + def defer(callback: Callable[[], object]) -> None: + cleanup.callback(callback) + + idp.provision(marker="complete", group="team", defer=defer) + assert deletions.get_nowait() == "/admin/realms/test/users/resource-1" + assert deletions.get_nowait() == "/admin/realms/test/groups/resource-1" + assert deletions.empty() + + +def test_cleanup_failure_is_visible() -> None: + with _idp_server(delete_status=500) as (idp, _): + with pytest.warns(RuntimeWarning, match="cleanup failed.*HTTP 500"): + idp.delete_group("group") + + +def test_expired_admin_credentials_do_not_abort_remaining_cleanups() -> None: + with _idp_server(admin_status=401) as (idp, _): + with pytest.warns(RuntimeWarning, match="cleanup could not authenticate") as warnings: + idp.delete_user("user") + idp.delete_group("group") + assert len(warnings) == 2 + + def test_new_users_are_born_fully_set_up() -> None: """A user without a profile or with a pending required action authenticates nowhere: Keycloak answers every grant with "Account is not fully set up".""" From f7b1fdc39c559ec5171bdfd5ed5382c924409812 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 11 Sep 2026 16:43:31 -0700 Subject: [PATCH 4/7] test(e2e): check cleanup warnings through one teardown action --- tests/e2e/test_idp.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/e2e/test_idp.py b/tests/e2e/test_idp.py index 90a85de8765..33a09a0f13a 100644 --- a/tests/e2e/test_idp.py +++ b/tests/e2e/test_idp.py @@ -136,9 +136,11 @@ def test_cleanup_failure_is_visible() -> None: def test_expired_admin_credentials_do_not_abort_remaining_cleanups() -> None: with _idp_server(admin_status=401) as (idp, _): + cleanup: Final = ExitStack() + cleanup.callback(idp.delete_group, "group") + cleanup.callback(idp.delete_user, "user") with pytest.warns(RuntimeWarning, match="cleanup could not authenticate") as warnings: - idp.delete_user("user") - idp.delete_group("group") + cleanup.close() assert len(warnings) == 2 From 77f406dc002190cf1717a865fe656c20a62175cf Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 11 Sep 2026 17:09:35 -0700 Subject: [PATCH 5/7] test(e2e): start persistent Keycloak in the changed-test runner --- .github/e2e-stack/down.sh | 2 +- .github/e2e-stack/select_tests.py | 1 + .github/e2e-stack/start-idp.sh | 45 ++++++++ .github/e2e-stack/up.sh | 9 ++ .github/workflows/test-e2e-changed.yml | 5 +- .../test_e2e_changed_gate.py | 2 + .../code_coverage_tests/test_e2e_idp_stack.py | 106 ++++++++++++++++++ tests/e2e/CONTRIBUTING.md | 4 +- tests/e2e/gateway/stage_mirror_ci_config.yml | 7 ++ 9 files changed, 177 insertions(+), 4 deletions(-) create mode 100644 .github/e2e-stack/start-idp.sh create mode 100644 tests/code_coverage_tests/test_e2e_idp_stack.py diff --git a/.github/e2e-stack/down.sh b/.github/e2e-stack/down.sh index 9f72f2d6e64..740d626beea 100755 --- a/.github/e2e-stack/down.sh +++ b/.github/e2e-stack/down.sh @@ -10,7 +10,7 @@ for pid_file in "${STACK_DIR}"/pids/*.pid; do rm -f "${pid_file}" done -for container in e2e-nginx e2e-valkey e2e-jaeger e2e-postgres; do +for container in e2e-nginx e2e-keycloak e2e-valkey e2e-jaeger e2e-postgres; do docker rm -f "${container}" >/dev/null 2>&1 done diff --git a/.github/e2e-stack/select_tests.py b/.github/e2e-stack/select_tests.py index a62358f81ff..238818a0d36 100644 --- a/.github/e2e-stack/select_tests.py +++ b/.github/e2e-stack/select_tests.py @@ -11,6 +11,7 @@ UNSUPPORTED: Final = re.compile( ) HARNESS: Final = re.compile( r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$" + r"|^tests/e2e/idp_realm\.json$" r"|^tests/e2e/gateway/" r"|^\.github/e2e-stack/" r"|^\.github/workflows/test-e2e-changed\.yml$" diff --git a/.github/e2e-stack/start-idp.sh b/.github/e2e-stack/start-idp.sh new file mode 100644 index 00000000000..e59a7ade34c --- /dev/null +++ b/.github/e2e-stack/start-idp.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +KEYCLOAK_IMAGE="${E2E_KEYCLOAK_IMAGE:-quay.io/keycloak/keycloak@sha256:ff4257d0d64efbe99ed1ddfaf07765cc3c36dc7518bf8324d41961327f441c54}" +KEYCLOAK_PORT="${E2E_KEYCLOAK_PORT:-8081}" +POSTGRES_IMAGE="${E2E_POSTGRES_IMAGE:-postgres:16.6}" +: "${DATABASE_HOST:?}" "${DATABASE_PORT:?}" "${DATABASE_USER:?}" "${DATABASE_PASSWORD:?}" "${DATABASE_NAME:?}" + +DB_HOST="${DATABASE_HOST}" +DB_NETWORK_ARGS=(--network bridge) +IDP_NETWORK_ARGS=(-p "127.0.0.1:${KEYCLOAK_PORT}:${KEYCLOAK_PORT}") +if [[ "$(uname)" == "Linux" ]]; then + DB_NETWORK_ARGS=(--network host) + IDP_NETWORK_ARGS=(--network host) +elif [[ "${DB_HOST}" == "127.0.0.1" || "${DB_HOST}" == "localhost" ]]; then + DB_HOST=host.docker.internal +fi + +docker run --rm "${DB_NETWORK_ARGS[@]}" -e "PGPASSWORD=${DATABASE_PASSWORD}" \ + "${POSTGRES_IMAGE}" psql -h "${DB_HOST}" -p "${DATABASE_PORT}" \ + -U "${DATABASE_USER}" -d "${DATABASE_NAME}" -v ON_ERROR_STOP=1 \ + -c 'CREATE SCHEMA IF NOT EXISTS keycloak' >/dev/null + +docker rm -f e2e-keycloak >/dev/null 2>&1 || true +docker run -d --name e2e-keycloak "${IDP_NETWORK_ARGS[@]}" --memory 1536m \ + -v "${REPO_ROOT}/tests/e2e/idp_realm.json:/opt/keycloak/data/import/realm.json:ro" \ + -e KC_DB=postgres -e "KC_DB_URL_HOST=${DB_HOST}" -e "KC_DB_URL_PORT=${DATABASE_PORT}" \ + -e "KC_DB_URL_DATABASE=${DATABASE_NAME}" -e KC_DB_SCHEMA=keycloak \ + -e "KC_DB_USERNAME=${DATABASE_USER}" -e "KC_DB_PASSWORD=${DATABASE_PASSWORD}" \ + -e KC_DB_POOL_INITIAL_SIZE=2 -e KC_DB_POOL_MIN_SIZE=2 -e KC_DB_POOL_MAX_SIZE=10 \ + -e "KC_HTTP_PORT=${KEYCLOAK_PORT}" -e KC_BOOTSTRAP_ADMIN_USERNAME=admin \ + -e KC_BOOTSTRAP_ADMIN_PASSWORD=e2e-ephemeral-idp-not-a-secret \ + "${KEYCLOAK_IMAGE}" start-dev --import-realm >/dev/null + +deadline=$((SECONDS + ${E2E_KEYCLOAK_STARTUP_TIMEOUT:-300})) +until curl -fsS --connect-timeout 2 --max-time 3 \ + "http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e/.well-known/openid-configuration" >/dev/null 2>&1; do + if ((SECONDS >= deadline)); then + echo 'e2e-stack: timed out waiting for the Keycloak realm' >&2 + exit 1 + fi + sleep 2 +done +echo 'e2e-stack: Keycloak realm is up' diff --git a/.github/e2e-stack/up.sh b/.github/e2e-stack/up.sh index 2f2e6c6f9a8..a789a570483 100755 --- a/.github/e2e-stack/up.sh +++ b/.github/e2e-stack/up.sh @@ -25,6 +25,7 @@ DATABASE_PASSWORD="${E2E_DATABASE_PASSWORD:-dbpassword9090}" DATABASE_NAME="${E2E_DATABASE_NAME:-litellm}" JAEGER_OTLP_PORT="${E2E_JAEGER_OTLP_PORT:-4318}" JAEGER_QUERY_PORT="${E2E_JAEGER_QUERY_PORT:-16686}" +KEYCLOAK_PORT="${E2E_KEYCLOAK_PORT:-8081}" MASTER_KEY="${LITELLM_MASTER_KEY:-sk-e2e-$(openssl rand -hex 16)}" @@ -124,6 +125,9 @@ SERVER_ENV=( "OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:${JAEGER_OTLP_PORT}" "SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem" "PYTHONPATH=${REPO_ROOT}" + "JWT_PUBLIC_KEY_URL=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e/protocol/openid-connect/certs" + "JWT_ISSUER=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e" + "JWT_AUDIENCE=litellm-e2e" ) if [[ -n "${VERTEXAI_CREDENTIALS:-}" ]]; then printf '%s' "${VERTEXAI_CREDENTIALS}" > "${STACK_DIR}/vertex-adc.json" @@ -132,6 +136,8 @@ fi cd "${REPO_ROOT}" +env "${SERVER_ENV[@]}" "E2E_KEYCLOAK_PORT=${KEYCLOAK_PORT}" bash .github/e2e-stack/start-idp.sh + log "running migrations" env "${SERVER_ENV[@]}" uv run --no-sync python migrations/run.py >"${LOGS_DIR}/migrations.log" 2>&1 @@ -200,6 +206,9 @@ LITELLM_MASTER_KEY=${MASTER_KEY} REDIS_HOST=127.0.0.1 REDIS_PORT=${REDIS_PORT} E2E_OTEL_QUERY_URL=http://127.0.0.1:${JAEGER_QUERY_PORT} +E2E_KEYCLOAK_URL=http://127.0.0.1:${KEYCLOAK_PORT} +E2E_KEYCLOAK_ADMIN_USER=admin +E2E_KEYCLOAK_ADMIN_PASSWORD=e2e-ephemeral-idp-not-a-secret SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem DATABASE_URL=postgresql://${DATABASE_USER}:${DATABASE_PASSWORD}@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_NAME} EOF diff --git a/.github/workflows/test-e2e-changed.yml b/.github/workflows/test-e2e-changed.yml index 23ab6dfcfe4..1db597ff673 100644 --- a/.github/workflows/test-e2e-changed.yml +++ b/.github/workflows/test-e2e-changed.yml @@ -27,6 +27,8 @@ jobs: sparse-checkout: | .github/e2e-stack tests/e2e/access_control + tests/e2e/management/test_jwt_management_e2e.py + tests/e2e/other/test_jwt_auth_e2e.py persist-credentials: false ref: ${{ github.sha }} @@ -45,7 +47,8 @@ jobs: --jq '.[] | select(.status != "removed") | .filename')" gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.head.sha' | grep -Fxq "${HEAD_SHA}" tests="$(printf '%s\n' "${files}" \ - | python3 .github/e2e-stack/select_tests.py tests/e2e/access_control/test_*.py)" + | python3 .github/e2e-stack/select_tests.py tests/e2e/access_control/test_*.py \ + tests/e2e/management/test_jwt_management_e2e.py tests/e2e/other/test_jwt_auth_e2e.py)" echo "tests=${tests}" >> "${GITHUB_OUTPUT}" if [ -n "${tests}" ]; then echo "any=true" >> "${GITHUB_OUTPUT}" diff --git a/tests/code_coverage_tests/test_e2e_changed_gate.py b/tests/code_coverage_tests/test_e2e_changed_gate.py index d2b842364a6..588402e3996 100644 --- a/tests/code_coverage_tests/test_e2e_changed_gate.py +++ b/tests/code_coverage_tests/test_e2e_changed_gate.py @@ -144,6 +144,8 @@ def test_changed_suite_files_are_selected_unless_the_stack_cannot_run_them( "tests/e2e/pytest.ini", "tests/e2e/gateway/stage_mirror_ci_config.yml", ".github/e2e-stack/up.sh", + ".github/e2e-stack/start-idp.sh", + "tests/e2e/idp_realm.json", ".github/workflows/test-e2e-changed.yml", ), ) diff --git a/tests/code_coverage_tests/test_e2e_idp_stack.py b/tests/code_coverage_tests/test_e2e_idp_stack.py new file mode 100644 index 00000000000..3b994fe521f --- /dev/null +++ b/tests/code_coverage_tests/test_e2e_idp_stack.py @@ -0,0 +1,106 @@ +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[2] +START_IDP = ROOT / ".github/e2e-stack/start-idp.sh" + + +def run_start(tmp_path: Path, *, platform: str = "Linux", failure: str = ""): + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + calls = tmp_path / "docker.jsonl" + programs = { + "docker": """import json, os, sys +with open(os.environ['DOCKER_LOG'], 'a') as out: + out.write(json.dumps(sys.argv[1:]) + '\\n') +if os.environ['FAILURE'] == 'schema' and 'psql' in sys.argv: + sys.exit(17) +if os.environ['FAILURE'] == 'launch' and '--name' in sys.argv: + sys.exit(18) +""", + "curl": "import os, sys; sys.exit(1 if os.environ['FAILURE'] == 'readiness' else 0)\n", + "uname": "import os; print(os.environ['PLATFORM'])\n", + } + for name, source in programs.items(): + program = bin_dir / name + program.write_text(f"#!{sys.executable}\n{source}") + program.chmod(0o755) + result = subprocess.run( + ["bash", str(START_IDP)], + env={ + **os.environ, + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "DOCKER_LOG": str(calls), + "PLATFORM": platform, + "FAILURE": failure, + "DATABASE_HOST": "127.0.0.1", + "DATABASE_PORT": "5544", + "DATABASE_USER": "fixture_user", + "DATABASE_PASSWORD": "fixture_password", + "DATABASE_NAME": "fixture_db", + "E2E_KEYCLOAK_PORT": "8181", + "E2E_KEYCLOAK_STARTUP_TIMEOUT": "0", + }, + capture_output=True, + text=True, + timeout=10, + ) + return result, [json.loads(line) for line in calls.read_text().splitlines()] + + +@pytest.mark.parametrize("platform", ("Linux", "Darwin")) +def test_idp_uses_existing_database_and_imports_runner_realm(tmp_path: Path, platform: str) -> None: + result, calls = run_start(tmp_path, platform=platform) + + assert result.returncode == 0, result.stderr + schema, _, launch = calls + host = "127.0.0.1" if platform == "Linux" else "host.docker.internal" + assert schema[schema.index("-h") + 1] == host + assert schema[schema.index("-p") + 1] == "5544" + assert "ON_ERROR_STOP=1" in schema + assert "CREATE SCHEMA IF NOT EXISTS keycloak" in schema + assert f"KC_DB_URL_HOST={host}" in launch + assert "KC_DB_URL_PORT=5544" in launch + assert "KC_DB_SCHEMA=keycloak" in launch + assert "KC_DB_POOL_MAX_SIZE=10" in launch + assert f"{ROOT}/tests/e2e/idp_realm.json:/opt/keycloak/data/import/realm.json:ro" in launch + assert "KC_HTTP_PORT=8181" in launch + if platform == "Linux": + assert launch[launch.index("--network") + 1] == "host" + else: + assert launch[launch.index("-p") + 1] == "127.0.0.1:8181:8181" + assert "Keycloak realm is up" in result.stdout + + +@pytest.mark.parametrize(("failure", "code"), (("schema", 17), ("launch", 18), ("readiness", 1))) +def test_idp_failure_stops_stack_startup(tmp_path: Path, failure: str, code: int) -> None: + result, calls = run_start(tmp_path, failure=failure) + + assert result.returncode == code + assert "Keycloak realm is up" not in result.stdout + if failure == "schema": + assert len(calls) == 1, "do not replace an IdP when its database is unavailable" + + +def test_proxy_and_test_runner_share_the_same_jwt_configuration() -> None: + config = yaml.safe_load((ROOT / "tests/e2e/gateway/stage_mirror_ci_config.yml").read_text()) + general = config["general_settings"] + assert general["enable_jwt_auth"] is True + assert general["litellm_jwtauth"] == { + "user_id_jwt_field": "sub", + "user_email_jwt_field": "email", + "team_ids_jwt_field": "groups", + "user_id_upsert": True, + } + up = (START_IDP.parent / "up.sh").read_text() + assert '"JWT_ISSUER=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e"' in up + assert '"JWT_AUDIENCE=litellm-e2e"' in up + assert "E2E_KEYCLOAK_URL=http://127.0.0.1:${KEYCLOAK_PORT}" in up + assert up.index("bash .github/e2e-stack/start-idp.sh") < up.index("start_server backend") + assert "e2e-keycloak" in (START_IDP.parent / "down.sh").read_text() diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 53fcdbfe381..948ad819a8c 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -69,7 +69,7 @@ The suites run against a live proxy, so bring one up first by running the litell uv run pytest tests/e2e/other/test_jwt_auth_e2e.py tests/e2e/management/test_jwt_management_e2e.py --reruns 0 -v ``` - CI runs this suite against a Keycloak deployed beside the ephemeral stack, and that deployment (the JWT config block, `JWT_PUBLIC_KEY_URL`, and the admin credential handed to the run pod) lives in the project-releaser repo, not here. CI fetches the realm from the test-runner revision even when it reuses a gateway image from another commit. Keycloak stores its realm, keys and users in a separate schema in the build's PostgreSQL, so replacing the IdP pod preserves token validity. Its startup probe waits for the imported realm. Losing the whole ephemeral database invalidates the stack. Keycloak skips imports into an existing realm, so changes to the realm export require a fresh stack (or deliberately replacing the local data volume). A stack without it fails the JWT tests rather than skipping them + Buildkite runs this suite against a Keycloak deployed beside the ephemeral stack by project-releaser. It fetches the realm from the test-runner revision even when it reuses a gateway image from another commit. The GitHub Actions changed-test stack starts the same digest-pinned Keycloak through `.github/e2e-stack/start-idp.sh`, imports the checked-out realm, and exports the IdP URL and credentials in `stack.env`. Both runners configure issuer/audience validation and store the realm, keys and users in a separate schema in the stack's PostgreSQL, so replacing Keycloak preserves token validity. Both wait for realm discovery before running tests. Losing the whole ephemeral database invalidates the stack. Keycloak skips imports into an existing realm, so changes to the realm export require a fresh stack (or deliberately replacing the local data volume). A stack without it fails the JWT tests rather than skipping them 4. Run a suite against it; the harness reads `LITELLM_PROXY_URL` (default `http://localhost:4000`): @@ -101,7 +101,7 @@ A couple of logging destinations are configured on the proxy rather than by the ### The pull request check -Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite as a canary, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Jaeger, and TLS cluster-mode Valkey. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start +Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch diff --git a/tests/e2e/gateway/stage_mirror_ci_config.yml b/tests/e2e/gateway/stage_mirror_ci_config.yml index 229e8514dee..8c8e64443cb 100644 --- a/tests/e2e/gateway/stage_mirror_ci_config.yml +++ b/tests/e2e/gateway/stage_mirror_ci_config.yml @@ -1,4 +1,11 @@ general_settings: + proxy_batch_write_at: 5 + enable_jwt_auth: true + litellm_jwtauth: + user_id_jwt_field: sub + user_email_jwt_field: email + team_ids_jwt_field: groups + user_id_upsert: true proxy_config_reload_interval_seconds: 7 store_prompts_in_spend_logs: true database_connection_pool_limit: 10 From ec64005ff7a5feeaa75fdd0ac17092e6ad5702c8 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 11 Sep 2026 17:17:03 -0700 Subject: [PATCH 6/7] ci: run the JWT stack startup checks --- .github/workflows/test-code-quality.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-code-quality.yml b/.github/workflows/test-code-quality.yml index e6d2264fbf0..9c7e0db7065 100644 --- a/.github/workflows/test-code-quality.yml +++ b/.github/workflows/test-code-quality.yml @@ -81,7 +81,7 @@ jobs: run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_workflow_job_name_collisions.py - name: test_e2e_changed_gate - run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py + run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py tests/code_coverage_tests/test_e2e_idp_stack.py - name: router_code_coverage run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py From 90bf2911dcf739337cb4801e0de73bdcb7a4220e Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 11 Sep 2026 17:25:38 -0700 Subject: [PATCH 7/7] test(e2e): verify IdP readiness through real HTTP --- .../code_coverage_tests/test_e2e_idp_stack.py | 47 +++++++++++-------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/tests/code_coverage_tests/test_e2e_idp_stack.py b/tests/code_coverage_tests/test_e2e_idp_stack.py index 3b994fe521f..93596af915b 100644 --- a/tests/code_coverage_tests/test_e2e_idp_stack.py +++ b/tests/code_coverage_tests/test_e2e_idp_stack.py @@ -2,16 +2,17 @@ import json import os import subprocess import sys +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path +from threading import Thread import pytest -import yaml ROOT = Path(__file__).resolve().parents[2] START_IDP = ROOT / ".github/e2e-stack/start-idp.sh" -def run_start(tmp_path: Path, *, platform: str = "Linux", failure: str = ""): +def run_start(tmp_path: Path, *, platform: str = "Linux", failure: str = "", port: int = 8181, real_curl: bool = False): bin_dir = tmp_path / "bin" bin_dir.mkdir() calls = tmp_path / "docker.jsonl" @@ -27,6 +28,8 @@ if os.environ['FAILURE'] == 'launch' and '--name' in sys.argv: "curl": "import os, sys; sys.exit(1 if os.environ['FAILURE'] == 'readiness' else 0)\n", "uname": "import os; print(os.environ['PLATFORM'])\n", } + if real_curl: + del programs["curl"] for name, source in programs.items(): program = bin_dir / name program.write_text(f"#!{sys.executable}\n{source}") @@ -44,7 +47,7 @@ if os.environ['FAILURE'] == 'launch' and '--name' in sys.argv: "DATABASE_USER": "fixture_user", "DATABASE_PASSWORD": "fixture_password", "DATABASE_NAME": "fixture_db", - "E2E_KEYCLOAK_PORT": "8181", + "E2E_KEYCLOAK_PORT": str(port), "E2E_KEYCLOAK_STARTUP_TIMEOUT": "0", }, capture_output=True, @@ -88,19 +91,25 @@ def test_idp_failure_stops_stack_startup(tmp_path: Path, failure: str, code: int assert len(calls) == 1, "do not replace an IdP when its database is unavailable" -def test_proxy_and_test_runner_share_the_same_jwt_configuration() -> None: - config = yaml.safe_load((ROOT / "tests/e2e/gateway/stage_mirror_ci_config.yml").read_text()) - general = config["general_settings"] - assert general["enable_jwt_auth"] is True - assert general["litellm_jwtauth"] == { - "user_id_jwt_field": "sub", - "user_email_jwt_field": "email", - "team_ids_jwt_field": "groups", - "user_id_upsert": True, - } - up = (START_IDP.parent / "up.sh").read_text() - assert '"JWT_ISSUER=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e"' in up - assert '"JWT_AUDIENCE=litellm-e2e"' in up - assert "E2E_KEYCLOAK_URL=http://127.0.0.1:${KEYCLOAK_PORT}" in up - assert up.index("bash .github/e2e-stack/start-idp.sh") < up.index("start_server backend") - assert "e2e-keycloak" in (START_IDP.parent / "down.sh").read_text() +def test_readiness_requires_the_imported_realm_on_the_configured_port(tmp_path: Path) -> None: + expected_path = "/realms/litellm-e2e/.well-known/openid-configuration" + observed_paths: list[str] = [] + + class Discovery(BaseHTTPRequestHandler): + def do_GET(self) -> None: + observed_paths.append(self.path) + self.send_response(200 if self.path == expected_path else 404) + self.end_headers() + + with ThreadingHTTPServer(("127.0.0.1", 0), Discovery) as server: + worker = Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + result, _ = run_start(tmp_path, port=server.server_port, real_curl=True) + finally: + server.shutdown() + worker.join(timeout=5) + + assert result.returncode == 0, result.stderr + assert observed_paths == [expected_path] + assert "Keycloak realm is up" in result.stdout