diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 2ae01823a96..b8633979854 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -48,7 +48,21 @@ jobs: const cosignSection = [ `## Verify Docker Image Signature`, ``, - `All LiteLLM Docker images are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). To verify the integrity of an image before deploying:`, + `All LiteLLM Docker images are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit \`0112e53\`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).`, + ``, + `**Verify using the pinned commit hash (recommended):**`, + ``, + `A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:`, + ``, + '```bash', + `cosign verify \\`, + ` --key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \\`, + ` ghcr.io/berriai/litellm:${tag}`, + '```', + ``, + `**Verify using the release tag (convenience):**`, + ``, + `Tags are protected in this repository and resolve to the same key. This option is easier to read but relies on tag protection rules:`, ``, '```bash', `cosign verify \\`, diff --git a/README.md b/README.md index 7d910617f7e..d7b8bad69f3 100644 --- a/README.md +++ b/README.md @@ -404,6 +404,32 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature 2. Install dependencies `npm install` 3. Run `npm run dev` to start the dashboard +# Verify Docker Image Signatures + +All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0). + +**Verify using the pinned commit hash (recommended):** + +A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key: + +```bash +cosign verify \ + --key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \ + ghcr.io/berriai/litellm: +``` + +**Verify using a release tag (convenience):** + +Tags are protected in this repository and resolve to the same key. This option is easier to read but relies on tag protection rules: + +```bash +cosign verify \ + --key https://raw.githubusercontent.com/BerriAI/litellm//cosign.pub \ + ghcr.io/berriai/litellm: +``` + +Replace `` with the version you are deploying (e.g. `v1.83.0-stable`). + # Enterprise For companies that need better security, user management and professional support diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 8e911e95ffa..f3c7728146d 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -41,22 +41,24 @@ COPY . . ENV LITELLM_NON_ROOT=true # Build Admin UI using the upstream command order while keeping a single RUN layer -# NOTE: .npmrc (which has ignore-scripts=true and min-release-age=3d) is temporarily -# renamed during npm install/ci. This is safe because npm ci installs from +# NOTE: .npmrc files (which may set ignore-scripts=true and min-release-age=3d) +# are temporarily renamed during npm install/ci so they don't block lifecycle +# scripts needed by the build. This is safe because npm ci installs from # package-lock.json with pinned versions + integrity hashes. RUN mkdir -p /var/lib/litellm/ui && \ - mv /app/.npmrc /app/.npmrc.bak && \ + ([ -f /app/.npmrc ] && mv /app/.npmrc /app/.npmrc.bak || true) && \ npm install -g npm@11.12.1 && \ npm install -g node-gyp@12.2.0 && \ - ln -sf /usr/local/lib/node_modules/node-gyp /usr/lib/node_modules/npm/node_modules/node-gyp && \ + ln -sf "$(npm root -g)/node-gyp" "$(npm root -g)/npm/node_modules/node-gyp" && \ npm cache clean --force && \ cd /app/ui/litellm-dashboard && \ if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \ cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \ fi && \ - mv .npmrc .npmrc.bak && \ + ([ -f .npmrc ] && mv .npmrc .npmrc.bak || true) && \ npm ci && \ - mv .npmrc.bak .npmrc && mv /app/.npmrc.bak /app/.npmrc && \ + ([ -f .npmrc.bak ] && mv .npmrc.bak .npmrc || true) && \ + ([ -f /app/.npmrc.bak ] && mv /app/.npmrc.bak /app/.npmrc || true) && \ npm run build && \ cp -r /app/ui/litellm-dashboard/out/* /var/lib/litellm/ui/ && \ mkdir -p /var/lib/litellm/assets && \ diff --git a/docs/my-website/blog/ci_cd_v2_improvements/index.md b/docs/my-website/blog/ci_cd_v2_improvements/index.md index fb9a6609c95..85581143969 100644 --- a/docs/my-website/blog/ci_cd_v2_improvements/index.md +++ b/docs/my-website/blog/ci_cd_v2_improvements/index.md @@ -31,7 +31,21 @@ Building on the roadmap from our [security incident](https://docs.litellm.ai/blo ## Verify Docker image signatures -Starting from `v1.83.0-nightly`, all LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). To verify the integrity of an image before deploying: +Starting from `v1.83.0-nightly`, all LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0). + +**Verify using the pinned commit hash (recommended):** + +A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key: + +```bash +cosign verify \ + --key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \ + ghcr.io/berriai/litellm: +``` + +**Verify using a release tag (convenience):** + +Tags are protected in this repository and resolve to the same key. This option is easier to read but relies on tag protection rules: ```bash cosign verify \ diff --git a/docs/my-website/blog/security_townhall_updates/index.md b/docs/my-website/blog/security_townhall_updates/index.md index 633e97df3e7..39db096c533 100644 --- a/docs/my-website/blog/security_townhall_updates/index.md +++ b/docs/my-website/blog/security_townhall_updates/index.md @@ -147,7 +147,21 @@ We believe that [Cosign](https://github.com/sigstore/cosign) is a good fit for t #### How to verify a Docker image with Cosign -Starting from `v1.83.0-nightly`, all LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). To verify the integrity of an image before deploying: +Starting from `v1.83.0-nightly`, all LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key that was introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0). + +**Verify using the pinned commit hash (recommended):** + +A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key: + +```bash +cosign verify \ + --key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \ + ghcr.io/berriai/litellm: +``` + +**Verify using a release tag (convenience):** + +Tags are protected in this repository and resolve to the same key. This option is easier to read but relies on tag protection rules: ```bash cosign verify \ diff --git a/docs/my-website/blog/security_update_march_2026/index.md b/docs/my-website/blog/security_update_march_2026/index.md index 628e26f0c65..6e7b77d1e40 100644 --- a/docs/my-website/blog/security_update_march_2026/index.md +++ b/docs/my-website/blog/security_update_march_2026/index.md @@ -710,7 +710,21 @@ The LiteLLM AI Gateway team has already taken the following steps: ## Verify Docker image signatures -Starting from `v1.83.0-nightly`, all LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). To verify the integrity of an image before deploying: +Starting from `v1.83.0-nightly`, all LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0). + +**Verify using the pinned commit hash (recommended):** + +A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key: + +```bash +cosign verify \ + --key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \ + ghcr.io/berriai/litellm: +``` + +**Verify using a release tag (convenience):** + +Tags are protected in this repository and resolve to the same key. This option is easier to read but relies on tag protection rules: ```bash cosign verify \ diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index cceab822505..ce840ffcf3b 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -238,7 +238,7 @@ router_settings: | public_routes | List[str] | (Enterprise Feature) Control list of public routes | | alert_types | List[str] | Control list of alert types to send to slack (Doc on alert types)[./alerting.md] | | enforced_params | List[str] | (Enterprise Feature) List of params that must be included in all requests to the proxy | -| enable_oauth2_auth | boolean | (Enterprise Feature) If true, enables oauth2.0 authentication | +| enable_oauth2_auth | boolean | (Enterprise Feature) If true, enables oauth2.0 authentication on LLM + info routes | | use_x_forwarded_for | str | If true, uses the X-Forwarded-For header to get the client IP address | | service_account_settings | List[Dict[str, Any]] | Set `service_account_settings` if you want to create settings that only apply to service account keys (Doc on service accounts)[./service_accounts.md] | | image_generation_model | str | The default model to use for image generation - ignores model set in request | diff --git a/docs/my-website/docs/proxy/deploy.md b/docs/my-website/docs/proxy/deploy.md index d4f02afbb13..4b087afd841 100644 --- a/docs/my-website/docs/proxy/deploy.md +++ b/docs/my-website/docs/proxy/deploy.md @@ -67,7 +67,21 @@ docker compose up ### Verify Docker image signatures -All LiteLLM Docker images are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). You can verify the integrity of an image before deploying: +All LiteLLM Docker images are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0). + +**Verify using the pinned commit hash (recommended):** + +A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key: + +```bash +cosign verify \ + --key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \ + ghcr.io/berriai/litellm: +``` + +**Verify using a release tag (convenience):** + +Tags are protected in this repository and resolve to the same key. This option is easier to read but relies on tag protection rules: ```bash cosign verify \ diff --git a/docs/my-website/docs/proxy/oauth2.md b/docs/my-website/docs/proxy/oauth2.md index c0597058cfd..9b94a017ca1 100644 --- a/docs/my-website/docs/proxy/oauth2.md +++ b/docs/my-website/docs/proxy/oauth2.md @@ -63,16 +63,19 @@ Start the LiteLLM Proxy with [`--detailed_debug` mode and you should see more ve ## Using OAuth2 + JWT Together -If both `enable_oauth2_auth` and `enable_jwt_auth` are enabled, LiteLLM can split auth paths: -- JWT validation for user tokens -- OAuth2 introspection for machine tokens +LiteLLM supports two OAuth2 + JWT modes: -For JWT-shaped machine tokens, configure `litellm_jwtauth.routing_overrides`: +1. **Global OAuth2 mode** (`enable_oauth2_auth: true`) + OAuth2 auth is enabled on LLM + info routes. +2. **Selective JWT override mode** (`enable_oauth2_auth: false`) + Only JWT-shaped tokens that match `litellm_jwtauth.routing_overrides` are routed to OAuth2 on LLM + info routes. + +For selective routing (OAuth2 only for specific JWTs), configure: ```yaml title="config.yaml" general_settings: enable_jwt_auth: true - enable_oauth2_auth: true + enable_oauth2_auth: false litellm_jwtauth: routing_overrides: - iss: "machine-issuer.example.com" diff --git a/docs/my-website/docs/proxy/token_auth.md b/docs/my-website/docs/proxy/token_auth.md index d37b05391b6..4d49a2445ef 100644 --- a/docs/my-website/docs/proxy/token_auth.md +++ b/docs/my-website/docs/proxy/token_auth.md @@ -792,16 +792,18 @@ litellm_jwtauth: ## Route JWT-Shaped Machine Tokens to OAuth2 -Use this when both are enabled: +Use this when: - `enable_jwt_auth: true` for standard JWT validation -- `enable_oauth2_auth: true` for OAuth2 introspection +- machine tokens are JWT-shaped and should be routed to OAuth2 based on claims -If some machine tokens are also JWT-shaped, configure `routing_overrides` to route matching tokens to OAuth2. +`routing_overrides` supports two operating modes: +- **Selective mode**: set `enable_oauth2_auth: false` to send only matching JWTs to OAuth2 on LLM + info routes +- **Global mode**: set `enable_oauth2_auth: true` to also enable OAuth2 on LLM + info routes ```yaml title="config.yaml" general_settings: enable_jwt_auth: true - enable_oauth2_auth: true + enable_oauth2_auth: false litellm_jwtauth: user_id_jwt_field: "sub" routing_overrides: @@ -822,7 +824,7 @@ general_settings: ```yaml title="config.yaml" general_settings: enable_jwt_auth: true - enable_oauth2_auth: true + enable_oauth2_auth: false litellm_jwtauth: routing_overrides: - iss: ["machine-issuer.example.com", "backup-issuer.example.com"] diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 8351f084f28..d781c91992d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -7822,8 +7822,8 @@ "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, + "max_output_tokens": 8192, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, @@ -7838,6 +7838,26 @@ "cache_read_input_token_cost": 3.6e-07, "cache_creation_input_token_cost": 4.5e-06 }, + "bedrock/us-gov-east-1/anthropic.claude-sonnet-4-5-20250929-v1:0": { + "cache_creation_input_token_cost": 4.125e-06, + "cache_read_input_token_cost": 3.3e-07, + "input_cost_per_token": 3.3e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.65e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.65e-06, "litellm_provider": "bedrock", @@ -7973,8 +7993,8 @@ "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, + "max_output_tokens": 8192, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, @@ -7989,6 +8009,26 @@ "cache_read_input_token_cost": 3.6e-07, "cache_creation_input_token_cost": 4.5e-06 }, + "bedrock/us-gov-west-1/anthropic.claude-sonnet-4-5-20250929-v1:0": { + "cache_creation_input_token_cost": 4.125e-06, + "cache_read_input_token_cost": 3.3e-07, + "input_cost_per_token": 3.3e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.65e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.65e-06, "litellm_provider": "bedrock", @@ -28945,6 +28985,32 @@ "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, + "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0": { + "cache_creation_input_token_cost": 4.125e-06, + "cache_read_input_token_cost": 3.3e-07, + "input_cost_per_token": 3.3e-06, + "input_cost_per_token_above_200k_tokens": 6.6e-06, + "output_cost_per_token_above_200k_tokens": 2.475e-05, + "cache_creation_input_token_cost_above_200k_tokens": 8.25e-06, + "cache_read_input_token_cost_above_200k_tokens": 6.6e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 1.65e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346, + "supports_native_structured_output": true + }, "au.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, "cache_read_input_token_cost": 1.1e-07, diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 046c39a9101..61c618eeb18 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -690,42 +690,39 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 ########## End of Route Checks Before Reading DB / Cache for "token" ######## - if general_settings.get("enable_oauth2_auth", False) is True: - # Only apply OAuth2 M2M authentication to LLM API routes and info routes, not UI/management routes - # This allows UI SSO to work separately from API M2M authentication - # Note: Info routes are already scoped to the user - if RouteChecks.is_llm_api_route(route=route) or RouteChecks.is_info_route( - route=route - ): - # When both OAuth2 and JWT auth are enabled, use token format to decide: - # - JWT tokens (3 dot-separated parts) -> skip OAuth2, fall through to JWT handler - # - Opaque tokens -> use OAuth2 handler - # This allows JWT for users and OAuth2 for M2M on the same instance - is_jwt = ( - jwt_handler.is_jwt(token=api_key) - if general_settings.get("enable_jwt_auth", False) is True - else False - ) - # Routing uses unverified JWT claims only to choose auth path. - # Final authentication is enforced by the selected validator. - route_jwt_to_oauth2 = ( - is_jwt - and _should_route_jwt_to_oauth2_override( - token=api_key, jwt_handler=jwt_handler - ) - ) - if not is_jwt or route_jwt_to_oauth2: - # return UserAPIKeyAuth object - # helper to check if the api_key is a valid oauth2 token - from litellm.proxy.proxy_server import premium_user + enable_oauth2_auth = general_settings.get("enable_oauth2_auth", False) is True + enable_jwt_auth = general_settings.get("enable_jwt_auth", False) is True + is_jwt = jwt_handler.is_jwt(token=api_key) if enable_jwt_auth else False - if premium_user is not True: - raise ValueError( - "Oauth2 token validation is only available for premium users" - + CommonProxyErrors.not_premium_user.value - ) + # Routing uses unverified JWT claims only to choose auth path. + # Final authentication is enforced by the selected validator. + route_jwt_to_oauth2 = ( + is_jwt + and _should_route_jwt_to_oauth2_override( + token=api_key, jwt_handler=jwt_handler + ) + ) - return await Oauth2Handler.check_oauth2_token(token=api_key) + # OAuth2 applies for: + # 1) when global OAuth2 auth is enabled on LLM + info routes + # 2) JWT tokens that explicitly match routing_overrides on LLM + info routes + should_apply_override_oauth2 = route_jwt_to_oauth2 and ( + RouteChecks.is_llm_api_route(route=route) + or RouteChecks.is_info_route(route=route) + ) + should_apply_global_oauth2 = enable_oauth2_auth and ( + RouteChecks.is_llm_api_route(route=route) + or RouteChecks.is_info_route(route=route) + ) + if (should_apply_global_oauth2 and not is_jwt) or should_apply_override_oauth2: + from litellm.proxy.proxy_server import premium_user + if premium_user is not True: + raise ValueError( + "Oauth2 token validation is only available for premium users" + + CommonProxyErrors.not_premium_user.value + ) + + return await Oauth2Handler.check_oauth2_token(token=api_key) if general_settings.get("enable_oauth2_proxy_auth", False) is True: return await handle_oauth2_proxy_request(request=request) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 00d8ce182ec..323ff7fd531 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -502,9 +502,7 @@ def _enforce_upperbound_key_params( for elem in data: key, value = elem - upperbound_value = getattr( - litellm.upperbound_key_generate_params, key, None - ) + upperbound_value = getattr(litellm.upperbound_key_generate_params, key, None) if upperbound_value is not None: if value is None: if fill_defaults: @@ -524,9 +522,7 @@ def _enforce_upperbound_key_params( }, ) elif key in ["budget_duration", "duration"]: - upperbound_duration = duration_in_seconds( - duration=upperbound_value - ) + upperbound_duration = duration_in_seconds(duration=upperbound_value) if value == "-1": user_duration = float("inf") else: @@ -1759,9 +1755,7 @@ async def _process_single_key_update( decision = result.get("decision", True) message = result.get("message", "Authentication Failed - Custom Auth Rule") if not decision: - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, detail=message - ) + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=message) # Enforce upperbound key params on update (don't fill defaults) _enforce_upperbound_key_params(update_key_request, fill_defaults=False) @@ -2638,22 +2632,39 @@ async def info_key_fn_v2( detail={"message": "Malformed request. No keys passed in."}, ) - key_info = await prisma_client.get_data( - token=data.keys, table_name="key", query_type="find_all" - ) - if key_info is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail={"message": "No keys found"}, + # Resolve key_aliases to tokens so we never pass token=None (unbounded query) + tokens_to_query = list(data.keys) if data.keys else [] + if data.key_aliases: + alias_rows = await prisma_client.db.litellm_verificationtoken.find_many( + where={"key_alias": {"in": data.key_aliases}}, + include={"litellm_budget_table": True}, ) + alias_tokens = [row.token for row in alias_rows if row.token] + tokens_to_query.extend(alias_tokens) + + if not tokens_to_query: + return {"key": data.keys, "info": []} + + key_info = await prisma_client.get_data( + token=tokens_to_query, table_name="key", query_type="find_all" + ) + if not key_info: + return {"key": data.keys, "info": []} + filtered_key_info = [] for k in key_info: + if not await _can_user_query_key_info( + user_api_key_dict=user_api_key_dict, + key=k.token, + key_info=k, + ): + continue try: - k = k.model_dump() # noqa + k_dict = k.model_dump() except Exception: - # if using pydantic v1 - k = k.dict() - filtered_key_info.append(k) + k_dict = k.dict() + k_dict.pop("token", None) + filtered_key_info.append(k_dict) return {"key": data.keys, "info": filtered_key_info} except Exception as e: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6da9a004b8a..cfdb2911fdf 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -7822,8 +7822,8 @@ "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, + "max_output_tokens": 8192, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, @@ -7838,6 +7838,26 @@ "cache_read_input_token_cost": 3.6e-07, "cache_creation_input_token_cost": 4.5e-06 }, + "bedrock/us-gov-east-1/anthropic.claude-sonnet-4-5-20250929-v1:0": { + "cache_creation_input_token_cost": 4.125e-06, + "cache_read_input_token_cost": 3.3e-07, + "input_cost_per_token": 3.3e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.65e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.65e-06, "litellm_provider": "bedrock", @@ -7973,8 +7993,8 @@ "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, + "max_output_tokens": 8192, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, @@ -7989,6 +8009,26 @@ "cache_read_input_token_cost": 3.6e-07, "cache_creation_input_token_cost": 4.5e-06 }, + "bedrock/us-gov-west-1/anthropic.claude-sonnet-4-5-20250929-v1:0": { + "cache_creation_input_token_cost": 4.125e-06, + "cache_read_input_token_cost": 3.3e-07, + "input_cost_per_token": 3.3e-06, + "litellm_provider": "bedrock", + "max_input_tokens": 200000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.65e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.65e-06, "litellm_provider": "bedrock", @@ -28930,6 +28970,32 @@ "tool_use_system_prompt_tokens": 346, "supports_native_structured_output": true }, + "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0": { + "cache_creation_input_token_cost": 4.125e-06, + "cache_read_input_token_cost": 3.3e-07, + "input_cost_per_token": 3.3e-06, + "input_cost_per_token_above_200k_tokens": 6.6e-06, + "output_cost_per_token_above_200k_tokens": 2.475e-05, + "cache_creation_input_token_cost_above_200k_tokens": 8.25e-06, + "cache_read_input_token_cost_above_200k_tokens": 6.6e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 1.65e-05, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 346, + "supports_native_structured_output": true + }, "au.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, "cache_read_input_token_cost": 1.1e-07, diff --git a/tests/proxy_unit_tests/test_check_responses_cost.py b/tests/proxy_unit_tests/test_check_responses_cost.py index 601df9c4c7f..6b64f52cd78 100644 --- a/tests/proxy_unit_tests/test_check_responses_cost.py +++ b/tests/proxy_unit_tests/test_check_responses_cost.py @@ -47,11 +47,15 @@ class TestCheckResponsesCost: CheckResponsesCost, ) - return CheckResponsesCost( + instance = CheckResponsesCost( proxy_logging_obj=mock_proxy_logging_obj, prisma_client=mock_prisma_client, llm_router=mock_llm_router, ) + # Mock _expire_stale_rows (raw SQL) so _cleanup_stale_managed_objects + # succeeds without a real DB. Individual tests can override this. + instance._expire_stale_rows = AsyncMock(return_value=0) + return instance def test_initialization(self, check_responses_cost_instance): """Test that CheckResponsesCost initializes correctly""" @@ -67,9 +71,6 @@ class TestCheckResponsesCost: mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock( return_value=[] ) - mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock( - return_value=0 - ) await check_responses_cost_instance.check_responses_cost() @@ -86,24 +87,20 @@ class TestCheckResponsesCost: async def test_cleanup_stale_managed_objects( self, check_responses_cost_instance, mock_prisma_client ): - """Stale rows (older than cutoff) are bulk-updated to stale_expired before polling.""" - mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock( - return_value=5 - ) + """Stale rows are expired via _expire_stale_rows before polling.""" + from litellm.constants import STALE_OBJECT_CLEANUP_BATCH_SIZE + + check_responses_cost_instance._expire_stale_rows = AsyncMock(return_value=5) mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock( return_value=[] ) await check_responses_cost_instance.check_responses_cost() - # The first update_many call should be the stale-row cleanup scoped to "response" - calls = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args_list - stale_call = calls[0] - assert stale_call[1]["data"] == {"status": "stale_expired"} - where = stale_call[1]["where"] - assert where["file_purpose"] == "response" - assert "stale_expired" in where["status"]["not_in"] - assert "created_at" in where + # _expire_stale_rows should have been called with a cutoff datetime and batch size + check_responses_cost_instance._expire_stale_rows.assert_called_once() + call_args = check_responses_cost_instance._expire_stale_rows.call_args + assert call_args[0][1] == STALE_OBJECT_CLEANUP_BATCH_SIZE @pytest.mark.asyncio async def test_check_responses_cost_with_completed_response( @@ -145,10 +142,10 @@ class TestCheckResponsesCost: await check_responses_cost_instance.check_responses_cost() - # calls[0] = stale cleanup, calls[1] = job completion + # update_many should only contain the job completion call calls = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args_list - assert len(calls) == 2 - completion_call = calls[1] + assert len(calls) == 1 + completion_call = calls[0] assert completion_call[1]["data"]["status"] == "completed" assert completion_call[1]["where"]["id"]["in"] == ["job-123"] @@ -188,10 +185,10 @@ class TestCheckResponsesCost: await check_responses_cost_instance.check_responses_cost() - # calls[0] = stale cleanup, calls[1] = job completion + # update_many should only contain the job completion call calls = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args_list - assert len(calls) == 2 - assert calls[1][1]["data"]["status"] == "completed" + assert len(calls) == 1 + assert calls[0][1]["data"]["status"] == "completed" @pytest.mark.asyncio async def test_check_responses_cost_with_cancelled_response( @@ -229,10 +226,10 @@ class TestCheckResponsesCost: await check_responses_cost_instance.check_responses_cost() - # calls[0] = stale cleanup, calls[1] = job completion + # update_many should only contain the job completion call calls = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args_list - assert len(calls) == 2 - assert calls[1][1]["data"]["status"] == "completed" + assert len(calls) == 1 + assert calls[0][1]["data"]["status"] == "completed" @pytest.mark.asyncio async def test_check_responses_cost_with_in_progress_response( @@ -270,10 +267,11 @@ class TestCheckResponsesCost: await check_responses_cost_instance.check_responses_cost() - # Only the stale-cleanup call should have fired — no completion update + # No job completion update_many — response is still in progress calls = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args_list - assert len(calls) == 1 - assert calls[0][1]["data"] == {"status": "stale_expired"} + assert len(calls) == 0 + # Stale cleanup still ran via _expire_stale_rows + check_responses_cost_instance._expire_stale_rows.assert_called_once() @pytest.mark.asyncio async def test_check_responses_cost_with_queued_response( @@ -311,10 +309,11 @@ class TestCheckResponsesCost: await check_responses_cost_instance.check_responses_cost() - # Only the stale-cleanup call should have fired — no completion update + # No job completion update_many — response is still queued calls = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args_list - assert len(calls) == 1 - assert calls[0][1]["data"] == {"status": "stale_expired"} + assert len(calls) == 0 + # Stale cleanup still ran via _expire_stale_rows + check_responses_cost_instance._expire_stale_rows.assert_called_once() @pytest.mark.asyncio async def test_check_responses_cost_with_exception( @@ -345,10 +344,11 @@ class TestCheckResponsesCost: # Should not raise, just skip the job await check_responses_cost_instance.check_responses_cost() - # Only the stale-cleanup call should have fired — no completion update + # No job completion update_many — exception skipped the job calls = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args_list - assert len(calls) == 1 - assert calls[0][1]["data"] == {"status": "stale_expired"} + assert len(calls) == 0 + # Stale cleanup still ran via _expire_stale_rows + check_responses_cost_instance._expire_stale_rows.assert_called_once() @pytest.mark.asyncio async def test_check_responses_cost_multiple_jobs( @@ -424,10 +424,10 @@ class TestCheckResponsesCost: await check_responses_cost_instance.check_responses_cost() - # calls[0] = stale cleanup, calls[1] = completion of 2 finished jobs + # update_many should only contain the job completion call calls = mock_prisma_client.db.litellm_managedobjecttable.update_many.call_args_list - assert len(calls) == 2 - completion_call = calls[1] + assert len(calls) == 1 + completion_call = calls[0] assert len(completion_call[1]["where"]["id"]["in"]) == 2 assert "job-1" in completion_call[1]["where"]["id"]["in"] assert "job-3" in completion_call[1]["where"]["id"]["in"] diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index 6e1b245b3de..14912b67ab8 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -713,6 +713,51 @@ class TestJWTOAuth2Coexistence: mock_jwt_auth.assert_not_called() assert result.user_id == "machine-client-1" + @pytest.mark.asyncio + async def test_oauth2_path_requires_premium_user(self): + """ + OAuth2 token validation should fail when enterprise premium is disabled. + """ + opaque_token = "some-opaque-m2m-oauth2-token" + general_settings = { + "enable_oauth2_auth": True, + "enable_jwt_auth": True, + } + + mock_request = MagicMock() + mock_request.url.path = "/v1/chat/completions" + mock_request.headers = {"authorization": f"Bearer {opaque_token}"} + mock_request.query_params = {} + + with patch( + "litellm.proxy.proxy_server.general_settings", general_settings + ), patch("litellm.proxy.proxy_server.premium_user", False), patch( + "litellm.proxy.proxy_server.master_key", "sk-master" + ), patch( + "litellm.proxy.proxy_server.prisma_client", None + ), patch( + "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", + new_callable=AsyncMock, + ) as mock_oauth2: + litellm.proxy.proxy_server.jwt_handler.update_environment( + prisma_client=None, + user_api_key_cache=DualCache(), + litellm_jwtauth=LiteLLM_JWTAuth(), + ) + + with pytest.raises(ProxyException) as exc_info: + await user_api_key_auth( + request=mock_request, + api_key=f"Bearer {opaque_token}", + ) + + assert exc_info.value.type == ProxyErrorTypes.auth_error + assert ( + "Oauth2 token validation is only available for premium users" + in exc_info.value.message + ) + mock_oauth2.assert_not_called() + @pytest.mark.asyncio async def test_both_enabled_jwt_token_skips_oauth2(self): """ @@ -974,6 +1019,248 @@ class TestJWTOAuth2Coexistence: mock_jwt_auth.assert_not_called() assert result.user_id == "machine-client-aud-list" + @pytest.mark.asyncio + async def test_routing_override_routes_jwt_to_oauth2_when_oauth2_globally_disabled( + self, + ): + """ + If enable_oauth2_auth is false, JWT tokens matching routing_overrides + should still route to OAuth2 introspection. + """ + jwt_token = ( + "eyJhbGciOiJSUzI1NiJ9." + "eyJpc3MiOiJtYWNoaW5lLWlzc3Vlci5leGFtcGxlLmNvbSIsImNsaWVudF9pZCI6Ik1JRF9MSVRFTExNIn0." + "c2ln" + ) + general_settings = { + "enable_oauth2_auth": False, + "enable_jwt_auth": True, + } + mock_oauth2_response = UserAPIKeyAuth( + api_key=jwt_token, + user_id="machine-client-override-oauth2-off", + ) + + mock_request = MagicMock() + mock_request.url.path = "/v1/chat/completions" + mock_request.headers = {"authorization": f"Bearer {jwt_token}"} + mock_request.query_params = {} + + with patch( + "litellm.proxy.proxy_server.general_settings", general_settings + ), patch("litellm.proxy.proxy_server.premium_user", True), patch( + "litellm.proxy.proxy_server.master_key", "sk-master" + ), patch( + "litellm.proxy.proxy_server.prisma_client", None + ), patch( + "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", + new_callable=AsyncMock, + return_value=mock_oauth2_response, + ) as mock_oauth2, patch( + "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", + new_callable=AsyncMock, + ) as mock_jwt_auth: + litellm.proxy.proxy_server.jwt_handler.update_environment( + prisma_client=None, + user_api_key_cache=DualCache(), + litellm_jwtauth=LiteLLM_JWTAuth( + routing_overrides=[ + JWTRoutingOverride( + iss="machine-issuer.example.com", + client_id="MID_LITELLM", + path="oauth2", + ) + ] + ), + ) + + result = await user_api_key_auth( + request=mock_request, + api_key=f"Bearer {jwt_token}", + ) + + mock_oauth2.assert_called_once_with(token=jwt_token) + mock_jwt_auth.assert_not_called() + assert result.user_id == "machine-client-override-oauth2-off" + + @pytest.mark.asyncio + async def test_opaque_token_does_not_use_oauth2_when_oauth2_globally_disabled( + self, + ): + """ + With enable_oauth2_auth=false, opaque tokens must not be sent to OAuth2. + """ + opaque_token = "sk-ui-session-token" + general_settings = { + "enable_oauth2_auth": False, + "enable_jwt_auth": True, + } + + mock_request = MagicMock() + mock_request.url.path = "/v1/chat/completions" + mock_request.headers = {"authorization": f"Bearer {opaque_token}"} + mock_request.query_params = {} + + with patch( + "litellm.proxy.proxy_server.general_settings", general_settings + ), patch("litellm.proxy.proxy_server.premium_user", True), patch( + "litellm.proxy.proxy_server.master_key", "sk-master" + ), patch( + "litellm.proxy.proxy_server.prisma_client", None + ), patch( + "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", + new_callable=AsyncMock, + ) as mock_oauth2: + with pytest.raises(ProxyException) as exc_info: + await user_api_key_auth( + request=mock_request, + api_key=f"Bearer {opaque_token}", + ) + + assert exc_info.value.type in ( + ProxyErrorTypes.auth_error, + ProxyErrorTypes.no_db_connection, + ) + mock_oauth2.assert_not_called() + + @pytest.mark.asyncio + async def test_routing_override_on_info_route_uses_oauth2_when_oauth2_globally_disabled( + self, + ): + """ + With enable_oauth2_auth=false, a JWT matching routing_overrides should + still route to OAuth2 on info routes. + """ + jwt_token = ( + "eyJhbGciOiJSUzI1NiJ9." + "eyJpc3MiOiJtYWNoaW5lLWlzc3Vlci5leGFtcGxlLmNvbSIsImNsaWVudF9pZCI6Ik1JRF9MSVRFTExNIn0." + "c2ln" + ) + general_settings = { + "enable_oauth2_auth": False, + "enable_jwt_auth": True, + } + mock_oauth2_response = UserAPIKeyAuth( + api_key=jwt_token, + user_id="machine-client-info-override-oauth2-off", + ) + + mock_request = MagicMock() + mock_request.url.path = "/team/list" + mock_request.headers = {"authorization": f"Bearer {jwt_token}"} + mock_request.query_params = {} + + with patch( + "litellm.proxy.proxy_server.general_settings", general_settings + ), patch("litellm.proxy.proxy_server.premium_user", True), patch( + "litellm.proxy.proxy_server.master_key", "sk-master" + ), patch( + "litellm.proxy.proxy_server.prisma_client", None + ), patch( + "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", + new_callable=AsyncMock, + return_value=mock_oauth2_response, + ) as mock_oauth2, patch( + "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", + new_callable=AsyncMock, + ) as mock_jwt_auth: + litellm.proxy.proxy_server.jwt_handler.update_environment( + prisma_client=None, + user_api_key_cache=DualCache(), + litellm_jwtauth=LiteLLM_JWTAuth( + routing_overrides=[ + JWTRoutingOverride( + iss="machine-issuer.example.com", + client_id="MID_LITELLM", + path="oauth2", + ) + ] + ), + ) + + result = await user_api_key_auth( + request=mock_request, + api_key=f"Bearer {jwt_token}", + ) + + mock_oauth2.assert_called_once_with(token=jwt_token) + mock_jwt_auth.assert_not_called() + assert result.user_id == "machine-client-info-override-oauth2-off" + + @pytest.mark.asyncio + async def test_routing_override_on_management_route_does_not_use_oauth2(self): + """ + JWT routing_overrides should not force OAuth2 on management routes. + """ + jwt_token = ( + "eyJhbGciOiJSUzI1NiJ9." + "eyJpc3MiOiJtYWNoaW5lLWlzc3Vlci5leGFtcGxlLmNvbSIsImNsaWVudF9pZCI6Ik1JRF9MSVRFTExNIn0." + "c2ln" + ) + general_settings = { + "enable_oauth2_auth": False, + "enable_jwt_auth": True, + } + mock_jwt_result = { + "is_proxy_admin": True, + "team_object": None, + "user_object": None, + "end_user_object": None, + "org_object": None, + "token": jwt_token, + "team_id": None, + "user_id": "jwt-admin-user", + "end_user_id": None, + "org_id": None, + "team_membership": None, + "jwt_claims": { + "iss": "machine-issuer.example.com", + "client_id": "MID_LITELLM", + }, + } + + mock_request = MagicMock() + mock_request.url.path = "/key/generate" + mock_request.headers = {"authorization": f"Bearer {jwt_token}"} + mock_request.query_params = {} + + with patch( + "litellm.proxy.proxy_server.general_settings", general_settings + ), patch("litellm.proxy.proxy_server.premium_user", True), patch( + "litellm.proxy.proxy_server.master_key", "sk-master" + ), patch( + "litellm.proxy.proxy_server.prisma_client", None + ), patch( + "litellm.proxy.auth.user_api_key_auth.Oauth2Handler.check_oauth2_token", + new_callable=AsyncMock, + ) as mock_oauth2, patch( + "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", + new_callable=AsyncMock, + return_value=mock_jwt_result, + ) as mock_jwt_auth: + litellm.proxy.proxy_server.jwt_handler.update_environment( + prisma_client=None, + user_api_key_cache=DualCache(), + litellm_jwtauth=LiteLLM_JWTAuth( + routing_overrides=[ + JWTRoutingOverride( + iss="machine-issuer.example.com", + client_id="MID_LITELLM", + path="oauth2", + ) + ] + ), + ) + + result = await user_api_key_auth( + request=mock_request, + api_key=f"Bearer {jwt_token}", + ) + + mock_oauth2.assert_not_called() + mock_jwt_auth.assert_called_once() + assert result.user_id == "jwt-admin-user" + @pytest.mark.asyncio async def test_only_oauth2_enabled_handles_all_tokens(self): """ diff --git a/tests/ui_e2e_tests/constants.ts b/tests/ui_e2e_tests/constants.ts new file mode 100644 index 00000000000..484cde30fc6 --- /dev/null +++ b/tests/ui_e2e_tests/constants.ts @@ -0,0 +1,40 @@ +export const ADMIN_STORAGE_PATH = "admin.storageState.json"; + +// Page enum — maps to ?page= query parameter values in the UI +export enum Page { + ApiKeys = "api-keys", + Teams = "teams", + AdminSettings = "settings", +} + +// Test user credentials — all users have password "test" (hashed in seed.sql) +export enum Role { + ProxyAdmin = "proxy_admin", + ProxyAdminViewer = "proxy_admin_viewer", + InternalUser = "internal_user", + InternalUserViewer = "internal_user_viewer", + TeamAdmin = "team_admin", +} + +export const users: Record = { + [Role.ProxyAdmin]: { + email: "admin", + password: process.env.LITELLM_MASTER_KEY || "sk-1234", + }, + [Role.ProxyAdminViewer]: { + email: "adminviewer@test.local", + password: "test", + }, + [Role.InternalUser]: { + email: "internal@test.local", + password: "test", + }, + [Role.InternalUserViewer]: { + email: "viewer@test.local", + password: "test", + }, + [Role.TeamAdmin]: { + email: "teamadmin@test.local", + password: "test", + }, +}; diff --git a/tests/ui_e2e_tests/fixtures/config.yml b/tests/ui_e2e_tests/fixtures/config.yml new file mode 100644 index 00000000000..438c236b03b --- /dev/null +++ b/tests/ui_e2e_tests/fixtures/config.yml @@ -0,0 +1,16 @@ +model_list: + - model_name: fake-openai-gpt-4 + litellm_params: + model: openai/fake-gpt-4 + api_base: os.environ/MOCK_LLM_URL + api_key: fake-key + - model_name: fake-anthropic-claude + litellm_params: + model: openai/fake-claude + api_base: os.environ/MOCK_LLM_URL + api_key: fake-key + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + database_url: os.environ/DATABASE_URL + store_prompts_in_spend_logs: true diff --git a/tests/ui_e2e_tests/fixtures/mock_llm_server/server.py b/tests/ui_e2e_tests/fixtures/mock_llm_server/server.py new file mode 100644 index 00000000000..7a0699016aa --- /dev/null +++ b/tests/ui_e2e_tests/fixtures/mock_llm_server/server.py @@ -0,0 +1,118 @@ +""" +Mock LLM server for UI e2e tests. +Responds to OpenAI-format endpoints with canned responses. +""" + +import time +import json +import uuid + +import uvicorn +from fastapi import FastAPI, Request +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import StreamingResponse + + +app = FastAPI(title="Mock LLM Server") +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"], +) + + +@app.get("/health") +async def health(): + return {"status": "ok"} + + +@app.get("/v1/models") +@app.get("/models") +async def list_models(): + return { + "object": "list", + "data": [ + {"id": "fake-gpt-4", "object": "model", "owned_by": "mock"}, + {"id": "fake-claude", "object": "model", "owned_by": "mock"}, + ], + } + + +@app.post("/v1/chat/completions") +@app.post("/chat/completions") +async def chat_completions(request: Request): + body = await request.json() + model = body.get("model", "mock-model") + stream = body.get("stream", False) + + response_id = f"chatcmpl-{uuid.uuid4().hex[:12]}" + created = int(time.time()) + + if stream: + async def stream_generator(): + chunk = { + "id": response_id, + "object": "chat.completion.chunk", + "created": created, + "model": model, + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "This is a mock response."}, + "finish_reason": None, + } + ], + } + yield f"data: {json.dumps(chunk)}\n\n" + + done_chunk = { + "id": response_id, + "object": "chat.completion.chunk", + "created": created, + "model": model, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + } + yield f"data: {json.dumps(done_chunk)}\n\n" + yield "data: [DONE]\n\n" + + return StreamingResponse( + stream_generator(), media_type="text/event-stream" + ) + + return { + "id": response_id, + "object": "chat.completion", + "created": created, + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "This is a mock response."}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 8, "total_tokens": 18}, + } + + +@app.post("/v1/embeddings") +@app.post("/embeddings") +async def embeddings(request: Request): + body = await request.json() + inputs = body.get("input", [""]) + if isinstance(inputs, str): + inputs = [inputs] + return { + "object": "list", + "data": [ + {"object": "embedding", "index": i, "embedding": [0.0] * 1536} + for i in range(len(inputs)) + ], + "model": body.get("model", "mock-embedding"), + "usage": {"prompt_tokens": 5, "total_tokens": 5}, + } + + +if __name__ == "__main__": + uvicorn.run(app, host="127.0.0.1", port=8090) diff --git a/tests/ui_e2e_tests/fixtures/seed.sql b/tests/ui_e2e_tests/fixtures/seed.sql new file mode 100644 index 00000000000..f271a748177 --- /dev/null +++ b/tests/ui_e2e_tests/fixtures/seed.sql @@ -0,0 +1,103 @@ +-- UI E2E Test Database Seed +-- Run with: psql $DATABASE_URL -f seed.sql + +-- ============================================================ +-- 1. Budget Table (must be first — referenced by org FK) +-- ============================================================ +INSERT INTO "LiteLLM_BudgetTable" ( + budget_id, max_budget, created_by, updated_by +) VALUES ( + 'e2e-budget-org', 1000.0, 'e2e-proxy-admin', 'e2e-proxy-admin' +) ON CONFLICT (budget_id) DO NOTHING; + +-- ============================================================ +-- 2. Organization +-- ============================================================ +INSERT INTO "LiteLLM_OrganizationTable" ( + organization_id, organization_alias, budget_id, metadata, models, spend, + model_spend, created_by, updated_by +) VALUES ( + 'e2e-org-main', 'E2E Organization', 'e2e-budget-org', '{}'::jsonb, + ARRAY[]::text[], 0.0, '{}'::jsonb, 'e2e-proxy-admin', 'e2e-proxy-admin' +) ON CONFLICT (organization_id) DO NOTHING; + +-- ============================================================ +-- 3. Users (password is scrypt hash of "test") +-- ============================================================ +INSERT INTO "LiteLLM_UserTable" ( + user_id, user_email, user_role, password, teams, models, metadata, + spend, model_spend, model_max_budget +) VALUES +( + 'e2e-proxy-admin', 'admin@test.local', 'proxy_admin', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr', + ARRAY['e2e-team-crud']::text[], ARRAY[]::text[], '{}'::jsonb, + 0.0, '{}'::jsonb, '{}'::jsonb +), +( + 'e2e-admin-viewer', 'adminviewer@test.local', 'proxy_admin_viewer', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr', + ARRAY[]::text[], ARRAY[]::text[], '{}'::jsonb, + 0.0, '{}'::jsonb, '{}'::jsonb +), +( + 'e2e-internal-user', 'internal@test.local', 'internal_user', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr', + ARRAY['e2e-team-crud', 'e2e-team-org']::text[], ARRAY[]::text[], '{}'::jsonb, + 0.0, '{}'::jsonb, '{}'::jsonb +), +( + 'e2e-internal-viewer', 'viewer@test.local', 'internal_user_viewer', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr', + ARRAY[]::text[], ARRAY[]::text[], '{}'::jsonb, + 0.0, '{}'::jsonb, '{}'::jsonb +), +( + 'e2e-team-admin', 'teamadmin@test.local', 'internal_user', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr', + ARRAY['e2e-team-crud', 'e2e-team-delete']::text[], ARRAY[]::text[], '{}'::jsonb, + 0.0, '{}'::jsonb, '{}'::jsonb +) +ON CONFLICT (user_id) DO NOTHING; + +-- ============================================================ +-- 4. Teams +-- ============================================================ +INSERT INTO "LiteLLM_TeamTable" ( + team_id, team_alias, organization_id, admins, members, + members_with_roles, metadata, models, spend, model_spend, + model_max_budget, blocked +) VALUES +( + 'e2e-team-crud', 'E2E Team CRUD', NULL, + ARRAY['e2e-team-admin']::text[], + ARRAY['e2e-team-admin', 'e2e-internal-user']::text[], + '[{"role": "admin", "user_id": "e2e-team-admin"}, {"role": "user", "user_id": "e2e-internal-user"}]'::jsonb, + '{}'::jsonb, + ARRAY['fake-openai-gpt-4', 'fake-anthropic-claude']::text[], + 0.0, '{}'::jsonb, '{}'::jsonb, false +), +( + 'e2e-team-delete', 'E2E Team Delete', NULL, + ARRAY['e2e-team-admin']::text[], + ARRAY['e2e-team-admin']::text[], + '[{"role": "admin", "user_id": "e2e-team-admin"}]'::jsonb, + '{}'::jsonb, + ARRAY['fake-openai-gpt-4']::text[], + 0.0, '{}'::jsonb, '{}'::jsonb, false +), +( + 'e2e-team-org', 'E2E Team In Org', 'e2e-org-main', + ARRAY[]::text[], + ARRAY['e2e-internal-user']::text[], + '[{"role": "user", "user_id": "e2e-internal-user"}]'::jsonb, + '{}'::jsonb, + ARRAY['fake-openai-gpt-4']::text[], + 0.0, '{}'::jsonb, '{}'::jsonb, false +) +ON CONFLICT (team_id) DO NOTHING; + +-- ============================================================ +-- 5. Team Memberships +-- ============================================================ +INSERT INTO "LiteLLM_TeamMembership" (user_id, team_id, spend) VALUES + ('e2e-team-admin', 'e2e-team-crud', 0.0), + ('e2e-internal-user', 'e2e-team-crud', 0.0), + ('e2e-team-admin', 'e2e-team-delete', 0.0), + ('e2e-internal-user', 'e2e-team-org', 0.0) +ON CONFLICT (user_id, team_id) DO NOTHING; diff --git a/tests/ui_e2e_tests/globalSetup.ts b/tests/ui_e2e_tests/globalSetup.ts new file mode 100644 index 00000000000..ff82cded2c3 --- /dev/null +++ b/tests/ui_e2e_tests/globalSetup.ts @@ -0,0 +1,32 @@ +import { chromium, expect } from "@playwright/test"; +import { users, Role, ADMIN_STORAGE_PATH } from "./constants"; +import * as fs from "fs"; + +async function globalSetup() { + const browser = await chromium.launch(); + const page = await browser.newPage(); + await page.goto("http://localhost:4000/ui/login"); + await page.getByPlaceholder("Enter your username").fill(users[Role.ProxyAdmin].email); + await page.getByPlaceholder("Enter your password").fill(users[Role.ProxyAdmin].password); + await page.getByRole("button", { name: "Login", exact: true }).click(); + try { + // Wait for navigation away from login page into the dashboard + await page.waitForURL( + (url) => url.pathname.startsWith("/ui") && !url.pathname.includes("/login"), + { timeout: 30_000 }, + ); + // Wait for sidebar to render as a signal that the dashboard is ready + await expect(page.getByRole("menuitem", { name: "Virtual Keys" })).toBeVisible({ timeout: 30_000 }); + } catch (e) { + // Save a screenshot for debugging before re-throwing + fs.mkdirSync("test-results", { recursive: true }); + await page.screenshot({ path: "test-results/global-setup-failure.png", fullPage: true }); + console.error("Global setup failed. Screenshot saved to test-results/global-setup-failure.png"); + console.error("Current URL:", page.url()); + throw e; + } + await page.context().storageState({ path: ADMIN_STORAGE_PATH }); + await browser.close(); +} + +export default globalSetup; diff --git a/tests/ui_e2e_tests/helpers/login.ts b/tests/ui_e2e_tests/helpers/login.ts new file mode 100644 index 00000000000..d1c1ac410fc --- /dev/null +++ b/tests/ui_e2e_tests/helpers/login.ts @@ -0,0 +1,16 @@ +import { Page as PlaywrightPage, expect } from "@playwright/test"; +import { users, Role } from "../constants"; + +export async function loginAs(page: PlaywrightPage, role: Role) { + const user = users[role]; + await page.goto("/ui/login"); + await page.getByPlaceholder("Enter your username").fill(user.email); + await page.getByPlaceholder("Enter your password").fill(user.password); + await page.getByRole("button", { name: "Login", exact: true }).click(); + // Wait for navigation away from login page into the dashboard + await page.waitForURL((url) => url.pathname.startsWith("/ui") && !url.pathname.includes("/login"), { + timeout: 30_000, + }); + // Wait for sidebar to render as a signal that the dashboard is ready + await expect(page.getByRole("menuitem", { name: "Virtual Keys" })).toBeVisible({ timeout: 30_000 }); +} diff --git a/tests/ui_e2e_tests/helpers/navigation.ts b/tests/ui_e2e_tests/helpers/navigation.ts new file mode 100644 index 00000000000..065e2180535 --- /dev/null +++ b/tests/ui_e2e_tests/helpers/navigation.ts @@ -0,0 +1,6 @@ +import { Page as PlaywrightPage } from "@playwright/test"; +import { Page } from "../constants"; + +export async function navigateToPage(page: PlaywrightPage, targetPage: Page) { + await page.goto(`/ui?page=${targetPage}`); +} diff --git a/tests/ui_e2e_tests/package-lock.json b/tests/ui_e2e_tests/package-lock.json new file mode 100644 index 00000000000..5ce8059b784 --- /dev/null +++ b/tests/ui_e2e_tests/package-lock.json @@ -0,0 +1,76 @@ +{ + "name": "litellm-ui-e2e-tests", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "litellm-ui-e2e-tests", + "devDependencies": { + "@playwright/test": "^1.50.0" + } + }, + "node_modules/@playwright/test": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.59.1.tgz", + "integrity": "sha512-PG6q63nQg5c9rIi4/Z5lR5IVF7yU5MqmKaPOe0HSc0O2cX1fPi96sUQu5j7eo4gKCkB2AnNGoWt7y4/Xx3Kcqg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.59.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.59.1.tgz", + "integrity": "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.59.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.59.1.tgz", + "integrity": "sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + } + } +} diff --git a/tests/ui_e2e_tests/package.json b/tests/ui_e2e_tests/package.json new file mode 100644 index 00000000000..b57fcd51e59 --- /dev/null +++ b/tests/ui_e2e_tests/package.json @@ -0,0 +1,12 @@ +{ + "name": "litellm-ui-e2e-tests", + "private": true, + "devDependencies": { + "@playwright/test": "^1.50.0" + }, + "scripts": { + "e2e": "playwright test", + "e2e:headed": "playwright test --headed", + "e2e:ui": "playwright test --ui" + } +} diff --git a/tests/ui_e2e_tests/playwright.config.ts b/tests/ui_e2e_tests/playwright.config.ts new file mode 100644 index 00000000000..6c6fd2d5e95 --- /dev/null +++ b/tests/ui_e2e_tests/playwright.config.ts @@ -0,0 +1,31 @@ +import { defineConfig, devices } from "@playwright/test"; + +const isCI = !!process.env.CI; + +export default defineConfig({ + testDir: "./tests", + testMatch: "**/*.spec.ts", + globalSetup: "./globalSetup.ts", + fullyParallel: false, + forbidOnly: isCI, + retries: isCI ? 2 : 0, + workers: 1, + reporter: isCI ? [["html", { open: "never" }]] : [["html"]], + timeout: 4 * 60 * 1000, + expect: { + timeout: 10_000, + }, + use: { + baseURL: "http://localhost:4000", + trace: "on-first-retry", + screenshot: "only-on-failure", + actionTimeout: 15_000, + navigationTimeout: 30_000, + }, + projects: [ + { + name: "chromium", + use: { ...devices["Desktop Chrome"] }, + }, + ], +}); diff --git a/tests/ui_e2e_tests/run_e2e.sh b/tests/ui_e2e_tests/run_e2e.sh new file mode 100755 index 00000000000..400e53258d9 --- /dev/null +++ b/tests/ui_e2e_tests/run_e2e.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ================================================================ +# UI E2E Test Runner +# Starts postgres, seeds DB, starts mock + proxy, runs Playwright. +# All credentials are generated per run — nothing is stored on disk. +# +# In CI (CI=true), expects: +# - PostgreSQL already running on 127.0.0.1:5432 +# - DATABASE_URL already set +# - Python/Poetry already installed +# - Node.js/npx already available +# ================================================================ + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +IS_CI="${CI:-false}" +CONTAINER_NAME="litellm-e2e-postgres-$$" +MOCK_PID="" +PROXY_PID="" + +# --- Ensure common tool paths are available (local dev only) --- +if [ "$IS_CI" = "false" ]; then + for p in /usr/local/bin /opt/homebrew/bin "$HOME/.local/bin" /opt/homebrew/opt/postgresql@14/bin /opt/homebrew/opt/libpq/bin; do + [ -d "$p" ] && export PATH="$p:$PATH" + done + [ -s "$HOME/.nvm/nvm.sh" ] && source "$HOME/.nvm/nvm.sh" +fi + +# --- Cleanup on exit --- +cleanup() { + echo "Cleaning up..." + [ -n "$MOCK_PID" ] && kill "$MOCK_PID" 2>/dev/null || true + [ -n "$PROXY_PID" ] && kill "$PROXY_PID" 2>/dev/null || true + if [ "$IS_CI" = "false" ]; then + docker stop "$CONTAINER_NAME" 2>/dev/null || true + fi + echo "Done." +} +trap cleanup EXIT INT TERM + +# --- Pre-flight checks --- +for cmd in python3 npx poetry; do + command -v "$cmd" >/dev/null 2>&1 || { echo "Error: $cmd not found."; exit 1; } +done + +# --- Database setup --- +if [ "$IS_CI" = "false" ]; then + # Local: spin up a postgres container + for cmd in docker psql; do + command -v "$cmd" >/dev/null 2>&1 || { echo "Error: $cmd not found."; exit 1; } + done + for port in 4000 5432 8090; do + if lsof -ti ":$port" >/dev/null 2>&1; then + echo "Error: port $port is in use" + exit 1 + fi + done + + export POSTGRES_USER="e2euser" + export POSTGRES_PASSWORD="$(openssl rand -hex 32)" + export POSTGRES_DB="litellm_e2e" + export DATABASE_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@127.0.0.1:5432/${POSTGRES_DB}" + + echo "=== Starting PostgreSQL ===" + docker run -d --rm --name "$CONTAINER_NAME" \ + -e POSTGRES_USER -e POSTGRES_PASSWORD -e POSTGRES_DB \ + -p 127.0.0.1:5432:5432 \ + postgres:16 + + echo "Waiting for PostgreSQL..." + for i in $(seq 1 30); do + if PGPASSWORD="$POSTGRES_PASSWORD" pg_isready -h 127.0.0.1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" >/dev/null 2>&1; then + break + fi + sleep 1 + done +else + # CI: postgres is already running as a service container + echo "=== Using CI PostgreSQL service ===" + : "${DATABASE_URL:?DATABASE_URL must be set in CI}" +fi + +# --- Credentials --- +export LITELLM_MASTER_KEY="sk-e2e-$(openssl rand -hex 32)" +export MOCK_LLM_URL="http://127.0.0.1:8090/v1" +export DISABLE_SCHEMA_UPDATE="true" + +# --- Python environment --- +echo "=== Setting up Python environment ===" +cd "$REPO_ROOT" +if ! poetry run python3 -c "import prisma" 2>/dev/null; then + echo "Installing Python dependencies (first run)..." + poetry install --with dev,proxy-dev --extras "proxy" --quiet + poetry run pip install nodejs-wheel-binaries 2>/dev/null || true + poetry run prisma generate --schema litellm/proxy/schema.prisma +fi + +echo "=== Pushing Prisma schema to database ===" +poetry run prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss + +# --- Mock LLM server --- +echo "=== Starting mock LLM server ===" +poetry run python3 "$SCRIPT_DIR/fixtures/mock_llm_server/server.py" & +MOCK_PID=$! + +for i in $(seq 1 15); do + if curl -sf http://127.0.0.1:8090/health >/dev/null 2>&1; then break; fi + sleep 1 +done + +# --- LiteLLM proxy --- +echo "=== Starting LiteLLM proxy ===" +cd "$REPO_ROOT" +poetry run python3 -m litellm.proxy.proxy_cli \ + --config "$SCRIPT_DIR/fixtures/config.yml" \ + --port 4000 & +PROXY_PID=$! + +echo "Waiting for proxy..." +PROXY_READY=0 +for i in $(seq 1 180); do + if ! kill -0 "$PROXY_PID" 2>/dev/null; then + echo "Error: proxy process exited unexpectedly" + exit 1 + fi + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer $LITELLM_MASTER_KEY" 2>/dev/null || true) + if [ "$HTTP_CODE" = "200" ]; then + PROXY_READY=1 + break + fi + sleep 1 +done +if [ "$PROXY_READY" -ne 1 ]; then + echo "Error: proxy did not become healthy within 180 seconds" + exit 1 +fi +echo "Proxy is ready." + +# --- Seed database --- +echo "=== Seeding database ===" +# Extract credentials from DATABASE_URL for psql +DB_USER=$(echo "$DATABASE_URL" | sed -n 's|.*://\([^:]*\):.*|\1|p') +DB_PASS=$(echo "$DATABASE_URL" | sed -n 's|.*://[^:]*:\([^@]*\)@.*|\1|p') +DB_HOST=$(echo "$DATABASE_URL" | sed -n 's|.*@\([^:]*\):.*|\1|p') +DB_PORT=$(echo "$DATABASE_URL" | sed -n 's|.*:\([0-9]*\)/.*|\1|p') +DB_NAME=$(echo "$DATABASE_URL" | sed -n 's|.*/\([^?]*\).*|\1|p') + +PGPASSWORD="$DB_PASS" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" \ + -f "$SCRIPT_DIR/fixtures/seed.sql" + +# --- Playwright --- +echo "=== Installing Playwright dependencies ===" +cd "$SCRIPT_DIR" +npm install --silent + +echo "=== Running Playwright tests ===" +npx playwright test "$@" +EXIT_CODE=$? + +exit $EXIT_CODE diff --git a/tests/ui_e2e_tests/tests/roles/admin-viewer.spec.ts b/tests/ui_e2e_tests/tests/roles/admin-viewer.spec.ts new file mode 100644 index 00000000000..6e29d7c22e2 --- /dev/null +++ b/tests/ui_e2e_tests/tests/roles/admin-viewer.spec.ts @@ -0,0 +1,10 @@ +import { test, expect } from "@playwright/test"; +import { Role } from "../../constants"; +import { loginAs } from "../../helpers/login"; + +test.describe("Admin Viewer Role", () => { + test("Should not see Test Key page", async ({ page }) => { + await loginAs(page, Role.ProxyAdminViewer); + await expect(page.getByRole("menuitem", { name: "Test Key" })).not.toBeVisible(); + }); +}); diff --git a/tests/ui_e2e_tests/tests/roles/internal-user.spec.ts b/tests/ui_e2e_tests/tests/roles/internal-user.spec.ts new file mode 100644 index 00000000000..4cb39a1ce7e --- /dev/null +++ b/tests/ui_e2e_tests/tests/roles/internal-user.spec.ts @@ -0,0 +1,12 @@ +import { test, expect } from "@playwright/test"; +import { Page, Role } from "../../constants"; +import { loginAs } from "../../helpers/login"; +import { navigateToPage } from "../../helpers/navigation"; + +test.describe("Internal User Role", () => { + test("Should not see litellm-dashboard keys", async ({ page }) => { + await loginAs(page, Role.InternalUser); + await navigateToPage(page, Page.ApiKeys); + await expect(page.getByText("litellm-dashboard")).not.toBeVisible(); + }); +}); diff --git a/tests/ui_e2e_tests/tests/roles/internal-viewer.spec.ts b/tests/ui_e2e_tests/tests/roles/internal-viewer.spec.ts new file mode 100644 index 00000000000..3ca17caff14 --- /dev/null +++ b/tests/ui_e2e_tests/tests/roles/internal-viewer.spec.ts @@ -0,0 +1,28 @@ +import { test, expect } from "@playwright/test"; +import { Page, Role } from "../../constants"; +import { loginAs } from "../../helpers/login"; +import { navigateToPage } from "../../helpers/navigation"; + +test.describe("Internal User Viewer Role", () => { + test("Can only see allowed pages", async ({ page }) => { + await loginAs(page, Role.InternalUserViewer); + await expect(page.getByRole("menuitem", { name: "Virtual Keys" })).toBeVisible(); + await expect(page.getByRole("menuitem", { name: "Admin Settings" })).not.toBeVisible(); + }); + + test("Cannot create keys", async ({ page }) => { + await loginAs(page, Role.InternalUserViewer); + await navigateToPage(page, Page.ApiKeys); + await expect(page.getByRole("button", { name: /Create New Key/i })).not.toBeVisible(); + }); + + test("Cannot edit or delete keys", async ({ page }) => { + await loginAs(page, Role.InternalUserViewer); + await navigateToPage(page, Page.ApiKeys); + // Ensure the keys table has loaded before asserting absence of actions + await expect(page.getByRole("menuitem", { name: "Virtual Keys" })).toBeVisible(); + await expect(page.getByRole("button", { name: /Edit Key/i })).not.toBeVisible(); + await expect(page.getByRole("button", { name: /Delete Key/i })).not.toBeVisible(); + await expect(page.getByRole("button", { name: /Regenerate Key/i })).not.toBeVisible(); + }); +}); diff --git a/tests/ui_e2e_tests/tests/roles/proxy-admin.spec.ts b/tests/ui_e2e_tests/tests/roles/proxy-admin.spec.ts new file mode 100644 index 00000000000..b94e2aefaa8 --- /dev/null +++ b/tests/ui_e2e_tests/tests/roles/proxy-admin.spec.ts @@ -0,0 +1,21 @@ +import { test, expect } from "@playwright/test"; +import { ADMIN_STORAGE_PATH, Page, Role, users } from "../../constants"; +import { navigateToPage } from "../../helpers/navigation"; + +test.describe("Proxy Admin Role", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + test("Can create keys", async ({ page }) => { + await navigateToPage(page, Page.ApiKeys); + await expect(page.getByRole("button", { name: /Create New Key/i })).toBeVisible(); + }); + + test("Can list teams via API", async ({ page }) => { + const response = await page.request.get("/team/list", { + headers: { + Authorization: `Bearer ${users[Role.ProxyAdmin].password}`, + }, + }); + expect(response.status()).toBe(200); + }); +}); diff --git a/tests/ui_e2e_tests/tests/roles/team-admin.spec.ts b/tests/ui_e2e_tests/tests/roles/team-admin.spec.ts new file mode 100644 index 00000000000..a12a5a04842 --- /dev/null +++ b/tests/ui_e2e_tests/tests/roles/team-admin.spec.ts @@ -0,0 +1,13 @@ +import { test, expect } from "@playwright/test"; +import { Page, Role } from "../../constants"; +import { loginAs } from "../../helpers/login"; +import { navigateToPage } from "../../helpers/navigation"; + +test.describe("Team Admin Role", () => { + test("Can view team keys but not admin settings", async ({ page }) => { + await loginAs(page, Role.TeamAdmin); + await navigateToPage(page, Page.ApiKeys); + await expect(page.getByRole("menuitem", { name: "Virtual Keys" })).toBeVisible(); + await expect(page.getByRole("menuitem", { name: "Admin Settings" })).not.toBeVisible(); + }); +}); diff --git a/tests/ui_e2e_tests/tests/security/login-logout.spec.ts b/tests/ui_e2e_tests/tests/security/login-logout.spec.ts new file mode 100644 index 00000000000..88ce3f1a6d0 --- /dev/null +++ b/tests/ui_e2e_tests/tests/security/login-logout.spec.ts @@ -0,0 +1,18 @@ +import { test, expect } from "@playwright/test"; +import { users, Role } from "../../constants"; + +test.describe("Authentication", () => { + test("Login with valid admin credentials", async ({ page }) => { + await page.goto("/ui/login"); + await page.getByPlaceholder("Enter your username").fill(users[Role.ProxyAdmin].email); + await page.getByPlaceholder("Enter your password").fill(users[Role.ProxyAdmin].password); + await page.getByRole("button", { name: "Login", exact: true }).click(); + await expect(page.getByRole("menuitem", { name: "Virtual Keys" })).toBeVisible(); + }); + + test("Unauthenticated user is redirected to login", async ({ page }) => { + await page.goto("/ui"); + await page.waitForURL(/\/ui\/login/); + await expect(page.getByRole("heading", { name: /Login/i })).toBeVisible(); + }); +}); diff --git a/tests/ui_e2e_tests/tsconfig.json b/tests/ui_e2e_tests/tsconfig.json new file mode 100644 index 00000000000..8d92d3764e5 --- /dev/null +++ b/tests/ui_e2e_tests/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "strict": true, + "esModuleInterop": true, + "outDir": "./dist", + "rootDir": "." + }, + "include": ["**/*.ts"] +} diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index dfad2a1e795..4392fd99d90 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -16,6 +16,7 @@ "format:check": "prettier --check .", "e2e": "playwright test --config e2e_tests/playwright.config.ts", "e2e:ui": "playwright test --ui --config e2e_tests/playwright.config.ts", + "e2e:psql": "../../tests/ui_e2e_tests/run_e2e.sh", "knip": "knip", "knip:fix": "knip --fix" },