Merge pull request #18501 from BerriAI/litellm_ui_e2e_login_session

[Infra] UI - E2E Tests Setup

.circleci/config.yml

@@ -3476,7 +3476,6 @@ jobs:
           name: Install Playwright Browsers
           command: |
             npx playwright install
-
       - run:
           name: Build Docker image
           command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
@@ -3515,7 +3514,10 @@ jobs:
       - run:
           name: Run Playwright Tests
           command: |
-            npx playwright test e2e_ui_tests/ --reporter=html --output=test-results
+            npx playwright test \
+              --config ui/litellm-dashboard/e2e_tests/playwright.config.ts \
+              --reporter=html \
+              --output=test-results
           no_output_timeout: 120m
       - store_artifacts:
           path: test-results
@@ -3973,4 +3975,4 @@ workflows:
             - proxy_pass_through_endpoint_tests
             - check_code_and_doc_quality
             - publish_proxy_extras
-            - guardrails_testing
+            - guardrails_testing

.gitignore

@@ -101,3 +101,5 @@ tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
 litellm/proxy/_experimental/out/guardrails/index.html
 scripts/test_vertex_ai_search.py
 LAZY_LOADING_IMPROVEMENTS.md
+**/test-results
+**/playwright-report

ui/litellm-dashboard/e2e_tests/auth/login.setup.ts

@@ -0,0 +1,20 @@
+import { test, expect } from "@playwright/test";
+
+test("login and save auth state", async ({ page }) => {
+  await page.goto("http://localhost:4000/ui");
+
+  await page.getByPlaceholder("Enter your username").fill("admin");
+  await page.getByPlaceholder("Enter your password").fill("gm");
+
+  const loginButton = page.getByRole("button", { name: "Login" });
+  await expect(loginButton).toBeEnabled();
+  await loginButton.click();
+
+  // Assert successful login (important)
+  await expect(page.getByText("AI Gateway")).toBeVisible();
+
+  // 🔐 Save auth state
+  await page.context().storageState({
+    path: "storageState.json",
+  });
+});

ui/litellm-dashboard/e2e_tests/playwright.config.ts

@@ -0,0 +1,47 @@
+import { defineConfig, devices } from "@playwright/test";
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: ".",
+  testMatch: ["**/*.spec.ts", "**/*.setup.ts"],
+  testIgnore: ["**/*.test.*"],
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: "html",
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    baseURL: "http://localhost:3000",
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: "on-first-retry",
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: "chromium",
+      use: { ...devices["Desktop Chrome"] },
+    },
+
+    {
+      name: "firefox",
+      use: { ...devices["Desktop Firefox"] },
+    },
+  ],
+
+  /* Timeout settings */
+  timeout: 4 * 60 * 1000,
+  expect: {
+    timeout: 10 * 1000,
+  },
+});

ui/litellm-dashboard/package-lock.json

@@ -39,6 +39,7 @@
         "uuid": "^11.1.0"
       },
       "devDependencies": {
+        "@playwright/test": "^1.57.0",
         "@tailwindcss/forms": "^0.5.7",
         "@testing-library/dom": "^10.4.1",
         "@testing-library/jest-dom": "^6.8.0",
@@ -4927,6 +4928,22 @@
         "node": ">=12.4.0"
       }
     },
+    "node_modules/@playwright/test": {
+      "version": "1.57.0",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.57.0.tgz",
+      "integrity": "sha512-6TyEnHgd6SArQO8UO2OMTxshln3QMWBtPGrOCgs3wVEmQmwyuNtB10IZMfmYDE0riwNR1cu4q+pPcxMVtaG3TA==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright": "1.57.0"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/@pnpm/config.env-replace": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/@pnpm/config.env-replace/-/config.env-replace-1.1.0.tgz",
@@ -19219,6 +19236,53 @@
         "pathe": "^2.0.3"
       }
     },
+    "node_modules/playwright": {
+      "version": "1.57.0",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.57.0.tgz",
+      "integrity": "sha512-ilYQj1s8sr2ppEJ2YVadYBN0Mb3mdo9J0wQ+UuDhzYqURwSoW4n1Xs5vs7ORwgDGmyEh33tRMeS8KhdkMoLXQw==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright-core": "1.57.0"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "optionalDependencies": {
+        "fsevents": "2.3.2"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.57.0",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.57.0.tgz",
+      "integrity": "sha512-agTcKlMw/mjBWOnD6kFZttAAGHgi/Nw0CZ2o6JqWSbMlI219lAFLZZCyqByTsvVAJq5XA5H8cA6PrvBRpBWEuQ==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/playwright/node_modules/fsevents": {
+      "version": "2.3.2",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+      "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+      "dev": true,
+      "hasInstallScript": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
+    },
     "node_modules/points-on-curve": {
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/points-on-curve/-/points-on-curve-0.2.0.tgz",

ui/litellm-dashboard/package.json

@@ -10,7 +10,9 @@
     "test": "vitest",
     "test:watch": "vitest -w",
     "format": "prettier --write .",
-    "format:check": "prettier --check ."
+    "format:check": "prettier --check .",
+    "e2e": "playwright test --config e2e_tests/playwright.config.ts",
+    "e2e:ui": "playwright test --ui --config e2e_tests/playwright.config.ts"
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.54.0",
@@ -44,6 +46,7 @@
     "uuid": "^11.1.0"
   },
   "devDependencies": {
+    "@playwright/test": "^1.57.0",
     "@tailwindcss/forms": "^0.5.7",
     "@testing-library/dom": "^10.4.1",
     "@testing-library/jest-dom": "^6.8.0",

ui/litellm-dashboard/storageState.json

@@ -0,0 +1,15 @@
+{
+  "cookies": [
+    {
+      "name": "token",
+      "value": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VyX2lkIjoiZGVmYXVsdF91c2VyX2lkIiwia2V5Ijoic2stUkJVZGFncVZuY0VWMDNVdnNDZkstdyIsInVzZXJfZW1haWwiOm51bGwsInVzZXJfcm9sZSI6InByb3h5X2FkbWluIiwibG9naW5fbWV0aG9kIjoidXNlcm5hbWVfcGFzc3dvcmQiLCJwcmVtaXVtX3VzZXIiOnRydWUsImF1dGhfaGVhZGVyX25hbWUiOiJBdXRob3JpemF0aW9uIiwiZGlzYWJsZWRfbm9uX2FkbWluX3BlcnNvbmFsX2tleV9jcmVhdGlvbiI6ZmFsc2UsInNlcnZlcl9yb290X3BhdGgiOiIvIn0.aW_1-qDI9HuZvDq53ufTX_HrE6EJ7XSCPBb-N136KjQ",
+      "domain": "localhost",
+      "path": "/",
+      "expires": -1,
+      "httpOnly": false,
+      "secure": false,
+      "sameSite": "Lax"
+    }
+  ],
+  "origins": []
+}

ui/litellm-dashboard/vitest.config.ts

@@ -8,6 +8,8 @@ export default defineConfig({
     globals: true,
     css: true, // lets you import CSS/modules without extra mocks
     coverage: { reporter: ["text", "lcov"] },
+    exclude: ["e2e_tests/**,", "node_modules/**"],
+    include: ["src/**/*.test.ts", "src/**/*.test.tsx", "tests/**/*.test.ts", "tests/**/*.test.tsx"],
   },
   resolve: {
     alias: {