diff --git a/.circleci/config.yml b/.circleci/config.yml index e30dc02b2ab..d98fdaa4a4d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3476,7 +3476,6 @@ jobs: name: Install Playwright Browsers command: | npx playwright install - - run: name: Build Docker image command: docker build -t my-app:latest -f ./docker/Dockerfile.database . @@ -3515,7 +3514,10 @@ jobs: - run: name: Run Playwright Tests command: | - npx playwright test e2e_ui_tests/ --reporter=html --output=test-results + npx playwright test \ + --config ui/litellm-dashboard/e2e_tests/playwright.config.ts \ + --reporter=html \ + --output=test-results no_output_timeout: 120m - store_artifacts: path: test-results @@ -3973,4 +3975,4 @@ workflows: - proxy_pass_through_endpoint_tests - check_code_and_doc_quality - publish_proxy_extras - - guardrails_testing \ No newline at end of file + - guardrails_testing diff --git a/.gitignore b/.gitignore index 8196d1d9f24..b6d4fd44190 100644 --- a/.gitignore +++ b/.gitignore @@ -101,3 +101,5 @@ tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py litellm/proxy/_experimental/out/guardrails/index.html scripts/test_vertex_ai_search.py LAZY_LOADING_IMPROVEMENTS.md +**/test-results +**/playwright-report \ No newline at end of file diff --git a/ui/litellm-dashboard/e2e_tests/auth/login.setup.ts b/ui/litellm-dashboard/e2e_tests/auth/login.setup.ts new file mode 100644 index 00000000000..712191f6843 --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/auth/login.setup.ts @@ -0,0 +1,20 @@ +import { test, expect } from "@playwright/test"; + +test("login and save auth state", async ({ page }) => { + await page.goto("http://localhost:4000/ui"); + + await page.getByPlaceholder("Enter your username").fill("admin"); + await page.getByPlaceholder("Enter your password").fill("gm"); + + const loginButton = page.getByRole("button", { name: "Login" }); + await expect(loginButton).toBeEnabled(); + await loginButton.click(); + + // Assert successful login (important) + await expect(page.getByText("AI Gateway")).toBeVisible(); + + // 🔐 Save auth state + await page.context().storageState({ + path: "storageState.json", + }); +}); diff --git a/ui/litellm-dashboard/e2e_tests/playwright.config.ts b/ui/litellm-dashboard/e2e_tests/playwright.config.ts new file mode 100644 index 00000000000..80d45a892c8 --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/playwright.config.ts @@ -0,0 +1,47 @@ +import { defineConfig, devices } from "@playwright/test"; + +/** + * See https://playwright.dev/docs/test-configuration. + */ +export default defineConfig({ + testDir: ".", + testMatch: ["**/*.spec.ts", "**/*.setup.ts"], + testIgnore: ["**/*.test.*"], + /* Run tests in files in parallel */ + fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: process.env.CI ? 2 : 0, + /* Opt out of parallel tests on CI. */ + workers: process.env.CI ? 1 : undefined, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: "html", + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + /* Base URL to use in actions like `await page.goto('/')`. */ + baseURL: "http://localhost:3000", + + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: "on-first-retry", + }, + + /* Configure projects for major browsers */ + projects: [ + { + name: "chromium", + use: { ...devices["Desktop Chrome"] }, + }, + + { + name: "firefox", + use: { ...devices["Desktop Firefox"] }, + }, + ], + + /* Timeout settings */ + timeout: 4 * 60 * 1000, + expect: { + timeout: 10 * 1000, + }, +}); diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index b65c35e78d8..cc21b516e24 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -39,6 +39,7 @@ "uuid": "^11.1.0" }, "devDependencies": { + "@playwright/test": "^1.57.0", "@tailwindcss/forms": "^0.5.7", "@testing-library/dom": "^10.4.1", "@testing-library/jest-dom": "^6.8.0", @@ -4927,6 +4928,22 @@ "node": ">=12.4.0" } }, + "node_modules/@playwright/test": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.57.0.tgz", + "integrity": "sha512-6TyEnHgd6SArQO8UO2OMTxshln3QMWBtPGrOCgs3wVEmQmwyuNtB10IZMfmYDE0riwNR1cu4q+pPcxMVtaG3TA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.57.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@pnpm/config.env-replace": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@pnpm/config.env-replace/-/config.env-replace-1.1.0.tgz", @@ -19219,6 +19236,53 @@ "pathe": "^2.0.3" } }, + "node_modules/playwright": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.57.0.tgz", + "integrity": "sha512-ilYQj1s8sr2ppEJ2YVadYBN0Mb3mdo9J0wQ+UuDhzYqURwSoW4n1Xs5vs7ORwgDGmyEh33tRMeS8KhdkMoLXQw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.57.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.57.0.tgz", + "integrity": "sha512-agTcKlMw/mjBWOnD6kFZttAAGHgi/Nw0CZ2o6JqWSbMlI219lAFLZZCyqByTsvVAJq5XA5H8cA6PrvBRpBWEuQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/points-on-curve": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/points-on-curve/-/points-on-curve-0.2.0.tgz", diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index ce42d0ba41a..a018a4e7a58 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -10,7 +10,9 @@ "test": "vitest", "test:watch": "vitest -w", "format": "prettier --write .", - "format:check": "prettier --check ." + "format:check": "prettier --check .", + "e2e": "playwright test --config e2e_tests/playwright.config.ts", + "e2e:ui": "playwright test --ui --config e2e_tests/playwright.config.ts" }, "dependencies": { "@anthropic-ai/sdk": "^0.54.0", @@ -44,6 +46,7 @@ "uuid": "^11.1.0" }, "devDependencies": { + "@playwright/test": "^1.57.0", "@tailwindcss/forms": "^0.5.7", "@testing-library/dom": "^10.4.1", "@testing-library/jest-dom": "^6.8.0", diff --git a/ui/litellm-dashboard/storageState.json b/ui/litellm-dashboard/storageState.json new file mode 100644 index 00000000000..f2f012b34f8 --- /dev/null +++ b/ui/litellm-dashboard/storageState.json @@ -0,0 +1,15 @@ +{ + "cookies": [ + { + "name": "token", + "value": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VyX2lkIjoiZGVmYXVsdF91c2VyX2lkIiwia2V5Ijoic2stUkJVZGFncVZuY0VWMDNVdnNDZkstdyIsInVzZXJfZW1haWwiOm51bGwsInVzZXJfcm9sZSI6InByb3h5X2FkbWluIiwibG9naW5fbWV0aG9kIjoidXNlcm5hbWVfcGFzc3dvcmQiLCJwcmVtaXVtX3VzZXIiOnRydWUsImF1dGhfaGVhZGVyX25hbWUiOiJBdXRob3JpemF0aW9uIiwiZGlzYWJsZWRfbm9uX2FkbWluX3BlcnNvbmFsX2tleV9jcmVhdGlvbiI6ZmFsc2UsInNlcnZlcl9yb290X3BhdGgiOiIvIn0.aW_1-qDI9HuZvDq53ufTX_HrE6EJ7XSCPBb-N136KjQ", + "domain": "localhost", + "path": "/", + "expires": -1, + "httpOnly": false, + "secure": false, + "sameSite": "Lax" + } + ], + "origins": [] +} \ No newline at end of file diff --git a/ui/litellm-dashboard/vitest.config.ts b/ui/litellm-dashboard/vitest.config.ts index 194b6fa691f..51e3df7a280 100644 --- a/ui/litellm-dashboard/vitest.config.ts +++ b/ui/litellm-dashboard/vitest.config.ts @@ -8,6 +8,8 @@ export default defineConfig({ globals: true, css: true, // lets you import CSS/modules without extra mocks coverage: { reporter: ["text", "lcov"] }, + exclude: ["e2e_tests/**,", "node_modules/**"], + include: ["src/**/*.test.ts", "src/**/*.test.tsx", "tests/**/*.test.ts", "tests/**/*.test.tsx"], }, resolve: { alias: {