From a1849a152ce40eaefd86f00d406b73dbaae8b089 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 29 Dec 2025 11:27:22 -0800 Subject: [PATCH 01/11] Playwright setup in UI directory --- .gitignore | 2 + .../e2e_tests/playwright.config.ts | 52 +++++++++++++++ .../e2e_tests/tests/basic.spec.ts | 10 +++ ui/litellm-dashboard/package-lock.json | 64 +++++++++++++++++++ ui/litellm-dashboard/package.json | 5 +- ui/litellm-dashboard/vitest.config.ts | 2 + 6 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 ui/litellm-dashboard/e2e_tests/playwright.config.ts create mode 100644 ui/litellm-dashboard/e2e_tests/tests/basic.spec.ts diff --git a/.gitignore b/.gitignore index 8196d1d9f24..b6d4fd44190 100644 --- a/.gitignore +++ b/.gitignore @@ -101,3 +101,5 @@ tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py litellm/proxy/_experimental/out/guardrails/index.html scripts/test_vertex_ai_search.py LAZY_LOADING_IMPROVEMENTS.md +**/test-results +**/playwright-report \ No newline at end of file diff --git a/ui/litellm-dashboard/e2e_tests/playwright.config.ts b/ui/litellm-dashboard/e2e_tests/playwright.config.ts new file mode 100644 index 00000000000..cce416075f4 --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/playwright.config.ts @@ -0,0 +1,52 @@ +import { defineConfig, devices } from "@playwright/test"; + +/** + * See https://playwright.dev/docs/test-configuration. + */ +export default defineConfig({ + testDir: "./tests", + testMatch: "**/*.spec.ts", + testIgnore: ["**/*.test.*"], + /* Run tests in files in parallel */ + fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: process.env.CI ? 2 : 0, + /* Opt out of parallel tests on CI. */ + workers: process.env.CI ? 1 : undefined, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: "html", + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + /* Base URL to use in actions like `await page.goto('/')`. */ + baseURL: "http://localhost:3000", + + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: "on-first-retry", + }, + + /* Configure projects for major browsers */ + projects: [ + { + name: "chromium", + use: { ...devices["Desktop Chrome"] }, + }, + + { + name: "firefox", + use: { ...devices["Desktop Firefox"] }, + }, + + { + name: "webkit", + use: { ...devices["Desktop Safari"] }, + }, + ], + + /* Timeout settings */ + timeout: 4 * 60 * 1000, + expect: { + timeout: 10 * 1000, + }, +}); diff --git a/ui/litellm-dashboard/e2e_tests/tests/basic.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/basic.spec.ts new file mode 100644 index 00000000000..9e990f60866 --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/tests/basic.spec.ts @@ -0,0 +1,10 @@ +import { test, expect } from "@playwright/test"; + +test("basic test to verify playwright setup", async ({ page }) => { + // Navigate to the base URL + await page.goto("/"); + + // Wait for the page to load and check if we can find some basic content + // This is a very basic test just to verify the setup works + await expect(page).toHaveTitle(/.*LiteLLM.*/); +}); diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index b65c35e78d8..cc21b516e24 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -39,6 +39,7 @@ "uuid": "^11.1.0" }, "devDependencies": { + "@playwright/test": "^1.57.0", "@tailwindcss/forms": "^0.5.7", "@testing-library/dom": "^10.4.1", "@testing-library/jest-dom": "^6.8.0", @@ -4927,6 +4928,22 @@ "node": ">=12.4.0" } }, + "node_modules/@playwright/test": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.57.0.tgz", + "integrity": "sha512-6TyEnHgd6SArQO8UO2OMTxshln3QMWBtPGrOCgs3wVEmQmwyuNtB10IZMfmYDE0riwNR1cu4q+pPcxMVtaG3TA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.57.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@pnpm/config.env-replace": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@pnpm/config.env-replace/-/config.env-replace-1.1.0.tgz", @@ -19219,6 +19236,53 @@ "pathe": "^2.0.3" } }, + "node_modules/playwright": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.57.0.tgz", + "integrity": "sha512-ilYQj1s8sr2ppEJ2YVadYBN0Mb3mdo9J0wQ+UuDhzYqURwSoW4n1Xs5vs7ORwgDGmyEh33tRMeS8KhdkMoLXQw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.57.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.57.0.tgz", + "integrity": "sha512-agTcKlMw/mjBWOnD6kFZttAAGHgi/Nw0CZ2o6JqWSbMlI219lAFLZZCyqByTsvVAJq5XA5H8cA6PrvBRpBWEuQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/points-on-curve": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/points-on-curve/-/points-on-curve-0.2.0.tgz", diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index ce42d0ba41a..a018a4e7a58 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -10,7 +10,9 @@ "test": "vitest", "test:watch": "vitest -w", "format": "prettier --write .", - "format:check": "prettier --check ." + "format:check": "prettier --check .", + "e2e": "playwright test --config e2e_tests/playwright.config.ts", + "e2e:ui": "playwright test --ui --config e2e_tests/playwright.config.ts" }, "dependencies": { "@anthropic-ai/sdk": "^0.54.0", @@ -44,6 +46,7 @@ "uuid": "^11.1.0" }, "devDependencies": { + "@playwright/test": "^1.57.0", "@tailwindcss/forms": "^0.5.7", "@testing-library/dom": "^10.4.1", "@testing-library/jest-dom": "^6.8.0", diff --git a/ui/litellm-dashboard/vitest.config.ts b/ui/litellm-dashboard/vitest.config.ts index 194b6fa691f..51e3df7a280 100644 --- a/ui/litellm-dashboard/vitest.config.ts +++ b/ui/litellm-dashboard/vitest.config.ts @@ -8,6 +8,8 @@ export default defineConfig({ globals: true, css: true, // lets you import CSS/modules without extra mocks coverage: { reporter: ["text", "lcov"] }, + exclude: ["e2e_tests/**,", "node_modules/**"], + include: ["src/**/*.test.ts", "src/**/*.test.tsx", "tests/**/*.test.ts", "tests/**/*.test.tsx"], }, resolve: { alias: { From 41fa2c0812d8aa143c0bf61d610850c537879b97 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 29 Dec 2025 12:00:03 -0800 Subject: [PATCH 02/11] migrate e2e tests to ui directory --- .circleci/config.yml | 4 ++-- .../e2e_tests/auth/login.setup.ts | 20 +++++++++++++++++++ .../e2e_tests/playwright.config.ts | 4 ++-- .../e2e_tests/tests/basic.spec.ts | 10 ---------- ui/litellm-dashboard/storageState.json | 15 ++++++++++++++ 5 files changed, 39 insertions(+), 14 deletions(-) create mode 100644 ui/litellm-dashboard/e2e_tests/auth/login.setup.ts delete mode 100644 ui/litellm-dashboard/e2e_tests/tests/basic.spec.ts create mode 100644 ui/litellm-dashboard/storageState.json diff --git a/.circleci/config.yml b/.circleci/config.yml index e30dc02b2ab..b1bee9675f3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3515,7 +3515,7 @@ jobs: - run: name: Run Playwright Tests command: | - npx playwright test e2e_ui_tests/ --reporter=html --output=test-results + cd ui/litellm-dashboard && npm run e2e no_output_timeout: 120m - store_artifacts: path: test-results @@ -3973,4 +3973,4 @@ workflows: - proxy_pass_through_endpoint_tests - check_code_and_doc_quality - publish_proxy_extras - - guardrails_testing \ No newline at end of file + - guardrails_testing diff --git a/ui/litellm-dashboard/e2e_tests/auth/login.setup.ts b/ui/litellm-dashboard/e2e_tests/auth/login.setup.ts new file mode 100644 index 00000000000..712191f6843 --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/auth/login.setup.ts @@ -0,0 +1,20 @@ +import { test, expect } from "@playwright/test"; + +test("login and save auth state", async ({ page }) => { + await page.goto("http://localhost:4000/ui"); + + await page.getByPlaceholder("Enter your username").fill("admin"); + await page.getByPlaceholder("Enter your password").fill("gm"); + + const loginButton = page.getByRole("button", { name: "Login" }); + await expect(loginButton).toBeEnabled(); + await loginButton.click(); + + // Assert successful login (important) + await expect(page.getByText("AI Gateway")).toBeVisible(); + + // 🔐 Save auth state + await page.context().storageState({ + path: "storageState.json", + }); +}); diff --git a/ui/litellm-dashboard/e2e_tests/playwright.config.ts b/ui/litellm-dashboard/e2e_tests/playwright.config.ts index cce416075f4..5ab68b3534f 100644 --- a/ui/litellm-dashboard/e2e_tests/playwright.config.ts +++ b/ui/litellm-dashboard/e2e_tests/playwright.config.ts @@ -4,8 +4,8 @@ import { defineConfig, devices } from "@playwright/test"; * See https://playwright.dev/docs/test-configuration. */ export default defineConfig({ - testDir: "./tests", - testMatch: "**/*.spec.ts", + testDir: ".", + testMatch: ["**/*.spec.ts", "**/*.setup.ts"], testIgnore: ["**/*.test.*"], /* Run tests in files in parallel */ fullyParallel: true, diff --git a/ui/litellm-dashboard/e2e_tests/tests/basic.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/basic.spec.ts deleted file mode 100644 index 9e990f60866..00000000000 --- a/ui/litellm-dashboard/e2e_tests/tests/basic.spec.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { test, expect } from "@playwright/test"; - -test("basic test to verify playwright setup", async ({ page }) => { - // Navigate to the base URL - await page.goto("/"); - - // Wait for the page to load and check if we can find some basic content - // This is a very basic test just to verify the setup works - await expect(page).toHaveTitle(/.*LiteLLM.*/); -}); diff --git a/ui/litellm-dashboard/storageState.json b/ui/litellm-dashboard/storageState.json new file mode 100644 index 00000000000..f2f012b34f8 --- /dev/null +++ b/ui/litellm-dashboard/storageState.json @@ -0,0 +1,15 @@ +{ + "cookies": [ + { + "name": "token", + "value": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VyX2lkIjoiZGVmYXVsdF91c2VyX2lkIiwia2V5Ijoic2stUkJVZGFncVZuY0VWMDNVdnNDZkstdyIsInVzZXJfZW1haWwiOm51bGwsInVzZXJfcm9sZSI6InByb3h5X2FkbWluIiwibG9naW5fbWV0aG9kIjoidXNlcm5hbWVfcGFzc3dvcmQiLCJwcmVtaXVtX3VzZXIiOnRydWUsImF1dGhfaGVhZGVyX25hbWUiOiJBdXRob3JpemF0aW9uIiwiZGlzYWJsZWRfbm9uX2FkbWluX3BlcnNvbmFsX2tleV9jcmVhdGlvbiI6ZmFsc2UsInNlcnZlcl9yb290X3BhdGgiOiIvIn0.aW_1-qDI9HuZvDq53ufTX_HrE6EJ7XSCPBb-N136KjQ", + "domain": "localhost", + "path": "/", + "expires": -1, + "httpOnly": false, + "secure": false, + "sameSite": "Lax" + } + ], + "origins": [] +} \ No newline at end of file From ab89587acac69ea1108411a277778df9bfbd4f09 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 29 Dec 2025 12:49:28 -0800 Subject: [PATCH 03/11] circle ci config change for e2e --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b1bee9675f3..723eba408c6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3515,7 +3515,7 @@ jobs: - run: name: Run Playwright Tests command: | - cd ui/litellm-dashboard && npm run e2e + cd ui/litellm-dashboard && npx playwright test --config=e2e_tests/playwright.config.ts no_output_timeout: 120m - store_artifacts: path: test-results From 657eb2b09b8b0042feecd182ac160fd3b6ae9827 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 29 Dec 2025 12:57:36 -0800 Subject: [PATCH 04/11] attempt fix 2 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 723eba408c6..f6100a0c432 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3475,7 +3475,7 @@ jobs: - run: name: Install Playwright Browsers command: | - npx playwright install + cd ui/litellm-dashboard && npx playwright install - run: name: Build Docker image From 108fc4a59f991090755b55425c858b54b39b6c3d Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 29 Dec 2025 13:10:50 -0800 Subject: [PATCH 05/11] Using microsoft official docker for playwright --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f6100a0c432..a777ec0255e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3423,7 +3423,7 @@ jobs: e2e_ui_testing: machine: - image: ubuntu-2204:2023.10.1 + image: mcr.microsoft.com/playwright:v1.57.0-noble resource_class: xlarge working_directory: ~/project steps: From 990b7ff93d8d00f5dcbde8bd774d7beadc3965a2 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 29 Dec 2025 13:18:48 -0800 Subject: [PATCH 06/11] Fixing CI/CD --- .circleci/config.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a777ec0255e..c18f5cd128a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3422,8 +3422,8 @@ jobs: --coverage.reportsDirectory=coverage/html e2e_ui_testing: - machine: - image: mcr.microsoft.com/playwright:v1.57.0-noble + docker: + - image: mcr.microsoft.com/playwright:v1.57.0-noble resource_class: xlarge working_directory: ~/project steps: @@ -3431,12 +3431,6 @@ jobs: - setup_google_dns - attach_workspace: at: ~/project - - run: - name: Upgrade Docker to v24.x (API 1.44+) - command: | - curl -fsSL https://get.docker.com | sh - sudo usermod -aG docker $USER - docker version - run: name: Install Python 3.9 command: | From ec29fe14720eae7d4dc853fc4777ca66bacc544c Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 29 Dec 2025 13:27:00 -0800 Subject: [PATCH 07/11] remove google dns for e2e --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c18f5cd128a..4caa5189cb0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3428,7 +3428,6 @@ jobs: working_directory: ~/project steps: - checkout - - setup_google_dns - attach_workspace: at: ~/project - run: From f8d90939fe76662aae002302b14dd0d3ae2a49ec Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 29 Dec 2025 13:40:36 -0800 Subject: [PATCH 08/11] Attempt fix --- .circleci/config.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4caa5189cb0..6d9a9ff5bee 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3433,14 +3433,10 @@ jobs: - run: name: Install Python 3.9 command: | - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" - conda init bash - source ~/.bashrc - conda create -n myenv python=3.9 -y - conda activate myenv - python --version + $HOME/miniconda/bin/conda create -y -n myenv python=3.9 + conda run -n myenv python --version - run: name: Install Dependencies command: | From a4341c6e5b3df5e51d2c3ca4f2ab4d14a6e304c2 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 29 Dec 2025 19:56:44 -0800 Subject: [PATCH 09/11] test removing python install step --- .circleci/config.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6d9a9ff5bee..a010d2a1c8b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3430,13 +3430,6 @@ jobs: - checkout - attach_workspace: at: ~/project - - run: - name: Install Python 3.9 - command: | - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - $HOME/miniconda/bin/conda create -y -n myenv python=3.9 - conda run -n myenv python --version - run: name: Install Dependencies command: | From 554830d2e2fb9d4089c2bb427e26d49bbe131e3a Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 29 Dec 2025 20:33:46 -0800 Subject: [PATCH 10/11] reverting --- .circleci/config.yml | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a010d2a1c8b..34ba0c6c3a8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3422,14 +3422,32 @@ jobs: --coverage.reportsDirectory=coverage/html e2e_ui_testing: - docker: - - image: mcr.microsoft.com/playwright:v1.57.0-noble + machine: + image: ubuntu-2204:2023.10.1 resource_class: xlarge working_directory: ~/project steps: - checkout + - setup_google_dns - attach_workspace: at: ~/project + - run: + name: Upgrade Docker to v24.x (API 1.44+) + command: | + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version + - run: + name: Install Python 3.9 + command: | + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + export PATH="$HOME/miniconda/bin:$PATH" + conda init bash + source ~/.bashrc + conda create -n myenv python=3.9 -y + conda activate myenv + python --version - run: name: Install Dependencies command: | @@ -3457,8 +3475,7 @@ jobs: - run: name: Install Playwright Browsers command: | - cd ui/litellm-dashboard && npx playwright install - + npx playwright install --with-deps - run: name: Build Docker image command: docker build -t my-app:latest -f ./docker/Dockerfile.database . @@ -3497,7 +3514,10 @@ jobs: - run: name: Run Playwright Tests command: | - cd ui/litellm-dashboard && npx playwright test --config=e2e_tests/playwright.config.ts + npx playwright test \ + --config ui/litellm-dashboard/e2e_tests/playwright.config.ts \ + --reporter=html \ + --output=test-results no_output_timeout: 120m - store_artifacts: path: test-results From 46ce68619c0762788c0c0fd4456c76e52729c737 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 29 Dec 2025 20:49:24 -0800 Subject: [PATCH 11/11] attempt fix 3 --- .circleci/config.yml | 2 +- ui/litellm-dashboard/e2e_tests/playwright.config.ts | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 34ba0c6c3a8..d98fdaa4a4d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3475,7 +3475,7 @@ jobs: - run: name: Install Playwright Browsers command: | - npx playwright install --with-deps + npx playwright install - run: name: Build Docker image command: docker build -t my-app:latest -f ./docker/Dockerfile.database . diff --git a/ui/litellm-dashboard/e2e_tests/playwright.config.ts b/ui/litellm-dashboard/e2e_tests/playwright.config.ts index 5ab68b3534f..80d45a892c8 100644 --- a/ui/litellm-dashboard/e2e_tests/playwright.config.ts +++ b/ui/litellm-dashboard/e2e_tests/playwright.config.ts @@ -37,11 +37,6 @@ export default defineConfig({ name: "firefox", use: { ...devices["Desktop Firefox"] }, }, - - { - name: "webkit", - use: { ...devices["Desktop Safari"] }, - }, ], /* Timeout settings */