From 4f364a8138c2ce991212c534656e3351da407382 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 11 Apr 2026 20:06:56 -0700 Subject: [PATCH 1/4] [Test] UI - Models: Add E2E tests for Add Model flow Add E2E tests covering: - Test connection with bad credentials shows failure modal - Adding a specific model and verifying it appears in All Models table - Adding a wildcard route and verifying it appears in All Models table - Verifying model dropdown shows provider-specific models (existing test updated) Added data-testid attributes to UI components to support stable test selectors. Tests verified passing 3/3 consecutive runs with zero flakiness. --- .../tests/modelsPage/addModel.spec.ts | 144 +++++++++++++++++- .../components/AllModelsTab.tsx | 3 +- .../src/components/add_model/AddModelForm.tsx | 4 +- .../add_model/litellm_model_name.tsx | 1 + .../add_model/model_connection_test.tsx | 4 +- 5 files changed, 143 insertions(+), 13 deletions(-) diff --git a/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts index 58a87ead3db..3c056a25811 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts @@ -1,24 +1,29 @@ import { test, expect } from "@playwright/test"; import { ADMIN_STORAGE_PATH, E2E_TEAM_CRUD_ID } from "../../constants"; import { Role, users } from "../../fixtures/users"; +import { navigateToPage } from "../../helpers/navigation"; +import { Page } from "../../fixtures/pages"; test.describe("Add Model", () => { test.use({ storageState: ADMIN_STORAGE_PATH }); test("Able to see all models for a specific provider in the model dropdown", async ({ page }) => { - await page.goto("/ui"); - - await page.getByText("Models + Endpoints").click(); + await navigateToPage(page, Page.Models); await page.getByRole("tab", { name: "Add Model" }).click(); - const providerInputDropdown = page.getByRole("combobox", { name: /Provider/i }); - await providerInputDropdown.fill("Anthropic"); + const providerDropdown = page.getByRole("combobox", { name: /Provider/i }); + await providerDropdown.fill("Anthropic"); await page.waitForTimeout(1000); - await providerInputDropdown.press("Enter"); + await providerDropdown.press("Enter"); await page.waitForTimeout(2000); - const providerModelsDropdown = page.locator(".ant-select-selection-overflow").first(); - await providerModelsDropdown.click(); + // The model field should be a multi-select dropdown (not a text input) + const modelSelect = page.getByTestId("model-name-select"); + await expect(modelSelect).toBeVisible({ timeout: 10_000 }); + + // Click to open the dropdown and verify provider-specific models are listed + const modelDropdown = page.locator(".ant-select-selection-overflow").first(); + await modelDropdown.click(); await expect(page.getByTitle("claude-haiku-4-5", { exact: true })).toBeVisible(); }); @@ -72,4 +77,127 @@ test.describe("Add Model", () => { await expect(page.getByText("999", { exact: true })).toBeVisible({ timeout: 10_000 }); await expect(page.getByText("888", { exact: true })).toBeVisible({ timeout: 10_000 }); }); + + test("Test connection with bad credentials shows failure", async ({ page }) => { + await navigateToPage(page, Page.Models); + await page.getByRole("tab", { name: "Add Model" }).click(); + + // Select provider: Anthropic + const providerDropdown = page.getByRole("combobox", { name: /Provider/i }); + await providerDropdown.fill("Anthropic"); + await page.waitForTimeout(1000); + await providerDropdown.press("Enter"); + await page.waitForTimeout(2000); + + // Select model: claude-haiku-4-5 + const modelDropdown = page.locator(".ant-select-selection-overflow").first(); + await modelDropdown.click(); + await page.getByTitle("claude-haiku-4-5", { exact: true }).click(); + await page.keyboard.press("Escape"); + + // Enter bad API key + const apiKeyInput = page.locator('input[type="password"]').first(); + await apiKeyInput.fill("sk-bad-key-12345"); + + // Click Test Connect + await page.getByTestId("test-connect-btn").click(); + + // Wait for modal to appear and connection test to complete + await expect(page.getByText("Connection Test Results")).toBeVisible({ timeout: 10_000 }); + + // Verify failure message appears (the test makes a real API call, so it will fail with bad creds) + await expect(page.getByTestId("connection-failure-msg")).toBeVisible({ timeout: 30_000 }); + await expect(page.getByTestId("connection-failure-msg")).toContainText("failed"); + }); + + test("Add specific model and verify it appears in All Models", async ({ page }) => { + await navigateToPage(page, Page.Models); + await page.getByRole("tab", { name: "Add Model" }).click(); + + // Select provider: Anthropic + const providerDropdown = page.getByRole("combobox", { name: /Provider/i }); + await providerDropdown.fill("Anthropic"); + await page.waitForTimeout(1000); + await providerDropdown.press("Enter"); + await page.waitForTimeout(2000); + + // Select model: claude-haiku-4-5 + const modelDropdown = page.locator(".ant-select-selection-overflow").first(); + await modelDropdown.click(); + await page.getByTitle("claude-haiku-4-5", { exact: true }).click(); + await page.keyboard.press("Escape"); + + // Enter any API key + const apiKeyInput = page.locator('input[type="password"]').first(); + await apiKeyInput.fill("sk-any-key-for-add-test"); + + // Click Add Model + await page.getByTestId("add-model-btn").click(); + + // Wait for success notification + await expect(page.getByText("created successfully")).toBeVisible({ timeout: 15_000 }); + + // Navigate to All Models tab + await page.getByRole("tab", { name: "All Models" }).click(); + await page.waitForLoadState("networkidle"); + await page.waitForTimeout(2000); + + // Search for the model we just added + await page.getByTestId("model-search-input").fill("claude-haiku-4-5"); + await page.waitForTimeout(1000); + + // Verify the model appears in the results count (not "Showing 0 results") + const resultsCount = page.getByTestId("models-results-count"); + await expect(resultsCount).not.toHaveText("Showing 0 results", { timeout: 15_000 }); + + // Verify the model name appears in the table body + const tableBody = page.locator("table tbody"); + await expect(tableBody.getByText("claude-haiku-4-5").first()).toBeVisible({ timeout: 15_000 }); + }); + + test("Add wildcard route and verify it appears in All Models", async ({ page }) => { + await navigateToPage(page, Page.Models); + await page.getByRole("tab", { name: "Add Model" }).click(); + + // Select provider: Cohere + const providerDropdown = page.getByRole("combobox", { name: /Provider/i }); + await providerDropdown.fill("Cohere"); + await page.waitForTimeout(1000); + await providerDropdown.press("Enter"); + await page.waitForTimeout(2000); + + // Select All Cohere Models (Wildcard) + const modelDropdown = page.locator(".ant-select-selection-overflow").first(); + await modelDropdown.click(); + const wildcardOption = page.getByTitle(/All .* Models \(Wildcard\)/); + await wildcardOption.click(); + await page.keyboard.press("Escape"); + + // Enter any API key + const apiKeyInput = page.locator('input[type="password"]').first(); + await apiKeyInput.fill("sk-any-key-for-wildcard-test"); + + // Click Add Model + await page.getByTestId("add-model-btn").click(); + + // Wait for success notification + await expect(page.getByText("created successfully")).toBeVisible({ timeout: 15_000 }); + + // Navigate to All Models tab + await page.getByRole("tab", { name: "All Models" }).click(); + await page.waitForLoadState("networkidle"); + await page.waitForTimeout(2000); + + // Search for the wildcard model + await page.getByTestId("model-search-input").fill("cohere"); + await page.waitForTimeout(1000); + + // Verify the model appears in the results count (not "Showing 0 results") + const resultsCount = page.getByTestId("models-results-count"); + await expect(resultsCount).not.toHaveText("Showing 0 results", { timeout: 15_000 }); + + // Verify the wildcard model appears in the table body (wildcard models show as "cohere/*") + const tableBody = page.locator("table tbody"); + await expect(tableBody.getByText("cohere/").first()).toBeVisible({ timeout: 15_000 }); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx index d7687def801..5431c196883 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx @@ -365,6 +365,7 @@ const AllModelsTab = ({ setModelNameSearch(e.target.value)} @@ -472,7 +473,7 @@ const AllModelsTab = ({ {isLoading ? ( ) : ( - + {paginationMeta.total_count > 0 ? `Showing ${((currentPage - 1) * pageSize) + 1} - ${Math.min(currentPage * pageSize, paginationMeta.total_count)} of ${paginationMeta.total_count} results` : "Showing 0 results"} diff --git a/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx b/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx index 969373421cf..f27d9bac4cd 100644 --- a/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx @@ -367,10 +367,10 @@ const AddModelForm: React.FC = ({ Need Help?
- - +
diff --git a/ui/litellm-dashboard/src/components/add_model/litellm_model_name.tsx b/ui/litellm-dashboard/src/components/add_model/litellm_model_name.tsx index 521fed7f7b5..56ecfcd3d76 100644 --- a/ui/litellm-dashboard/src/components/add_model/litellm_model_name.tsx +++ b/ui/litellm-dashboard/src/components/add_model/litellm_model_name.tsx @@ -126,6 +126,7 @@ const LiteLLMModelNameField: React.FC = ({ ) : providerModels.length > 0 ? ( - + Connection to {modelName} successful! @@ -190,7 +190,7 @@ ${formattedBody}
- + Connection to {modelName} failed
From 5e07c1cbc9131e25be07a4476ddba7de64d04e1b Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 11 Apr 2026 20:21:37 -0700 Subject: [PATCH 2/4] address greptile review feedback (greploop iteration 1) Add cleanup helper to delete models created during tests, preventing stale data accumulation across repeated test runs. --- .../tests/modelsPage/addModel.spec.ts | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts index 3c056a25811..c07fd827b39 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts @@ -4,6 +4,34 @@ import { Role, users } from "../../fixtures/users"; import { navigateToPage } from "../../helpers/navigation"; import { Page } from "../../fixtures/pages"; +/** + * Helper to delete a model by searching for it via the API and deleting matching entries. + * Accepts a partial model name to match against. + */ +async function cleanupModels(request: any, searchTerm: string) { + try { + const response = await request.get("/v2/model/info?include_team_models=true&page=1&size=100", { + headers: { Authorization: "Bearer sk-1234" }, + }); + const data = await response.json(); + const models = data?.data || []; + for (const model of models) { + const name = model.model_name || ""; + if (name.includes(searchTerm)) { + await request.post("/model/delete", { + headers: { + Authorization: "Bearer sk-1234", + "Content-Type": "application/json", + }, + data: { id: model.model_info?.id }, + }); + } + } + } catch { + // Best-effort cleanup; don't fail the test + } +} + test.describe("Add Model", () => { test.use({ storageState: ADMIN_STORAGE_PATH }); @@ -110,7 +138,9 @@ test.describe("Add Model", () => { await expect(page.getByTestId("connection-failure-msg")).toContainText("failed"); }); - test("Add specific model and verify it appears in All Models", async ({ page }) => { + test("Add specific model and verify it appears in All Models", async ({ page, request }) => { + // Clean up any leftover models from previous runs + await cleanupModels(request, "claude-haiku-4-5"); await navigateToPage(page, Page.Models); await page.getByRole("tab", { name: "Add Model" }).click(); @@ -155,7 +185,9 @@ test.describe("Add Model", () => { await expect(tableBody.getByText("claude-haiku-4-5").first()).toBeVisible({ timeout: 15_000 }); }); - test("Add wildcard route and verify it appears in All Models", async ({ page }) => { + test("Add wildcard route and verify it appears in All Models", async ({ page, request }) => { + // Clean up any leftover models from previous runs + await cleanupModels(request, "cohere/"); await navigateToPage(page, Page.Models); await page.getByRole("tab", { name: "Add Model" }).click(); From cce716334806d0c1f554bf0d206958c739af6872 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 11 Apr 2026 20:34:50 -0700 Subject: [PATCH 3/4] fix CI: replace data-testid selectors with text/role-based selectors The data-testid attributes added to React components are not present in the CI-built UI output. Switch to using getByRole and getByText selectors which work with the rendered DOM regardless of build cache. --- .../tests/modelsPage/addModel.spec.ts | 71 ++++++++----------- 1 file changed, 29 insertions(+), 42 deletions(-) diff --git a/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts index c07fd827b39..1a37caf62fd 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts @@ -32,6 +32,17 @@ async function cleanupModels(request: any, searchTerm: string) { } } +/** + * Helper to select a provider from the Add Model form dropdown. + */ +async function selectProvider(page: any, providerName: string) { + const providerDropdown = page.getByRole("combobox", { name: /Provider/i }); + await providerDropdown.fill(providerName); + await page.waitForTimeout(1000); + await providerDropdown.press("Enter"); + await page.waitForTimeout(2000); +} + test.describe("Add Model", () => { test.use({ storageState: ADMIN_STORAGE_PATH }); @@ -39,19 +50,13 @@ test.describe("Add Model", () => { await navigateToPage(page, Page.Models); await page.getByRole("tab", { name: "Add Model" }).click(); - const providerDropdown = page.getByRole("combobox", { name: /Provider/i }); - await providerDropdown.fill("Anthropic"); - await page.waitForTimeout(1000); - await providerDropdown.press("Enter"); - await page.waitForTimeout(2000); + await selectProvider(page, "Anthropic"); - // The model field should be a multi-select dropdown (not a text input) - const modelSelect = page.getByTestId("model-name-select"); - await expect(modelSelect).toBeVisible({ timeout: 10_000 }); - - // Click to open the dropdown and verify provider-specific models are listed + // The model field should be a multi-select dropdown; click to open it const modelDropdown = page.locator(".ant-select-selection-overflow").first(); await modelDropdown.click(); + + // Verify provider-specific models are listed await expect(page.getByTitle("claude-haiku-4-5", { exact: true })).toBeVisible(); }); @@ -110,12 +115,7 @@ test.describe("Add Model", () => { await navigateToPage(page, Page.Models); await page.getByRole("tab", { name: "Add Model" }).click(); - // Select provider: Anthropic - const providerDropdown = page.getByRole("combobox", { name: /Provider/i }); - await providerDropdown.fill("Anthropic"); - await page.waitForTimeout(1000); - await providerDropdown.press("Enter"); - await page.waitForTimeout(2000); + await selectProvider(page, "Anthropic"); // Select model: claude-haiku-4-5 const modelDropdown = page.locator(".ant-select-selection-overflow").first(); @@ -127,15 +127,14 @@ test.describe("Add Model", () => { const apiKeyInput = page.locator('input[type="password"]').first(); await apiKeyInput.fill("sk-bad-key-12345"); - // Click Test Connect - await page.getByTestId("test-connect-btn").click(); + // Click Test Connect button by its text + await page.getByRole("button", { name: "Test Connect" }).click(); // Wait for modal to appear and connection test to complete await expect(page.getByText("Connection Test Results")).toBeVisible({ timeout: 10_000 }); // Verify failure message appears (the test makes a real API call, so it will fail with bad creds) - await expect(page.getByTestId("connection-failure-msg")).toBeVisible({ timeout: 30_000 }); - await expect(page.getByTestId("connection-failure-msg")).toContainText("failed"); + await expect(page.getByText(/Connection to .* failed/)).toBeVisible({ timeout: 30_000 }); }); test("Add specific model and verify it appears in All Models", async ({ page, request }) => { @@ -144,12 +143,7 @@ test.describe("Add Model", () => { await navigateToPage(page, Page.Models); await page.getByRole("tab", { name: "Add Model" }).click(); - // Select provider: Anthropic - const providerDropdown = page.getByRole("combobox", { name: /Provider/i }); - await providerDropdown.fill("Anthropic"); - await page.waitForTimeout(1000); - await providerDropdown.press("Enter"); - await page.waitForTimeout(2000); + await selectProvider(page, "Anthropic"); // Select model: claude-haiku-4-5 const modelDropdown = page.locator(".ant-select-selection-overflow").first(); @@ -161,8 +155,8 @@ test.describe("Add Model", () => { const apiKeyInput = page.locator('input[type="password"]').first(); await apiKeyInput.fill("sk-any-key-for-add-test"); - // Click Add Model - await page.getByTestId("add-model-btn").click(); + // Click Add Model button by its text + await page.getByRole("button", { name: "Add Model" }).last().click(); // Wait for success notification await expect(page.getByText("created successfully")).toBeVisible({ timeout: 15_000 }); @@ -173,12 +167,11 @@ test.describe("Add Model", () => { await page.waitForTimeout(2000); // Search for the model we just added - await page.getByTestId("model-search-input").fill("claude-haiku-4-5"); + await page.locator('input[placeholder="Search model names..."]').fill("claude-haiku-4-5"); await page.waitForTimeout(1000); // Verify the model appears in the results count (not "Showing 0 results") - const resultsCount = page.getByTestId("models-results-count"); - await expect(resultsCount).not.toHaveText("Showing 0 results", { timeout: 15_000 }); + await expect(page.getByText(/Showing \d+ - \d+ of \d+ results/)).toBeVisible({ timeout: 15_000 }); // Verify the model name appears in the table body const tableBody = page.locator("table tbody"); @@ -191,12 +184,7 @@ test.describe("Add Model", () => { await navigateToPage(page, Page.Models); await page.getByRole("tab", { name: "Add Model" }).click(); - // Select provider: Cohere - const providerDropdown = page.getByRole("combobox", { name: /Provider/i }); - await providerDropdown.fill("Cohere"); - await page.waitForTimeout(1000); - await providerDropdown.press("Enter"); - await page.waitForTimeout(2000); + await selectProvider(page, "Cohere"); // Select All Cohere Models (Wildcard) const modelDropdown = page.locator(".ant-select-selection-overflow").first(); @@ -209,8 +197,8 @@ test.describe("Add Model", () => { const apiKeyInput = page.locator('input[type="password"]').first(); await apiKeyInput.fill("sk-any-key-for-wildcard-test"); - // Click Add Model - await page.getByTestId("add-model-btn").click(); + // Click Add Model button by its text + await page.getByRole("button", { name: "Add Model" }).last().click(); // Wait for success notification await expect(page.getByText("created successfully")).toBeVisible({ timeout: 15_000 }); @@ -221,12 +209,11 @@ test.describe("Add Model", () => { await page.waitForTimeout(2000); // Search for the wildcard model - await page.getByTestId("model-search-input").fill("cohere"); + await page.locator('input[placeholder="Search model names..."]').fill("cohere"); await page.waitForTimeout(1000); // Verify the model appears in the results count (not "Showing 0 results") - const resultsCount = page.getByTestId("models-results-count"); - await expect(resultsCount).not.toHaveText("Showing 0 results", { timeout: 15_000 }); + await expect(page.getByText(/Showing \d+ - \d+ of \d+ results/)).toBeVisible({ timeout: 15_000 }); // Verify the wildcard model appears in the table body (wildcard models show as "cohere/*") const tableBody = page.locator("table tbody"); From 9b74ff3ef7f9fe924b69e3be6eb961a3e777ed4b Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 11 Apr 2026 20:50:34 -0700 Subject: [PATCH 4/4] remove unnecessary cleanup helper The database is freshly seeded on every test run via seed.sql, so per-test cleanup is not needed. --- .../tests/modelsPage/addModel.spec.ts | 36 ++----------------- 1 file changed, 2 insertions(+), 34 deletions(-) diff --git a/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts index 1a37caf62fd..8834724f76b 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts @@ -4,34 +4,6 @@ import { Role, users } from "../../fixtures/users"; import { navigateToPage } from "../../helpers/navigation"; import { Page } from "../../fixtures/pages"; -/** - * Helper to delete a model by searching for it via the API and deleting matching entries. - * Accepts a partial model name to match against. - */ -async function cleanupModels(request: any, searchTerm: string) { - try { - const response = await request.get("/v2/model/info?include_team_models=true&page=1&size=100", { - headers: { Authorization: "Bearer sk-1234" }, - }); - const data = await response.json(); - const models = data?.data || []; - for (const model of models) { - const name = model.model_name || ""; - if (name.includes(searchTerm)) { - await request.post("/model/delete", { - headers: { - Authorization: "Bearer sk-1234", - "Content-Type": "application/json", - }, - data: { id: model.model_info?.id }, - }); - } - } - } catch { - // Best-effort cleanup; don't fail the test - } -} - /** * Helper to select a provider from the Add Model form dropdown. */ @@ -137,9 +109,7 @@ test.describe("Add Model", () => { await expect(page.getByText(/Connection to .* failed/)).toBeVisible({ timeout: 30_000 }); }); - test("Add specific model and verify it appears in All Models", async ({ page, request }) => { - // Clean up any leftover models from previous runs - await cleanupModels(request, "claude-haiku-4-5"); + test("Add specific model and verify it appears in All Models", async ({ page }) => { await navigateToPage(page, Page.Models); await page.getByRole("tab", { name: "Add Model" }).click(); @@ -178,9 +148,7 @@ test.describe("Add Model", () => { await expect(tableBody.getByText("claude-haiku-4-5").first()).toBeVisible({ timeout: 15_000 }); }); - test("Add wildcard route and verify it appears in All Models", async ({ page, request }) => { - // Clean up any leftover models from previous runs - await cleanupModels(request, "cohere/"); + test("Add wildcard route and verify it appears in All Models", async ({ page }) => { await navigateToPage(page, Page.Models); await page.getByRole("tab", { name: "Add Model" }).click();