diff --git a/tests/e2e/ui/helpers/mcp.ts b/tests/e2e/ui/helpers/mcp.ts new file mode 100644 index 00000000000..a2e7a601527 --- /dev/null +++ b/tests/e2e/ui/helpers/mcp.ts @@ -0,0 +1,60 @@ +import { expect, Page as PwPage } from "@playwright/test"; +import { navigateToPage } from "./navigation"; +import { Page } from "../fixtures/pages"; + +/** + * Creates an MCP server through the UI's discovery -> custom form flow and + * returns the name it was given. + * + * Transport is always Streamable HTTP and auth is always None: those are the + * only combination the rest of this file's callers exercise, and "http" is the + * only transport value the proxy actually accepts. + * + * mcpServers.spec.ts still carries its own inline copy of this flow. It + * predates this helper and is green, so it was left alone rather than + * refactored blind; fold it in the next time that spec is touched. + */ +export async function createMcpServer(page: PwPage, url: string): Promise { + await navigateToPage(page, Page.McpServers); + + await page.getByRole("button", { name: /Add New MCP Server/i }).click(); + const discovery = page.getByRole("dialog").filter({ hasText: "Add MCP Server" }); + await expect(discovery).toBeVisible({ timeout: 5_000 }); + await discovery.getByRole("button", { name: /Custom Server/i }).click(); + + const formModal = page.locator(".ant-modal:visible").filter({ hasText: "MCP Server Name" }); + await expect(formModal).toBeVisible({ timeout: 5_000 }); + + // validateMCPServerName rejects spaces and hyphens. The worker index is in + // the name because at workers>1 two workers can reach Date.now() in the same + // millisecond, and a duplicate name makes the create fail rather than the + // assertion. + const name = `e2e_mcp_${process.env.TEST_WORKER_INDEX ?? "0"}_${Date.now()}`; + await formModal.locator('input[id="server_name"]').fill(name); + + const transportField = formModal.locator(".ant-form-item", { hasText: "Transport Type" }); + await transportField.locator(".ant-select").click(); + await page.locator(".ant-select-dropdown:visible").getByText("Streamable HTTP").click(); + + await formModal.locator('input[id="url"]').fill(url); + + // The auth_type Form.Item has no label prop (CreateMCPServer.tsx), so anchor + // on the enclosing Collapse panel instead of label text. + const authSection = formModal.locator(".ant-collapse-item", { hasText: /^Authentication/ }); + await authSection.locator(".ant-form-item").first().locator(".ant-select").click(); + await page.locator(".ant-select-dropdown:visible").getByText("None", { exact: true }).click(); + + await formModal.getByRole("button", { name: /^Add MCP Server$/ }).click(); + await expect(page.getByText("MCP Server created successfully").first()).toBeVisible({ timeout: 15_000 }); + + const card = page.getByTestId("mcp-servers-grid").getByText(name).first(); + await expect(card).toBeVisible({ timeout: 10_000 }); + return name; +} + +/** Opens a server from the grid and switches to its MCP Tools tab. */ +export async function openMcpToolsTab(page: PwPage, serverName: string): Promise { + await page.getByTestId("mcp-servers-grid").getByText(serverName).first().click(); + await expect(page.getByRole("button", { name: /Back to All Servers/i })).toBeVisible({ timeout: 10_000 }); + await page.getByRole("tab", { name: "MCP Tools" }).click(); +} diff --git a/tests/e2e/ui/tests/mcp/mcpTools.spec.ts b/tests/e2e/ui/tests/mcp/mcpTools.spec.ts new file mode 100644 index 00000000000..4f7fdaed9b2 --- /dev/null +++ b/tests/e2e/ui/tests/mcp/mcpTools.spec.ts @@ -0,0 +1,100 @@ +import { test, expect, Locator } from "@playwright/test"; +import { ADMIN_STORAGE_PATH } from "../../constants"; +import { createMcpServer, openMcpToolsTab } from "../../helpers/mcp"; + +// Covers the two MCP manual-QA items that mcpServers.spec.ts cannot: listing a +// server's tools, and actually calling one. Both need an MCP server that really +// answers, which the create-only spec deliberately avoids (it points at an +// unreachable .test.local URL). +// +// THIS SPEC MAKES A NETWORK CALL TO A THIRD PARTY, and that is a real cost, so +// it is stated plainly rather than buried: +// +// - The upstream is DeepWiki's public MCP server. It needs no credentials +// (Streamable HTTP, auth None), which is what makes it usable from a public +// repo -- there is no secret to leak. +// - The call is made by the PROXY, not the browser, so the proxy pod is what +// needs egress. Nothing in the e2e chart restricts that: the only +// NetworkPolicies the stack renders belong to the bundled postgresql and +// redis subcharts. +// - It is read-only (read_wiki_structure returns a repo's doc outline) and +// answered in well under a second when measured directly. +// +// If DeepWiki has an outage this spec goes red for a reason that is not a +// litellm regression. That failure is left VISIBLE by default rather than +// auto-skipped, because a spec that silently skips on connection trouble also +// silently skips when the proxy's MCP client breaks -- which is the exact +// regression it exists to catch. Set E2E_SKIP_EXTERNAL_MCP=1 to opt out +// explicitly when the upstream is known-bad. +const MCP_SERVER_URL = "https://mcp.deepwiki.com/mcp"; +const TOOL_NAME = "read_wiki_structure"; +const TOOL_ARG_REPO = "BerriAI/litellm"; + +// Each tool card renders its name in an `h4.font-mono` and its description in a +// sibling

. Matching the heading rather than page text keeps a tool whose +// DESCRIPTION mentions another tool's name from tripping strict mode. +const toolCard = (list: Locator, name: string): Locator => + list.locator("h4.font-mono").filter({ hasText: new RegExp(`^${name}$`) }); + +test.describe("MCP Tools", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + test.skip(!!process.env.E2E_SKIP_EXTERNAL_MCP, "E2E_SKIP_EXTERNAL_MCP is set"); + + let serverName: string; + + test.beforeEach(async ({ page }) => { + serverName = await createMcpServer(page, MCP_SERVER_URL); + await openMcpToolsTab(page, serverName); + }); + + test("MCP Tools tab lists the tools the upstream server advertises", async ({ page }) => { + // The list is fetched through the proxy on tab mount, so allow for a cold + // upstream connection rather than the ~250ms a warm direct call takes. + const toolList = page.locator(".mcp-tools-scrollable"); + await expect(toolList).toBeVisible({ timeout: 30_000 }); + + // Assert on tool names DeepWiki actually advertises. Asserting merely that + // the list is non-empty would still pass if the proxy returned some other + // server's tools. + await expect(toolCard(toolList, TOOL_NAME)).toBeVisible(); + await expect(toolCard(toolList, "ask_question")).toBeVisible(); + await expect(toolCard(toolList, "read_wiki_contents")).toBeVisible(); + + // Search narrows the list. No other tool's name or description contains + // this string, so exactly one card must survive. + await page.getByPlaceholder("Search tools...").fill(TOOL_NAME); + await expect(toolList.locator("h4.font-mono")).toHaveCount(1); + await expect(toolCard(toolList, TOOL_NAME)).toBeVisible(); + }); + + test("Calling a tool from the Test Tool panel returns the upstream result", async ({ page }) => { + const toolList = page.locator(".mcp-tools-scrollable"); + await expect(toolList).toBeVisible({ timeout: 30_000 }); + + await toolCard(toolList, TOOL_NAME).click(); + + // Selecting a tool swaps the right-hand pane in for the empty state. + await expect(page.getByText("Test Tool:", { exact: true })).toBeVisible({ timeout: 10_000 }); + await expect(page.getByText("Ready to Call Tool")).toBeVisible(); + + // The arguments form is generated from the tool's own inputSchema -- + // MCPToolArgumentsForm renders one antd Form.Item per property, named after + // the property, so `repoName` is proof the schema round-tripped through the + // proxy rather than the panel falling back to its generic "input" field. + const repoInput = page.locator('input[id="repoName"]'); + await expect(repoInput).toBeVisible(); + await repoInput.fill(TOOL_ARG_REPO); + + await page.getByRole("button", { name: "Call Tool", exact: true }).click(); + + await expect(page.getByText("Tool executed successfully")).toBeVisible({ timeout: 60_000 }); + // Content assertion, not just the success chrome: read_wiki_structure + // answers with the repo's page outline, so the result pane must contain + // the repo it was asked about. + await expect(page.getByText(TOOL_ARG_REPO).first()).toBeVisible(); + + // A second call is offered rather than the button resetting to its + // first-run label. + await expect(page.getByRole("button", { name: "Call Again", exact: true })).toBeVisible(); + }); +});