scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 35

test(e2e-ui): cover listing and calling MCP tools

Browse source 
Covers the two MCP manual-QA items the create-only spec cannot reach:
opening a server's tool list, and calling a tool and seeing its result.

Both need an MCP server that actually answers, so this points at DeepWiki's
public MCP server -- Streamable HTTP, auth None, so there is no credential to
hold and nothing to leak from a public repo. The call is made by the proxy,
not the browser; nothing in the e2e chart restricts that egress.

The external dependency is real and is left visible: an upstream outage turns
these red rather than auto-skipping, because a spec that skips itself on
connection trouble also skips when the proxy's MCP client is what broke.
E2E_SKIP_EXTERNAL_MCP=1 is the explicit opt-out.

Not yet executed against a live stack.
This commit is contained in:
Yuneng Jiang 2026-08-08 20:15:49 -07:00
parent 30968fb721
commit 5e189e9b1a
2 changed files with 160 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

60
tests/e2e/ui/helpers/mcp.ts Normal file
View file

@ -0,0 +1,60 @@
import { expect, Page as PwPage } from "@playwright/test";
import { navigateToPage } from "./navigation";
import { Page } from "../fixtures/pages";
/**
* Creates an MCP server through the UI's discovery -> custom form flow and
* returns the name it was given.
*
* Transport is always Streamable HTTP and auth is always None: those are the
* only combination the rest of this file's callers exercise, and "http" is the
* only transport value the proxy actually accepts.
*
* mcpServers.spec.ts still carries its own inline copy of this flow. It
* predates this helper and is green, so it was left alone rather than
* refactored blind; fold it in the next time that spec is touched.
*/
export async function createMcpServer(page: PwPage, url: string): Promise<string> {
await navigateToPage(page, Page.McpServers);
await page.getByRole("button", { name: /Add New MCP Server/i }).click();
const discovery = page.getByRole("dialog").filter({ hasText: "Add MCP Server" });
await expect(discovery).toBeVisible({ timeout: 5_000 });
await discovery.getByRole("button", { name: /Custom Server/i }).click();
const formModal = page.locator(".ant-modal:visible").filter({ hasText: "MCP Server Name" });
await expect(formModal).toBeVisible({ timeout: 5_000 });
// validateMCPServerName rejects spaces and hyphens. The worker index is in
// the name because at workers>1 two workers can reach Date.now() in the same
// millisecond, and a duplicate name makes the create fail rather than the
// assertion.
const name = `e2e_mcp_${process.env.TEST_WORKER_INDEX ?? "0"}_${Date.now()}`;
await formModal.locator('input[id="server_name"]').fill(name);
const transportField = formModal.locator(".ant-form-item", { hasText: "Transport Type" });
await transportField.locator(".ant-select").click();
await page.locator(".ant-select-dropdown:visible").getByText("Streamable HTTP").click();
await formModal.locator('input[id="url"]').fill(url);
// The auth_type Form.Item has no label prop (CreateMCPServer.tsx), so anchor
// on the enclosing Collapse panel instead of label text.
const authSection = formModal.locator(".ant-collapse-item", { hasText: /^Authentication/ });
await authSection.locator(".ant-form-item").first().locator(".ant-select").click();
await page.locator(".ant-select-dropdown:visible").getByText("None", { exact: true }).click();
await formModal.getByRole("button", { name: /^Add MCP Server$/ }).click();
await expect(page.getByText("MCP Server created successfully").first()).toBeVisible({ timeout: 15_000 });
const card = page.getByTestId("mcp-servers-grid").getByText(name).first();
await expect(card).toBeVisible({ timeout: 10_000 });
return name;
}
/** Opens a server from the grid and switches to its MCP Tools tab. */
export async function openMcpToolsTab(page: PwPage, serverName: string): Promise<void> {
await page.getByTestId("mcp-servers-grid").getByText(serverName).first().click();
await expect(page.getByRole("button", { name: /Back to All Servers/i })).toBeVisible({ timeout: 10_000 });
await page.getByRole("tab", { name: "MCP Tools" }).click();
}

100
tests/e2e/ui/tests/mcp/mcpTools.spec.ts Normal file
View file

@ -0,0 +1,100 @@
import { test, expect, Locator } from "@playwright/test";
import { ADMIN_STORAGE_PATH } from "../../constants";
import { createMcpServer, openMcpToolsTab } from "../../helpers/mcp";
// Covers the two MCP manual-QA items that mcpServers.spec.ts cannot: listing a
// server's tools, and actually calling one. Both need an MCP server that really
// answers, which the create-only spec deliberately avoids (it points at an
// unreachable .test.local URL).
//
// THIS SPEC MAKES A NETWORK CALL TO A THIRD PARTY, and that is a real cost, so
// it is stated plainly rather than buried:
//
// - The upstream is DeepWiki's public MCP server. It needs no credentials
// (Streamable HTTP, auth None), which is what makes it usable from a public
// repo -- there is no secret to leak.
// - The call is made by the PROXY, not the browser, so the proxy pod is what
// needs egress. Nothing in the e2e chart restricts that: the only
// NetworkPolicies the stack renders belong to the bundled postgresql and
// redis subcharts.
// - It is read-only (read_wiki_structure returns a repo's doc outline) and
// answered in well under a second when measured directly.
//
// If DeepWiki has an outage this spec goes red for a reason that is not a
// litellm regression. That failure is left VISIBLE by default rather than
// auto-skipped, because a spec that silently skips on connection trouble also
// silently skips when the proxy's MCP client breaks -- which is the exact
// regression it exists to catch. Set E2E_SKIP_EXTERNAL_MCP=1 to opt out
// explicitly when the upstream is known-bad.
const MCP_SERVER_URL = "https://mcp.deepwiki.com/mcp";
const TOOL_NAME = "read_wiki_structure";
const TOOL_ARG_REPO = "BerriAI/litellm";
// Each tool card renders its name in an `h4.font-mono` and its description in a
// sibling <p>. Matching the heading rather than page text keeps a tool whose
// DESCRIPTION mentions another tool's name from tripping strict mode.
const toolCard = (list: Locator, name: string): Locator =>
list.locator("h4.font-mono").filter({ hasText: new RegExp(`^${name}$`) });
test.describe("MCP Tools", () => {
test.use({ storageState: ADMIN_STORAGE_PATH });
test.skip(!!process.env.E2E_SKIP_EXTERNAL_MCP, "E2E_SKIP_EXTERNAL_MCP is set");
let serverName: string;
test.beforeEach(async ({ page }) => {
serverName = await createMcpServer(page, MCP_SERVER_URL);
await openMcpToolsTab(page, serverName);
});
test("MCP Tools tab lists the tools the upstream server advertises", async ({ page }) => {
// The list is fetched through the proxy on tab mount, so allow for a cold
// upstream connection rather than the ~250ms a warm direct call takes.
const toolList = page.locator(".mcp-tools-scrollable");
await expect(toolList).toBeVisible({ timeout: 30_000 });
// Assert on tool names DeepWiki actually advertises. Asserting merely that
// the list is non-empty would still pass if the proxy returned some other
// server's tools.
await expect(toolCard(toolList, TOOL_NAME)).toBeVisible();
await expect(toolCard(toolList, "ask_question")).toBeVisible();
await expect(toolCard(toolList, "read_wiki_contents")).toBeVisible();
// Search narrows the list. No other tool's name or description contains
// this string, so exactly one card must survive.
await page.getByPlaceholder("Search tools...").fill(TOOL_NAME);
await expect(toolList.locator("h4.font-mono")).toHaveCount(1);
await expect(toolCard(toolList, TOOL_NAME)).toBeVisible();
});
test("Calling a tool from the Test Tool panel returns the upstream result", async ({ page }) => {
const toolList = page.locator(".mcp-tools-scrollable");
await expect(toolList).toBeVisible({ timeout: 30_000 });
await toolCard(toolList, TOOL_NAME).click();
// Selecting a tool swaps the right-hand pane in for the empty state.
await expect(page.getByText("Test Tool:", { exact: true })).toBeVisible({ timeout: 10_000 });
await expect(page.getByText("Ready to Call Tool")).toBeVisible();
// The arguments form is generated from the tool's own inputSchema --
// MCPToolArgumentsForm renders one antd Form.Item per property, named after
// the property, so `repoName` is proof the schema round-tripped through the
// proxy rather than the panel falling back to its generic "input" field.
const repoInput = page.locator('input[id="repoName"]');
await expect(repoInput).toBeVisible();
await repoInput.fill(TOOL_ARG_REPO);
await page.getByRole("button", { name: "Call Tool", exact: true }).click();
await expect(page.getByText("Tool executed successfully")).toBeVisible({ timeout: 60_000 });
// Content assertion, not just the success chrome: read_wiki_structure
// answers with the repo's page outline, so the result pane must contain
// the repo it was asked about.
await expect(page.getByText(TOOL_ARG_REPO).first()).toBeVisible();
// A second call is offered rather than the button resetting to its
// first-run label.
await expect(page.getByRole("button", { name: "Call Again", exact: true })).toBeVisible();
});
});