From 5bcced0187d8091c4cb21553c6c18b7cabfa3f41 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 16 Oct 2026 14:04:36 -0700 Subject: [PATCH] feat(ui): redesign the playground chat around a single composer and inspector (#46934) * feat(ui): add clean-slate playground chat prototype * feat(ui): replace the playground chat with the redesigned composer and inspector * fix(ui): close the playground chat gaps found in self-review Resend only complete text exchanges, so empty replies, MCP direct calls and transcriptions no longer leak into chat history. A stopped request can no longer append output, attach its gateway trace to a newer reply or clear the newer request's loading state. MCP selections are only pruned once both the server and toolset lists load. The MCP tool picker lists toolsets again and calls the server that backs the chosen tool, and a saved model the current key cannot use is cleared when the model list loads, matching the old chat Also vendors the scroll-fade-x utility the attachment primitive uses * fix(ui): resolve playground toolset tools per server and drop late session ids A toolset now lists only its own tools, each resolved on the server the toolset names, so the args form and the call always target the same tool. A saved toolset no longer asks for tools with its own id while the toolset list loads, and a failed tool list says so instead of claiming the server has no tools A stopped request can no longer overwrite the Responses session id, starting a new chat drops a request that is still running, embeddings replies are no longer resent as chat history, and a saved model missing from an already cached model list is cleared too. A failing toolset list falls back right away instead of retrying while the MCP pickers show loading * fix(ui): guard playground sends and label inspector stats Pressing Enter twice no longer sends the same message twice, an attachment the new endpoint cannot take is dropped when the endpoint changes, a stopped Responses request can no longer show its code output on a newer reply, and a reply that broke mid-stream is no longer resent as history The inspector's speed, cost and routing stats get the same icons as the reply footer, and the footer's metrics now line up with its copy and Inspect buttons * fix(ui): write playground key settings only through the encoded storage path Key source, virtual key and custom headers now go through their own write loop, separate from the plain settings, so a key value can never take the plain sessionStorage branch * fix(ui): restore playground BYOK connect and reconnect Picking a BYOK server for a direct tool call now shows a key button next to the server picker, so the key can be connected without leaving the MCP endpoint. A connected BYOK server gets a Replace key action in the MCP dialog, also shown when its tools fail to load, and saving a key reloads that server's tool list * revert: restore the single playground settings storage loop Reverts ca4e742115. The CodeQL finding covers the same encoded sessionStorage handling the old chat used, so it is left as is * test(ui): type the BYOK credential mock without deep instantiation * fix(ui): let users back out of replacing a BYOK key The replace-key form in the MCP dialog now has Cancel, and it resets when another server is picked or the dialog closes, so a working key is never forced to be overwritten * refactor(ui): name the MCP dialog row focus handler * fix(ui): accept exact custom models, tags and decimal temperatures in the playground A typed model or tag that is a prefix of a listed one could not be used, because Enter picked the partial match and the custom action only showed when nothing matched. The custom value is now a list option whenever no exact match exists. Temperature gets an editable input beside the slider so values finer than the 0.1 step can be sent again. * fix(ui): stop claiming no guardrails ran when the gateway sent no headers Direct MCP calls, A2A agents and any request that fails before headers arrive built their trace from an empty header set, so the inspector said no guardrails or policies ran. The gateway fields are now left unset in that case and the inspector says the endpoint does not report them. * fix(ui): leave policies unknown when the gateway headers are hidden The proxy exposes x-litellm-applied-guardrails to other origins but not x-litellm-applied-policies, so a dashboard on another origin read every request as having run no policies. Policies are now unknown unless the call id header is readable, and the inspector says so. --- tests/e2e/ui/helpers/playground.ts | 55 +- .../guardrails/presidioUserStory.spec.ts | 10 +- .../tests/internal-user/internalUser.spec.ts | 9 +- .../ui/tests/playground/playground.spec.ts | 23 +- .../e2e/ui/tests/team-admin/teamAdmin.spec.ts | 15 +- ui/litellm-dashboard/eslint-suppressions.json | 15 + .../components/chat/ByokConnect.tsx | 152 ++++ .../components/chat/Composer.test.tsx | 61 ++ .../playground/components/chat/Composer.tsx | 196 ++++++ .../components/chat/ComposerControls.tsx | 120 ++++ .../components/chat/Conversation.tsx | 388 +++++++++++ .../playground/components/chat/Inspector.tsx | 292 ++++++++ .../playground/components/chat/McpDialog.tsx | 391 +++++++++++ .../playground/components/chat/ModelName.tsx | 74 ++ .../components/chat/Pickers.test.tsx | 83 +++ .../playground/components/chat/Pickers.tsx | 193 +++++ .../chat/PlaygroundChat.integration.test.tsx | 658 ++++++++++++++++++ .../components/chat/PlaygroundChat.tsx | 489 +++++++++++++ .../components/chat/ReplyContent.tsx | 92 +++ .../components/chat/SettingsPopover.tsx | 395 +++++++++++ .../components/chat/composerFiles.ts | 52 ++ .../components/chat/dispatchRequest.test.ts | 58 ++ .../components/chat/dispatchRequest.ts | 295 ++++++++ .../components/chat/endpointCapabilities.ts | 104 +++ .../chat/playgroundSettings.test.ts | 149 ++++ .../components/chat/playgroundSettings.ts | 192 +++++ .../playground/components/chat/useIsWide.ts | 16 + .../components/chat/usePlaygroundCatalog.ts | 147 ++++ .../components/chat/usePlaygroundSend.ts | 263 +++++++ .../components/chat/usePlaygroundSettings.ts | 38 + .../llm_calls/anthropic_messages.tsx | 3 + .../playground/llm_calls/audio_speech.tsx | 3 + .../llm_calls/audio_transcriptions.tsx | 3 + .../playground/llm_calls/embeddings_api.tsx | 3 + .../playground/llm_calls/image_edits.tsx | 3 + .../playground/llm_calls/image_generation.tsx | 3 + .../playground/llm_calls/interactions_api.tsx | 3 + .../app/(dashboard)/playground/page.test.tsx | 4 +- .../src/app/(dashboard)/playground/page.tsx | 87 ++- ui/litellm-dashboard/src/app/globals.css | 196 ++++++ .../src/components/chat_ui/types.ts | 24 + .../components/llm_calls/chat_completion.tsx | 23 +- .../components/llm_calls/reply_trace.test.ts | 64 ++ .../src/components/llm_calls/reply_trace.ts | 49 ++ .../components/llm_calls/responses_api.tsx | 18 +- .../src/components/ui/accordion.tsx | 59 ++ .../src/components/ui/attachment.tsx | 185 +++++ .../src/components/ui/item.tsx | 179 +++++ .../src/components/ui/kbd.tsx | 20 + .../src/components/ui/marker.tsx | 67 ++ .../src/components/ui/message-scroller.tsx | 117 ++++ .../src/components/ui/message.tsx | 74 ++ .../src/components/ui/spinner.tsx | 16 + .../src/components/ui/toggle-group.tsx | 87 +++ .../src/components/ui/toggle.tsx | 37 + .../src/lib/http/observeResponseHeaders.ts | 9 + 56 files changed, 6245 insertions(+), 116 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/ByokConnect.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Composer.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Composer.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/ComposerControls.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Conversation.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Inspector.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/McpDialog.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/ModelName.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Pickers.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Pickers.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/PlaygroundChat.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/PlaygroundChat.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/ReplyContent.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/SettingsPopover.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/composerFiles.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/dispatchRequest.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/dispatchRequest.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/endpointCapabilities.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/playgroundSettings.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/playgroundSettings.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/useIsWide.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/usePlaygroundCatalog.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/usePlaygroundSend.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/usePlaygroundSettings.ts create mode 100644 ui/litellm-dashboard/src/components/llm_calls/reply_trace.test.ts create mode 100644 ui/litellm-dashboard/src/components/llm_calls/reply_trace.ts create mode 100644 ui/litellm-dashboard/src/components/ui/accordion.tsx create mode 100644 ui/litellm-dashboard/src/components/ui/attachment.tsx create mode 100644 ui/litellm-dashboard/src/components/ui/item.tsx create mode 100644 ui/litellm-dashboard/src/components/ui/kbd.tsx create mode 100644 ui/litellm-dashboard/src/components/ui/marker.tsx create mode 100644 ui/litellm-dashboard/src/components/ui/message-scroller.tsx create mode 100644 ui/litellm-dashboard/src/components/ui/message.tsx create mode 100644 ui/litellm-dashboard/src/components/ui/spinner.tsx create mode 100644 ui/litellm-dashboard/src/components/ui/toggle-group.tsx create mode 100644 ui/litellm-dashboard/src/components/ui/toggle.tsx create mode 100644 ui/litellm-dashboard/src/lib/http/observeResponseHeaders.ts diff --git a/tests/e2e/ui/helpers/playground.ts b/tests/e2e/ui/helpers/playground.ts index 866334e1c94..b46a2263072 100644 --- a/tests/e2e/ui/helpers/playground.ts +++ b/tests/e2e/ui/helpers/playground.ts @@ -2,42 +2,57 @@ import { expect, type Locator, type Page as PlaywrightPage } from "@playwright/t import { navigateToPage, dismissFeedbackPopup } from "./navigation"; import { Page } from "../fixtures/pages"; -/** Controls for the Test Key / Playground page, shared with the router-fallback specs. */ - -/** - * The configuration panel is rendered twice, docked and overlay, with one visible at a time. - * Every control is narrowed to the visible copy or it trips strict mode against its hidden twin. - */ export const onlyVisible = (locator: Locator): Locator => locator.filter({ visible: true }).first(); -/** The model combobox, addressed by the placeholder its search input shows before selection. */ -export const modelSelect = (page: PlaywrightPage): Locator => onlyVisible(page.getByPlaceholder("Select a Model")); +export const composer = (page: PlaywrightPage): Locator => + onlyVisible(page.getByRole("group").filter({ has: page.getByRole("textbox", { name: "Message" }) })); + +export const modelSelect = (page: PlaywrightPage): Locator => + onlyVisible(page.getByRole("combobox", { name: "Model", exact: true })); export const sendButton = (page: PlaywrightPage): Locator => - onlyVisible(page.getByRole("button", { name: "Send message" })); + composer(page).getByRole("button", { name: "Send message" }); -/** The Virtual Key Source dropdown, addressed by the accessible name on its trigger. */ -export const keySourceSelect = (page: PlaywrightPage): Locator => - onlyVisible(page.getByLabel("Virtual Key Source")); +export const keySourceSelect = (page: PlaywrightPage): Locator => onlyVisible(page.getByLabel("Send as")); export async function openPlayground(page: PlaywrightPage): Promise { await navigateToPage(page, Page.LlmPlayground); await dismissFeedbackPopup(page); - await expect(onlyVisible(page.getByText("Virtual Key Source"))).toBeVisible({ - timeout: 20_000, - }); + await expect(modelSelect(page)).toBeVisible({ timeout: 20_000 }); + const searchHint = page.getByRole("button", { name: "Dismiss search hint" }); + if (await searchHint.isVisible()) { + await searchHint.click(); + } +} + +export async function openRequestSettings(page: PlaywrightPage): Promise { + await composer(page).getByRole("button", { name: "Settings", exact: true }).click(); + await expect(keySourceSelect(page)).toBeVisible({ timeout: 10_000 }); +} + +export async function enterVirtualKey(page: PlaywrightPage, key: string): Promise { + await openRequestSettings(page); + await keySourceSelect(page).click(); + await onlyVisible(page.getByRole("option", { name: "Virtual key", exact: true })).click({ timeout: 15_000 }); + const keyInput = onlyVisible(page.getByLabel("Virtual key", { exact: true })); + await expect(keyInput).toBeVisible({ timeout: 10_000 }); + await keyInput.fill(key); + await page.keyboard.press("Escape"); +} + +export async function searchModels(page: PlaywrightPage, query: string): Promise { + await modelSelect(page).click(); + await onlyVisible(page.getByPlaceholder("Search models")).fill(query); } export async function selectModel(page: PlaywrightPage, model: string): Promise { - const select = modelSelect(page); - await select.click(); - // Virtualized: options outside the rendered window are absent from the DOM, so search first. - await select.fill(model); + await searchModels(page, model); await onlyVisible(page.getByRole("option", { name: model, exact: true })).click({ timeout: 15_000 }); + await expect(modelSelect(page)).toContainText(model); } export async function sendMessage(page: PlaywrightPage, message: string): Promise { - const input = onlyVisible(page.getByPlaceholder("Type your message", { exact: false })); + const input = composer(page).getByRole("textbox", { name: "Message" }); await expect(input).toBeVisible({ timeout: 15_000 }); await input.fill(message); await sendButton(page).click(); diff --git a/tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts b/tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts index b4dc7c4e8da..c57a9147ab0 100644 --- a/tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts +++ b/tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts @@ -3,7 +3,7 @@ import { ADMIN_STORAGE_PATH, MOCK_PRESIDIO_URL, PROPAGATION_TIMEOUT_MS } from ". import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation"; import { Page } from "../../fixtures/pages"; import { CHAT_MODEL_A, masterKey, rootPath, uniqueSuffix, waitForSpendLog } from "../../helpers/traffic"; -import { openPlayground, selectModel, sendButton, onlyVisible } from "../../helpers/playground"; +import { composer, openPlayground, selectModel, sendButton, onlyVisible } from "../../helpers/playground"; const RAW_EMAIL = "jane.doe@example.com"; const RAW_PHONE = "555-867-5309"; @@ -123,14 +123,12 @@ test.describe("Presidio PII guardrail, end to end from the dashboard", () => { await openPlayground(page); await selectModel(page, CHAT_MODEL_A); - const guardrailSelect = onlyVisible(page.getByPlaceholder("Select guardrails")); - await expect(guardrailSelect).toBeVisible({ timeout: 20_000 }); - await guardrailSelect.click(); - await guardrailSelect.fill(guardrailName); + await composer(page).getByRole("combobox", { name: "Guardrails" }).click(); + await onlyVisible(page.getByPlaceholder("Search guardrails")).fill(guardrailName); await onlyVisible(page.getByRole("option", { name: guardrailName, exact: true })).click({ timeout: 20_000 }); await page.keyboard.press("Escape"); - const input = onlyVisible(page.getByPlaceholder("Type your message", { exact: false })); + const input = composer(page).getByRole("textbox", { name: "Message" }); await expect(input).toBeVisible({ timeout: 15_000 }); await input.fill(prompt); diff --git a/tests/e2e/ui/tests/internal-user/internalUser.spec.ts b/tests/e2e/ui/tests/internal-user/internalUser.spec.ts index f392c5104da..d34bc034453 100644 --- a/tests/e2e/ui/tests/internal-user/internalUser.spec.ts +++ b/tests/e2e/ui/tests/internal-user/internalUser.spec.ts @@ -9,7 +9,7 @@ import { import { Page } from "../../fixtures/pages"; import { navigateToPage, clickTeamId } from "../../helpers/navigation"; import { CHAT_MODEL_A, MOCK_RESPONSE_TEXT, masterKey } from "../../helpers/traffic"; -import { keySourceSelect, onlyVisible, openPlayground, selectModel, sendMessage } from "../../helpers/playground"; +import { openPlayground, selectModel, sendMessage, enterVirtualKey } from "../../helpers/playground"; test.describe("Internal User", () => { test.use({ storageState: INTERNAL_USER_STORAGE_PATH }); @@ -73,12 +73,7 @@ test.describe("Internal User", () => { try { await openPlayground(page); - await keySourceSelect(page).click(); - await onlyVisible(page.getByRole("option", { name: "Virtual Key" })).click({ timeout: 15_000 }); - - const keyInput = onlyVisible(page.getByPlaceholder("Enter custom Virtual Key")); - await expect(keyInput).toBeVisible({ timeout: 10_000 }); - await keyInput.fill(apiKey); + await enterVirtualKey(page, apiKey); await selectModel(page, CHAT_MODEL_A); await sendMessage(page, `internal user team key ping ${keyName}`); diff --git a/tests/e2e/ui/tests/playground/playground.spec.ts b/tests/e2e/ui/tests/playground/playground.spec.ts index 9a9ecd539b3..205cc57ae46 100644 --- a/tests/e2e/ui/tests/playground/playground.spec.ts +++ b/tests/e2e/ui/tests/playground/playground.spec.ts @@ -1,7 +1,14 @@ import { test, expect } from "@playwright/test"; import { ADMIN_STORAGE_PATH } from "../../constants"; import { CHAT_MODEL_A, CHAT_MODEL_B, MOCK_RESPONSE_TEXT, createVirtualKey } from "../../helpers/traffic"; -import { keySourceSelect, onlyVisible, openPlayground, selectModel, sendMessage } from "../../helpers/playground"; +import { + keySourceSelect, + openPlayground, + openRequestSettings, + selectModel, + sendMessage, + enterVirtualKey, +} from "../../helpers/playground"; /** * The one flow that exercises the dashboard's own LLM call path rather than an admin CRUD endpoint, @@ -14,8 +21,9 @@ test.describe("Playground", () => { test(`chats with ${model} using the current UI session`, async ({ page }) => { await openPlayground(page); - // "Current UI Session" is the default: the logged-in admin's key, nothing pasted. - await expect(keySourceSelect(page)).toContainText("Current UI Session"); + await openRequestSettings(page); + await expect(keySourceSelect(page)).toContainText("Current UI session"); + await page.keyboard.press("Escape"); await selectModel(page, model); const prompt = `playground ping for ${model}`; @@ -34,14 +42,7 @@ test.describe("Playground", () => { await openPlayground(page); - // Switch the source to "Virtual Key" and paste the key we just minted. - await keySourceSelect(page).click(); - await onlyVisible(page.getByRole("option", { name: "Virtual Key" })).click({ timeout: 15_000 }); - - const keyInput = onlyVisible(page.getByPlaceholder("Enter custom Virtual Key")); - await expect(keyInput).toBeVisible({ timeout: 10_000 }); - await keyInput.fill(key); - + await enterVirtualKey(page, key); await selectModel(page, CHAT_MODEL_A); await sendMessage(page, "playground ping via virtual key"); diff --git a/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts b/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts index f3c031f0172..ff41778b2c9 100644 --- a/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts +++ b/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts @@ -10,7 +10,7 @@ import { Page } from "../../fixtures/pages"; import { navigateToPage, dismissFeedbackPopup, clickTeamId } from "../../helpers/navigation"; import { captureRequestBody, readBack } from "../../helpers/roundTrip"; import { CHAT_MODEL_A, masterKey } from "../../helpers/traffic"; -import { keySourceSelect, modelSelect, onlyVisible, openPlayground } from "../../helpers/playground"; +import { onlyVisible, openPlayground, searchModels, enterVirtualKey } from "../../helpers/playground"; /** * Every identifier a roster is addressable by. Which of user_id / user_email is populated depends on @@ -228,21 +228,14 @@ test.describe("Team Admin", () => { .toBe(true); await openPlayground(page); - await keySourceSelect(page).click(); - await onlyVisible(page.getByRole("option", { name: "Virtual Key" })).click({ timeout: 15_000 }); + await enterVirtualKey(page, teamKey); - const keyInput = onlyVisible(page.getByPlaceholder("Enter custom Virtual Key")); - await expect(keyInput).toBeVisible({ timeout: 10_000 }); - await keyInput.fill(teamKey); - - const select = modelSelect(page); - await select.click(); - await select.fill(teamModelName); + await searchModels(page, teamModelName); await expect(onlyVisible(page.getByRole("option", { name: teamModelName }))).toBeVisible({ timeout: 15_000, }); - await select.fill(CHAT_MODEL_A); + await onlyVisible(page.getByPlaceholder("Search models")).fill(CHAT_MODEL_A); await expect(onlyVisible(page.getByRole("option", { name: CHAT_MODEL_A }))).toBeVisible({ timeout: 15_000, }); diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index b1b6cef7320..9eead41a102 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -2090,11 +2090,26 @@ "count": 1 } }, + "src/components/ui/attachment.tsx": { + "local/no-ad-hoc-z-index": { + "count": 2 + } + }, + "src/components/ui/message-scroller.tsx": { + "local/no-noop-hover-variant": { + "count": 1 + } + }, "src/components/ui/slider.tsx": { "no-nested-ternary": { "count": 1 } }, + "src/components/ui/toggle-group.tsx": { + "local/no-ad-hoc-z-index": { + "count": 2 + } + }, "src/components/update_model_credentials_modal.tsx": { "local/filename-pascal-case": { "count": 1 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/ByokConnect.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/ByokConnect.tsx new file mode 100644 index 00000000000..f94c43b720a --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/ByokConnect.tsx @@ -0,0 +1,152 @@ +"use client"; + +import { useId, useState } from "react"; +import { Check, ExternalLink, KeyRound } from "lucide-react"; +import { PasswordInput } from "@/components/shared/PasswordInput"; +import { Button } from "@/components/ui/button"; +import { InputGroupButton } from "@/components/ui/input-group"; +import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; +import { Field, FieldDescription, FieldGroup, FieldLabel } from "@/components/ui/field"; +import { Spinner } from "@/components/ui/spinner"; +import { Switch } from "@/components/ui/switch"; +import type { MCPServer } from "@/components/mcp_tools/types"; +import { fetchClient } from "@/lib/http/api"; +import { ApiError } from "@/lib/http/client"; +import { toast } from "@/lib/toast"; + +const saveErrorMessage = (error: unknown): string => { + if (error instanceof ApiError) { + const detail = (error.body as { detail?: { error?: string } } | null)?.detail?.error; + if (detail) return detail; + } + return error instanceof Error && error.message ? error.message : "Failed to connect"; +}; + +export function ByokConnect({ + server, + name, + onConnected, + onCancel, +}: { + server: MCPServer; + name: string; + onConnected: () => void; + onCancel?: () => void; +}) { + const [credential, setCredential] = useState(""); + const [save, setSave] = useState(true); + const [isSaving, setIsSaving] = useState(false); + const keyId = useId(); + const saveId = useId(); + + const connect = async () => { + if (!credential.trim()) { + toast.error("Enter your API key"); + return; + } + setIsSaving(true); + try { + await fetchClient.POST("/v1/mcp/server/{server_id}/user-credential", { + params: { path: { server_id: server.server_id } }, + body: { credential: credential.trim(), save }, + }); + toast.success(`Connected to ${name}`); + setCredential(""); + onConnected(); + } catch (error) { + toast.error(saveErrorMessage(error)); + } finally { + setIsSaving(false); + } + }; + + return ( + +
+

+ + {server.has_user_credential ? `Replace your ${name} API key` : `${name} needs your API key`} +

+

+ {server.has_user_credential + ? `The new key replaces the one the gateway uses for ${name}.` + : `The gateway calls ${name} with your own key. Connect it to list and use its tools.`} +

+
+ {server.byok_description && server.byok_description.length > 0 && ( +
    + {server.byok_description.map((item) => ( +
  • + + {item} +
    • + ))} +
+ )} + + {name} API key + setCredential(event.target.value)} + placeholder="Paste your API key" + /> + {server.byok_api_key_help_url && ( + + + Where to find this key + + + + )} + + + + Save the key for next time + + + +
+ + {onCancel && ( + + )} +
+ + ); +} + +export function ByokKeyButton({ + server, + name, + onConnected, +}: { + server: MCPServer; + name: string; + onConnected: () => void; +}) { + const [open, setOpen] = useState(false); + return ( + + }> + + {server.has_user_credential ? "Replace key" : "Connect key"} + + + { + setOpen(false); + onConnected(); + }} + /> + + + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Composer.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Composer.test.tsx new file mode 100644 index 00000000000..c62b839e5b3 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Composer.test.tsx @@ -0,0 +1,61 @@ +import { createRef } from "react"; +import { createPortal } from "react-dom"; +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; +import { EndpointType } from "@/components/chat_ui/mode_endpoint_mapping"; +import type { MCPToolArgumentsFormRef } from "@/components/mcp_tools/MCPToolArgumentsForm"; +import { Composer, type ChecklistProps, type DraftState } from "./Composer"; +import { capabilitiesFor } from "./endpointCapabilities"; + +const checklist: ChecklistProps = { groups: [], value: [], onValueChange: vi.fn(), isLoading: false }; + +const draft: DraftState = { + text: "", + setText: vi.fn(), + files: [], + attach: vi.fn(), + removeFile: vi.fn(), + isLoading: false, + send: vi.fn().mockResolvedValue(undefined), + cancel: vi.fn(), +}; + +const renderComposer = () => + render( + } + draft={draft} + directTool={null} + toolArgsForm={createRef()} + mcpControl={null} + vectorStores={checklist} + guardrails={checklist} + codeInterpreter={{ enabled: false, onToggle: vi.fn() }} + settingsControl={createPortal(, document.body)} + sendDisabled + />, + ); + +describe("Composer", () => { + it("keeps focus in a popover field opened from the composer toolbar", async () => { + const user = userEvent.setup(); + renderComposer(); + + await user.click(screen.getByLabelText("Request tags")); + + expect(screen.getByLabelText("Request tags")).toHaveFocus(); + }); + + it("focuses the message box when empty toolbar space is clicked", async () => { + const user = userEvent.setup(); + const { container } = renderComposer(); + const toolbar = container.querySelectorAll('[data-slot="input-group-addon"]'); + + await user.click(toolbar[toolbar.length - 1]); + + expect(screen.getByLabelText("Message")).toHaveFocus(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Composer.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Composer.tsx new file mode 100644 index 00000000000..a9fbb777393 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Composer.tsx @@ -0,0 +1,196 @@ +"use client"; + +import { useRef, type DragEvent, type KeyboardEvent, type MouseEvent, type ReactNode, type RefObject } from "react"; +import { ArrowUp, Code2, Database, Paperclip, Shield, Square } from "lucide-react"; +import { EndpointType } from "@/components/chat_ui/mode_endpoint_mapping"; +import MCPToolArgumentsForm, { type MCPToolArgumentsFormRef } from "@/components/mcp_tools/MCPToolArgumentsForm"; +import type { MCPTool } from "@/components/mcp_tools/types"; +import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupTextarea } from "@/components/ui/input-group"; +import { Toggle } from "@/components/ui/toggle"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { DraftAttachments } from "./ComposerControls"; +import type { DraftFile } from "./composerFiles"; +import { ACCEPT_FOR, ATTACH_LABEL, type Capabilities } from "./endpointCapabilities"; +import { ChecklistMenu, type OptionGroup } from "./Pickers"; + +export type ChecklistProps = Readonly<{ + groups: readonly OptionGroup[]; + value: readonly string[]; + onValueChange: (value: readonly string[]) => void; + isLoading: boolean; +}>; + +export type DraftState = Readonly<{ + text: string; + setText: (text: string) => void; + files: readonly DraftFile[]; + attach: (files: readonly File[]) => void; + removeFile: (index: number) => void; + isLoading: boolean; + send: () => Promise; + cancel: () => void; +}>; + +function AttachButton({ + capabilities, + attach, +}: { + capabilities: Capabilities; + attach: (files: readonly File[]) => void; +}) { + const input = useRef(null); + if (!capabilities.attach) return null; + const label = ATTACH_LABEL[capabilities.attach]; + return ( + <> + + input.current?.click()} />} + > + + + {label} + + { + attach(Array.from(event.target.files ?? [])); + event.target.value = ""; + }} + /> + + ); +} + +function SendButton({ draft, disabled }: { draft: DraftState; disabled: boolean }) { + if (draft.isLoading) + return ( + + + + ); + return ( + void draft.send()} + > + + + ); +} + +export function Composer({ + capabilities, + endpoint, + targetRow, + draft, + directTool, + toolArgsForm, + mcpControl, + vectorStores, + guardrails, + codeInterpreter, + settingsControl, + sendDisabled, +}: { + capabilities: Capabilities; + endpoint: EndpointType; + targetRow: ReactNode; + draft: DraftState; + directTool: MCPTool | null; + toolArgsForm: RefObject; + mcpControl: ReactNode; + vectorStores: ChecklistProps; + guardrails: ChecklistProps; + codeInterpreter: Readonly<{ enabled: boolean; onToggle: (enabled: boolean) => void }>; + settingsControl: ReactNode; + sendDisabled: boolean; +}) { + const isDirectCall = endpoint === EndpointType.MCP; + const isRealtime = endpoint === EndpointType.REALTIME; + const message = useRef(null); + const focusMessage = (event: MouseEvent) => { + const target = event.target as HTMLElement; + if (!event.currentTarget.contains(target) || target.closest("button, input, textarea")) return; + message.current?.focus(); + }; + const handleDrop = (event: DragEvent) => { + if (!capabilities.attach) return; + event.preventDefault(); + draft.attach(Array.from(event.dataTransfer.files)); + }; + const handleKeyDown = (event: KeyboardEvent) => { + if (event.key !== "Enter" || event.shiftKey || event.nativeEvent.isComposing) return; + event.preventDefault(); + if (!sendDisabled) void draft.send(); + }; + const directCallBody = ( +
+ {directTool ? ( + + ) : ( +

Pick a server and a tool above to fill in its arguments

+ )} +
+ ); + const textBody = ( + draft.setText(event.target.value)} + onKeyDown={handleKeyDown} + placeholder={capabilities.placeholder} + rows={2} + aria-label="Message" + className="max-h-60" + /> + ); + return ( + capabilities.attach && event.preventDefault()} onDrop={handleDrop}> + + {targetRow} + + {draft.files.length > 0 && ( + + + + )} + {!isRealtime && (isDirectCall ? directCallBody : textBody)} + + {capabilities.mcp && mcpControl} + {capabilities.vectorStores && ( + } emptyText="No vector stores yet" {...vectorStores} /> + )} + {capabilities.guardrails && ( + } emptyText="No guardrails yet" {...guardrails} /> + )} + {capabilities.codeInterpreter && ( + + + Code interpreter + + )} + + {settingsControl} + + {!isRealtime && } + + + + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/ComposerControls.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/ComposerControls.tsx new file mode 100644 index 00000000000..e3e52be3b7b --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/ComposerControls.tsx @@ -0,0 +1,120 @@ +"use client"; + +import { ChevronDown, FileText, Music, X } from "lucide-react"; +import { EndpointType } from "@/components/chat_ui/mode_endpoint_mapping"; +import { + Attachment, + AttachmentAction, + AttachmentActions, + AttachmentContent, + AttachmentDescription, + AttachmentGroup, + AttachmentMedia, + AttachmentTitle, +} from "@/components/ui/attachment"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuLabel, + DropdownMenuRadioGroup, + DropdownMenuRadioItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { InputGroupButton } from "@/components/ui/input-group"; +import { ENDPOINT_OPTIONS, OPEN_AI_VOICE_SELECT_OPTIONS, type OpenAIVoice } from "../chat_ui/chatConstants"; +import { formatBytes, type DraftFile } from "./composerFiles"; + +const shortEndpoint = (label: string): string => label.replace(/^\/v1(beta)?\//, ""); + +export function EndpointMenu({ + value, + onValueChange, +}: { + value: EndpointType; + onValueChange: (value: EndpointType) => void; +}) { + const current = ENDPOINT_OPTIONS.find((option) => option.value === value); + return ( + + }> + {shortEndpoint(current?.label ?? value)} + + + + onValueChange(next as EndpointType)}> + Endpoint + {ENDPOINT_OPTIONS.map((option) => ( + + {option.label} + + ))} + + + + ); +} + +export function VoiceMenu({ + value, + onValueChange, +}: { + value: OpenAIVoice; + onValueChange: (value: OpenAIVoice) => void; +}) { + const current = OPEN_AI_VOICE_SELECT_OPTIONS.find((option) => option.value === value); + return ( + + }> + Voice: {current?.label.split(" - ")[0] ?? value} + + + + onValueChange(next as OpenAIVoice)}> + Voice + {OPEN_AI_VOICE_SELECT_OPTIONS.map((option) => ( + + {option.label} + + ))} + + + + ); +} + +function FileMedia({ draft }: { draft: DraftFile }) { + if (draft.previewUrl) + return ( + + + + ); + return {draft.file.type.startsWith("audio/") ? : }; +} + +export function DraftAttachments({ + files, + onRemove, +}: { + files: readonly DraftFile[]; + onRemove: (index: number) => void; +}) { + return ( + + {files.map((draft, index) => ( + + + + {draft.file.name} + {formatBytes(draft.file.size)} + + + onRemove(index)}> + + + + + ))} + + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Conversation.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Conversation.tsx new file mode 100644 index 00000000000..31eb395408a --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Conversation.tsx @@ -0,0 +1,388 @@ +"use client"; + +import type { ReactNode } from "react"; +import { + ArrowDownToLine, + ArrowUpFromLine, + Brain, + Clock, + Coins, + DatabaseZap, + FileText, + Music, + PlayCircle, + Repeat, + ScanSearch, + Timer, + Wrench, +} from "lucide-react"; +import { EndpointType } from "@/components/chat_ui/mode_endpoint_mapping"; +import MCPEventsDisplay from "@/components/chat_ui/MCPEventsDisplay"; +import ReasoningContent from "@/components/chat_ui/ReasoningContent"; +import type { MessageAttachment, MessageType } from "@/components/chat_ui/types"; +import type { CodeInterpreterResult } from "@/components/llm_calls/code_interpreter_handler"; +import type { MCPEvent } from "@/components/mcp_tools/types"; +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; +import { + Attachment, + AttachmentContent, + AttachmentDescription, + AttachmentGroup, + AttachmentMedia, + AttachmentTitle, +} from "@/components/ui/attachment"; +import { Badge } from "@/components/ui/badge"; +import { Bubble, BubbleContent } from "@/components/ui/Bubble"; +import { Button } from "@/components/ui/button"; +import { Empty, EmptyContent, EmptyDescription, EmptyHeader, EmptyMedia, EmptyTitle } from "@/components/ui/empty"; +import { Marker, MarkerContent, MarkerIcon } from "@/components/ui/marker"; +import { Message, MessageContent, MessageFooter, MessageHeader } from "@/components/ui/message"; +import { + MessageScroller, + MessageScrollerButton, + MessageScrollerContent, + MessageScrollerItem, + MessageScrollerProvider, + MessageScrollerViewport, +} from "@/components/ui/message-scroller"; +import { Spinner } from "@/components/ui/spinner"; +import { Toggle } from "@/components/ui/toggle"; +import A2AMetrics from "../chat_ui/A2AMetrics"; +import CodeInterpreterOutput from "../chat_ui/CodeInterpreterOutput"; +import { SearchResultsDisplay } from "../chat_ui/SearchResultsDisplay"; +import { formatBytes } from "./composerFiles"; +import { CopyIconButton, FallbackBadge, LongValue } from "./ModelName"; +import { ReplyBody } from "./ReplyContent"; + +export const formatSeconds = (ms: number): string => `${(ms / 1000).toFixed(2)}s`; + +export const formatCost = (cost: number): string => + `$${cost < 0.0001 && cost > 0 ? cost.toPrecision(2) : cost.toFixed(4)}`; + +const MCP_EVENT_ENDPOINTS: readonly EndpointType[] = [EndpointType.CHAT, EndpointType.RESPONSES]; + +const textOf = (message: MessageType): string => (typeof message.content === "string" ? message.content : ""); + +const fellBack = (message: MessageType): boolean => (message.trace?.attemptedFallbacks ?? 0) > 0; + +export const servedModel = (message: MessageType): string | undefined => + message.trace?.modelGroup ?? (message.isError ? undefined : message.model ?? message.trace?.requestedModel); + +const replyModel = (message: MessageType): Readonly<{ value: string; detail: string }> | null => { + const served = servedModel(message); + if (served) return { value: served, detail: fellBack(message) ? "Served by, after a fallback" : "Served by" }; + const requested = message.trace?.requestedModel ?? message.model; + return requested ? { value: requested, detail: "Requested model" } : null; +}; + +function Metric({ icon, value, label }: { icon: ReactNode; value: string; label?: string }) { + return ( + + {icon} + {value} + {label} + + ); +} + +function ReplyMetrics({ message }: { message: MessageType }) { + const usage = message.usage; + const cost = usage?.cost ?? message.trace?.cost; + const cached = usage?.servedFromResponseCache || message.trace?.cacheHit; + return ( +
+ {message.timeToFirstToken !== undefined && ( + } value={formatSeconds(message.timeToFirstToken)} label="to first token" /> + )} + {message.totalLatency !== undefined && ( + } value={formatSeconds(message.totalLatency)} label="total" /> + )} + {usage?.promptTokens !== undefined && ( + } value={String(usage.promptTokens)} label="in" /> + )} + {usage?.completionTokens !== undefined && ( + } value={String(usage.completionTokens)} label="out" /> + )} + {usage?.reasoningTokens ? ( + } value={String(usage.reasoningTokens)} label="reasoning" /> + ) : null} + {usage?.cacheReadTokens ? ( + } value={String(usage.cacheReadTokens)} label="cached" /> + ) : null} + {cost !== undefined && } value={formatCost(cost)} />} + {message.toolName && } value={message.toolName} />} + {cached && Cached response} +
+ ); +} + +function AttachmentMediaFor({ attachment }: { attachment: MessageAttachment }) { + if (attachment.previewUrl) + return ( + + + + ); + return {attachment.kind === "audio" ? : }; +} + +function UserTurn({ message }: { message: MessageType }) { + const text = textOf(message); + return ( + + + {message.attachments && message.attachments.length > 0 && ( + + {message.attachments.map((attachment, index) => ( + + + + {attachment.name} + {formatBytes(attachment.size)} + + + ))} + + )} + {message.mcpToolCall && ( + + + + + Called {message.mcpToolCall.tool} + + 
+                {JSON.stringify(message.mcpToolCall.arguments, null, 2)}
+
+ + + )} + {text && ( + + {text} + + )} + + + ); +} + +export type LastReplyExtras = Readonly<{ + endpoint: EndpointType; + mcpEvents: readonly MCPEvent[]; + codeInterpreter: CodeInterpreterResult | null; + accessToken: string; +}>; + +function AssistantReply({ + message, + isLast, + extras, + inspected, + onInspect, +}: { + message: MessageType; + isLast: boolean; + extras: LastReplyExtras; + inspected: boolean; + onInspect: () => void; +}) { + const model = replyModel(message); + const showMcpEvents = isLast && extras.mcpEvents.length > 0 && MCP_EVENT_ENDPOINTS.includes(extras.endpoint); + const showCode = isLast && extras.codeInterpreter && extras.endpoint === EndpointType.RESPONSES; + return ( + + + {model && ( + + + {fellBack(message) && } + + )} + {message.reasoningContent && } + {showMcpEvents && } + {message.searchResults && } + {showCode && extras.codeInterpreter && ( + + )} + {message.isError ? ( + + The request failed + {textOf(message)} + + ) : ( + + )} + +
+ {message.a2aMetadata ? ( + + ) : ( + + )} +
+ + {!message.isImage && !message.isAudio && } + {message.trace && ( + + + Inspect + + )} + + + + + ); +} + +function WaitingTurn({ target }: { target: string | undefined }) { + return ( + + +
+ + {target ? `Waiting for ${target}` : "Waiting for a reply"} +
+ + + ); +} + +function SwitchMarker({ model }: { model: string }) { + return ( + + + + + Switched to {model} + + ); +} + +function EmptyConversation({ + suggestions, + onPick, +}: { + suggestions: readonly string[]; + onPick: (text: string) => void; +}) { + return ( + + + + + + Send a test request + + Pick a model and anything the request should go through, then inspect what the gateway did with it + + + {suggestions.length > 0 && ( + + {suggestions.map((suggestion) => ( + + ))} + + )} + + ); +} + +const switchedModel = (messages: readonly MessageType[], index: number): string | null => { + const message = messages[index]; + if (message?.role !== "user" || !message.model) return null; + const previous = messages.slice(0, index).findLast((candidate) => candidate.role === "user"); + return previous?.model && previous.model !== message.model ? message.model : null; +}; + +export function Conversation({ + messages, + isLoading, + waitingFor, + extras, + inspectedIndex, + onInspect, + suggestions, + onPickSuggestion, +}: { + messages: readonly MessageType[]; + isLoading: boolean; + waitingFor: string | undefined; + extras: LastReplyExtras; + inspectedIndex: number | null; + onInspect: (index: number) => void; + suggestions: readonly string[]; + onPickSuggestion: (text: string) => void; +}) { + if (messages.length === 0) return ; + const lastIndex = messages.length - 1; + const waiting = isLoading && messages[lastIndex]?.role === "user"; + return ( + + + + + {messages.flatMap((message, index) => { + const switched = switchedModel(messages, index); + const turn = ( + + {message.role === "user" ? ( + + ) : ( + onInspect(index)} + /> + )} + + ); + return switched + ? [ + + + , + turn, + ] + : [turn]; + })} + {waiting && ( + + + + )} + + + + + + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Inspector.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Inspector.tsx new file mode 100644 index 00000000000..3a9919ce38f --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Inspector.tsx @@ -0,0 +1,292 @@ +"use client"; + +import { useState, type ReactNode } from "react"; +import { + ArrowDownToLine, + ArrowUpFromLine, + Brain, + Clock, + Coins, + DatabaseZap, + Gauge, + RotateCw, + Server, + Shuffle, + Timer, +} from "lucide-react"; +import type { MessageType } from "@/components/chat_ui/types"; +import { Accordion, AccordionContent, AccordionItem, AccordionTrigger } from "@/components/ui/accordion"; +import { Badge } from "@/components/ui/badge"; +import { Empty, EmptyDescription, EmptyHeader, EmptyTitle } from "@/components/ui/empty"; +import { FieldDescription } from "@/components/ui/field"; +import { Item, ItemActions, ItemContent, ItemDescription, ItemGroup, ItemTitle } from "@/components/ui/item"; +import { ScrollArea, ScrollBar } from "@/components/ui/scroll-area"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { ToggleGroup, ToggleGroupItem } from "@/components/ui/toggle-group"; +import { formatCost, formatSeconds, servedModel } from "./Conversation"; +import { CopyIconButton, FallbackBadge, LongValue } from "./ModelName"; + +export type CodeSdk = "openai" | "azure"; + +function TraceRow({ title, description, children }: { title: string; description?: string; children?: ReactNode }) { + return ( + + + {title} + {description && {description}} + + {children && {children}} + + ); +} + +type StatValue = Readonly<{ label: string; icon: ReactNode; value: string | undefined }>; + +function StatGrid({ stats }: { stats: readonly StatValue[] }) { + const shown = stats.filter((stat): stat is StatValue & { value: string } => stat.value !== undefined); + if (shown.length === 0) return null; + return ( + + {shown.map((stat) => ( + + + {stat.icon} + {stat.label} + + {stat.value} + + ))} + + ); +} + +function TraceSection({ value, title, children }: { value: string; title: string; children: ReactNode }) { + return ( + + {title} + + {children} + + + ); +} + +function Muted({ children }: { children: ReactNode }) { + return

{children}

; +} + +const count = (value: number | undefined): string | undefined => (value === undefined ? undefined : String(value)); +const millis = (value: number | undefined): string | undefined => + value === undefined ? undefined : `${Math.round(value)}ms`; + +function Routing({ message }: { message: MessageType }) { + const trace = message.trace; + const served = servedModel(message); + const fellBack = (trace?.attemptedFallbacks ?? 0) > 0; + const reported = Boolean(trace?.callId || trace?.modelGroup || trace?.providerModel); + return ( + + {trace?.requestedModel && ( + + + + )} + {served && ( + + {fellBack && } + + + )} + {trace?.providerModel && ( + + + + )} + {trace?.apiBase && ( + + + + )} + {trace?.callId && ( + + {trace.callId} + + + )} + {trace?.responseId && ( + + {trace.responseId} + + + )} + , value: count(trace?.attemptedFallbacks) }, + { label: "Retries", icon: , value: count(trace?.attemptedRetries) }, + { label: "Overhead", icon: , value: millis(trace?.overheadMs) }, + ]} + /> + {!reported && ( + + The gateway did not share routing details with this page. Open the dashboard from the proxy to see them + + )} + + ); +} + +const noneRan = (policies: readonly string[] | undefined): string | null => { + if (policies === undefined) return "No guardrails ran on this request"; + return policies.length === 0 ? "No guardrails or policies ran on this request" : null; +}; + +function Guardrails({ message }: { message: MessageType }) { + const guardrails = message.trace?.appliedGuardrails; + const policies = message.trace?.appliedPolicies; + if (guardrails === undefined) { + return ( + + This endpoint does not report which guardrails or policies ran + + ); + } + const emptyNote = guardrails.length === 0 ? noneRan(policies) : null; + return ( + + {guardrails.map((name) => ( + + Ran + + ))} + {policies?.map((name) => ( + + Ran + + ))} + {emptyNote && {emptyNote}} + {policies === undefined && Policies are only reported when the dashboard is opened from the proxy} + + ); +} + +function SpeedAndCost({ message }: { message: MessageType }) { + const usage = message.usage; + const cost = usage?.cost ?? message.trace?.cost; + return ( + + , + value: message.timeToFirstToken === undefined ? undefined : formatSeconds(message.timeToFirstToken), + }, + { + label: "Total", + icon: , + value: message.totalLatency === undefined ? undefined : formatSeconds(message.totalLatency), + }, + { label: "Cost", icon: , value: cost === undefined ? undefined : formatCost(cost) }, + { label: "Tokens in", icon: , value: count(usage?.promptTokens) }, + { label: "Tokens out", icon: , value: count(usage?.completionTokens) }, + { label: "Cached", icon: , value: count(usage?.cacheReadTokens) }, + { label: "Reasoning", icon: , value: count(usage?.reasoningTokens) }, + { label: "Gateway time", icon: , value: millis(message.trace?.responseMs) }, + ]} + /> + + ); +} + +function NothingYet() { + return ( + + + Nothing to inspect yet + + Each reply shows here with the deployment that served it, fallbacks, guardrails, timing and cost + + + + ); +} + +function Trace({ message }: { message: MessageType | null }) { + if (!message) return ; + return ( + + + + + + ); +} + +function CodeTab({ code }: { code: (sdk: CodeSdk) => string }) { + const [sdk, setSdk] = useState("openai"); + const snippet = code(sdk); + return ( +
+
+ value[0] && setSdk(value[0] as CodeSdk)} + variant="outline" + size="sm" + spacing={0} + aria-label="SDK" + > + + OpenAI SDK + + + Azure SDK + + + + + +
+ + Send this request from your own app. Replace YOUR_LITELLM_API_KEY with a virtual key + + + 
{snippet}
+ + +
+ ); +} + +export function InspectorPanel({ message, code }: { message: MessageType | null; code: (sdk: CodeSdk) => string }) { + return ( + +
+ + Trace + Code + +
+ + + + + + + + + + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/McpDialog.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/McpDialog.tsx new file mode 100644 index 00000000000..e5e0828825d --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/McpDialog.tsx @@ -0,0 +1,391 @@ +"use client"; + +import { useId, useState, type ReactNode } from "react"; +import { Blocks, KeyRound, Search } from "lucide-react"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Checkbox } from "@/components/ui/checkbox"; +import { + Dialog, + DialogClose, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, + DialogTrigger, +} from "@/components/ui/dialog"; +import { Empty, EmptyContent, EmptyDescription, EmptyHeader, EmptyTitle } from "@/components/ui/empty"; +import { Field, FieldLabel, FieldLegend, FieldSet } from "@/components/ui/field"; +import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from "@/components/ui/input-group"; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { Spinner } from "@/components/ui/spinner"; +import type { MCPServer, MCPToolset } from "@/components/mcp_tools/types"; +import { uiHref } from "@/utils/uiHref"; +import { ByokConnect } from "./ByokConnect"; +import { + ALL_MCP_SERVERS, + TOOLSET_PREFIX, + allowedTools, + toggleMcpItem, + toggleToolLimit, + withoutStaleLimits, + type ToolLimits, +} from "./playgroundSettings"; +import { useServerTools } from "./usePlaygroundCatalog"; + +type Row = Readonly<{ value: string; label: string }>; + +export const serverName = (server: MCPServer): string => server.alias || server.server_name || server.server_id; + +const matches = (value: string, query: string): boolean => value.toLowerCase().includes(query.trim().toLowerCase()); + +const needsKey = (server: MCPServer | undefined): boolean => Boolean(server?.is_byok && !server.has_user_credential); + +function SearchField({ + value, + onChange, + placeholder, + autoFocus, +}: { + value: string; + onChange: (value: string) => void; + placeholder: string; + autoFocus: boolean; +}) { + return ( + + + + + onChange(event.target.value)} + placeholder={placeholder} + aria-label={placeholder} + autoFocus={autoFocus} + /> + + ); +} + +function MasterRow({ + row, + checked, + onCheckedChange, + active, + onSelect, + summary, +}: { + row: Row; + checked: boolean; + onCheckedChange: (checked: boolean) => void; + active: boolean; + onSelect: () => void; + summary?: string; +}) { + return ( +
+ + +
+ ); +} + +function ToolRow({ + label, + checked, + disabled, + onCheckedChange, +}: { + label: string; + checked: boolean; + disabled: boolean; + onCheckedChange: (checked: boolean) => void; +}) { + const id = useId(); + return ( + + + + {label} + + + ); +} + +function Note({ title, children, action }: { title: string; children: ReactNode; action?: ReactNode }) { + return ( + + + {title} + {children} + + {action && {action}} + + ); +} + +function RowGroup({ title, rows, render }: { title: string; rows: readonly Row[]; render: (row: Row) => ReactNode }) { + if (rows.length === 0) return null; + return ( +
+ + {title} + + {rows.map(render)} +
+ ); +} + +export function McpDialog({ + requestKey, + servers, + toolsets, + isLoading, + selection, + toolLimits, + onChange, + onCredentialSaved, + defaultOpen = false, +}: { + requestKey: string | null; + servers: readonly MCPServer[]; + toolsets: readonly MCPToolset[]; + isLoading: boolean; + selection: readonly string[]; + toolLimits: ToolLimits; + onChange: (selection: readonly string[], limits: ToolLimits) => void; + onCredentialSaved: () => void; + defaultOpen?: boolean; +}) { + const [focused, setFocused] = useState(selection.find((item) => item !== ALL_MCP_SERVERS) ?? ALL_MCP_SERVERS); + const [query, setQuery] = useState(""); + const [toolQuery, setToolQuery] = useState(""); + const [replacingKey, setReplacingKey] = useState(null); + const serverById = new Map(servers.map((server) => [server.server_id, server])); + const selectedServers = selection.filter((item) => serverById.has(item)); + const toolsByServer = useServerTools( + requestKey, + [...new Set([...selectedServers, ...(serverById.has(focused) ? [focused] : [])])].filter( + (id) => !needsKey(serverById.get(id)), + ), + ); + const toolNames = (serverId: string): readonly string[] => + (toolsByServer[serverId]?.tools ?? []).map((tool) => tool.name); + + const serverRows: Row[] = servers.map((server) => ({ value: server.server_id, label: serverName(server) })); + const toolsetRows: Row[] = toolsets.map((toolset) => ({ + value: `${TOOLSET_PREFIX}${toolset.toolset_id}`, + label: toolset.toolset_name, + })); + const visible = (rows: readonly Row[]) => rows.filter((row) => matches(row.label, query)); + + const toggle = (item: string, on: boolean) => { + const next = toggleMcpItem(selection, item, on); + onChange(next, withoutStaleLimits(toolLimits, next)); + if (on) setFocused(item); + }; + + const summary = (row: Row): string | undefined => { + if (!selection.includes(row.value)) return undefined; + const server = serverById.get(row.value); + if (!server) return undefined; + if (needsKey(server)) return "Needs your key"; + const tools = toolNames(row.value); + if (tools.length === 0) return undefined; + const allowed = allowedTools(toolLimits, row.value, tools).length; + return allowed === tools.length ? `All ${tools.length} tools` : `${allowed} of ${tools.length} tools`; + }; + + const focus = (item: string) => { + setFocused(item); + setToolQuery(""); + setReplacingKey(null); + }; + + const masterRow = (row: Row) => ( + toggle(row.value, on)} + active={row.value === focused} + onSelect={() => focus(row.value)} + summary={summary(row)} + /> + ); + + const detail = (): ReactNode => { + if (focused === ALL_MCP_SERVERS) + return Every server this key can reach, with all of its tools; + const toolset = toolsets.find((item) => `${TOOLSET_PREFIX}${item.toolset_id}` === focused); + if (toolset) + return ( + +
+

+ {toolset.description || "A toolset's tools are fixed by whoever created it"} +

+
+ {toolset.tools.map((tool) => ( + + {tool.tool_name} + + ))} +
+
+ + ); + const server = serverById.get(focused); + if (!server) return Pick a server or toolset on the left; + const name = serverName(server); + if (needsKey(server) || replacingKey === focused) + return ( + { + setReplacingKey(null); + onCredentialSaved(); + }} + onCancel={needsKey(server) ? undefined : () => setReplacingKey(null)} + /> + ); + const replaceKey = server.is_byok ? ( + + ) : undefined; + if (!selection.includes(focused)) + return ( + + Check this server to choose which of its tools the model can call + + ); + const state = toolsByServer[focused]; + if (!state || state.isLoading) + return ( +
+ +
+ ); + if (state.failed) + return ( + + Couldn't load this server's tools + + ); + const tools = toolNames(focused); + if (tools.length === 0) return This server has no tools; + const allowed = allowedTools(toolLimits, focused, tools); + return ( +
+
+ + + {replaceKey} +
+ +
+ {tools + .filter((tool) => matches(tool, toolQuery)) + .map((tool) => ( + + onChange(selection, toggleToolLimit(toolLimits, { server: focused, tools }, tool, on)) + } + /> + ))} +
+ +
+ ); + }; + + const nothingAvailable = !isLoading && servers.length === 0 && toolsets.length === 0; + + return ( + !open && setReplacingKey(null)}> + }> + + MCP + {selection.length > 0 && ( + + {selection.length} + + )} + + + + MCP servers + + Pick the servers the model can use, then limit their tools. A server allows every tool until you uncheck + some. + + + {nothingAvailable ? ( +
+ + Add one on the MCP servers page, or send as a key that can reach one + +
+ ) : ( +
+
+
+ +
+ + {isLoading ? ( +
+ +
+ ) : ( +
+ + +
+ )} + +
+
{detail()}
+
+ )} + + }>Done + + + + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/ModelName.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/ModelName.tsx new file mode 100644 index 00000000000..bbea909711a --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/ModelName.tsx @@ -0,0 +1,74 @@ +"use client"; + +import { Copy, Split } from "lucide-react"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { HoverCard, HoverCardContent, HoverCardTrigger } from "@/components/ui/hover-card"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { cn } from "@/lib/cva.config"; +import { toast } from "@/lib/toast"; + +export function copyText(value: string, label: string): void { + void navigator.clipboard.writeText(value).then( + () => toast.success(`${label} copied`), + () => toast.error("Unable to copy to clipboard"), + ); +} + +export function CopyIconButton({ value, label }: { value: string; label: string }) { + return ( + + copyText(value, label)} /> + } + > + + + Copy {label} + + ); +} + +export function LongValue({ + value, + detail, + copyLabel, + className, + align = "start", +}: { + value: string; + detail: string; + copyLabel: string; + className?: string; + align?: "start" | "end"; +}) { + return ( + + } + > + {value} + + +
+
+ {detail} + {value} +
+ +
+ + + ); +} + +export function FallbackBadge() { + return ( + + + Fallback + + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Pickers.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Pickers.test.tsx new file mode 100644 index 00000000000..92258ed1428 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Pickers.test.tsx @@ -0,0 +1,83 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; +import { ChecklistMenu, TargetPicker, type OptionGroup } from "./Pickers"; + +const GROUPS: readonly OptionGroup[] = [ + { + label: "openai", + options: [ + { value: "gpt-fast", label: "gpt-fast" }, + { value: "gpt-smart", label: "gpt-smart" }, + ], + }, +]; + +const renderTargetPicker = (onValueChange = vi.fn()) => { + render( + , + ); + return onValueChange; +}; + +describe("TargetPicker", () => { + it("selects the first match when Enter is pressed after searching", async () => { + const user = userEvent.setup(); + const onValueChange = renderTargetPicker(); + + await user.click(screen.getByRole("combobox", { name: "Model" })); + await user.type(await screen.findByPlaceholderText("Search models"), "smart{Enter}"); + + expect(onValueChange).toHaveBeenCalledWith("gpt-smart"); + }); + + it("uses a typed model name that matches nothing when Enter is pressed", async () => { + const user = userEvent.setup(); + const onValueChange = renderTargetPicker(); + + await user.click(screen.getByRole("combobox", { name: "Model" })); + await user.type(await screen.findByPlaceholderText("Search models"), "my-custom-model{Enter}"); + + expect(onValueChange).toHaveBeenCalledWith("my-custom-model"); + }); +}); + +describe("ChecklistMenu", () => { + it("toggles the first match when Enter is pressed after searching", async () => { + const user = userEvent.setup(); + const onValueChange = vi.fn(); + render( + , + ); + + await user.click(screen.getByRole("combobox", { name: "Guardrails" })); + await user.type(await screen.findByPlaceholderText("Search guardrails"), "tox{Enter}"); + + expect(onValueChange).toHaveBeenLastCalledWith(["pii-mask", "toxicity"]); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Pickers.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Pickers.tsx new file mode 100644 index 00000000000..28ce6fe2f4a --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/Pickers.tsx @@ -0,0 +1,193 @@ +"use client"; + +import { useState, type ReactNode } from "react"; +import { Plus } from "lucide-react"; +import { Badge } from "@/components/ui/badge"; +import { + Combobox, + ComboboxCollection, + ComboboxContent, + ComboboxEmpty, + ComboboxGroup, + ComboboxInput, + ComboboxItem, + ComboboxLabel, + ComboboxList, + ComboboxTrigger, +} from "@/components/ui/combobox"; +import { InputGroupButton } from "@/components/ui/input-group"; +import { Spinner } from "@/components/ui/spinner"; +import { cn } from "@/lib/cva.config"; +import type { PickerOption } from "./usePlaygroundCatalog"; + +export type OptionGroup = Readonly<{ label: string; options: readonly PickerOption[] }>; + +type GroupItems = Readonly<{ value: string; items: readonly string[] }>; + +const toItems = (groups: readonly OptionGroup[]): GroupItems[] => + groups + .filter((group) => group.options.length > 0) + .map((group) => ({ value: group.label, items: group.options.map((option) => option.value) })); + +const isListed = (index: ReadonlyMap, name: string): boolean => + index.has(name) || [...index.values()].some((option) => option.label === name); + +const optionIndex = (groups: readonly OptionGroup[]): ReadonlyMap => + new Map(groups.flatMap((group) => group.options.map((option) => [option.value, option] as const))); + +function GroupedOptions({ index, mono = false }: { index: ReadonlyMap; mono?: boolean }) { + return ( + + {(group: GroupItems) => ( + + {group.value} + + {(value: string) => { + const option = index.get(value); + if (!option) return ; + return ( + + + {option.label} + {option.description && ( + {option.description} + )} + + + ); + }} + + + )} + + ); +} + +function CustomItem({ value }: { value: string }) { + return ( + + + + Use {value} + + + ); +} + +function CountBadge({ count }: { count: number }) { + if (count === 0) return null; + return ( + + {count} + + ); +} + +export function ChecklistMenu({ + label, + icon, + groups, + value, + onValueChange, + isLoading, + emptyText, + defaultOpen = false, +}: { + label: string; + icon: ReactNode; + groups: readonly OptionGroup[]; + value: readonly string[]; + onValueChange: (value: readonly string[]) => void; + isLoading: boolean; + emptyText: string; + defaultOpen?: boolean; +}) { + const index = optionIndex(groups); + const items = toItems(groups); + return ( + onValueChange(next)} + itemToStringLabel={(item: string) => index.get(item)?.label ?? item} + defaultOpen={defaultOpen} + autoHighlight + > + }> + {icon} + {label} + + + + + + {isLoading ? : items.length === 0 && emptyText} + {!isLoading && items.length > 0 && "Nothing matches"} + + + + + ); +} + +export function TargetPicker({ + label, + placeholder, + groups, + value, + onValueChange, + isLoading, + emptyText, + allowCustom = false, + defaultOpen = false, +}: { + label: string; + placeholder: string; + groups: readonly OptionGroup[]; + value: string | null; + onValueChange: (value: string) => void; + isLoading: boolean; + emptyText: string; + allowCustom?: boolean; + defaultOpen?: boolean; +}) { + const [query, setQuery] = useState(""); + const index = optionIndex(groups); + const custom = query.trim(); + const offerCustom = allowCustom && custom !== "" && !isListed(index, custom); + const items = [...toItems(groups), ...(offerCustom ? [{ value: "Custom name", items: [custom] }] : [])]; + const unsetText = isLoading ? `Loading ${label.toLowerCase()}s` : placeholder; + return ( + next && onValueChange(next)} + inputValue={query} + onInputValueChange={setQuery} + onOpenChange={(open) => !open && setQuery("")} + itemToStringLabel={(item: string) => index.get(item)?.label ?? item} + defaultOpen={defaultOpen} + autoHighlight + > + }> + {isLoading && !value ? : null} + + {value ? index.get(value)?.label ?? value : unsetText} + + + + + + {items.length === 0 && !custom ? emptyText : `No matching ${label.toLowerCase()}`} + + + + + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/PlaygroundChat.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/PlaygroundChat.integration.test.tsx new file mode 100644 index 00000000000..c80f47045c2 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/PlaygroundChat.integration.test.tsx @@ -0,0 +1,658 @@ +import { act, fireEvent, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { renderWithProviders as render, testQueryClient } from "@/../tests/test-utils"; +import { makeOpenAIChatCompletionRequest } from "@/components/llm_calls/chat_completion"; +import { fetchAvailableModels } from "@/components/llm_calls/fetch_models"; +import { makeOpenAIResponsesRequest } from "@/components/llm_calls/responses_api"; +import { callMCPTool, fetchMCPServers, fetchMCPToolsets, listMCPTools, tagListCall } from "@/components/networking"; +import { setSecureItem } from "@/utils/secureStorage"; +import { fetchClient } from "@/lib/http/api"; +import { PlaygroundChat } from "./PlaygroundChat"; + +vi.mock("@/app/(dashboard)/hooks/useCan", () => ({ default: () => true })); + +vi.mock("@/lib/http/api", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, fetchClient: { ...actual.fetchClient, POST: vi.fn() } }; +}); + +vi.mock("@/components/llm_calls/fetch_models", () => ({ fetchAvailableModels: vi.fn() })); + +vi.mock("@/components/llm_calls/chat_completion", () => ({ makeOpenAIChatCompletionRequest: vi.fn() })); + +vi.mock("@/components/llm_calls/responses_api", () => ({ makeOpenAIResponsesRequest: vi.fn() })); + +vi.mock("@/components/networking", () => ({ + tagListCall: vi.fn().mockResolvedValue({}), + vectorStoreListCall: vi.fn().mockResolvedValue({ data: [] }), + getGuardrailsList: vi.fn().mockResolvedValue({ guardrails: [{ guardrail_name: "pii-mask" }] }), + getPoliciesList: vi.fn().mockResolvedValue({ policies: [] }), + fetchMCPServers: vi.fn().mockResolvedValue([]), + fetchMCPToolsets: vi.fn().mockResolvedValue([]), + listMCPTools: vi.fn().mockResolvedValue({ tools: [] }), + callMCPTool: vi.fn(), + getGatewayBaseUrl: () => "http://gateway.test", + getGlobalLitellmHeaderName: () => "Authorization", +})); + +const chatRequest = vi.mocked(makeOpenAIChatCompletionRequest); + +const responsesRequest = vi.mocked(makeOpenAIResponsesRequest); + +const credentialPost = fetchClient.POST as unknown as ReturnType; + +type ChatArgs = Parameters; + +type ResponsesArgs = Parameters; + +const sentMessages = (call: number) => (chatRequest.mock.calls[call] as ChatArgs)[0]; + +const replyWith = + (text: string) => + async (...args: ChatArgs) => { + args[1](text, args[2]); + }; + +const preload = (values: Readonly>) => + Object.entries(values).forEach(([key, value]) => sessionStorage.setItem(key, value)); + +const deferred = () => { + const control: { release: () => void } = { release: () => undefined }; + const promise = new Promise((resolve) => { + control.release = resolve; + }); + return { promise, release: () => control.release() }; +}; + +const mcpTool = (name: string, required: readonly string[] = []) => ({ + name, + inputSchema: { + type: "object", + properties: Object.fromEntries(required.map((field) => [field, { type: "string" }])), + required: [...required], + }, +}); + +const settled = () => waitFor(() => expect(testQueryClient.isFetching()).toBe(0)); + +const renderChat = () => + render(); + +const send = async (user: ReturnType, text: string) => { + await user.type(screen.getByRole("textbox", { name: "Message" }), text); + await user.click(screen.getByRole("button", { name: "Send message" })); +}; + +describe("PlaygroundChat", () => { + beforeEach(() => { + vi.clearAllMocks(); + sessionStorage.clear(); + testQueryClient.clear(); + Element.prototype.scrollIntoView = vi.fn(); + chatRequest.mockReset(); + responsesRequest.mockReset(); + vi.mocked(callMCPTool).mockReset(); + credentialPost.mockReset().mockResolvedValue({ data: {}, response: new Response() }); + vi.mocked(fetchMCPServers).mockReset().mockResolvedValue([]); + vi.mocked(fetchMCPToolsets).mockReset().mockResolvedValue([]); + vi.mocked(listMCPTools).mockReset().mockResolvedValue({ tools: [] }); + vi.mocked(fetchAvailableModels).mockResolvedValue([ + { model_group: "gpt-fast", mode: "chat", providers: ["openai"] }, + ]); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("sends the chosen model and guardrails with the session key", async () => { + const user = userEvent.setup(); + chatRequest.mockImplementation(replyWith("Hi there")); + renderChat(); + + await user.click(screen.getByRole("combobox", { name: "Model" })); + await user.click(await screen.findByRole("option", { name: "gpt-fast" })); + await user.click(screen.getByRole("combobox", { name: "Guardrails" })); + await user.click(await screen.findByRole("option", { name: "pii-mask" })); + await user.keyboard("{Escape}"); + await send(user, "hello"); + + expect(await screen.findByText("Hi there")).toBeInTheDocument(); + const [messages, , model, apiKey, , , , , , , , guardrails] = chatRequest.mock.calls[0] as ChatArgs; + expect(messages).toEqual([{ role: "user", content: "hello" }]); + expect(model).toBe("gpt-fast"); + expect(apiKey).toBe("session-token"); + expect(guardrails).toEqual(["pii-mask"]); + }); + + it("keeps answered turns in the history but never resends a failed one", async () => { + const user = userEvent.setup(); + chatRequest + .mockImplementationOnce(replyWith("Hi there")) + .mockRejectedValueOnce(new Error("rate limited")) + .mockImplementationOnce(replyWith("Back again")); + renderChat(); + + await user.click(screen.getByRole("combobox", { name: "Model" })); + await user.click(await screen.findByRole("option", { name: "gpt-fast" })); + await send(user, "hello"); + expect(await screen.findByText("Hi there")).toBeInTheDocument(); + await send(user, "this one fails"); + expect(await screen.findByText("The request failed")).toBeInTheDocument(); + await send(user, "try again"); + + await waitFor(() => expect(chatRequest).toHaveBeenCalledTimes(3)); + expect(sentMessages(2)).toEqual([ + { role: "user", content: "hello" }, + { role: "assistant", content: "Hi there" }, + { role: "user", content: "try again" }, + ]); + }); + + it("sends every saved request setting with the chat request", async () => { + const user = userEvent.setup(); + chatRequest.mockImplementation(replyWith("Hi there")); + vi.mocked(fetchMCPServers).mockResolvedValue([{ server_id: "srv-1", alias: "docs" }]); + preload({ + selectedModel: "gpt-fast", + selectedTags: JSON.stringify(["team-a"]), + selectedVectorStores: JSON.stringify(["vs-1"]), + selectedPolicies: JSON.stringify(["strict"]), + selectedMCPServers: JSON.stringify(["srv-1"]), + streamingEnabled: "false", + }); + setSecureItem("customHeaders", JSON.stringify([["x-team", "blue"]])); + renderChat(); + + await waitFor(() => expect(fetchMCPServers).toHaveBeenCalled()); + await send(user, "hello"); + + await waitFor(() => expect(chatRequest).toHaveBeenCalledTimes(1)); + const args = chatRequest.mock.calls[0] as ChatArgs; + expect(args[2]).toBe("gpt-fast"); + expect(args[4]).toEqual(["team-a"]); + expect(args[10]).toEqual(["vs-1"]); + expect(args[12]).toEqual(["strict"]); + expect(args[13]).toEqual(["srv-1"]); + expect(args[25]).toBe(false); + expect(args[26]).toMatchObject({ "x-team": "blue" }); + }); + + it("sends a custom model and tag that are prefixes of listed ones", async () => { + const user = userEvent.setup(); + chatRequest.mockImplementation(replyWith("Hi there")); + vi.mocked(tagListCall).mockResolvedValueOnce({ + production: { name: "production", models: [], created_at: "", updated_at: "" }, + }); + renderChat(); + + await user.click(screen.getByRole("combobox", { name: "Model" })); + await user.type(await screen.findByPlaceholderText("Search models"), "gpt-f"); + await user.click(await screen.findByRole("option", { name: "Use gpt-f" })); + await user.click(screen.getByRole("button", { name: "Settings" })); + await user.type(await screen.findByLabelText("Tags"), "prod"); + await user.click(await screen.findByRole("option", { name: "Add prod" })); + await send(user, "hello"); + + await waitFor(() => expect(chatRequest).toHaveBeenCalledTimes(1)); + const args = chatRequest.mock.calls[0] as ChatArgs; + expect(args[2]).toBe("gpt-f"); + expect(args[4]).toEqual(["prod"]); + }); + + it("sends a temperature typed with two decimals", async () => { + const user = userEvent.setup(); + chatRequest.mockImplementation(replyWith("Hi there")); + preload({ selectedModel: "gpt-fast" }); + renderChat(); + + await user.click(screen.getByRole("button", { name: "Settings" })); + await user.click(await screen.findByRole("button", { name: "Advanced" })); + await user.type(screen.getByLabelText("Temperature value"), "0.05"); + await user.keyboard("{Escape}"); + await send(user, "hello"); + + await waitFor(() => expect(chatRequest).toHaveBeenCalledTimes(1)); + expect((chatRequest.mock.calls[0] as ChatArgs)[16]).toBe(0.05); + }); + + it("keeps the saved MCP selection when the server list cannot be fetched", async () => { + const user = userEvent.setup(); + chatRequest.mockImplementation(replyWith("Hi there")); + vi.mocked(fetchMCPServers).mockRejectedValue(new Error("forbidden")); + preload({ selectedModel: "gpt-fast", selectedMCPServers: JSON.stringify(["srv-1"]) }); + renderChat(); + + await waitFor(() => expect(fetchMCPServers).toHaveBeenCalled()); + await settled(); + await send(user, "hello"); + + await waitFor(() => expect(chatRequest).toHaveBeenCalledTimes(1)); + expect((chatRequest.mock.calls[0] as ChatArgs)[13]).toEqual(["srv-1"]); + }); + + it("drops a saved model that the current key cannot use", async () => { + const user = userEvent.setup(); + chatRequest.mockImplementation(replyWith("Hi there")); + preload({ selectedModel: "retired-model" }); + renderChat(); + + await waitFor(() => expect(fetchAvailableModels).toHaveBeenCalled()); + await waitFor(() => expect(screen.getByRole("combobox", { name: "Model" })).toHaveTextContent("Select a model")); + await user.click(screen.getByRole("combobox", { name: "Model" })); + await user.click(await screen.findByRole("option", { name: "gpt-fast" })); + await send(user, "hello"); + + await waitFor(() => expect(chatRequest).toHaveBeenCalledTimes(1)); + expect((chatRequest.mock.calls[0] as ChatArgs)[2]).toBe("gpt-fast"); + }); + + it("calls a toolset tool on the server that backs it", async () => { + const user = userEvent.setup(); + vi.mocked(fetchMCPToolsets).mockResolvedValue([ + { toolset_id: "ts-1", toolset_name: "Research", tools: [{ server_id: "srv-2", tool_name: "search" }] }, + ]); + vi.mocked(listMCPTools).mockResolvedValue({ + tools: [{ name: "search", inputSchema: { type: "object", properties: {} } }], + }); + vi.mocked(callMCPTool).mockResolvedValue({ content: [{ type: "text", text: "found it" }] }); + preload({ endpointType: "mcp", selectedMCPServers: JSON.stringify(["toolset:ts-1"]) }); + renderChat(); + + await user.click(await screen.findByRole("combobox", { name: "Tool" })); + await user.click(await screen.findByRole("option", { name: /search/ })); + await user.click(screen.getByRole("button", { name: "Send message" })); + + await waitFor(() => expect(callMCPTool).toHaveBeenCalledTimes(1)); + expect(listMCPTools).toHaveBeenCalledWith("session-token", "srv-2"); + expect(listMCPTools).not.toHaveBeenCalledWith("session-token", "toolset:ts-1"); + expect(vi.mocked(callMCPTool).mock.calls[0]?.slice(1, 3)).toEqual(["srv-2", "search"]); + }); + + it("offers toolsets as targets and lists only the toolset's own tools", async () => { + const user = userEvent.setup(); + vi.mocked(fetchMCPToolsets).mockResolvedValue([ + { + toolset_id: "ts-1", + toolset_name: "Research", + tools: [ + { server_id: "srv-a", tool_name: "search" }, + { server_id: "srv-b", tool_name: "fetch" }, + { server_id: "srv-b", tool_name: "search" }, + ], + }, + ]); + vi.mocked(listMCPTools).mockImplementation(async (_key: string, server: string) => ({ + tools: + server === "srv-a" ? [mcpTool("search"), mcpTool("fetch", ["a_arg"])] : [mcpTool("search"), mcpTool("fetch")], + })); + vi.mocked(callMCPTool).mockResolvedValue({ content: [{ type: "text", text: "fetched" }] }); + preload({ endpointType: "mcp" }); + renderChat(); + + await user.click(await screen.findByRole("combobox", { name: "Server" })); + await user.click(await screen.findByRole("option", { name: /Research/ })); + await settled(); + await user.click(screen.getByRole("combobox", { name: "Tool" })); + await waitFor(() => + expect(screen.getAllByRole("option").map((option) => option.textContent)).toEqual(["search", "fetch"]), + ); + await user.click(screen.getByRole("option", { name: "fetch" })); + await user.click(screen.getByRole("button", { name: "Send message" })); + + await waitFor(() => expect(callMCPTool).toHaveBeenCalledTimes(1)); + expect(vi.mocked(callMCPTool).mock.calls[0]?.slice(1, 4)).toEqual(["srv-b", "fetch", {}]); + }); + + it("says the tools could not be loaded when the server's tool list fails", async () => { + const user = userEvent.setup(); + vi.mocked(fetchMCPServers).mockResolvedValue([{ server_id: "srv-1", alias: "docs" }]); + vi.mocked(listMCPTools).mockResolvedValue({ error: "server unreachable" }); + preload({ endpointType: "mcp", selectedMCPServers: JSON.stringify(["srv-1"]) }); + renderChat(); + + await settled(); + await user.click(screen.getByRole("combobox", { name: "Tool" })); + + expect(await screen.findByText("Unable to load tools for this server")).toBeInTheDocument(); + }); + + it("keeps the saved MCP selection when the toolset list cannot be fetched", async () => { + const user = userEvent.setup(); + chatRequest.mockImplementation(replyWith("Hi there")); + vi.mocked(fetchMCPServers).mockResolvedValue([{ server_id: "srv-1", alias: "docs" }]); + vi.mocked(fetchMCPToolsets).mockRejectedValue(new Error("database not connected")); + preload({ selectedModel: "gpt-fast", selectedMCPServers: JSON.stringify(["srv-1", "srv-gone"]) }); + renderChat(); + + await waitFor(() => expect(fetchMCPToolsets).toHaveBeenCalled()); + await settled(); + await send(user, "hello"); + + await waitFor(() => expect(chatRequest).toHaveBeenCalledTimes(1)); + expect((chatRequest.mock.calls[0] as ChatArgs)[13]).toEqual(["srv-1", "srv-gone"]); + }); + + it("drops a saved model that is missing from an already loaded model list", async () => { + const first = renderChat(); + await waitFor(() => expect(fetchAvailableModels).toHaveBeenCalled()); + await settled(); + first.unmount(); + sessionStorage.setItem("selectedModel", "retired-model"); + + renderChat(); + + await waitFor(() => expect(screen.getByRole("combobox", { name: "Model" })).toHaveTextContent("Select a model")); + expect(fetchAvailableModels).toHaveBeenCalledTimes(1); + }); + + it("drops a request that is still running when a new chat starts", async () => { + const user = userEvent.setup(); + const first = deferred(); + chatRequest.mockImplementationOnce(async (...args: ChatArgs) => { + await first.promise; + args[1]("late reply", args[2]); + }); + preload({ selectedModel: "gpt-fast" }); + renderChat(); + + await send(user, "first"); + expect(await screen.findByRole("button", { name: "Stop" })).toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "New chat" })); + await act(async () => { + first.release(); + await chatRequest.mock.results[0]?.value; + }); + + expect(chatRequest).toHaveBeenCalledTimes(1); + expect(screen.queryByRole("button", { name: "Stop" })).not.toBeInTheDocument(); + expect(screen.queryByText("late reply")).not.toBeInTheDocument(); + }); + + it("keeps a stopped request's response id out of the next request", async () => { + const user = userEvent.setup(); + const first = deferred(); + responsesRequest + .mockImplementationOnce(async (...args: ResponsesArgs) => { + await first.promise; + args[15]?.("resp-stopped"); + }) + .mockImplementationOnce(async (...args: ResponsesArgs) => { + args[15]?.("resp-fresh"); + args[1]("assistant", "fresh reply", args[2]); + }) + .mockImplementationOnce(async (...args: ResponsesArgs) => { + args[1]("assistant", "third reply", args[2]); + }); + preload({ endpointType: "responses", selectedModel: "gpt-fast", useApiSessionManagement: "true" }); + renderChat(); + + await send(user, "first"); + await user.click(await screen.findByRole("button", { name: "Stop" })); + await send(user, "second"); + expect(await screen.findByText("fresh reply")).toBeInTheDocument(); + await act(async () => { + first.release(); + await responsesRequest.mock.results[0]?.value; + }); + await send(user, "third"); + + await waitFor(() => expect(responsesRequest).toHaveBeenCalledTimes(3)); + expect((responsesRequest.mock.calls[2] as ResponsesArgs)[14]).toBe("resp-fresh"); + }); + + it("ignores a stopped request that finishes after a newer one started", async () => { + const user = userEvent.setup(); + const first = deferred(); + const second = deferred(); + chatRequest + .mockImplementationOnce(async (...args: ChatArgs) => { + await first.promise; + args[1]("late reply", args[2]); + }) + .mockImplementationOnce(async (...args: ChatArgs) => { + await second.promise; + args[1]("fresh reply", args[2]); + }); + preload({ selectedModel: "gpt-fast" }); + renderChat(); + + await send(user, "first"); + await user.click(await screen.findByRole("button", { name: "Stop" })); + await send(user, "second"); + first.release(); + + await waitFor(() => expect(chatRequest).toHaveBeenCalledTimes(2)); + expect(screen.getByRole("button", { name: "Stop" })).toBeInTheDocument(); + expect(screen.queryByText("late reply")).not.toBeInTheDocument(); + second.release(); + expect(await screen.findByText("fresh reply")).toBeInTheDocument(); + expect(screen.queryByText("late reply")).not.toBeInTheDocument(); + }); + + it("only says no guardrails ran when the gateway reported its headers", async () => { + const user = userEvent.setup(); + vi.stubGlobal("matchMedia", (query: string) => ({ + matches: true, + media: query, + addEventListener: () => undefined, + removeEventListener: () => undefined, + })); + chatRequest + .mockImplementationOnce(replyWith("unreported reply")) + .mockImplementationOnce(async (...args: ChatArgs) => { + args[27]?.(new Headers({ "x-litellm-call-id": "call-reported" })); + args[1]("reported reply", args[2]); + }); + preload({ selectedModel: "gpt-fast", selectedGuardrails: JSON.stringify(["pii-mask"]) }); + renderChat(); + + await send(user, "first"); + expect(await screen.findByText("unreported reply")).toBeInTheDocument(); + expect(screen.getByText("This endpoint does not report which guardrails or policies ran")).toBeInTheDocument(); + expect(screen.queryByText("No guardrails or policies ran on this request")).not.toBeInTheDocument(); + + await send(user, "second"); + expect(await screen.findByText("call-reported")).toBeInTheDocument(); + expect(screen.getByText("No guardrails or policies ran on this request")).toBeInTheDocument(); + }); + + it("does not claim policies were skipped when the gateway's own headers are hidden", async () => { + const user = userEvent.setup(); + vi.stubGlobal("matchMedia", (query: string) => ({ + matches: true, + media: query, + addEventListener: () => undefined, + removeEventListener: () => undefined, + })); + chatRequest.mockImplementationOnce(async (...args: ChatArgs) => { + args[27]?.(new Headers({ "content-type": "application/json" })); + args[1]("cross-origin reply", args[2]); + }); + preload({ selectedModel: "gpt-fast" }); + renderChat(); + + await send(user, "hello"); + expect(await screen.findByText("cross-origin reply")).toBeInTheDocument(); + expect( + screen.getByText("Policies are only reported when the dashboard is opened from the proxy"), + ).toBeInTheDocument(); + expect(screen.queryByText(/policies ran on this request/)).not.toBeInTheDocument(); + }); + + it("keeps a stopped request's gateway details off the newer reply", async () => { + const user = userEvent.setup(); + vi.stubGlobal("matchMedia", (query: string) => ({ + matches: true, + media: query, + addEventListener: () => undefined, + removeEventListener: () => undefined, + })); + const first = deferred(); + chatRequest + .mockImplementationOnce(async (...args: ChatArgs) => { + args[27]?.(new Headers({ "x-litellm-call-id": "call-stopped" })); + await first.promise; + }) + .mockImplementationOnce(async (...args: ChatArgs) => { + args[27]?.(new Headers({ "x-litellm-call-id": "call-fresh" })); + args[1]("fresh reply", args[2]); + }); + preload({ selectedModel: "gpt-fast" }); + renderChat(); + + await send(user, "first"); + await user.click(await screen.findByRole("button", { name: "Stop" })); + await send(user, "second"); + expect(await screen.findByText("call-fresh")).toBeInTheDocument(); + await act(async () => { + first.release(); + await chatRequest.mock.results[0]?.value; + }); + + expect(screen.queryByRole("button", { name: "Stop" })).not.toBeInTheDocument(); + expect(screen.getByText("call-fresh")).toBeInTheDocument(); + expect(screen.queryByText("call-stopped")).not.toBeInTheDocument(); + }); + + it("sends once when Enter is pressed twice before the first request starts", async () => { + const user = userEvent.setup(); + const first = deferred(); + chatRequest.mockImplementation(async (...args: ChatArgs) => { + await first.promise; + args[1]("Hi there", args[2]); + }); + preload({ selectedModel: "gpt-fast" }); + renderChat(); + + const message = screen.getByRole("textbox", { name: "Message" }); + await user.type(message, "hello"); + fireEvent.keyDown(message, { key: "Enter" }); + fireEvent.keyDown(message, { key: "Enter" }); + + await waitFor(() => expect(chatRequest).toHaveBeenCalled()); + await act(async () => { + first.release(); + await chatRequest.mock.results[0]?.value; + }); + expect(chatRequest).toHaveBeenCalledTimes(1); + expect(screen.getAllByText("hello")).toHaveLength(1); + }); + + it("drops a chat attachment when the endpoint cannot use it", async () => { + const user = userEvent.setup(); + preload({ selectedModel: "gpt-fast" }); + renderChat(); + + fireEvent.drop(screen.getByRole("textbox", { name: "Message" }), { + dataTransfer: { files: [new File(["%PDF"], "notes.pdf", { type: "application/pdf" })] }, + }); + expect(await screen.findByRole("button", { name: "Remove notes.pdf" })).toBeInTheDocument(); + + await user.click(screen.getByRole("button", { name: "Endpoint" })); + await user.click(await screen.findByRole("menuitemradio", { name: "/v1/audio/transcriptions" })); + + await waitFor(() => + expect(screen.getByRole("button", { name: "Endpoint" })).toHaveTextContent("audio/transcriptions"), + ); + expect(screen.queryByRole("button", { name: "Remove notes.pdf" })).not.toBeInTheDocument(); + }); + + it("keeps a stopped request's code output off the newer reply", async () => { + const user = userEvent.setup(); + const first = deferred(); + responsesRequest + .mockImplementationOnce(async (...args: ResponsesArgs) => { + await first.promise; + args[18]?.({ code: "print('stopped run')", containerId: "c-1", annotations: [] }); + }) + .mockImplementationOnce(async (...args: ResponsesArgs) => { + args[1]("assistant", "fresh reply", args[2]); + }); + preload({ endpointType: "responses", selectedModel: "gpt-fast" }); + renderChat(); + + await send(user, "first"); + await user.click(await screen.findByRole("button", { name: "Stop" })); + await send(user, "second"); + expect(await screen.findByText("fresh reply")).toBeInTheDocument(); + await act(async () => { + first.release(); + await responsesRequest.mock.results[0]?.value; + }); + + expect(screen.queryByText("Python Code Executed")).not.toBeInTheDocument(); + }); + + it("lets a user connect a key for a BYOK server picked for a direct tool call", async () => { + const user = userEvent.setup(); + vi.mocked(fetchMCPServers).mockResolvedValue([ + { server_id: "srv-byok", alias: "Linear", is_byok: true, has_user_credential: false }, + ]); + preload({ endpointType: "mcp", selectedMCPServers: JSON.stringify(["srv-byok"]) }); + renderChat(); + + await user.click(await screen.findByRole("button", { name: "Connect key" })); + await user.type(await screen.findByLabelText("Linear API key"), "lin-key-1"); + await user.click(screen.getByRole("button", { name: "Connect" })); + + await waitFor(() => expect(credentialPost).toHaveBeenCalledTimes(1)); + expect(credentialPost.mock.calls[0]?.[1]).toMatchObject({ + params: { path: { server_id: "srv-byok" } }, + body: { credential: "lin-key-1" }, + }); + await waitFor(() => expect(fetchMCPServers).toHaveBeenCalledTimes(2)); + }); + + it("lets a user replace the key of a connected BYOK server whose tools fail to load", async () => { + const user = userEvent.setup(); + vi.mocked(fetchMCPServers).mockResolvedValue([ + { server_id: "srv-byok", alias: "Linear", is_byok: true, has_user_credential: true }, + ]); + vi.mocked(listMCPTools).mockResolvedValue({ error: "401 from upstream" }); + preload({ selectedModel: "gpt-fast", selectedMCPServers: JSON.stringify(["srv-byok"]) }); + renderChat(); + + await settled(); + await user.click(screen.getByRole("button", { name: "MCP" })); + await user.click(await screen.findByRole("button", { name: "Replace key" })); + await user.type(await screen.findByLabelText("Linear API key"), "lin-key-2"); + await user.click(screen.getByRole("button", { name: "Connect" })); + + await waitFor(() => expect(credentialPost).toHaveBeenCalledTimes(1)); + expect(credentialPost.mock.calls[0]?.[1]).toMatchObject({ body: { credential: "lin-key-2" } }); + await waitFor(() => + expect(vi.mocked(listMCPTools).mock.calls.filter(([, server]) => server === "srv-byok").length).toBeGreaterThan( + 1, + ), + ); + }); + + it("leaves replace-key mode on cancel or when the MCP dialog closes", async () => { + const user = userEvent.setup(); + vi.mocked(fetchMCPServers).mockResolvedValue([ + { server_id: "srv-byok", alias: "Linear", is_byok: true, has_user_credential: true }, + ]); + vi.mocked(listMCPTools).mockResolvedValue({ error: "401 from upstream" }); + preload({ selectedModel: "gpt-fast", selectedMCPServers: JSON.stringify(["srv-byok"]) }); + renderChat(); + + await settled(); + await user.click(screen.getByRole("button", { name: "MCP" })); + await user.click(await screen.findByRole("button", { name: "Replace key" })); + expect(await screen.findByLabelText("Linear API key")).toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Cancel" })); + expect(screen.queryByLabelText("Linear API key")).not.toBeInTheDocument(); + + await user.click(screen.getByRole("button", { name: "Replace key" })); + expect(await screen.findByLabelText("Linear API key")).toBeInTheDocument(); + await user.keyboard("{Escape}"); + await waitFor(() => expect(screen.queryByLabelText("Linear API key")).not.toBeInTheDocument()); + await user.click(screen.getByRole("button", { name: "MCP" })); + + expect(await screen.findByRole("button", { name: "Replace key" })).toBeInTheDocument(); + expect(screen.queryByLabelText("Linear API key")).not.toBeInTheDocument(); + expect(credentialPost).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/PlaygroundChat.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/PlaygroundChat.tsx new file mode 100644 index 00000000000..5b2de80315d --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/PlaygroundChat.tsx @@ -0,0 +1,489 @@ +"use client"; + +import { useRef, useState } from "react"; +import { ChevronDown, PanelRight, RotateCcw, SquarePen } from "lucide-react"; +import useCan from "@/app/(dashboard)/hooks/useCan"; +import { generateCodeSnippet } from "@/components/chat_ui/CodeSnippets"; +import { EndpointType, getEndpointType } from "@/components/chat_ui/mode_endpoint_mapping"; +import type { MessageType } from "@/components/chat_ui/types"; +import type { ModelGroup } from "@/components/llm_calls/fetch_models"; +import { customHeadersFromPairs } from "@/components/llm_calls/request_headers"; +import type { MCPToolArgumentsFormRef } from "@/components/mcp_tools/MCPToolArgumentsForm"; +import type { MCPServer, MCPTool, MCPToolset } from "@/components/mcp_tools/types"; +import { Button } from "@/components/ui/button"; +import { ButtonGroup } from "@/components/ui/button-group"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { Sheet, SheetContent, SheetTitle } from "@/components/ui/sheet"; +import { toast } from "@/lib/toast"; +import { useChatHistory } from "../../hooks/useChatHistory"; +import { useCodeInterpreter } from "../../hooks/useCodeInterpreter"; +import { filterModelsForEndpoint, isModelCompatibleWithEndpoint } from "../chat_ui/EndpointUtils"; +import RealtimePlayground from "../chat_ui/RealtimePlayground"; +import { Composer } from "./Composer"; +import { EndpointMenu, VoiceMenu } from "./ComposerControls"; +import { Conversation, formatCost } from "./Conversation"; +import { capabilitiesFor, CODE_INTERPRETER_SUGGESTIONS, SUGGESTIONS } from "./endpointCapabilities"; +import { InspectorPanel, type CodeSdk } from "./Inspector"; +import { ByokKeyButton } from "./ByokConnect"; +import { McpDialog, serverName } from "./McpDialog"; +import { TargetPicker, type OptionGroup } from "./Pickers"; +import { TOOLSET_PREFIX, withEndpoint, withReachableMcp } from "./playgroundSettings"; +import { SettingsPopover } from "./SettingsPopover"; +import { + useGuardrailOptions, + useMcpCatalog, + usePlaygroundAgents, + usePlaygroundModels, + usePolicyOptions, + useServerTools, + useTagOptions, + useVectorStoreOptions, + type PickerOption, + type ServerTools, +} from "./usePlaygroundCatalog"; +import { usePlaygroundSend, type SendDeps } from "./usePlaygroundSend"; +import { usePlaygroundSettings } from "./usePlaygroundSettings"; +import { useIsWide } from "./useIsWide"; + +export type ProxySettings = Readonly<{ PROXY_BASE_URL?: string; LITELLM_UI_API_DOC_BASE_URL?: string | null }>; + +const POLICY_PREFIX = "policy:"; + +const providerOf = (model: ModelGroup): string => model.providers?.[0] ?? "other"; + +const modelGroups = (models: readonly ModelGroup[]): readonly OptionGroup[] => + [...new Set(models.map(providerOf))].sort().map((provider) => ({ + label: provider, + options: models + .filter((model) => providerOf(model) === provider) + .map((model) => ({ value: model.model_group, label: model.model_group })), + })); + +const modelEmptyText = (failed: boolean, hasKey: boolean): string => { + if (failed) return "Unable to load models for this key"; + if (!hasKey) return "Enter a virtual key to load models"; + return "No models available for this endpoint"; +}; + +const reachableMcp = (servers: readonly MCPServer[], toolsets: readonly MCPToolset[]): ReadonlySet => + new Set([ + ...servers.map((server) => server.server_id), + ...toolsets.map((toolset) => `${TOOLSET_PREFIX}${toolset.toolset_id}`), + ]); + +const toolsetFor = (selection: string | null, toolsets: readonly MCPToolset[]): MCPToolset | null => + toolsets.find((toolset) => `${TOOLSET_PREFIX}${toolset.toolset_id}` === selection) ?? null; + +const backingServers = (selection: string | null, toolset: MCPToolset | null): readonly string[] => { + if (toolset) return [...new Set(toolset.tools.map((tool) => tool.server_id))]; + return selection && !selection.startsWith(TOOLSET_PREFIX) ? [selection] : []; +}; + +type DirectTarget = Readonly<{ server: string; tool: MCPTool }>; + +const toolsetTargets = (toolset: MCPToolset, loaded: Readonly>): DirectTarget[] => + toolset.tools + .flatMap((entry) => { + const tool = loaded[entry.server_id]?.tools.find((candidate) => candidate.name === entry.tool_name); + return tool ? [{ server: entry.server_id, tool }] : []; + }) + .filter((target, index, targets) => targets.findIndex((other) => other.tool.name === target.tool.name) === index); + +const directTargets = ( + servers: readonly string[], + toolset: MCPToolset | null, + loaded: Readonly>, +): readonly DirectTarget[] => { + if (toolset) return toolsetTargets(toolset, loaded); + return servers.flatMap((server) => (loaded[server]?.tools ?? []).map((tool) => ({ server, tool }))); +}; + +const prefixed = (options: readonly PickerOption[]): readonly PickerOption[] => + options.map((option) => ({ ...option, value: `${POLICY_PREFIX}${option.value}` })); + +const lastTracedReply = (messages: readonly MessageType[]): number => + messages.findLastIndex((message) => message.role === "assistant" && message.trace !== undefined); + +const totalCost = (messages: readonly MessageType[]): number => + messages.reduce((sum, message) => sum + (message.usage?.cost ?? message.trace?.cost ?? 0), 0); + +const conversationSummary = (messages: readonly MessageType[]): string => { + const requests = messages.filter((message) => message.role === "user").length; + if (requests === 0) return "New chat"; + const label = requests === 1 ? "1 request" : `${requests} requests`; + const cost = totalCost(messages); + return cost > 0 ? `${label}, ${formatCost(cost)}` : label; +}; + +export function PlaygroundChat({ + accessToken, + disabledPersonalKeyCreation, + proxySettings, +}: { + accessToken: string | null; + disabledPersonalKeyCreation: boolean; + proxySettings: ProxySettings | undefined; +}) { + const { settings: stored, update, reset } = usePlaygroundSettings(disabledPersonalKeyCreation); + const canViewPolicies = useCan("viewPolicies"); + const apiKey = (stored.keySource === "session" ? accessToken : stored.virtualKey.trim()) || null; + const mcp = useMcpCatalog(apiKey); + const settings = withReachableMcp(stored, mcp.isReady ? reachableMcp(mcp.servers, mcp.toolsets) : null); + const capabilities = capabilitiesFor(settings.endpoint); + const chat = useChatHistory({ simplified: false }); + const codeInterpreter = useCodeInterpreter(); + const toolArgsForm = useRef(null); + const [inspected, setInspected] = useState(null); + const [sheetOpen, setSheetOpen] = useState(false); + const isWide = useIsWide(); + + const models = usePlaygroundModels(apiKey); + const [checkedModels, setCheckedModels] = useState(undefined); + if (models.data && models.data !== checkedModels) { + setCheckedModels(models.data); + const loaded = models.data; + update((current) => + current.model && !loaded.some((option) => option.model_group === current.model) + ? { ...current, model: null } + : current, + ); + } + const agents = usePlaygroundAgents(apiKey, settings.proxyBaseUrl, settings.endpoint === EndpointType.A2A_AGENTS); + const guardrails = useGuardrailOptions(accessToken); + const policies = usePolicyOptions(accessToken, canViewPolicies); + const vectorStores = useVectorStoreOptions(accessToken); + const tags = useTagOptions(accessToken); + + const directSelection = settings.endpoint === EndpointType.MCP ? settings.mcpSelection[0] ?? null : null; + const directToolset = toolsetFor(directSelection, mcp.toolsets); + const directServers = backingServers(directSelection, directToolset); + const directServerTools = useServerTools(apiKey, directServers); + const directByok = directToolset ? undefined : mcp.servers.find((server) => server.server_id === directSelection); + const targets = directTargets(directServers, directToolset, directServerTools); + const directTarget = targets.find((target) => target.tool.name === settings.directTool) ?? null; + const directTool = directTarget?.tool ?? null; + const toolsFailed = directServers.some((server) => directServerTools[server]?.failed); + const customHeaders = customHeadersFromPairs(settings.headerPairs); + + const sendDeps: SendDeps = { + settings, + capabilities, + apiKey, + signedIn: Boolean(accessToken), + customHeaders, + mcpServers: mcp.servers, + mcpToolsets: mcp.toolsets, + directCall: () => (directTarget ? { serverId: directTarget.server, tool: directTarget.tool.name } : null), + toolArgsForm, + chat, + codeInterpreter, + }; + const draft = usePlaygroundSend(sendDeps); + + const messages = chat.chatHistory; + const inspectedIndex = inspected ?? lastTracedReply(messages); + const inspectedMessage = inspectedIndex >= 0 ? messages[inspectedIndex] ?? null : null; + const modelList = models.data ?? []; + + const selectModel = (model: string) => { + const info = modelList.find((option) => option.model_group === model); + const switchTo = + info?.mode && !isModelCompatibleWithEndpoint(info, settings.endpoint) ? getEndpointType(info.mode) : null; + update((current) => ({ ...(switchTo ? withEndpoint(current, switchTo) : current), model })); + }; + + const inspect = (index: number) => { + setInspected(index); + if (!isWide) setSheetOpen(true); + }; + + const clearChat = () => { + chat.clearChatHistory(); + draft.clearDraft(); + codeInterpreter.clearResult(); + setInspected(null); + }; + + const newChat = () => { + clearChat(); + toast.success("Chat history cleared."); + }; + + const resetAll = () => { + clearChat(); + reset(); + toast.success("Chat and settings reset"); + }; + + const code = (sdk: CodeSdk): string => { + const snippet: Parameters[0] = { + apiKeySource: "custom", + accessToken: null, + apiKey: "", + inputMessage: draft.text, + chatHistory: messages, + selectedTags: [...settings.tags], + selectedVectorStores: [...settings.vectorStores], + selectedGuardrails: [...settings.guardrails], + selectedPolicies: [...settings.policies], + selectedMCPServers: [...settings.mcpSelection], + mcpServers: [...mcp.servers], + mcpServerToolRestrictions: Object.fromEntries( + Object.entries(settings.toolLimits).map(([server, tools]) => [server, [...tools]]), + ), + endpointType: settings.endpoint, + selectedModel: settings.model ?? undefined, + selectedSdk: sdk, + selectedVoice: settings.voice, + proxySettings, + customHeaders, + }; + return generateCodeSnippet(snippet); + }; + + const targetRow = ( + <> + {capabilities.target === "model" && ( + + )} + {capabilities.target === "agent" && ( + ({ + value: agent.agent_name, + label: agent.agent_name || agent.agent_id, + description: agent.agent_card_params?.description, + })), + }, + ]} + value={settings.agent} + onValueChange={(agent) => update({ agent })} + isLoading={agents.isLoading} + emptyText="No agents available for this key" + /> + )} + {capabilities.target === "mcpTool" && ( + <> + ({ value: server.server_id, label: serverName(server) })), + }, + { + label: "Toolsets", + options: mcp.toolsets.map((toolset) => ({ + value: `${TOOLSET_PREFIX}${toolset.toolset_id}`, + label: toolset.toolset_name, + description: toolset.description, + })), + }, + ]} + value={directSelection} + onValueChange={(server) => update({ mcpSelection: [server], directTool: null })} + isLoading={mcp.isLoading} + emptyText="No MCP servers available for this key" + /> + {directByok?.is_byok && ( + + )} + {directSelection && ( + ({ + value: tool.name, + label: tool.name, + description: tool.description, + })), + }, + ]} + value={settings.directTool} + onValueChange={(tool) => update({ directTool: tool })} + isLoading={directServers.some((server) => directServerTools[server]?.isLoading)} + emptyText={toolsFailed ? "Unable to load tools for this server" : "This server has no tools"} + /> + )} + + )} + update((current) => withEndpoint(current, endpoint))} + /> + {capabilities.voice && update({ voice })} />} + + ); + + const guardrailGroups: readonly OptionGroup[] = [ + { label: "Guardrails", options: guardrails.data ?? [] }, + ...(canViewPolicies ? [{ label: "Policies", options: prefixed(policies.data ?? []) }] : []), + ]; + + const composer = ( + update({ mcpSelection, toolLimits })} + onCredentialSaved={mcp.refreshCredentials} + /> + } + vectorStores={{ + groups: [{ label: "Vector stores", options: vectorStores.data ?? [] }], + value: settings.vectorStores, + onValueChange: (value) => update({ vectorStores: value }), + isLoading: vectorStores.isLoading, + }} + guardrails={{ + groups: guardrailGroups, + value: [...settings.guardrails, ...settings.policies.map((policy) => `${POLICY_PREFIX}${policy}`)], + onValueChange: (value) => + update({ + guardrails: value.filter((item) => !item.startsWith(POLICY_PREFIX)), + policies: value + .filter((item) => item.startsWith(POLICY_PREFIX)) + .map((item) => item.slice(POLICY_PREFIX.length)), + }), + isLoading: guardrails.isLoading || policies.isLoading, + }} + codeInterpreter={{ enabled: codeInterpreter.enabled, onToggle: codeInterpreter.setEnabled }} + settingsControl={ + + } + sendDisabled={draft.isLoading} + /> + ); + + const suggestions = + settings.endpoint === EndpointType.RESPONSES && codeInterpreter.enabled + ? CODE_INTERPRETER_SUGGESTIONS + : SUGGESTIONS[settings.endpoint] ?? []; + + const inspector = ; + + return ( +
+
+
+ {conversationSummary(messages)} + + {!isWide && ( + + )} + + + + } + > + + + + + + New chat, keep settings + + + + Reset chat and settings + + + + + +
+ {settings.endpoint === EndpointType.REALTIME ? ( +
+ 0 ? [...settings.guardrails] : undefined} + /> +
+ ) : ( + + )} +
{composer}
+
+ {isWide ? ( + + ) : ( + + + Inspector + {inspector} + + + )} +
+ ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/ReplyContent.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/ReplyContent.tsx new file mode 100644 index 00000000000..5733ede7168 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat/ReplyContent.tsx @@ -0,0 +1,92 @@ +"use client"; + +import ReactMarkdown from "react-markdown"; +import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; +import { coy } from "react-syntax-highlighter/dist/esm/styles/prism"; +import remarkGfm from "remark-gfm"; +import type { MessageType } from "@/components/chat_ui/types"; +import { ScrollArea, ScrollBar } from "@/components/ui/scroll-area"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import { useSyntaxTheme } from "@/hooks/useSyntaxTheme"; +import AudioRenderer from "../chat_ui/AudioRenderer"; + +const textOf = (message: MessageType): string => (typeof message.content === "string" ? message.content : ""); + +function CodeBlock({ language, code }: { language: string; code: string }) { + const theme = useSyntaxTheme(coy); + return ( + + {code} + + ); +} + +export function Markdown({ children }: { children: string }) { + return ( +

, + ul: ({ node: _node, ...props }) =>