diff --git a/ui/litellm-dashboard/src/app/chat/page.integration.test.tsx b/ui/litellm-dashboard/src/app/chat/page.integration.test.tsx index 2020a66bd1c..8f1e7d5ea5a 100644 --- a/ui/litellm-dashboard/src/app/chat/page.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/chat/page.integration.test.tsx @@ -1,13 +1,19 @@ -import { fireEvent, screen, waitFor, within } from "@testing-library/react"; +import { act, fireEvent, screen, waitFor, within } from "@testing-library/react"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { useChatHistory } from "@/components/chat/useChatHistory"; import ChatConversationPage from "./page"; import { renderWithProviders } from "@/../tests/test-utils"; import type { OnUrlUpdateFunction } from "nuqs/adapters/testing"; +import { fetchAvailableModels } from "@/components/llm_calls/fetch_models"; +import type { makeOpenAIResponsesRequest } from "@/components/llm_calls/responses_api"; +import type { MCPEvent } from "@/components/chat/types"; const { mockMakeOpenAIResponsesRequest, shellState } = vi.hoisted(() => ({ mockMakeOpenAIResponsesRequest: vi.fn(), - shellState: { storageUnavailable: false }, + shellState: { + activeConversationIdOverride: null as string | null, + storageUnavailable: false, + }, })); vi.mock("next/navigation", () => ({ @@ -41,6 +47,16 @@ vi.mock("react-syntax-highlighter/dist/esm/styles/prism", () => ({ coy: {}, oneD vi.mock("@/contexts/ChatShellContext", () => ({ useChatShell: () => { const history = useChatHistory(null, "metrics-test-user"); + const activeConversationId = shellState.activeConversationIdOverride ?? history.currentActiveId; + const activeConversation = shellState.activeConversationIdOverride + ? { + id: shellState.activeConversationIdOverride, + title: "Second conversation", + createdAt: Date.now(), + updatedAt: Date.now(), + messages: [], + } + : history.activeConversation; return { accessToken: "sk-test", userId: "metrics-test-user", @@ -50,8 +66,8 @@ vi.mock("@/contexts/ChatShellContext", () => ({ selectedMCPServers: [], setSelectedMCPServers: vi.fn(), conversations: history.conversations, - activeConversation: history.activeConversation, - activeConversationId: history.currentActiveId, + activeConversation, + activeConversationId, storageUnavailable: shellState.storageUnavailable, staleId: false, createConversation: history.createConversation, @@ -69,21 +85,214 @@ const ON_TIMING_DATA_INDEX = 7; const ON_USAGE_DATA_INDEX = 8; const ON_TOTAL_LATENCY_INDEX = 24; -async function sendOneMessage(onUrlUpdate?: OnUrlUpdateFunction): Promise { - renderWithProviders(, { onUrlUpdate }); +async function sendOneMessage(onUrlUpdate?: OnUrlUpdateFunction) { + const result = renderWithProviders(, { onUrlUpdate }); expect(await screen.findByRole("button", { name: /gpt-5\.4-mini/ })).toBeInTheDocument(); fireEvent.change(screen.getByPlaceholderText("How can I help you today?"), { target: { value: "How much did this cost?" }, }); fireEvent.click(screen.getByRole("button", { name: "Send" })); await waitFor(() => expect(mockMakeOpenAIResponsesRequest).toHaveBeenCalledTimes(1)); + return result; } describe("/ui/chat request metrics", () => { beforeEach(() => { localStorage.clear(); mockMakeOpenAIResponsesRequest.mockReset(); + shellState.activeConversationIdOverride = null; shellState.storageUnavailable = false; + vi.mocked(fetchAvailableModels).mockResolvedValue([{ model_group: "gpt-5.4-mini" }]); + }); + + it.each([ + { providers: ["openrouter"], stateless: true }, + { providers: ["openai", "openrouter"], stateless: true }, + { providers: ["openai"], stateless: false }, + { providers: undefined, stateless: false }, + ])("continues a conversation correctly with providers=$providers", async ({ providers, stateless }) => { + vi.mocked(fetchAvailableModels).mockResolvedValue([{ model_group: "gpt-5.4-mini", providers }]); + mockMakeOpenAIResponsesRequest.mockImplementation(async (...args: unknown[]) => { + await new Promise((resolve) => setTimeout(resolve, 0)); + (args[1] as (role: string, delta: string) => void)("assistant", "First answer"); + (args[15] as (id: string) => void)("response-first"); + }); + + await sendOneMessage(); + expect(await screen.findByText("First answer")).toBeVisible(); + fireEvent.change(screen.getByRole("textbox"), { + target: { value: "Follow up" }, + }); + await waitFor(() => expect(screen.getByRole("button", { name: "Send" })).toBeEnabled()); + fireEvent.click(screen.getByRole("button", { name: "Send" })); + await waitFor(() => expect(mockMakeOpenAIResponsesRequest).toHaveBeenCalledTimes(2)); + + const [first, second] = mockMakeOpenAIResponsesRequest.mock.calls; + expect(first[14]).toBeNull(); + expect(second[14]).toBe(stateless ? null : "response-first"); + expect(second[0]).toEqual( + stateless + ? [ + { role: "user", content: "How much did this cost?" }, + { role: "assistant", content: "First answer" }, + { role: "user", content: "Follow up" }, + ] + : [{ role: "user", content: "Follow up" }], + ); + }); + + it.each([ + { providers: ["openrouter"], stateless: true }, + { providers: ["openai", "openrouter"], stateless: true }, + { providers: ["openai"], stateless: false }, + ])("preserves MCP context with providers=$providers", async ({ providers, stateless }) => { + const event: MCPEvent = { + type: "response.output_item.done", + item: { + type: "mcp_call", + id: "mcp-inventory", + name: "lookup_inventory", + arguments: "{}", + output: '{"quantity":17}', + }, + }; + vi.mocked(fetchAvailableModels).mockResolvedValue([{ model_group: "gpt-5.4-mini", providers }]); + mockMakeOpenAIResponsesRequest.mockImplementationOnce( + async (...args: Parameters) => { + await new Promise((resolve) => setTimeout(resolve, 0)); + args[1]("assistant", "Lookup completed"); + args[15]?.("response-first"); + args[16]?.(event); + }, + ); + mockMakeOpenAIResponsesRequest.mockResolvedValue(undefined); + + await sendOneMessage(); + expect(await screen.findByText("Lookup completed")).toBeVisible(); + fireEvent.change(screen.getByRole("textbox"), { target: { value: "What was the quantity?" } }); + await waitFor(() => expect(screen.getByRole("button", { name: "Send" })).toBeEnabled()); + fireEvent.click(screen.getByRole("button", { name: "Send" })); + await waitFor(() => expect(mockMakeOpenAIResponsesRequest).toHaveBeenCalledTimes(2)); + + const second = mockMakeOpenAIResponsesRequest.mock.calls[1]; + expect(second[14]).toBe(stateless ? null : "response-first"); + expect(second[0]).toEqual( + stateless + ? [ + { role: "user", content: "How much did this cost?" }, + { role: "assistant", content: "Lookup completed", mcpEvents: [event] }, + { role: "user", content: "What was the quantity?" }, + ] + : [{ role: "user", content: "What was the quantity?" }], + ); + + fireEvent.mouseOver(screen.getByText("What was the quantity?")); + fireEvent.click(await screen.findByRole("button", { name: "Edit message" })); + fireEvent.change(screen.getByRole("textbox", { name: "Edit message" }), { + target: { value: "Repeat the inventory quantity" }, + }); + fireEvent.click(screen.getByRole("button", { name: "Save & Send" })); + await waitFor(() => expect(mockMakeOpenAIResponsesRequest).toHaveBeenCalledTimes(3)); + + const edited = mockMakeOpenAIResponsesRequest.mock.calls[2]; + expect(edited[14]).toBeNull(); + expect(edited[0]).toEqual([ + { role: "user", content: "How much did this cost?" }, + { role: "assistant", content: "Lookup completed", mcpEvents: [event] }, + { role: "user", content: "Repeat the inventory quantity" }, + ]); + }); + + it("resends history instead of reusing another model's response ID after changing models", async () => { + vi.mocked(fetchAvailableModels).mockResolvedValue([ + { model_group: "gpt-5.4-mini", providers: ["openai"] }, + { model_group: "second-model", providers: ["openai"] }, + ]); + mockMakeOpenAIResponsesRequest.mockImplementation(async (...args: unknown[]) => { + await new Promise((resolve) => setTimeout(resolve, 0)); + (args[1] as (role: string, delta: string) => void)("assistant", "First answer"); + (args[15] as (id: string) => void)("response-first"); + }); + + await sendOneMessage(); + expect(await screen.findByText("First answer")).toBeVisible(); + fireEvent.click(screen.getByRole("button", { name: /gpt-5\.4-mini/ })); + fireEvent.click(await screen.findByRole("button", { name: "second-model" })); + fireEvent.change(screen.getByRole("textbox"), { target: { value: "Follow up" } }); + await waitFor(() => expect(screen.getByRole("button", { name: "Send" })).toBeEnabled()); + fireEvent.click(screen.getByRole("button", { name: "Send" })); + await waitFor(() => expect(mockMakeOpenAIResponsesRequest).toHaveBeenCalledTimes(2)); + + const second = mockMakeOpenAIResponsesRequest.mock.calls[1]; + expect(second[2]).toBe("second-model"); + expect(second[14]).toBeNull(); + expect(second[0]).toEqual([ + { role: "user", content: "How much did this cost?" }, + { role: "assistant", content: "First answer" }, + { role: "user", content: "Follow up" }, + ]); + }); + + it.each([false, true])("ignores late response IDs after model changes ($0)", async (switchBack) => { + vi.mocked(fetchAvailableModels).mockResolvedValue([ + { model_group: "gpt-5.4-mini", providers: ["openai"] }, + { model_group: "second-model", providers: ["openai"] }, + ]); + const firstCompletion = Promise.withResolvers(); + mockMakeOpenAIResponsesRequest.mockImplementationOnce(async (...args: unknown[]) => { + await firstCompletion.promise; + (args[1] as (role: string, delta: string) => void)("assistant", "First answer"); + (args[15] as (id: string) => void)("response-first"); + }); + mockMakeOpenAIResponsesRequest.mockResolvedValue(undefined); + + await sendOneMessage(); + fireEvent.click(screen.getByRole("button", { name: /gpt-5\.4-mini/ })); + fireEvent.click(await screen.findByRole("button", { name: "second-model" })); + if (switchBack) { + fireEvent.click(screen.getByRole("button", { name: "second-model" })); + fireEvent.click(await screen.findByRole("button", { name: "gpt-5.4-mini" })); + } + + await act(async () => firstCompletion.resolve()); + expect(await screen.findByText("First answer")).toBeVisible(); + fireEvent.change(screen.getByRole("textbox"), { target: { value: "Follow up" } }); + await waitFor(() => expect(screen.getByRole("button", { name: "Send" })).toBeEnabled()); + fireEvent.click(screen.getByRole("button", { name: "Send" })); + await waitFor(() => expect(mockMakeOpenAIResponsesRequest).toHaveBeenCalledTimes(2)); + + const second = mockMakeOpenAIResponsesRequest.mock.calls[1]; + expect(second[2]).toBe(switchBack ? "gpt-5.4-mini" : "second-model"); + expect(second[14]).toBeNull(); + expect(second[0]).toEqual([ + { role: "user", content: "How much did this cost?" }, + { role: "assistant", content: "First answer" }, + { role: "user", content: "Follow up" }, + ]); + }); + + it("ignores late response IDs after changing conversations", async () => { + const firstCompletion = Promise.withResolvers(); + mockMakeOpenAIResponsesRequest.mockImplementationOnce(async (...args: unknown[]) => { + await firstCompletion.promise; + (args[1] as (role: string, delta: string) => void)("assistant", "First answer"); + (args[15] as (id: string) => void)("response-first"); + }); + mockMakeOpenAIResponsesRequest.mockResolvedValue(undefined); + + const { rerender } = await sendOneMessage(); + shellState.activeConversationIdOverride = "conversation-b"; + rerender(); + + await act(async () => firstCompletion.resolve()); + fireEvent.change(screen.getByRole("textbox"), { target: { value: "Follow up" } }); + await waitFor(() => expect(screen.getByRole("button", { name: "Send" })).toBeEnabled()); + fireEvent.click(screen.getByRole("button", { name: "Send" })); + await waitFor(() => expect(mockMakeOpenAIResponsesRequest).toHaveBeenCalledTimes(2)); + + const second = mockMakeOpenAIResponsesRequest.mock.calls[1]; + expect(second[14]).toBeNull(); + expect(second[0]).toEqual([{ role: "user", content: "Follow up" }]); }); it("renders latency, TTFT, token counts and cost reported for the assistant turn", async () => { diff --git a/ui/litellm-dashboard/src/app/chat/page.tsx b/ui/litellm-dashboard/src/app/chat/page.tsx index fdf6a1e02a3..5cf605eec2a 100644 --- a/ui/litellm-dashboard/src/app/chat/page.tsx +++ b/ui/litellm-dashboard/src/app/chat/page.tsx @@ -14,7 +14,7 @@ import { getChatRoutes } from "@/components/chat/ChatShell"; import ChatMessages from "@/components/chat/ChatMessages"; import MCPConnectPicker from "@/components/chat/MCPConnectPicker"; import { fetchAvailableModels } from "@/components/llm_calls/fetch_models"; -import { makeOpenAIResponsesRequest } from "@/components/llm_calls/responses_api"; +import { makeOpenAIResponsesRequest, type ResponsesMessage } from "@/components/llm_calls/responses_api"; import type { TokenUsage } from "@/components/chat_ui/ResponseMetrics"; import type { MCPEvent } from "@/components/chat/types"; import { getProviderLogoAndName } from "@/components/provider_info_helpers"; @@ -71,11 +71,14 @@ export default function ChatConversationPage() { const [selectedModel, setSelectedModel] = useState(null); const [models, setModels] = useState([]); + const [statelessModelGroups, setStatelessModelGroups] = useState>(new Set()); const [isLoadingModels, setIsLoadingModels] = useState(true); const [modelSelectorOpen, setModelSelectorOpen] = useState(false); const [modelSearchText, setModelSearchText] = useState(""); const [responsesSessionId, setResponsesSessionId] = useState(null); + const responsesSessionGenerationRef = useRef(0); + const previousResponsesConversationRef = useRef(activeConversationId); const [prevConversationIdForSessionReset, setPrevConversationIdForSessionReset] = useState(activeConversationId); const [isStreaming, setIsStreaming] = useState(false); const [inputText, setInputText] = useState(""); @@ -99,6 +102,9 @@ export default function ChatConversationPage() { .then((data) => { const names = (data || []).map((m: { model_group?: string }) => m.model_group ?? "").filter(Boolean); setModels(names); + setStatelessModelGroups( + new Set(data.filter((model) => model.providers?.includes("openrouter")).map((model) => model.model_group)), + ); try { const saved = localStorage.getItem(LOCALSTORAGE_MODEL_KEY); if (saved && names.includes(saved)) { @@ -117,6 +123,16 @@ export default function ChatConversationPage() { .finally(() => setIsLoadingModels(false)); }, [accessToken]); + useLayoutEffect(() => { + if ( + previousResponsesConversationRef.current !== null && + previousResponsesConversationRef.current !== activeConversationId + ) { + responsesSessionGenerationRef.current += 1; + } + previousResponsesConversationRef.current = activeConversationId; + }, [activeConversationId]); + // Reset the responses session when switching between conversations so that // previous_response_id from conversation A is never sent for conversation B. if (activeConversationId !== prevConversationIdForSessionReset) { @@ -125,17 +141,20 @@ export default function ChatConversationPage() { } const selectModel = useCallback((model: string) => { + responsesSessionGenerationRef.current += 1; setSelectedModel(model); + setResponsesSessionId(null); localStorage.setItem(LOCALSTORAGE_MODEL_KEY, model); setModelSelectorOpen(false); setModelSearchText(""); }, []); const handleSend = useCallback( - async (text: string, historyOverride?: Array<{ role: "user" | "assistant"; content: string }>) => { + async (text: string, historyOverride?: ResponsesMessage[]) => { const trimmed = text.trim(); if (!trimmed || !selectedModel || isStreaming) return; const model = selectedModel; + const sessionGeneration = responsesSessionGenerationRef.current; setInputText(""); let convId = activeConversationId; @@ -160,14 +179,9 @@ export default function ChatConversationPage() { setResponsesSessionId(null); } - // On a normal continuation turn with an active session, the Responses API - // already holds the prior context server-side, so we only pass the new - // user message (sending the full history would double-count it). - // - // On the very first turn (no session yet), we send the full history. - const previousResponseId = historyOverride ? null : responsesSessionId; + const previousResponseId = historyOverride || statelessModelGroups.has(model) ? null : responsesSessionId; - const history: Array<{ role: "user" | "assistant"; content: string }> = historyOverride + const history: ResponsesMessage[] = historyOverride ? [...historyOverride, { role: "user" as const, content: trimmed }] : previousResponseId ? [{ role: "user" as const, content: trimmed }] @@ -178,7 +192,7 @@ export default function ChatConversationPage() { .filter( (m): m is typeof m & { role: "user" | "assistant" } => m.role === "user" || m.role === "assistant", ) - .map((m) => ({ role: m.role, content: m.content })), + .map((m) => ({ role: m.role, content: m.content, mcpEvents: m.mcpEvents })), { role: "user" as const, content: trimmed }, ]; @@ -213,7 +227,11 @@ export default function ChatConversationPage() { undefined, selectedMCPServers.length > 0 ? selectedMCPServers : undefined, previousResponseId, - (id: string) => setResponsesSessionId(id), + (id: string) => { + if (responsesSessionGenerationRef.current === sessionGeneration) { + setResponsesSessionId(id); + } + }, (event: MCPEvent) => { // Accumulate locally only — persisted once in finally to avoid // one full localStorage write per MCP event during streaming. @@ -260,6 +278,7 @@ export default function ChatConversationPage() { updateLastAssistantMessage, isStreaming, responsesSessionId, + statelessModelGroups, setConversationIdInUrl, ], ); @@ -277,7 +296,7 @@ export default function ChatConversationPage() { const idx = msgs.findIndex((m) => m.id === messageId); const priorMessages = (idx === -1 ? msgs : msgs.slice(0, idx)) .filter((m) => m.role === "user" || m.role === "assistant") - .map((m) => ({ role: m.role as "user" | "assistant", content: m.content })); + .map((m) => ({ role: m.role, content: m.content, mcpEvents: m.mcpEvents })); truncateFromMessage(activeConversationId, messageId); handleSend(newContent, priorMessages); }, diff --git a/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.test.tsx b/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.test.tsx index 882623b7de7..a9c9fb6240f 100644 --- a/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.test.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.test.tsx @@ -63,7 +63,7 @@ describe("ViewSwitcher", () => { fireEvent.click(button); }); expect(await screen.findByText("Chat")).toBeInTheDocument(); - expect(screen.getByText(/Admins can enable in Settings/i)).toBeInTheDocument(); + expect(screen.getByText(/Settings > Admin Settings > UI Settings/i)).toBeInTheDocument(); act(() => { fireEvent.click(screen.getByText("Chat")); @@ -113,7 +113,7 @@ describe("ViewSwitcher", () => { fireEvent.click(screen.getByRole("button")); }); expect(await screen.findByText("Chat")).toBeInTheDocument(); - expect(screen.queryByText(/Admins can enable in Settings/i)).not.toBeInTheDocument(); + expect(screen.queryByText(/Settings > Admin Settings > UI Settings/i)).not.toBeInTheDocument(); act(() => { fireEvent.click(screen.getByText("Chat")); @@ -224,7 +224,7 @@ describe("ViewSwitcher", () => { }); expect(await screen.findByText("Observability")).toBeInTheDocument(); expect(screen.getByText("Chat")).toBeInTheDocument(); - expect(screen.getByText(/Admins can enable in Settings/i)).toBeInTheDocument(); + expect(screen.getByText(/Settings > Admin Settings > UI Settings/i)).toBeInTheDocument(); act(() => { fireEvent.click(screen.getByText("Chat")); diff --git a/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.tsx b/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.tsx index 24d9e332ad4..db92967de62 100644 --- a/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/ViewSwitcher.tsx @@ -78,7 +78,7 @@ export default function ViewSwitcher() {
Chat - Admins can enable in Settings + Admins can enable in Settings > Admin Settings > UI Settings
), diff --git a/ui/litellm-dashboard/src/components/chat/useChatHistory.test.ts b/ui/litellm-dashboard/src/components/chat/useChatHistory.test.ts index a26a9b0ae56..14fa53b6c1e 100644 --- a/ui/litellm-dashboard/src/components/chat/useChatHistory.test.ts +++ b/ui/litellm-dashboard/src/components/chat/useChatHistory.test.ts @@ -1,5 +1,5 @@ import { renderHook, act } from "@testing-library/react"; -import { describe, it, expect, beforeEach } from "vitest"; +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; import { useChatHistory } from "./useChatHistory"; describe("useChatHistory", () => { @@ -7,6 +7,32 @@ describe("useChatHistory", () => { localStorage.clear(); }); + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("creates unique conversation and message IDs without crypto.randomUUID", () => { + vi.stubGlobal("crypto", { getRandomValues: crypto.getRandomValues.bind(crypto) }); + const { result } = renderHook(() => useChatHistory(null, "http-user")); + + act(() => { + result.current.createConversation("test-model"); + }); + const conversationId = result.current.currentActiveId!; + act(() => { + result.current.appendMessage(conversationId, { role: "user", content: "Hello" }); + result.current.appendMessage(conversationId, { role: "assistant", content: "Hi" }); + }); + + const conversation = result.current.activeConversation!; + const ids = [conversation.id, ...conversation.messages.map((message) => message.id)]; + expect(new Set(ids).size).toBe(3); + for (const id of ids) { + expect(id).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i); + } + expect(conversation.messages.map((message) => message.content)).toEqual(["Hello", "Hi"]); + }); + it("does not leak conversations between different users on the same browser", () => { const { result: alice } = renderHook(({ userId }) => useChatHistory(null, userId), { initialProps: { userId: "alice" }, diff --git a/ui/litellm-dashboard/src/components/chat/useChatHistory.ts b/ui/litellm-dashboard/src/components/chat/useChatHistory.ts index b99ef52d18f..41ee6192a1d 100644 --- a/ui/litellm-dashboard/src/components/chat/useChatHistory.ts +++ b/ui/litellm-dashboard/src/components/chat/useChatHistory.ts @@ -1,4 +1,5 @@ import { useCallback, useEffect, useState } from "react"; +import { v4 as uuidv4 } from "uuid"; import { AssistantMessageUpdate, ChatMessage, Conversation } from "./types"; const STORAGE_KEY_PREFIX = "litellm_chat_history_v1"; @@ -106,7 +107,7 @@ export function useChatHistory( }, [conversations, userId, storageUnavailable]); const createConversation = useCallback((model: string): string => { - const id = crypto.randomUUID(); + const id = uuidv4(); const now = Date.now(); const newConversation: Conversation = { id, @@ -125,7 +126,7 @@ export function useChatHistory( const appendMessage = useCallback((conversationId: string, message: Omit) => { const newMessage: ChatMessage = { ...message, - id: crypto.randomUUID(), + id: uuidv4(), timestamp: Date.now(), }; diff --git a/ui/litellm-dashboard/src/components/chat_ui/MCPEventsDisplay.integration.test.tsx b/ui/litellm-dashboard/src/components/chat_ui/MCPEventsDisplay.integration.test.tsx new file mode 100644 index 00000000000..df391546279 --- /dev/null +++ b/ui/litellm-dashboard/src/components/chat_ui/MCPEventsDisplay.integration.test.tsx @@ -0,0 +1,63 @@ +import { fireEvent, render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import type { MCPEvent } from "@/components/mcp_tools/types"; +import MCPEventsDisplay from "./MCPEventsDisplay"; + +const toolsEvent: MCPEvent = { + type: "response.output_item.done", + item: { + type: "mcp_list_tools", + tools: [{ name: "available_lookup", description: "Look up a document" }], + }, +}; + +const callEvent: MCPEvent = { + type: "response.output_item.done", + item: { + type: "mcp_call", + name: "document_lookup", + arguments: '{"query":"chat"}', + output: "Found the requested document", + }, +}; + +describe("MCPEventsDisplay", () => { + it("collapses discovery while showing tool calls and lets users expand discovery", async () => { + render(); + + const listTools = screen.getByRole("button", { name: "List tools" }); + expect(listTools).toHaveAttribute("aria-expanded", "false"); + expect(screen.queryByText("available_lookup")).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "document_lookup" })).toHaveAttribute("aria-expanded", "true"); + expect(screen.getByText("Found the requested document")).toBeVisible(); + + fireEvent.click(listTools); + expect(await screen.findByText("available_lookup")).toBeVisible(); + expect(listTools).toHaveAttribute("aria-expanded", "true"); + }); + + it("keeps a tool call expanded when no discovery event is present", () => { + render(); + expect(screen.getByRole("button", { name: "document_lookup" })).toHaveAttribute("aria-expanded", "true"); + expect(screen.getByText("Found the requested document")).toBeVisible(); + }); + + it("opens streamed tool calls without overriding the user's collapsed panels", () => { + const { rerender } = render(); + rerender(); + + const toolCall = screen.getByRole("button", { name: "document_lookup" }); + expect(toolCall).toHaveAttribute("aria-expanded", "true"); + fireEvent.click(toolCall); + + const nextCall: MCPEvent = { + ...callEvent, + item: { ...callEvent.item, name: "next_lookup" }, + }; + rerender(); + + expect(toolCall).toHaveAttribute("aria-expanded", "false"); + expect(screen.getByRole("button", { name: "List tools" })).toHaveAttribute("aria-expanded", "false"); + expect(screen.getByRole("button", { name: "next_lookup" })).toHaveAttribute("aria-expanded", "true"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/chat_ui/MCPEventsDisplay.tsx b/ui/litellm-dashboard/src/components/chat_ui/MCPEventsDisplay.tsx index 6d7437a388d..809e08f5ff9 100644 --- a/ui/litellm-dashboard/src/components/chat_ui/MCPEventsDisplay.tsx +++ b/ui/litellm-dashboard/src/components/chat_ui/MCPEventsDisplay.tsx @@ -45,13 +45,9 @@ const MCPEventsDisplay: React.FC = ({ events, className } return null; } - const defaultOpenKeys = new Set( - toolsEvent ? ["list-tools"] : mcpCallEvents.map((_, index) => `mcp-call-${index}`), - ); - return (
- +
); }; @@ -59,22 +55,13 @@ const MCPEventsDisplay: React.FC = ({ events, className } interface MCPEventsPanelsProps { toolsEvent: MCPEvent | undefined; mcpCallEvents: MCPEvent[]; - defaultOpenKeys: Set; } -function MCPEventsPanels({ toolsEvent, mcpCallEvents, defaultOpenKeys }: MCPEventsPanelsProps) { - const [openKeys, setOpenKeys] = useState>(defaultOpenKeys); +function MCPEventsPanels({ toolsEvent, mcpCallEvents }: MCPEventsPanelsProps) { + const [openPanels, setOpenPanels] = useState>({}); const toggleKey = (key: string, open: boolean) => { - setOpenKeys((prev) => { - const next = new Set(prev); - if (open) { - next.add(key); - } else { - next.delete(key); - } - return next; - }); + setOpenPanels((prev) => ({ ...prev, [key]: open })); }; return ( @@ -86,7 +73,7 @@ function MCPEventsPanels({ toolsEvent, mcpCallEvents, defaultOpenKeys }: MCPEven toggleKey("list-tools", open)} >
@@ -109,7 +96,7 @@ function MCPEventsPanels({ toolsEvent, mcpCallEvents, defaultOpenKeys }: MCPEven key={key} panelKey={key} title={callEvent.item?.name || "Tool call"} - open={openKeys.has(key)} + open={openPanels[key] ?? true} onOpenChange={(open) => toggleKey(key, open)} >
diff --git a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.test.tsx index 41c673ddd2c..a1655baa8ce 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.test.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.test.tsx @@ -38,6 +38,21 @@ describe("fetchAvailableModels", () => { vi.clearAllMocks(); }); + it("preserves provider metadata for aliased and mixed-provider model groups", async () => { + modelHubCallMock.mockResolvedValue({ + data: [ + { model_group: "alias", providers: ["openrouter"] }, + { model_group: "mixed", providers: ["openai", "openrouter"] }, + { model_group: "unknown", providers: null }, + ], + }); + expect(await fetchAvailableModels("token")).toEqual([ + { model_group: "alias", providers: ["openrouter"] }, + { model_group: "mixed", providers: ["openai", "openrouter"] }, + { model_group: "unknown" }, + ]); + }); + it("retains provider identity without inventing it for an unknown route", async () => { modelHubCallMock.mockResolvedValue({ data: [ diff --git a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx index d38db631c6e..949f42e3191 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx @@ -27,7 +27,9 @@ const toModelGroup = (item: AvailableModel): ModelGroup => { const groupName = (item.model_group || item.id || item.model_name) ?? ""; return { model_group: groupName, - ...(item.providers && { providers: item.providers }), + ...(Array.isArray(item.providers) && { + providers: item.providers.filter((provider) => typeof provider === "string"), + }), ...(item.mode && { mode: item.mode }), ...(item.supports_reasoning === true && { supports_reasoning: true }), ...(item.supports_fast_mode === true && { supports_fast_mode: true }), diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx index 2d98b29173d..47a1e3608c5 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx @@ -3,6 +3,7 @@ import openai from "openai"; import { makeOpenAIResponsesRequest } from "./responses_api"; import { MessageType } from "../chat_ui/types"; import type { TokenUsage } from "../chat_ui/ResponseMetrics"; +import type { MCPEvent } from "../mcp_tools/types"; vi.mock("@/components/networking", () => ({ getProxyBaseUrl: vi.fn(() => "https://backend.example.com"), @@ -79,6 +80,74 @@ describe("responses_api", () => { expect(mockUpdateTextUI).toHaveBeenCalledWith("assistant", "Hi", "gpt-4"); }); + it("replays completed MCP calls and results in conversation order without partial or discovery events", async () => { + const item = { + type: "mcp_call", + id: "mcp-inventory", + name: "lookup_inventory", + arguments: '{"location":"warehouse"}', + output: '{"quantity":17}', + }; + const events: MCPEvent[] = [ + { type: "response.output_item.done", item: { type: "mcp_list_tools", tools: [] } }, + { type: "response.output_item.added", item }, + { type: "response.mcp_call.in_progress", item_id: item.id }, + { type: "response.output_item.done", item }, + { + type: "response.output_item.done", + item_id: "mcp-empty", + item: { ...item, id: undefined, output: "" }, + }, + { type: "response.output_item.done", item: { ...item, id: undefined } }, + { type: "response.output_item.done", item: { ...item, name: undefined } }, + { type: "response.output_item.done", item: { ...item, arguments: undefined } }, + { type: "response.output_item.done", item: { ...item, output: undefined } }, + ]; + const originalEvents = structuredClone(events); + const history = [ + { role: "user", content: "Check inventory", mcpEvents: events }, + { role: "assistant", content: "Lookup completed", mcpEvents: events }, + { role: "user", content: "What was the quantity?" }, + ]; + + await makeOpenAIResponsesRequest(history, mockUpdateTextUI, "model", "test-token"); + + expect(mockResponsesCreate.mock.calls[0][0].input).toEqual([ + { role: "user", content: "Check inventory", type: "message" }, + { type: "function_call", call_id: item.id, name: item.name, arguments: item.arguments }, + { type: "function_call_output", call_id: item.id, output: item.output }, + { type: "function_call", call_id: "mcp-empty", name: item.name, arguments: item.arguments }, + { type: "function_call_output", call_id: "mcp-empty", output: "" }, + { role: "assistant", content: "Lookup completed", type: "message" }, + { role: "user", content: "What was the quantity?", type: "message" }, + ]); + expect(events).toEqual(originalEvents); + }); + + it("replays an MCP-only assistant turn without an empty message", async () => { + const history = [ + { + role: "assistant", + content: "", + mcpEvents: [ + { + type: "response.output_item.done", + item: { type: "mcp_call", id: "mcp-only", name: "lookup", arguments: "{}", output: "17" }, + }, + ], + }, + { role: "user", content: "What was the quantity?" }, + ]; + + await makeOpenAIResponsesRequest(history, mockUpdateTextUI, "model", "test-token"); + + expect(mockResponsesCreate.mock.calls[0][0].input).toEqual([ + { type: "function_call", call_id: "mcp-only", name: "lookup", arguments: "{}" }, + { type: "function_call_output", call_id: "mcp-only", output: "17" }, + { role: "user", content: "What was the quantity?", type: "message" }, + ]); + }); + it("should send a non-streaming request and render the whole output at once when streaming is disabled", async () => { mockResponsesCreate.mockReturnValueOnce( nonStreamingResponse({ diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx index 9eb85bc624e..426abd4e172 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx @@ -1,5 +1,6 @@ import { observeResponseHeaders, type ResponseHeadersListener } from "@/lib/http/observeResponseHeaders"; import openai from "openai"; +import type { ResponseFunctionToolCall, ResponseInputItem } from "openai/resources/responses/responses"; import { MessageType } from "../chat_ui/types"; import { TokenUsage } from "../chat_ui/ResponseMetrics"; import { getGatewayBaseUrl, getProxyBaseUrl } from "@/components/networking"; @@ -18,6 +19,10 @@ import { export type { CodeInterpreterResult } from "./code_interpreter_handler"; +export interface ResponsesMessage extends MessageType { + mcpEvents?: MCPEvent[]; +} + interface ResponseOutputPart { type?: string; text?: string; @@ -62,7 +67,7 @@ const responseAsEvents = (response: NonStreamedResponse): SynthesizedResponseEve }; export async function makeOpenAIResponsesRequest( - messages: MessageType[], + messages: ResponsesMessage[], updateTextUI: (role: string, delta: string, model?: string) => void, selectedModel: string, accessToken: string | null, @@ -121,22 +126,32 @@ export async function makeOpenAIResponsesRequest( let firstTokenReceived = false; let servedFromResponseCache = false; - // Format messages for the API - const formattedInput = messages.map((message) => { - // If content is already an array (multimodal), use it directly - if (Array.isArray(message.content)) { - return { - role: message.role, - content: message.content, - type: "message", - }; - } - // Otherwise, wrap text content in the expected format - return { - role: message.role, - content: message.content, - type: "message", - }; + const formattedInput = messages.flatMap((message) => { + const mcpInput = + message.role === "assistant" + ? (message.mcpEvents ?? []).flatMap( + (event) => { + const item = event.item; + const callId = item?.id || event.item_id; + if (event.type !== "response.output_item.done" || item?.type !== "mcp_call" || !callId) { + return []; + } + if (!item.name || typeof item.arguments !== "string" || typeof item.output !== "string") { + return []; + } + return [ + { type: "function_call", call_id: callId, name: item.name, arguments: item.arguments }, + { type: "function_call_output", call_id: callId, output: item.output }, + ]; + }, + ) + : []; + return [ + ...mcpInput, + ...(mcpInput.length > 0 && message.content === "" + ? [] + : [{ role: message.role, content: message.content, type: "message" }]), + ]; }); // Build tools array