diff --git a/ui/litellm-dashboard/src/app/chat/page.integration.test.tsx b/ui/litellm-dashboard/src/app/chat/page.integration.test.tsx index 301724cbb3b..8f1e7d5ea5a 100644 --- a/ui/litellm-dashboard/src/app/chat/page.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/chat/page.integration.test.tsx @@ -5,6 +5,8 @@ import ChatConversationPage from "./page"; import { renderWithProviders } from "@/../tests/test-utils"; import type { OnUrlUpdateFunction } from "nuqs/adapters/testing"; import { fetchAvailableModels } from "@/components/llm_calls/fetch_models"; +import type { makeOpenAIResponsesRequest } from "@/components/llm_calls/responses_api"; +import type { MCPEvent } from "@/components/chat/types"; const { mockMakeOpenAIResponsesRequest, shellState } = vi.hoisted(() => ({ mockMakeOpenAIResponsesRequest: vi.fn(), @@ -139,6 +141,68 @@ describe("/ui/chat request metrics", () => { ); }); + it.each([ + { providers: ["openrouter"], stateless: true }, + { providers: ["openai", "openrouter"], stateless: true }, + { providers: ["openai"], stateless: false }, + ])("preserves MCP context with providers=$providers", async ({ providers, stateless }) => { + const event: MCPEvent = { + type: "response.output_item.done", + item: { + type: "mcp_call", + id: "mcp-inventory", + name: "lookup_inventory", + arguments: "{}", + output: '{"quantity":17}', + }, + }; + vi.mocked(fetchAvailableModels).mockResolvedValue([{ model_group: "gpt-5.4-mini", providers }]); + mockMakeOpenAIResponsesRequest.mockImplementationOnce( + async (...args: Parameters) => { + await new Promise((resolve) => setTimeout(resolve, 0)); + args[1]("assistant", "Lookup completed"); + args[15]?.("response-first"); + args[16]?.(event); + }, + ); + mockMakeOpenAIResponsesRequest.mockResolvedValue(undefined); + + await sendOneMessage(); + expect(await screen.findByText("Lookup completed")).toBeVisible(); + fireEvent.change(screen.getByRole("textbox"), { target: { value: "What was the quantity?" } }); + await waitFor(() => expect(screen.getByRole("button", { name: "Send" })).toBeEnabled()); + fireEvent.click(screen.getByRole("button", { name: "Send" })); + await waitFor(() => expect(mockMakeOpenAIResponsesRequest).toHaveBeenCalledTimes(2)); + + const second = mockMakeOpenAIResponsesRequest.mock.calls[1]; + expect(second[14]).toBe(stateless ? null : "response-first"); + expect(second[0]).toEqual( + stateless + ? [ + { role: "user", content: "How much did this cost?" }, + { role: "assistant", content: "Lookup completed", mcpEvents: [event] }, + { role: "user", content: "What was the quantity?" }, + ] + : [{ role: "user", content: "What was the quantity?" }], + ); + + fireEvent.mouseOver(screen.getByText("What was the quantity?")); + fireEvent.click(await screen.findByRole("button", { name: "Edit message" })); + fireEvent.change(screen.getByRole("textbox", { name: "Edit message" }), { + target: { value: "Repeat the inventory quantity" }, + }); + fireEvent.click(screen.getByRole("button", { name: "Save & Send" })); + await waitFor(() => expect(mockMakeOpenAIResponsesRequest).toHaveBeenCalledTimes(3)); + + const edited = mockMakeOpenAIResponsesRequest.mock.calls[2]; + expect(edited[14]).toBeNull(); + expect(edited[0]).toEqual([ + { role: "user", content: "How much did this cost?" }, + { role: "assistant", content: "Lookup completed", mcpEvents: [event] }, + { role: "user", content: "Repeat the inventory quantity" }, + ]); + }); + it("resends history instead of reusing another model's response ID after changing models", async () => { vi.mocked(fetchAvailableModels).mockResolvedValue([ { model_group: "gpt-5.4-mini", providers: ["openai"] }, diff --git a/ui/litellm-dashboard/src/app/chat/page.tsx b/ui/litellm-dashboard/src/app/chat/page.tsx index 7c1a79d6057..5cf605eec2a 100644 --- a/ui/litellm-dashboard/src/app/chat/page.tsx +++ b/ui/litellm-dashboard/src/app/chat/page.tsx @@ -14,7 +14,7 @@ import { getChatRoutes } from "@/components/chat/ChatShell"; import ChatMessages from "@/components/chat/ChatMessages"; import MCPConnectPicker from "@/components/chat/MCPConnectPicker"; import { fetchAvailableModels } from "@/components/llm_calls/fetch_models"; -import { makeOpenAIResponsesRequest } from "@/components/llm_calls/responses_api"; +import { makeOpenAIResponsesRequest, type ResponsesMessage } from "@/components/llm_calls/responses_api"; import type { TokenUsage } from "@/components/chat_ui/ResponseMetrics"; import type { MCPEvent } from "@/components/chat/types"; import { getProviderLogoAndName } from "@/components/provider_info_helpers"; @@ -150,7 +150,7 @@ export default function ChatConversationPage() { }, []); const handleSend = useCallback( - async (text: string, historyOverride?: Array<{ role: "user" | "assistant"; content: string }>) => { + async (text: string, historyOverride?: ResponsesMessage[]) => { const trimmed = text.trim(); if (!trimmed || !selectedModel || isStreaming) return; const model = selectedModel; @@ -181,7 +181,7 @@ export default function ChatConversationPage() { const previousResponseId = historyOverride || statelessModelGroups.has(model) ? null : responsesSessionId; - const history: Array<{ role: "user" | "assistant"; content: string }> = historyOverride + const history: ResponsesMessage[] = historyOverride ? [...historyOverride, { role: "user" as const, content: trimmed }] : previousResponseId ? [{ role: "user" as const, content: trimmed }] @@ -192,7 +192,7 @@ export default function ChatConversationPage() { .filter( (m): m is typeof m & { role: "user" | "assistant" } => m.role === "user" || m.role === "assistant", ) - .map((m) => ({ role: m.role, content: m.content })), + .map((m) => ({ role: m.role, content: m.content, mcpEvents: m.mcpEvents })), { role: "user" as const, content: trimmed }, ]; @@ -296,7 +296,7 @@ export default function ChatConversationPage() { const idx = msgs.findIndex((m) => m.id === messageId); const priorMessages = (idx === -1 ? msgs : msgs.slice(0, idx)) .filter((m) => m.role === "user" || m.role === "assistant") - .map((m) => ({ role: m.role as "user" | "assistant", content: m.content })); + .map((m) => ({ role: m.role, content: m.content, mcpEvents: m.mcpEvents })); truncateFromMessage(activeConversationId, messageId); handleSend(newContent, priorMessages); }, diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx index 2d98b29173d..47a1e3608c5 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx @@ -3,6 +3,7 @@ import openai from "openai"; import { makeOpenAIResponsesRequest } from "./responses_api"; import { MessageType } from "../chat_ui/types"; import type { TokenUsage } from "../chat_ui/ResponseMetrics"; +import type { MCPEvent } from "../mcp_tools/types"; vi.mock("@/components/networking", () => ({ getProxyBaseUrl: vi.fn(() => "https://backend.example.com"), @@ -79,6 +80,74 @@ describe("responses_api", () => { expect(mockUpdateTextUI).toHaveBeenCalledWith("assistant", "Hi", "gpt-4"); }); + it("replays completed MCP calls and results in conversation order without partial or discovery events", async () => { + const item = { + type: "mcp_call", + id: "mcp-inventory", + name: "lookup_inventory", + arguments: '{"location":"warehouse"}', + output: '{"quantity":17}', + }; + const events: MCPEvent[] = [ + { type: "response.output_item.done", item: { type: "mcp_list_tools", tools: [] } }, + { type: "response.output_item.added", item }, + { type: "response.mcp_call.in_progress", item_id: item.id }, + { type: "response.output_item.done", item }, + { + type: "response.output_item.done", + item_id: "mcp-empty", + item: { ...item, id: undefined, output: "" }, + }, + { type: "response.output_item.done", item: { ...item, id: undefined } }, + { type: "response.output_item.done", item: { ...item, name: undefined } }, + { type: "response.output_item.done", item: { ...item, arguments: undefined } }, + { type: "response.output_item.done", item: { ...item, output: undefined } }, + ]; + const originalEvents = structuredClone(events); + const history = [ + { role: "user", content: "Check inventory", mcpEvents: events }, + { role: "assistant", content: "Lookup completed", mcpEvents: events }, + { role: "user", content: "What was the quantity?" }, + ]; + + await makeOpenAIResponsesRequest(history, mockUpdateTextUI, "model", "test-token"); + + expect(mockResponsesCreate.mock.calls[0][0].input).toEqual([ + { role: "user", content: "Check inventory", type: "message" }, + { type: "function_call", call_id: item.id, name: item.name, arguments: item.arguments }, + { type: "function_call_output", call_id: item.id, output: item.output }, + { type: "function_call", call_id: "mcp-empty", name: item.name, arguments: item.arguments }, + { type: "function_call_output", call_id: "mcp-empty", output: "" }, + { role: "assistant", content: "Lookup completed", type: "message" }, + { role: "user", content: "What was the quantity?", type: "message" }, + ]); + expect(events).toEqual(originalEvents); + }); + + it("replays an MCP-only assistant turn without an empty message", async () => { + const history = [ + { + role: "assistant", + content: "", + mcpEvents: [ + { + type: "response.output_item.done", + item: { type: "mcp_call", id: "mcp-only", name: "lookup", arguments: "{}", output: "17" }, + }, + ], + }, + { role: "user", content: "What was the quantity?" }, + ]; + + await makeOpenAIResponsesRequest(history, mockUpdateTextUI, "model", "test-token"); + + expect(mockResponsesCreate.mock.calls[0][0].input).toEqual([ + { type: "function_call", call_id: "mcp-only", name: "lookup", arguments: "{}" }, + { type: "function_call_output", call_id: "mcp-only", output: "17" }, + { role: "user", content: "What was the quantity?", type: "message" }, + ]); + }); + it("should send a non-streaming request and render the whole output at once when streaming is disabled", async () => { mockResponsesCreate.mockReturnValueOnce( nonStreamingResponse({ diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx index 1f717e488e9..5ca19edc273 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx @@ -1,4 +1,5 @@ import openai from "openai"; +import type { ResponseFunctionToolCall, ResponseInputItem } from "openai/resources/responses/responses"; import { MessageType } from "../chat_ui/types"; import { TokenUsage } from "../chat_ui/ResponseMetrics"; import { getGatewayBaseUrl, getProxyBaseUrl } from "@/components/networking"; @@ -17,6 +18,10 @@ import { export type { CodeInterpreterResult } from "./code_interpreter_handler"; +export interface ResponsesMessage extends MessageType { + mcpEvents?: MCPEvent[]; +} + interface ResponseOutputPart { type?: string; text?: string; @@ -61,7 +66,7 @@ const responseAsEvents = (response: NonStreamedResponse): SynthesizedResponseEve }; export async function makeOpenAIResponsesRequest( - messages: MessageType[], + messages: ResponsesMessage[], updateTextUI: (role: string, delta: string, model?: string) => void, selectedModel: string, accessToken: string | null, @@ -118,22 +123,32 @@ export async function makeOpenAIResponsesRequest( let firstTokenReceived = false; let servedFromResponseCache = false; - // Format messages for the API - const formattedInput = messages.map((message) => { - // If content is already an array (multimodal), use it directly - if (Array.isArray(message.content)) { - return { - role: message.role, - content: message.content, - type: "message", - }; - } - // Otherwise, wrap text content in the expected format - return { - role: message.role, - content: message.content, - type: "message", - }; + const formattedInput = messages.flatMap((message) => { + const mcpInput = + message.role === "assistant" + ? (message.mcpEvents ?? []).flatMap( + (event) => { + const item = event.item; + const callId = item?.id || event.item_id; + if (event.type !== "response.output_item.done" || item?.type !== "mcp_call" || !callId) { + return []; + } + if (!item.name || typeof item.arguments !== "string" || typeof item.output !== "string") { + return []; + } + return [ + { type: "function_call", call_id: callId, name: item.name, arguments: item.arguments }, + { type: "function_call_output", call_id: callId, output: item.output }, + ]; + }, + ) + : []; + return [ + ...mcpInput, + ...(mcpInput.length > 0 && message.content === "" + ? [] + : [{ role: message.role, content: message.content, type: "message" }]), + ]; }); // Build tools array