From 41e44089061d3e03cdc969cd9772d4f36681dd4c Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 3 Aug 2026 12:55:39 -0700 Subject: [PATCH] feat(playground): add non-streaming response toggle (#35560) Adds a Stream responses checkbox (default on) to the playground Model Settings popover. When unchecked, chat completions and responses API requests are sent with stream: false and the full reply renders at once. The non-streamed result is replayed through the existing streaming handlers as synthesized chunks/events so MCP events, vector store results, usage and response ids behave identically in both modes. TTFT is suppressed when not streaming; total latency now also reported for the responses API. The toggle is scoped to the chat and responses endpoints, persists via sessionStorage, and is isolated from the simplified Agent Builder chat. Resolves LIT-3251 --- .../chat_ui/AdditionalModelSettings.test.tsx | 38 ++++ .../chat_ui/AdditionalModelSettings.tsx | 122 ++++++++----- .../components/chat_ui/ChatUI.test.tsx | 167 ++++++++++++++++++ .../playground/components/chat_ui/ChatUI.tsx | 17 +- .../llm_calls/chat_completion.test.tsx | 137 ++++++++++++++ .../components/llm_calls/chat_completion.tsx | 60 ++++--- .../llm_calls/responses_api.test.tsx | 152 ++++++++++++++++ .../components/llm_calls/responses_api.tsx | 80 +++++++-- 8 files changed, 685 insertions(+), 88 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AdditionalModelSettings.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AdditionalModelSettings.test.tsx index d6f29b469b8..1b443e98495 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AdditionalModelSettings.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AdditionalModelSettings.test.tsx @@ -48,6 +48,44 @@ describe("AdditionalModelSettings", () => { expect(maxTokensSlider).not.toBeDisabled(); }); + it("should not show Stream responses when onStreamingChange is not provided", () => { + render(); + expect(screen.queryByText(/Stream responses/i)).not.toBeInTheDocument(); + }); + + it("should render Stream responses checked by default and report unchecking it", async () => { + const user = userEvent.setup(); + const onStreamingChange = vi.fn(); + + render(); + + const streamingCheckbox = screen.getByRole("checkbox", { name: /Stream responses/i }); + expect(streamingCheckbox).toBeChecked(); + + await act(async () => { + await user.click(streamingCheckbox); + }); + + await waitFor(() => { + expect(onStreamingChange).toHaveBeenCalledWith(false); + }); + }); + + it("should keep the streaming toggle but drop advanced params when showAdvancedParams is false", () => { + render(); + + expect(screen.getByRole("checkbox", { name: /Stream responses/i })).toBeInTheDocument(); + expect(screen.queryByText("Use Advanced Parameters")).not.toBeInTheDocument(); + expect(screen.queryByText("Temperature")).not.toBeInTheDocument(); + expect(screen.queryByText("Max Tokens")).not.toBeInTheDocument(); + }); + + it("should reflect a disabled streaming setting from props", () => { + render(); + + expect(screen.getByRole("checkbox", { name: /Stream responses/i })).not.toBeChecked(); + }); + it("should not show Simulate failure to test fallbacks when onMockTestFallbacksChange is not provided", () => { render(); expect(screen.queryByText(/Simulate failure to test fallbacks/i)).not.toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AdditionalModelSettings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AdditionalModelSettings.tsx index 078c1b66afb..d4320110c4c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AdditionalModelSettings.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AdditionalModelSettings.tsx @@ -12,6 +12,9 @@ interface AdditionalModelSettingsProps { onUseAdvancedParamsChange?: (value: boolean) => void; mockTestFallbacks?: boolean; onMockTestFallbacksChange?: (value: boolean) => void; + streamingEnabled?: boolean; + onStreamingChange?: (value: boolean) => void; + showAdvancedParams?: boolean; } const AdditionalModelSettings: React.FC = ({ @@ -23,6 +26,9 @@ const AdditionalModelSettings: React.FC = ({ onUseAdvancedParamsChange, mockTestFallbacks, onMockTestFallbacksChange, + streamingEnabled = true, + onStreamingChange, + showAdvancedParams = true, }) => { const [internalUseAdvancedParams, setInternalUseAdvancedParams] = useState(false); const useAdvancedParams = @@ -64,9 +70,25 @@ const AdditionalModelSettings: React.FC = ({ return (
- handleUseAdvancedParamsChange(e.target.checked)}> - Use Advanced Parameters - + {onStreamingChange && ( +
+ onStreamingChange(e.target.checked)}> + Stream responses + + + + +
+ )} + + {showAdvancedParams && ( + handleUseAdvancedParamsChange(e.target.checked)}> + Use Advanced Parameters + + )} {onMockTestFallbacksChange && (
@@ -104,72 +126,74 @@ const AdditionalModelSettings: React.FC = ({
)} -
-
-
-
- Temperature - - - + {showAdvancedParams && ( +
+
+
+
+ Temperature + + + +
+
-
- -
-
-
-
- Max Tokens - - - +
+
+
+ Max Tokens + + + +
+
-
-
-
+ )}
); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.test.tsx index 9da3e3a4a08..b5f2bf7b10c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.test.tsx @@ -2,12 +2,17 @@ import { act, fireEvent, render, screen, waitFor } from "@testing-library/react" import { beforeEach, describe, expect, it, vi } from "vitest"; import ChatUI from "./ChatUI"; import * as fetchModelsModule from "@/components/llm_calls/fetch_models"; +import { makeOpenAIChatCompletionRequest } from "@/components/llm_calls/chat_completion"; // Mock the fetchAvailableModels function vi.mock("@/components/llm_calls/fetch_models", () => ({ fetchAvailableModels: vi.fn(), })); +vi.mock("@/components/llm_calls/chat_completion", () => ({ + makeOpenAIChatCompletionRequest: vi.fn().mockResolvedValue(undefined), +})); + // Mock other networking functions that cause errors vi.mock("@/components/networking", () => ({ tagListCall: vi.fn().mockResolvedValue({ data: [] }), @@ -21,6 +26,9 @@ beforeEach(() => { Element.prototype.scrollIntoView = () => {}; }); +const CHAT_REQUEST_ARG_COUNT = 26; +const STREAMING_ENABLED_ARG_INDEX = 25; + describe("ChatUI", () => { beforeEach(() => { // Reset mocks before each test @@ -334,6 +342,165 @@ describe("ChatUI", () => { }); }); + it("should send the chat request non-streaming after Stream responses is unchecked", async () => { + render( + , + ); + + await waitFor(() => { + expect(screen.getByText("Test Key")).toBeInTheDocument(); + }); + + const selectModelLabel = screen.getByText("Select Model"); + const modelSelect = selectModelLabel.closest("div")?.querySelector(".ant-select-selector"); + await act(async () => { + fireEvent.mouseDown(modelSelect!); + }); + + await waitFor(() => { + expect(screen.getAllByText("Model 1").length).toBeGreaterThan(0); + }); + + const model1Options = screen.getAllByText("Model 1"); + await act(async () => { + fireEvent.click(model1Options[model1Options.length - 1]); + }); + + await waitFor(() => { + expect(screen.getByTestId("model-settings-button")).toBeInTheDocument(); + }); + + await act(async () => { + fireEvent.click(screen.getByTestId("model-settings-button")); + }); + + const streamingCheckbox = await screen.findByRole("checkbox", { name: /Stream responses/i }); + expect(streamingCheckbox).toBeChecked(); + + await act(async () => { + fireEvent.click(streamingCheckbox); + }); + + await waitFor(() => { + expect(screen.getByRole("checkbox", { name: /Stream responses/i })).not.toBeChecked(); + }); + + const messageInput = screen.getByPlaceholderText("Type your message... (Shift+Enter for new line)"); + await act(async () => { + fireEvent.change(messageInput, { target: { value: "hello" } }); + }); + await act(async () => { + fireEvent.keyDown(messageInput, { key: "Enter", code: "Enter" }); + }); + + await waitFor(() => { + expect(makeOpenAIChatCompletionRequest).toHaveBeenCalledTimes(1); + }); + + const requestArgs = vi.mocked(makeOpenAIChatCompletionRequest).mock.calls[0]; + expect(requestArgs).toHaveLength(CHAT_REQUEST_ARG_COUNT); + expect(requestArgs[STREAMING_ENABLED_ARG_INDEX]).toBe(false); + }); + + it("should force streaming in simplified mode even when the playground setting is off", async () => { + sessionStorage.setItem("streamingEnabled", "false"); + + render( + , + ); + + await waitFor(() => { + expect(screen.getByText("Chat")).toBeInTheDocument(); + }); + + const messageInput = screen.getByPlaceholderText("Type your message... (Shift+Enter for new line)"); + await act(async () => { + fireEvent.change(messageInput, { target: { value: "hello" } }); + }); + await act(async () => { + fireEvent.keyDown(messageInput, { key: "Enter", code: "Enter" }); + }); + + await waitFor(() => { + expect(makeOpenAIChatCompletionRequest).toHaveBeenCalledTimes(1); + }); + + const requestArgs = vi.mocked(makeOpenAIChatCompletionRequest).mock.calls[0]; + expect(requestArgs).toHaveLength(CHAT_REQUEST_ARG_COUNT); + expect(requestArgs[STREAMING_ENABLED_ARG_INDEX]).toBe(true); + expect(sessionStorage.getItem("streamingEnabled")).toBe("false"); + }); + + it("should offer the streaming toggle for a responses-only model without advanced params", async () => { + (fetchModelsModule.fetchAvailableModels as any).mockResolvedValue([ + { model_group: "ResponsesModel", mode: "responses" }, + ]); + + render( + , + ); + + await waitFor(() => { + expect(screen.getByText("Test Key")).toBeInTheDocument(); + }); + + const endpointTypeText = screen.getByText("Endpoint Type"); + const endpointSelect = endpointTypeText.parentElement?.querySelector(".ant-select-selector"); + await act(async () => { + fireEvent.mouseDown(endpointSelect!); + }); + await act(async () => { + fireEvent.click(screen.getByText("/v1/responses")); + }); + + const selectModelLabel = screen.getByText("Select Model"); + const modelSelect = selectModelLabel.closest("div")?.querySelector(".ant-select-selector"); + await act(async () => { + fireEvent.mouseDown(modelSelect!); + }); + + await waitFor(() => { + expect(screen.getAllByText("ResponsesModel").length).toBeGreaterThan(0); + }); + + const modelOptions = screen.getAllByText("ResponsesModel"); + await act(async () => { + fireEvent.click(modelOptions[modelOptions.length - 1]); + }); + + await waitFor(() => { + expect(screen.getByTestId("model-settings-button")).toBeInTheDocument(); + }); + + await act(async () => { + fireEvent.click(screen.getByTestId("model-settings-button")); + }); + + expect(await screen.findByRole("checkbox", { name: /Stream responses/i })).toBeChecked(); + expect(screen.queryByText("Temperature")).not.toBeInTheDocument(); + expect(screen.queryByText("Use Advanced Parameters")).not.toBeInTheDocument(); + }); + it("should show Fill button and populate customProxyBaseUrl when proxySettings.LITELLM_UI_API_DOC_BASE_URL is provided", async () => { const testProxyUrl = "http://localhost:5000"; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx index d2cf27e0c8b..684814bfe5b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx @@ -261,6 +261,11 @@ const ChatUI: React.FC = ({ const [maxTokens, setMaxTokens] = useState(2048); const [useAdvancedParams, setUseAdvancedParams] = useState(false); const [mockTestFallbacks, setMockTestFallbacks] = useState(false); + const [streamingEnabled, setStreamingEnabled] = useState(() => { + if (simplified) return true; + const saved = sessionStorage.getItem("streamingEnabled"); + return saved === null ? true : saved === "true"; + }); // Code Interpreter state (using custom hook) const codeInterpreter = useCodeInterpreter(); @@ -372,6 +377,7 @@ const ChatUI: React.FC = ({ sessionStorage.removeItem("selectedMCPTools"); // Clean up old key if (!simplified) { + sessionStorage.setItem("streamingEnabled", JSON.stringify(streamingEnabled)); if (selectedModel) { sessionStorage.setItem("selectedModel", selectedModel); } else { @@ -392,6 +398,7 @@ const ChatUI: React.FC = ({ selectedMCPServers, mcpServerToolRestrictions, selectedVoice, + streamingEnabled, ]); useEffect(() => { @@ -771,6 +778,7 @@ const ChatUI: React.FC = ({ handleMCPEvent, mockTestFallbacks, mcpToolsets, + streamingEnabled, ); } else if (endpointType === EndpointType.IMAGE) { // For image generation @@ -852,6 +860,8 @@ const ChatUI: React.FC = ({ mcpServers, mcpServerToolRestrictions, mcpToolsets, + streamingEnabled, + updateTotalLatency, ); } else if (endpointType === EndpointType.ANTHROPIC_MESSAGES) { const apiChatHistory = [ @@ -1035,6 +1045,8 @@ const ChatUI: React.FC = ({ return !model.mode || model.mode === "chat"; }; + const supportsStreamingToggle = endpointType === EndpointType.CHAT || endpointType === EndpointType.RESPONSES; + const antIcon = ; return ( @@ -1184,10 +1196,11 @@ const ChatUI: React.FC = ({ Select Model - {isChatModel() ? ( + {isChatModel() || supportsStreamingToggle ? ( = ({ onUseAdvancedParamsChange={setUseAdvancedParams} mockTestFallbacks={mockTestFallbacks} onMockTestFallbacksChange={setMockTestFallbacks} + streamingEnabled={streamingEnabled} + onStreamingChange={supportsStreamingToggle ? setStreamingEnabled : undefined} /> } title="Model Settings" diff --git a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx index 8649834b318..10252adecd0 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx @@ -224,6 +224,143 @@ describe("chat_completion", () => { expect(callArgs.mock_testing_fallbacks).toBe(true); }); + it("should send a non-streaming request and render the whole message at once when streaming is disabled", async () => { + mockCreate.mockResolvedValueOnce({ + id: "chatcmpl-1", + object: "chat.completion", + created: 1, + model: "gpt-4", + choices: [ + { + index: 0, + finish_reason: "stop", + message: { role: "assistant", content: "Hello there" }, + }, + ], + usage: { + completion_tokens: 2, + prompt_tokens: 5, + total_tokens: 7, + cost: 0.25, + }, + }); + + const onTimingData = vi.fn(); + const onUsageData = vi.fn(); + const onTotalLatency = vi.fn(); + + await makeOpenAIChatCompletionRequest( + mockChatHistory, + mockUpdateUI, + "gpt-4", + "test-token", + undefined, // tags + undefined, // signal + undefined, // onReasoningContent + onTimingData, + onUsageData, + undefined, // traceId + undefined, // vector_store_ids + undefined, // guardrails + undefined, // policies + undefined, // selectedMCPServers + undefined, // onImageGenerated + undefined, // onSearchResults + undefined, // temperature + undefined, // max_tokens + onTotalLatency, + undefined, // customBaseUrl + undefined, // mcpServers + undefined, // mcpServerToolRestrictions + undefined, // onMCPEvent + undefined, // mockTestFallbacks + undefined, // mcpToolsets + false, // streamingEnabled + ); + + expect(mockCreate).toHaveBeenCalledTimes(1); + const callArgs = mockCreate.mock.calls[0][0]; + expect(callArgs.stream).toBe(false); + expect(callArgs).not.toHaveProperty("stream_options"); + + expect(mockUpdateUI).toHaveBeenCalledTimes(1); + expect(mockUpdateUI).toHaveBeenCalledWith("Hello there", "gpt-4"); + + expect(onUsageData).toHaveBeenCalledWith({ + completionTokens: 2, + promptTokens: 5, + totalTokens: 7, + cost: 0.25, + }); + expect(onTimingData).not.toHaveBeenCalled(); + expect(onTotalLatency).toHaveBeenCalledWith(expect.any(Number)); + }); + + it("should surface reasoning content and MCP metadata from a non-streaming response", async () => { + mockCreate.mockResolvedValueOnce({ + model: "gpt-4", + choices: [ + { + index: 0, + finish_reason: "stop", + message: { + role: "assistant", + content: "done", + reasoning_content: "thinking", + provider_specific_fields: { + mcp_tool_calls: [{ id: "call_1", function: { name: "search_docs", arguments: "{}" } }], + mcp_call_results: [{ tool_call_id: "call_1", result: "found it" }], + }, + }, + }, + ], + }); + + const onReasoningContent = vi.fn(); + const onMCPEvent = vi.fn(); + + await makeOpenAIChatCompletionRequest( + mockChatHistory, + mockUpdateUI, + "gpt-4", + "test-token", + undefined, // tags + undefined, // signal + onReasoningContent, + undefined, // onTimingData + undefined, // onUsageData + undefined, // traceId + undefined, // vector_store_ids + undefined, // guardrails + undefined, // policies + undefined, // selectedMCPServers + undefined, // onImageGenerated + undefined, // onSearchResults + undefined, // temperature + undefined, // max_tokens + undefined, // onTotalLatency + undefined, // customBaseUrl + undefined, // mcpServers + undefined, // mcpServerToolRestrictions + onMCPEvent, + undefined, // mockTestFallbacks + undefined, // mcpToolsets + false, // streamingEnabled + ); + + expect(onReasoningContent).toHaveBeenCalledWith("thinking"); + expect(onMCPEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: "response.output_item.done", + item: expect.objectContaining({ + type: "mcp_call", + name: "search_docs", + output: "found it", + }), + }), + ); + }); + it("should not include mock_testing_fallbacks in request body when mockTestFallbacks is false or undefined", async () => { await makeOpenAIChatCompletionRequest( mockChatHistory, diff --git a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx index 66be2fc7893..c20d758fe91 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx @@ -1,10 +1,26 @@ import openai from "openai"; -import { ChatCompletionMessageParam } from "openai/resources/chat/completions"; +import { ChatCompletion, ChatCompletionChunk, ChatCompletionMessageParam } from "openai/resources/chat/completions"; import { TokenUsage } from "../chat_ui/ResponseMetrics"; import { VectorStoreSearchResponse } from "../chat_ui/types"; import { getProxyBaseUrl } from "@/components/networking"; import { MCPServer, MCPToolset, type MCPEvent } from "@/components/mcp_tools/types"; +const completionAsSingleChunk = (completion: ChatCompletion): ChatCompletionChunk => + ({ + id: completion.id, + object: "chat.completion.chunk", + created: completion.created, + model: completion.model, + usage: completion.usage, + choices: [ + { + index: 0, + finish_reason: completion.choices[0]?.finish_reason ?? null, + delta: completion.choices[0]?.message ?? {}, + }, + ], + }) as unknown as ChatCompletionChunk; + export async function makeOpenAIChatCompletionRequest( chatHistory: { role: string; content: string | any[] }[], updateUI: (chunk: string, model?: string) => void, @@ -31,6 +47,7 @@ export async function makeOpenAIChatCompletionRequest( onMCPEvent?: (event: MCPEvent) => void, mockTestFallbacks?: boolean, mcpToolsets?: MCPToolset[], + streamingEnabled: boolean = true, ) { // base url should be the current base_url const isLocal = process.env.NODE_ENV === "development"; @@ -111,26 +128,25 @@ export async function makeOpenAIChatCompletionRequest( } } - // @ts-ignore - const response = await client.chat.completions.create( - { - model: selectedModel, - stream: true, - stream_options: { - include_usage: true, - }, - litellm_trace_id: traceId, - messages: chatHistory as ChatCompletionMessageParam[], - ...(vector_store_ids ? { vector_store_ids } : {}), - ...(guardrails ? { guardrails } : {}), - ...(policies ? { policies } : {}), - ...(tools.length > 0 ? { tools, tool_choice: "auto" } : {}), - ...(temperature !== undefined ? { temperature } : {}), - ...(max_tokens !== undefined ? { max_tokens } : {}), - ...(mockTestFallbacks ? { mock_testing_fallbacks: true } : {}), - }, - { signal }, - ); + const requestBody = { + model: selectedModel, + litellm_trace_id: traceId, + messages: chatHistory as ChatCompletionMessageParam[], + ...(vector_store_ids ? { vector_store_ids } : {}), + ...(guardrails ? { guardrails } : {}), + ...(policies ? { policies } : {}), + ...(tools.length > 0 ? { tools, tool_choice: "auto" as const } : {}), + ...(temperature !== undefined ? { temperature } : {}), + ...(max_tokens !== undefined ? { max_tokens } : {}), + ...(mockTestFallbacks ? { mock_testing_fallbacks: true } : {}), + }; + + const response: AsyncIterable | ChatCompletionChunk[] = streamingEnabled + ? await client.chat.completions.create( + { ...requestBody, stream: true, stream_options: { include_usage: true } }, + { signal }, + ) + : [completionAsSingleChunk(await client.chat.completions.create({ ...requestBody, stream: false }, { signal }))]; for await (const chunk of response) { // Process content and measure time to first token @@ -142,7 +158,7 @@ export async function makeOpenAIChatCompletionRequest( if (!firstTokenReceived && (chunk.choices[0]?.delta?.content || (delta && delta.reasoning_content))) { firstTokenReceived = true; timeToFirstToken = Date.now() - startTime; - if (onTimingData) { + if (onTimingData && streamingEnabled) { onTimingData(timeToFirstToken); } } diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx index 77ff5fd00bb..a897e6fc4cc 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx @@ -69,6 +69,158 @@ describe("responses_api", () => { expect(mockUpdateTextUI).toHaveBeenCalledWith("assistant", "Hi", "gpt-4"); }); + it("should send a non-streaming request and render the whole output at once when streaming is disabled", async () => { + mockResponsesCreate.mockResolvedValueOnce({ + id: "resp_456", + output: [ + { + type: "message", + content: [ + { type: "output_text", text: "Full " }, + { type: "output_text", text: "answer" }, + ], + }, + ], + usage: { output_tokens: 3, input_tokens: 4, total_tokens: 7 }, + }); + + const onTimingData = vi.fn(); + const onUsageData = vi.fn(); + const onResponseId = vi.fn(); + + await makeOpenAIResponsesRequest( + messages, + mockUpdateTextUI, + "gpt-4", + "test-token", + undefined, // tags + undefined, // signal + undefined, // onReasoningContent + onTimingData, + onUsageData, + undefined, // traceId + undefined, // vector_store_ids + undefined, // guardrails + undefined, // policies + undefined, // selectedMCPServers + undefined, // previousResponseId + onResponseId, + undefined, // onMCPEvent + undefined, // codeInterpreterEnabled + undefined, // onCodeInterpreterResult + undefined, // customBaseUrl + undefined, // mcpServers + undefined, // mcpServerToolRestrictions + undefined, // mcpToolsets + false, // streamingEnabled + ); + + expect(mockResponsesCreate).toHaveBeenCalledTimes(1); + expect(mockResponsesCreate.mock.calls[0][0].stream).toBe(false); + + expect(mockUpdateTextUI).toHaveBeenCalledTimes(1); + expect(mockUpdateTextUI).toHaveBeenCalledWith("assistant", "Full answer", "gpt-4"); + + expect(onUsageData).toHaveBeenCalledWith({ completionTokens: 3, promptTokens: 4, totalTokens: 7 }, ""); + expect(onResponseId).toHaveBeenCalledWith("resp_456"); + expect(onTimingData).not.toHaveBeenCalled(); + }); + + it("should report total latency in both streaming and non-streaming modes", async () => { + const onTotalLatency = vi.fn(); + const callWithStreaming = (streamingEnabled: boolean) => + makeOpenAIResponsesRequest( + messages, + mockUpdateTextUI, + "gpt-4", + "test-token", + undefined, // tags + undefined, // signal + undefined, // onReasoningContent + undefined, // onTimingData + undefined, // onUsageData + undefined, // traceId + undefined, // vector_store_ids + undefined, // guardrails + undefined, // policies + undefined, // selectedMCPServers + undefined, // previousResponseId + undefined, // onResponseId + undefined, // onMCPEvent + undefined, // codeInterpreterEnabled + undefined, // onCodeInterpreterResult + undefined, // customBaseUrl + undefined, // mcpServers + undefined, // mcpServerToolRestrictions + undefined, // mcpToolsets + streamingEnabled, + onTotalLatency, + ); + + await callWithStreaming(true); + expect(onTotalLatency).toHaveBeenCalledTimes(1); + expect(onTotalLatency).toHaveBeenLastCalledWith(expect.any(Number)); + + mockResponsesCreate.mockResolvedValueOnce({ + id: "resp_latency", + output: [{ type: "message", content: [{ type: "output_text", text: "Answer" }] }], + }); + + await callWithStreaming(false); + expect(onTotalLatency).toHaveBeenCalledTimes(2); + expect(onTotalLatency).toHaveBeenLastCalledWith(expect.any(Number)); + }); + + it("should replay MCP output items as events for a non-streaming response", async () => { + mockResponsesCreate.mockResolvedValueOnce({ + id: "resp_789", + output: [ + { type: "mcp_call", id: "mcp_1", name: "search_docs", arguments: "{}", output: "found it" }, + { type: "message", content: [{ type: "output_text", text: "Answer" }] }, + ], + usage: { output_tokens: 1, input_tokens: 1, total_tokens: 2 }, + }); + + const onMCPEvent = vi.fn(); + const onUsageData = vi.fn(); + + await makeOpenAIResponsesRequest( + messages, + mockUpdateTextUI, + "gpt-4", + "test-token", + undefined, // tags + undefined, // signal + undefined, // onReasoningContent + undefined, // onTimingData + onUsageData, + undefined, // traceId + undefined, // vector_store_ids + undefined, // guardrails + undefined, // policies + undefined, // selectedMCPServers + undefined, // previousResponseId + undefined, // onResponseId + onMCPEvent, + undefined, // codeInterpreterEnabled + undefined, // onCodeInterpreterResult + undefined, // customBaseUrl + undefined, // mcpServers + undefined, // mcpServerToolRestrictions + undefined, // mcpToolsets + false, // streamingEnabled + ); + + expect(onMCPEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: "response.output_item.done", + item_id: "mcp_1", + item: expect.objectContaining({ type: "mcp_call", name: "search_docs", output: "found it" }), + }), + ); + expect(onUsageData).toHaveBeenCalledWith(expect.anything(), "search_docs"); + }); + it("should configure MCP tools per server with restrictions", async () => { const selectedMCPServers = ["server-1", "server-2"]; const mcpServers = [ diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx index ef510d86b94..f356b2cb2c3 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx @@ -14,6 +14,49 @@ import { export type { CodeInterpreterResult } from "./code_interpreter_handler"; +interface ResponseOutputPart { + type?: string; + text?: string; +} + +interface ResponseOutputItem { + type?: string; + content?: ResponseOutputPart[]; + summary?: ResponseOutputPart[]; +} + +interface NonStreamedResponse { + output?: ResponseOutputItem[]; +} + +type SynthesizedResponseEvent = + | { type: "response.output_item.done"; item: ResponseOutputItem } + | { type: "response.reasoning.delta"; delta: string } + | { type: "response.output_text.delta"; delta: string } + | { type: "response.completed"; response: NonStreamedResponse }; + +const responseAsEvents = (response: NonStreamedResponse): SynthesizedResponseEvent[] => { + const outputItems = response.output ?? []; + const outputText = outputItems + .filter((item) => item.type === "message") + .flatMap((item) => item.content ?? []) + .filter((part) => part.type === "output_text") + .map((part) => part.text ?? "") + .join(""); + const reasoningText = outputItems + .filter((item) => item.type === "reasoning") + .flatMap((item) => item.summary ?? []) + .map((part) => part.text ?? "") + .join(""); + + return [ + ...outputItems.map((item) => ({ type: "response.output_item.done" as const, item })), + ...(reasoningText ? [{ type: "response.reasoning.delta" as const, delta: reasoningText }] : []), + ...(outputText ? [{ type: "response.output_text.delta" as const, delta: outputText }] : []), + { type: "response.completed" as const, response }, + ]; +}; + export async function makeOpenAIResponsesRequest( messages: MessageType[], updateTextUI: (role: string, delta: string, model?: string) => void, @@ -38,6 +81,8 @@ export async function makeOpenAIResponsesRequest( mcpServers?: MCPServer[], mcpServerToolRestrictions?: Record, mcpToolsets?: MCPToolset[], + streamingEnabled: boolean = true, + onTotalLatency?: (latency: number) => void, ) { if (!accessToken) { throw new Error("Virtual Key is required"); @@ -143,27 +188,26 @@ export async function makeOpenAIResponsesRequest( }); } + const requestBody = { + model: selectedModel, + input: formattedInput, + litellm_trace_id: traceId, + ...(previousResponseId ? { previous_response_id: previousResponseId } : {}), + ...(vector_store_ids ? { vector_store_ids } : {}), + ...(guardrails ? { guardrails } : {}), + ...(policies ? { policies } : {}), + ...(tools.length > 0 ? { tools, tool_choice: "auto" } : {}), + }; + // Create request to OpenAI responses API // Use 'any' type to avoid TypeScript issues with the experimental API - const response = await (client as any).responses.create( - { - model: selectedModel, - input: formattedInput, - stream: true, - litellm_trace_id: traceId, - ...(previousResponseId ? { previous_response_id: previousResponseId } : {}), - ...(vector_store_ids ? { vector_store_ids } : {}), - ...(guardrails ? { guardrails } : {}), - ...(policies ? { policies } : {}), - ...(tools.length > 0 ? { tools, tool_choice: "auto" } : {}), - }, - { signal }, - ); + const response = await (client as any).responses.create({ ...requestBody, stream: streamingEnabled }, { signal }); + const events = streamingEnabled ? response : responseAsEvents(response); let mcpToolUsed = ""; let codeInterpreterState: CodeInterpreterState = { code: "", containerId: "" }; - for await (const event of response) { + for await (const event of events) { // Use a type-safe approach to handle events if (typeof event === "object" && event !== null) { // Handle MCP events first @@ -215,7 +259,7 @@ export async function makeOpenAIResponsesRequest( firstTokenReceived = true; const timeToFirstToken = Date.now() - startTime; - if (onTimingData) { + if (onTimingData && streamingEnabled) { onTimingData(timeToFirstToken); } } @@ -259,6 +303,10 @@ export async function makeOpenAIResponsesRequest( } } + if (onTotalLatency) { + onTotalLatency(Date.now() - startTime); + } + return response; } catch (error) { if (signal?.aborted) {