-
Temperature
-
-
-
+ {showAdvancedParams && (
+
+
+
+
+ Temperature
+
+
+
+
+
-
-
-
-
);
};
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.test.tsx
index 9da3e3a4a08..b5f2bf7b10c 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.test.tsx
@@ -2,12 +2,17 @@ import { act, fireEvent, render, screen, waitFor } from "@testing-library/react"
import { beforeEach, describe, expect, it, vi } from "vitest";
import ChatUI from "./ChatUI";
import * as fetchModelsModule from "@/components/llm_calls/fetch_models";
+import { makeOpenAIChatCompletionRequest } from "@/components/llm_calls/chat_completion";
// Mock the fetchAvailableModels function
vi.mock("@/components/llm_calls/fetch_models", () => ({
fetchAvailableModels: vi.fn(),
}));
+vi.mock("@/components/llm_calls/chat_completion", () => ({
+ makeOpenAIChatCompletionRequest: vi.fn().mockResolvedValue(undefined),
+}));
+
// Mock other networking functions that cause errors
vi.mock("@/components/networking", () => ({
tagListCall: vi.fn().mockResolvedValue({ data: [] }),
@@ -21,6 +26,9 @@ beforeEach(() => {
Element.prototype.scrollIntoView = () => {};
});
+const CHAT_REQUEST_ARG_COUNT = 26;
+const STREAMING_ENABLED_ARG_INDEX = 25;
+
describe("ChatUI", () => {
beforeEach(() => {
// Reset mocks before each test
@@ -334,6 +342,165 @@ describe("ChatUI", () => {
});
});
+ it("should send the chat request non-streaming after Stream responses is unchecked", async () => {
+ render(
+
,
+ );
+
+ await waitFor(() => {
+ expect(screen.getByText("Test Key")).toBeInTheDocument();
+ });
+
+ const selectModelLabel = screen.getByText("Select Model");
+ const modelSelect = selectModelLabel.closest("div")?.querySelector(".ant-select-selector");
+ await act(async () => {
+ fireEvent.mouseDown(modelSelect!);
+ });
+
+ await waitFor(() => {
+ expect(screen.getAllByText("Model 1").length).toBeGreaterThan(0);
+ });
+
+ const model1Options = screen.getAllByText("Model 1");
+ await act(async () => {
+ fireEvent.click(model1Options[model1Options.length - 1]);
+ });
+
+ await waitFor(() => {
+ expect(screen.getByTestId("model-settings-button")).toBeInTheDocument();
+ });
+
+ await act(async () => {
+ fireEvent.click(screen.getByTestId("model-settings-button"));
+ });
+
+ const streamingCheckbox = await screen.findByRole("checkbox", { name: /Stream responses/i });
+ expect(streamingCheckbox).toBeChecked();
+
+ await act(async () => {
+ fireEvent.click(streamingCheckbox);
+ });
+
+ await waitFor(() => {
+ expect(screen.getByRole("checkbox", { name: /Stream responses/i })).not.toBeChecked();
+ });
+
+ const messageInput = screen.getByPlaceholderText("Type your message... (Shift+Enter for new line)");
+ await act(async () => {
+ fireEvent.change(messageInput, { target: { value: "hello" } });
+ });
+ await act(async () => {
+ fireEvent.keyDown(messageInput, { key: "Enter", code: "Enter" });
+ });
+
+ await waitFor(() => {
+ expect(makeOpenAIChatCompletionRequest).toHaveBeenCalledTimes(1);
+ });
+
+ const requestArgs = vi.mocked(makeOpenAIChatCompletionRequest).mock.calls[0];
+ expect(requestArgs).toHaveLength(CHAT_REQUEST_ARG_COUNT);
+ expect(requestArgs[STREAMING_ENABLED_ARG_INDEX]).toBe(false);
+ });
+
+ it("should force streaming in simplified mode even when the playground setting is off", async () => {
+ sessionStorage.setItem("streamingEnabled", "false");
+
+ render(
+
,
+ );
+
+ await waitFor(() => {
+ expect(screen.getByText("Chat")).toBeInTheDocument();
+ });
+
+ const messageInput = screen.getByPlaceholderText("Type your message... (Shift+Enter for new line)");
+ await act(async () => {
+ fireEvent.change(messageInput, { target: { value: "hello" } });
+ });
+ await act(async () => {
+ fireEvent.keyDown(messageInput, { key: "Enter", code: "Enter" });
+ });
+
+ await waitFor(() => {
+ expect(makeOpenAIChatCompletionRequest).toHaveBeenCalledTimes(1);
+ });
+
+ const requestArgs = vi.mocked(makeOpenAIChatCompletionRequest).mock.calls[0];
+ expect(requestArgs).toHaveLength(CHAT_REQUEST_ARG_COUNT);
+ expect(requestArgs[STREAMING_ENABLED_ARG_INDEX]).toBe(true);
+ expect(sessionStorage.getItem("streamingEnabled")).toBe("false");
+ });
+
+ it("should offer the streaming toggle for a responses-only model without advanced params", async () => {
+ (fetchModelsModule.fetchAvailableModels as any).mockResolvedValue([
+ { model_group: "ResponsesModel", mode: "responses" },
+ ]);
+
+ render(
+
,
+ );
+
+ await waitFor(() => {
+ expect(screen.getByText("Test Key")).toBeInTheDocument();
+ });
+
+ const endpointTypeText = screen.getByText("Endpoint Type");
+ const endpointSelect = endpointTypeText.parentElement?.querySelector(".ant-select-selector");
+ await act(async () => {
+ fireEvent.mouseDown(endpointSelect!);
+ });
+ await act(async () => {
+ fireEvent.click(screen.getByText("/v1/responses"));
+ });
+
+ const selectModelLabel = screen.getByText("Select Model");
+ const modelSelect = selectModelLabel.closest("div")?.querySelector(".ant-select-selector");
+ await act(async () => {
+ fireEvent.mouseDown(modelSelect!);
+ });
+
+ await waitFor(() => {
+ expect(screen.getAllByText("ResponsesModel").length).toBeGreaterThan(0);
+ });
+
+ const modelOptions = screen.getAllByText("ResponsesModel");
+ await act(async () => {
+ fireEvent.click(modelOptions[modelOptions.length - 1]);
+ });
+
+ await waitFor(() => {
+ expect(screen.getByTestId("model-settings-button")).toBeInTheDocument();
+ });
+
+ await act(async () => {
+ fireEvent.click(screen.getByTestId("model-settings-button"));
+ });
+
+ expect(await screen.findByRole("checkbox", { name: /Stream responses/i })).toBeChecked();
+ expect(screen.queryByText("Temperature")).not.toBeInTheDocument();
+ expect(screen.queryByText("Use Advanced Parameters")).not.toBeInTheDocument();
+ });
+
it("should show Fill button and populate customProxyBaseUrl when proxySettings.LITELLM_UI_API_DOC_BASE_URL is provided", async () => {
const testProxyUrl = "http://localhost:5000";
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx
index d2cf27e0c8b..684814bfe5b 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx
@@ -261,6 +261,11 @@ const ChatUI: React.FC
= ({
const [maxTokens, setMaxTokens] = useState(2048);
const [useAdvancedParams, setUseAdvancedParams] = useState(false);
const [mockTestFallbacks, setMockTestFallbacks] = useState(false);
+ const [streamingEnabled, setStreamingEnabled] = useState(() => {
+ if (simplified) return true;
+ const saved = sessionStorage.getItem("streamingEnabled");
+ return saved === null ? true : saved === "true";
+ });
// Code Interpreter state (using custom hook)
const codeInterpreter = useCodeInterpreter();
@@ -372,6 +377,7 @@ const ChatUI: React.FC = ({
sessionStorage.removeItem("selectedMCPTools"); // Clean up old key
if (!simplified) {
+ sessionStorage.setItem("streamingEnabled", JSON.stringify(streamingEnabled));
if (selectedModel) {
sessionStorage.setItem("selectedModel", selectedModel);
} else {
@@ -392,6 +398,7 @@ const ChatUI: React.FC = ({
selectedMCPServers,
mcpServerToolRestrictions,
selectedVoice,
+ streamingEnabled,
]);
useEffect(() => {
@@ -771,6 +778,7 @@ const ChatUI: React.FC = ({
handleMCPEvent,
mockTestFallbacks,
mcpToolsets,
+ streamingEnabled,
);
} else if (endpointType === EndpointType.IMAGE) {
// For image generation
@@ -852,6 +860,8 @@ const ChatUI: React.FC = ({
mcpServers,
mcpServerToolRestrictions,
mcpToolsets,
+ streamingEnabled,
+ updateTotalLatency,
);
} else if (endpointType === EndpointType.ANTHROPIC_MESSAGES) {
const apiChatHistory = [
@@ -1035,6 +1045,8 @@ const ChatUI: React.FC = ({
return !model.mode || model.mode === "chat";
};
+ const supportsStreamingToggle = endpointType === EndpointType.CHAT || endpointType === EndpointType.RESPONSES;
+
const antIcon = ;
return (
@@ -1184,10 +1196,11 @@ const ChatUI: React.FC = ({
Select Model
- {isChatModel() ? (
+ {isChatModel() || supportsStreamingToggle ? (
= ({
onUseAdvancedParamsChange={setUseAdvancedParams}
mockTestFallbacks={mockTestFallbacks}
onMockTestFallbacksChange={setMockTestFallbacks}
+ streamingEnabled={streamingEnabled}
+ onStreamingChange={supportsStreamingToggle ? setStreamingEnabled : undefined}
/>
}
title="Model Settings"
diff --git a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx
index 8649834b318..10252adecd0 100644
--- a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx
+++ b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx
@@ -224,6 +224,143 @@ describe("chat_completion", () => {
expect(callArgs.mock_testing_fallbacks).toBe(true);
});
+ it("should send a non-streaming request and render the whole message at once when streaming is disabled", async () => {
+ mockCreate.mockResolvedValueOnce({
+ id: "chatcmpl-1",
+ object: "chat.completion",
+ created: 1,
+ model: "gpt-4",
+ choices: [
+ {
+ index: 0,
+ finish_reason: "stop",
+ message: { role: "assistant", content: "Hello there" },
+ },
+ ],
+ usage: {
+ completion_tokens: 2,
+ prompt_tokens: 5,
+ total_tokens: 7,
+ cost: 0.25,
+ },
+ });
+
+ const onTimingData = vi.fn();
+ const onUsageData = vi.fn();
+ const onTotalLatency = vi.fn();
+
+ await makeOpenAIChatCompletionRequest(
+ mockChatHistory,
+ mockUpdateUI,
+ "gpt-4",
+ "test-token",
+ undefined, // tags
+ undefined, // signal
+ undefined, // onReasoningContent
+ onTimingData,
+ onUsageData,
+ undefined, // traceId
+ undefined, // vector_store_ids
+ undefined, // guardrails
+ undefined, // policies
+ undefined, // selectedMCPServers
+ undefined, // onImageGenerated
+ undefined, // onSearchResults
+ undefined, // temperature
+ undefined, // max_tokens
+ onTotalLatency,
+ undefined, // customBaseUrl
+ undefined, // mcpServers
+ undefined, // mcpServerToolRestrictions
+ undefined, // onMCPEvent
+ undefined, // mockTestFallbacks
+ undefined, // mcpToolsets
+ false, // streamingEnabled
+ );
+
+ expect(mockCreate).toHaveBeenCalledTimes(1);
+ const callArgs = mockCreate.mock.calls[0][0];
+ expect(callArgs.stream).toBe(false);
+ expect(callArgs).not.toHaveProperty("stream_options");
+
+ expect(mockUpdateUI).toHaveBeenCalledTimes(1);
+ expect(mockUpdateUI).toHaveBeenCalledWith("Hello there", "gpt-4");
+
+ expect(onUsageData).toHaveBeenCalledWith({
+ completionTokens: 2,
+ promptTokens: 5,
+ totalTokens: 7,
+ cost: 0.25,
+ });
+ expect(onTimingData).not.toHaveBeenCalled();
+ expect(onTotalLatency).toHaveBeenCalledWith(expect.any(Number));
+ });
+
+ it("should surface reasoning content and MCP metadata from a non-streaming response", async () => {
+ mockCreate.mockResolvedValueOnce({
+ model: "gpt-4",
+ choices: [
+ {
+ index: 0,
+ finish_reason: "stop",
+ message: {
+ role: "assistant",
+ content: "done",
+ reasoning_content: "thinking",
+ provider_specific_fields: {
+ mcp_tool_calls: [{ id: "call_1", function: { name: "search_docs", arguments: "{}" } }],
+ mcp_call_results: [{ tool_call_id: "call_1", result: "found it" }],
+ },
+ },
+ },
+ ],
+ });
+
+ const onReasoningContent = vi.fn();
+ const onMCPEvent = vi.fn();
+
+ await makeOpenAIChatCompletionRequest(
+ mockChatHistory,
+ mockUpdateUI,
+ "gpt-4",
+ "test-token",
+ undefined, // tags
+ undefined, // signal
+ onReasoningContent,
+ undefined, // onTimingData
+ undefined, // onUsageData
+ undefined, // traceId
+ undefined, // vector_store_ids
+ undefined, // guardrails
+ undefined, // policies
+ undefined, // selectedMCPServers
+ undefined, // onImageGenerated
+ undefined, // onSearchResults
+ undefined, // temperature
+ undefined, // max_tokens
+ undefined, // onTotalLatency
+ undefined, // customBaseUrl
+ undefined, // mcpServers
+ undefined, // mcpServerToolRestrictions
+ onMCPEvent,
+ undefined, // mockTestFallbacks
+ undefined, // mcpToolsets
+ false, // streamingEnabled
+ );
+
+ expect(onReasoningContent).toHaveBeenCalledWith("thinking");
+ expect(onMCPEvent).toHaveBeenCalledWith(
+ expect.objectContaining({
+ type: "response.output_item.done",
+ item: expect.objectContaining({
+ type: "mcp_call",
+ name: "search_docs",
+ output: "found it",
+ }),
+ }),
+ );
+ });
+
it("should not include mock_testing_fallbacks in request body when mockTestFallbacks is false or undefined", async () => {
await makeOpenAIChatCompletionRequest(
mockChatHistory,
diff --git a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx
index 66be2fc7893..c20d758fe91 100644
--- a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx
+++ b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx
@@ -1,10 +1,26 @@
import openai from "openai";
-import { ChatCompletionMessageParam } from "openai/resources/chat/completions";
+import { ChatCompletion, ChatCompletionChunk, ChatCompletionMessageParam } from "openai/resources/chat/completions";
import { TokenUsage } from "../chat_ui/ResponseMetrics";
import { VectorStoreSearchResponse } from "../chat_ui/types";
import { getProxyBaseUrl } from "@/components/networking";
import { MCPServer, MCPToolset, type MCPEvent } from "@/components/mcp_tools/types";
+const completionAsSingleChunk = (completion: ChatCompletion): ChatCompletionChunk =>
+ ({
+ id: completion.id,
+ object: "chat.completion.chunk",
+ created: completion.created,
+ model: completion.model,
+ usage: completion.usage,
+ choices: [
+ {
+ index: 0,
+ finish_reason: completion.choices[0]?.finish_reason ?? null,
+ delta: completion.choices[0]?.message ?? {},
+ },
+ ],
+ }) as unknown as ChatCompletionChunk;
+
export async function makeOpenAIChatCompletionRequest(
chatHistory: { role: string; content: string | any[] }[],
updateUI: (chunk: string, model?: string) => void,
@@ -31,6 +47,7 @@ export async function makeOpenAIChatCompletionRequest(
onMCPEvent?: (event: MCPEvent) => void,
mockTestFallbacks?: boolean,
mcpToolsets?: MCPToolset[],
+ streamingEnabled: boolean = true,
) {
// base url should be the current base_url
const isLocal = process.env.NODE_ENV === "development";
@@ -111,26 +128,25 @@ export async function makeOpenAIChatCompletionRequest(
}
}
- // @ts-ignore
- const response = await client.chat.completions.create(
- {
- model: selectedModel,
- stream: true,
- stream_options: {
- include_usage: true,
- },
- litellm_trace_id: traceId,
- messages: chatHistory as ChatCompletionMessageParam[],
- ...(vector_store_ids ? { vector_store_ids } : {}),
- ...(guardrails ? { guardrails } : {}),
- ...(policies ? { policies } : {}),
- ...(tools.length > 0 ? { tools, tool_choice: "auto" } : {}),
- ...(temperature !== undefined ? { temperature } : {}),
- ...(max_tokens !== undefined ? { max_tokens } : {}),
- ...(mockTestFallbacks ? { mock_testing_fallbacks: true } : {}),
- },
- { signal },
- );
+ const requestBody = {
+ model: selectedModel,
+ litellm_trace_id: traceId,
+ messages: chatHistory as ChatCompletionMessageParam[],
+ ...(vector_store_ids ? { vector_store_ids } : {}),
+ ...(guardrails ? { guardrails } : {}),
+ ...(policies ? { policies } : {}),
+ ...(tools.length > 0 ? { tools, tool_choice: "auto" as const } : {}),
+ ...(temperature !== undefined ? { temperature } : {}),
+ ...(max_tokens !== undefined ? { max_tokens } : {}),
+ ...(mockTestFallbacks ? { mock_testing_fallbacks: true } : {}),
+ };
+
+ const response: AsyncIterable | ChatCompletionChunk[] = streamingEnabled
+ ? await client.chat.completions.create(
+ { ...requestBody, stream: true, stream_options: { include_usage: true } },
+ { signal },
+ )
+ : [completionAsSingleChunk(await client.chat.completions.create({ ...requestBody, stream: false }, { signal }))];
for await (const chunk of response) {
// Process content and measure time to first token
@@ -142,7 +158,7 @@ export async function makeOpenAIChatCompletionRequest(
if (!firstTokenReceived && (chunk.choices[0]?.delta?.content || (delta && delta.reasoning_content))) {
firstTokenReceived = true;
timeToFirstToken = Date.now() - startTime;
- if (onTimingData) {
+ if (onTimingData && streamingEnabled) {
onTimingData(timeToFirstToken);
}
}
diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx
index 77ff5fd00bb..a897e6fc4cc 100644
--- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx
+++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx
@@ -69,6 +69,158 @@ describe("responses_api", () => {
expect(mockUpdateTextUI).toHaveBeenCalledWith("assistant", "Hi", "gpt-4");
});
+ it("should send a non-streaming request and render the whole output at once when streaming is disabled", async () => {
+ mockResponsesCreate.mockResolvedValueOnce({
+ id: "resp_456",
+ output: [
+ {
+ type: "message",
+ content: [
+ { type: "output_text", text: "Full " },
+ { type: "output_text", text: "answer" },
+ ],
+ },
+ ],
+ usage: { output_tokens: 3, input_tokens: 4, total_tokens: 7 },
+ });
+
+ const onTimingData = vi.fn();
+ const onUsageData = vi.fn();
+ const onResponseId = vi.fn();
+
+ await makeOpenAIResponsesRequest(
+ messages,
+ mockUpdateTextUI,
+ "gpt-4",
+ "test-token",
+ undefined, // tags
+ undefined, // signal
+ undefined, // onReasoningContent
+ onTimingData,
+ onUsageData,
+ undefined, // traceId
+ undefined, // vector_store_ids
+ undefined, // guardrails
+ undefined, // policies
+ undefined, // selectedMCPServers
+ undefined, // previousResponseId
+ onResponseId,
+ undefined, // onMCPEvent
+ undefined, // codeInterpreterEnabled
+ undefined, // onCodeInterpreterResult
+ undefined, // customBaseUrl
+ undefined, // mcpServers
+ undefined, // mcpServerToolRestrictions
+ undefined, // mcpToolsets
+ false, // streamingEnabled
+ );
+
+ expect(mockResponsesCreate).toHaveBeenCalledTimes(1);
+ expect(mockResponsesCreate.mock.calls[0][0].stream).toBe(false);
+
+ expect(mockUpdateTextUI).toHaveBeenCalledTimes(1);
+ expect(mockUpdateTextUI).toHaveBeenCalledWith("assistant", "Full answer", "gpt-4");
+
+ expect(onUsageData).toHaveBeenCalledWith({ completionTokens: 3, promptTokens: 4, totalTokens: 7 }, "");
+ expect(onResponseId).toHaveBeenCalledWith("resp_456");
+ expect(onTimingData).not.toHaveBeenCalled();
+ });
+
+ it("should report total latency in both streaming and non-streaming modes", async () => {
+ const onTotalLatency = vi.fn();
+ const callWithStreaming = (streamingEnabled: boolean) =>
+ makeOpenAIResponsesRequest(
+ messages,
+ mockUpdateTextUI,
+ "gpt-4",
+ "test-token",
+ undefined, // tags
+ undefined, // signal
+ undefined, // onReasoningContent
+ undefined, // onTimingData
+ undefined, // onUsageData
+ undefined, // traceId
+ undefined, // vector_store_ids
+ undefined, // guardrails
+ undefined, // policies
+ undefined, // selectedMCPServers
+ undefined, // previousResponseId
+ undefined, // onResponseId
+ undefined, // onMCPEvent
+ undefined, // codeInterpreterEnabled
+ undefined, // onCodeInterpreterResult
+ undefined, // customBaseUrl
+ undefined, // mcpServers
+ undefined, // mcpServerToolRestrictions
+ undefined, // mcpToolsets
+ streamingEnabled,
+ onTotalLatency,
+ );
+
+ await callWithStreaming(true);
+ expect(onTotalLatency).toHaveBeenCalledTimes(1);
+ expect(onTotalLatency).toHaveBeenLastCalledWith(expect.any(Number));
+
+ mockResponsesCreate.mockResolvedValueOnce({
+ id: "resp_latency",
+ output: [{ type: "message", content: [{ type: "output_text", text: "Answer" }] }],
+ });
+
+ await callWithStreaming(false);
+ expect(onTotalLatency).toHaveBeenCalledTimes(2);
+ expect(onTotalLatency).toHaveBeenLastCalledWith(expect.any(Number));
+ });
+
+ it("should replay MCP output items as events for a non-streaming response", async () => {
+ mockResponsesCreate.mockResolvedValueOnce({
+ id: "resp_789",
+ output: [
+ { type: "mcp_call", id: "mcp_1", name: "search_docs", arguments: "{}", output: "found it" },
+ { type: "message", content: [{ type: "output_text", text: "Answer" }] },
+ ],
+ usage: { output_tokens: 1, input_tokens: 1, total_tokens: 2 },
+ });
+
+ const onMCPEvent = vi.fn();
+ const onUsageData = vi.fn();
+
+ await makeOpenAIResponsesRequest(
+ messages,
+ mockUpdateTextUI,
+ "gpt-4",
+ "test-token",
+ undefined, // tags
+ undefined, // signal
+ undefined, // onReasoningContent
+ undefined, // onTimingData
+ onUsageData,
+ undefined, // traceId
+ undefined, // vector_store_ids
+ undefined, // guardrails
+ undefined, // policies
+ undefined, // selectedMCPServers
+ undefined, // previousResponseId
+ undefined, // onResponseId
+ onMCPEvent,
+ undefined, // codeInterpreterEnabled
+ undefined, // onCodeInterpreterResult
+ undefined, // customBaseUrl
+ undefined, // mcpServers
+ undefined, // mcpServerToolRestrictions
+ undefined, // mcpToolsets
+ false, // streamingEnabled
+ );
+
+ expect(onMCPEvent).toHaveBeenCalledWith(
+ expect.objectContaining({
+ type: "response.output_item.done",
+ item_id: "mcp_1",
+ item: expect.objectContaining({ type: "mcp_call", name: "search_docs", output: "found it" }),
+ }),
+ );
+ expect(onUsageData).toHaveBeenCalledWith(expect.anything(), "search_docs");
+ });
+
it("should configure MCP tools per server with restrictions", async () => {
const selectedMCPServers = ["server-1", "server-2"];
const mcpServers = [
diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx
index ef510d86b94..f356b2cb2c3 100644
--- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx
+++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx
@@ -14,6 +14,49 @@ import {
export type { CodeInterpreterResult } from "./code_interpreter_handler";
+interface ResponseOutputPart {
+ type?: string;
+ text?: string;
+}
+
+interface ResponseOutputItem {
+ type?: string;
+ content?: ResponseOutputPart[];
+ summary?: ResponseOutputPart[];
+}
+
+interface NonStreamedResponse {
+ output?: ResponseOutputItem[];
+}
+
+type SynthesizedResponseEvent =
+ | { type: "response.output_item.done"; item: ResponseOutputItem }
+ | { type: "response.reasoning.delta"; delta: string }
+ | { type: "response.output_text.delta"; delta: string }
+ | { type: "response.completed"; response: NonStreamedResponse };
+
+const responseAsEvents = (response: NonStreamedResponse): SynthesizedResponseEvent[] => {
+ const outputItems = response.output ?? [];
+ const outputText = outputItems
+ .filter((item) => item.type === "message")
+ .flatMap((item) => item.content ?? [])
+ .filter((part) => part.type === "output_text")
+ .map((part) => part.text ?? "")
+ .join("");
+ const reasoningText = outputItems
+ .filter((item) => item.type === "reasoning")
+ .flatMap((item) => item.summary ?? [])
+ .map((part) => part.text ?? "")
+ .join("");
+
+ return [
+ ...outputItems.map((item) => ({ type: "response.output_item.done" as const, item })),
+ ...(reasoningText ? [{ type: "response.reasoning.delta" as const, delta: reasoningText }] : []),
+ ...(outputText ? [{ type: "response.output_text.delta" as const, delta: outputText }] : []),
+ { type: "response.completed" as const, response },
+ ];
+};
+
export async function makeOpenAIResponsesRequest(
messages: MessageType[],
updateTextUI: (role: string, delta: string, model?: string) => void,
@@ -38,6 +81,8 @@ export async function makeOpenAIResponsesRequest(
mcpServers?: MCPServer[],
mcpServerToolRestrictions?: Record,
mcpToolsets?: MCPToolset[],
+ streamingEnabled: boolean = true,
+ onTotalLatency?: (latency: number) => void,
) {
if (!accessToken) {
throw new Error("Virtual Key is required");
@@ -143,27 +188,26 @@ export async function makeOpenAIResponsesRequest(
});
}
+ const requestBody = {
+ model: selectedModel,
+ input: formattedInput,
+ litellm_trace_id: traceId,
+ ...(previousResponseId ? { previous_response_id: previousResponseId } : {}),
+ ...(vector_store_ids ? { vector_store_ids } : {}),
+ ...(guardrails ? { guardrails } : {}),
+ ...(policies ? { policies } : {}),
+ ...(tools.length > 0 ? { tools, tool_choice: "auto" } : {}),
+ };
+
// Create request to OpenAI responses API
// Use 'any' type to avoid TypeScript issues with the experimental API
- const response = await (client as any).responses.create(
- {
- model: selectedModel,
- input: formattedInput,
- stream: true,
- litellm_trace_id: traceId,
- ...(previousResponseId ? { previous_response_id: previousResponseId } : {}),
- ...(vector_store_ids ? { vector_store_ids } : {}),
- ...(guardrails ? { guardrails } : {}),
- ...(policies ? { policies } : {}),
- ...(tools.length > 0 ? { tools, tool_choice: "auto" } : {}),
- },
- { signal },
- );
+ const response = await (client as any).responses.create({ ...requestBody, stream: streamingEnabled }, { signal });
+ const events = streamingEnabled ? response : responseAsEvents(response);
let mcpToolUsed = "";
let codeInterpreterState: CodeInterpreterState = { code: "", containerId: "" };
- for await (const event of response) {
+ for await (const event of events) {
// Use a type-safe approach to handle events
if (typeof event === "object" && event !== null) {
// Handle MCP events first
@@ -215,7 +259,7 @@ export async function makeOpenAIResponsesRequest(
firstTokenReceived = true;
const timeToFirstToken = Date.now() - startTime;
- if (onTimingData) {
+ if (onTimingData && streamingEnabled) {
onTimingData(timeToFirstToken);
}
}
@@ -259,6 +303,10 @@ export async function makeOpenAIResponsesRequest(
}
}
+ if (onTotalLatency) {
+ onTotalLatency(Date.now() - startTime);
+ }
+
return response;
} catch (error) {
if (signal?.aborted) {