mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-10 03:28:53 +00:00
feat(playground): add non-streaming response toggle (#35560)
Adds a Stream responses checkbox (default on) to the playground Model Settings popover. When unchecked, chat completions and responses API requests are sent with stream: false and the full reply renders at once. The non-streamed result is replayed through the existing streaming handlers as synthesized chunks/events so MCP events, vector store results, usage and response ids behave identically in both modes. TTFT is suppressed when not streaming; total latency now also reported for the responses API. The toggle is scoped to the chat and responses endpoints, persists via sessionStorage, and is isolated from the simplified Agent Builder chat. Resolves LIT-3251
This commit is contained in:
ryan-crabbe-berri
GitHub
parent
5b6194f427
commit
41e4408906
8 changed files with 685 additions and 88 deletions
|
|
@ -48,6 +48,44 @@ describe("AdditionalModelSettings", () => {
|
|||
expect(maxTokensSlider).not.toBeDisabled();
|
||||
});
|
||||
|
||||
it("should not show Stream responses when onStreamingChange is not provided", () => {
|
||||
render(<AdditionalModelSettings />);
|
||||
expect(screen.queryByText(/Stream responses/i)).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should render Stream responses checked by default and report unchecking it", async () => {
|
||||
const user = userEvent.setup();
|
||||
const onStreamingChange = vi.fn();
|
||||
|
||||
render(<AdditionalModelSettings onStreamingChange={onStreamingChange} />);
|
||||
|
||||
const streamingCheckbox = screen.getByRole("checkbox", { name: /Stream responses/i });
|
||||
expect(streamingCheckbox).toBeChecked();
|
||||
|
||||
await act(async () => {
|
||||
await user.click(streamingCheckbox);
|
||||
});
|
||||
|
||||
await waitFor(() => {
|
||||
expect(onStreamingChange).toHaveBeenCalledWith(false);
|
||||
});
|
||||
});
|
||||
|
||||
it("should keep the streaming toggle but drop advanced params when showAdvancedParams is false", () => {
|
||||
render(<AdditionalModelSettings showAdvancedParams={false} onStreamingChange={vi.fn()} />);
|
||||
|
||||
expect(screen.getByRole("checkbox", { name: /Stream responses/i })).toBeInTheDocument();
|
||||
expect(screen.queryByText("Use Advanced Parameters")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("Temperature")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("Max Tokens")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should reflect a disabled streaming setting from props", () => {
|
||||
render(<AdditionalModelSettings streamingEnabled={false} onStreamingChange={vi.fn()} />);
|
||||
|
||||
expect(screen.getByRole("checkbox", { name: /Stream responses/i })).not.toBeChecked();
|
||||
});
|
||||
|
||||
it("should not show Simulate failure to test fallbacks when onMockTestFallbacksChange is not provided", () => {
|
||||
render(<AdditionalModelSettings />);
|
||||
expect(screen.queryByText(/Simulate failure to test fallbacks/i)).not.toBeInTheDocument();
|
||||
|
|
|
|||
|
|
@ -12,6 +12,9 @@ interface AdditionalModelSettingsProps {
|
|||
onUseAdvancedParamsChange?: (value: boolean) => void;
|
||||
mockTestFallbacks?: boolean;
|
||||
onMockTestFallbacksChange?: (value: boolean) => void;
|
||||
streamingEnabled?: boolean;
|
||||
onStreamingChange?: (value: boolean) => void;
|
||||
showAdvancedParams?: boolean;
|
||||
}
|
||||
|
||||
const AdditionalModelSettings: React.FC<AdditionalModelSettingsProps> = ({
|
||||
|
|
@ -23,6 +26,9 @@ const AdditionalModelSettings: React.FC<AdditionalModelSettingsProps> = ({
|
|||
onUseAdvancedParamsChange,
|
||||
mockTestFallbacks,
|
||||
onMockTestFallbacksChange,
|
||||
streamingEnabled = true,
|
||||
onStreamingChange,
|
||||
showAdvancedParams = true,
|
||||
}) => {
|
||||
const [internalUseAdvancedParams, setInternalUseAdvancedParams] = useState(false);
|
||||
const useAdvancedParams =
|
||||
|
|
@ -64,9 +70,25 @@ const AdditionalModelSettings: React.FC<AdditionalModelSettingsProps> = ({
|
|||
|
||||
return (
|
||||
<div className="space-y-4 p-4 w-80">
|
||||
<Checkbox checked={useAdvancedParams} onChange={(e) => handleUseAdvancedParamsChange(e.target.checked)}>
|
||||
<span className="font-medium">Use Advanced Parameters</span>
|
||||
</Checkbox>
|
||||
{onStreamingChange && (
|
||||
<div className="flex items-center gap-1">
|
||||
<Checkbox checked={streamingEnabled} onChange={(e) => onStreamingChange(e.target.checked)}>
|
||||
<span className="font-medium">Stream responses</span>
|
||||
</Checkbox>
|
||||
<Tooltip title="Streams the answer token by token. Uncheck to send a non-streaming request and render the full response at once.">
|
||||
<InfoCircleOutlined
|
||||
className="text-xs text-gray-400 cursor-pointer shrink-0 hover:text-gray-600"
|
||||
aria-label="Help: Stream responses"
|
||||
/>
|
||||
</Tooltip>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{showAdvancedParams && (
|
||||
<Checkbox checked={useAdvancedParams} onChange={(e) => handleUseAdvancedParamsChange(e.target.checked)}>
|
||||
<span className="font-medium">Use Advanced Parameters</span>
|
||||
</Checkbox>
|
||||
)}
|
||||
|
||||
{onMockTestFallbacksChange && (
|
||||
<div className="flex items-center gap-1">
|
||||
|
|
@ -104,72 +126,74 @@ const AdditionalModelSettings: React.FC<AdditionalModelSettingsProps> = ({
|
|||
</div>
|
||||
)}
|
||||
|
||||
<div className="space-y-4 transition-opacity duration-200" style={{ opacity: disabledOpacity }}>
|
||||
<div>
|
||||
<div className="flex items-center justify-between mb-2">
|
||||
<div className="flex items-center gap-1">
|
||||
<Text className={`text-sm ${disabledTextColor}`}>Temperature</Text>
|
||||
<Tooltip title="Controls randomness. Lower values make output more deterministic, higher values more creative.">
|
||||
<InfoCircleOutlined className={`text-xs ${disabledTextColor} cursor-help`} />
|
||||
</Tooltip>
|
||||
{showAdvancedParams && (
|
||||
<div className="space-y-4 transition-opacity duration-200" style={{ opacity: disabledOpacity }}>
|
||||
<div>
|
||||
<div className="flex items-center justify-between mb-2">
|
||||
<div className="flex items-center gap-1">
|
||||
<Text className={`text-sm ${disabledTextColor}`}>Temperature</Text>
|
||||
<Tooltip title="Controls randomness. Lower values make output more deterministic, higher values more creative.">
|
||||
<InfoCircleOutlined className={`text-xs ${disabledTextColor} cursor-help`} />
|
||||
</Tooltip>
|
||||
</div>
|
||||
<InputNumber
|
||||
min={0}
|
||||
max={2}
|
||||
step={0.1}
|
||||
value={localTemperature}
|
||||
onChange={handleTemperatureChange}
|
||||
disabled={!useAdvancedParams}
|
||||
precision={1}
|
||||
className="w-20"
|
||||
/>
|
||||
</div>
|
||||
<InputNumber
|
||||
<Slider
|
||||
min={0}
|
||||
max={2}
|
||||
step={0.1}
|
||||
value={localTemperature}
|
||||
onChange={handleTemperatureChange}
|
||||
disabled={!useAdvancedParams}
|
||||
precision={1}
|
||||
className="w-20"
|
||||
marks={{
|
||||
0: "0",
|
||||
1: "1.0",
|
||||
2: "2.0",
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
<Slider
|
||||
min={0}
|
||||
max={2}
|
||||
step={0.1}
|
||||
value={localTemperature}
|
||||
onChange={handleTemperatureChange}
|
||||
disabled={!useAdvancedParams}
|
||||
marks={{
|
||||
0: "0",
|
||||
1: "1.0",
|
||||
2: "2.0",
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<div className="flex items-center justify-between mb-2">
|
||||
<div className="flex items-center gap-1">
|
||||
<Text className={`text-sm ${disabledTextColor}`}>Max Tokens</Text>
|
||||
<Tooltip title="Maximum number of tokens to generate in the response.">
|
||||
<InfoCircleOutlined className={`text-xs ${disabledTextColor} cursor-help`} />
|
||||
</Tooltip>
|
||||
<div>
|
||||
<div className="flex items-center justify-between mb-2">
|
||||
<div className="flex items-center gap-1">
|
||||
<Text className={`text-sm ${disabledTextColor}`}>Max Tokens</Text>
|
||||
<Tooltip title="Maximum number of tokens to generate in the response.">
|
||||
<InfoCircleOutlined className={`text-xs ${disabledTextColor} cursor-help`} />
|
||||
</Tooltip>
|
||||
</div>
|
||||
<InputNumber
|
||||
min={1}
|
||||
max={32768}
|
||||
step={1}
|
||||
value={localMaxTokens}
|
||||
onChange={handleMaxTokensChange}
|
||||
disabled={!useAdvancedParams}
|
||||
/>
|
||||
</div>
|
||||
<InputNumber
|
||||
<Slider
|
||||
min={1}
|
||||
max={32768}
|
||||
step={1}
|
||||
value={localMaxTokens}
|
||||
onChange={handleMaxTokensChange}
|
||||
disabled={!useAdvancedParams}
|
||||
marks={{
|
||||
1: "1",
|
||||
32768: "32768",
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
<Slider
|
||||
min={1}
|
||||
max={32768}
|
||||
step={1}
|
||||
value={localMaxTokens}
|
||||
onChange={handleMaxTokensChange}
|
||||
disabled={!useAdvancedParams}
|
||||
marks={{
|
||||
1: "1",
|
||||
32768: "32768",
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -2,12 +2,17 @@ import { act, fireEvent, render, screen, waitFor } from "@testing-library/react"
|
|||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import ChatUI from "./ChatUI";
|
||||
import * as fetchModelsModule from "@/components/llm_calls/fetch_models";
|
||||
import { makeOpenAIChatCompletionRequest } from "@/components/llm_calls/chat_completion";
|
||||
|
||||
// Mock the fetchAvailableModels function
|
||||
vi.mock("@/components/llm_calls/fetch_models", () => ({
|
||||
fetchAvailableModels: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/components/llm_calls/chat_completion", () => ({
|
||||
makeOpenAIChatCompletionRequest: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
// Mock other networking functions that cause errors
|
||||
vi.mock("@/components/networking", () => ({
|
||||
tagListCall: vi.fn().mockResolvedValue({ data: [] }),
|
||||
|
|
@ -21,6 +26,9 @@ beforeEach(() => {
|
|||
Element.prototype.scrollIntoView = () => {};
|
||||
});
|
||||
|
||||
const CHAT_REQUEST_ARG_COUNT = 26;
|
||||
const STREAMING_ENABLED_ARG_INDEX = 25;
|
||||
|
||||
describe("ChatUI", () => {
|
||||
beforeEach(() => {
|
||||
// Reset mocks before each test
|
||||
|
|
@ -334,6 +342,165 @@ describe("ChatUI", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("should send the chat request non-streaming after Stream responses is unchecked", async () => {
|
||||
render(
|
||||
<ChatUI
|
||||
accessToken="1234567890"
|
||||
token="1234567890"
|
||||
userRole="user"
|
||||
userID="1234567890"
|
||||
disabledPersonalKeyCreation={false}
|
||||
/>,
|
||||
);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText("Test Key")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
const selectModelLabel = screen.getByText("Select Model");
|
||||
const modelSelect = selectModelLabel.closest("div")?.querySelector(".ant-select-selector");
|
||||
await act(async () => {
|
||||
fireEvent.mouseDown(modelSelect!);
|
||||
});
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getAllByText("Model 1").length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
const model1Options = screen.getAllByText("Model 1");
|
||||
await act(async () => {
|
||||
fireEvent.click(model1Options[model1Options.length - 1]);
|
||||
});
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByTestId("model-settings-button")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
await act(async () => {
|
||||
fireEvent.click(screen.getByTestId("model-settings-button"));
|
||||
});
|
||||
|
||||
const streamingCheckbox = await screen.findByRole("checkbox", { name: /Stream responses/i });
|
||||
expect(streamingCheckbox).toBeChecked();
|
||||
|
||||
await act(async () => {
|
||||
fireEvent.click(streamingCheckbox);
|
||||
});
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByRole("checkbox", { name: /Stream responses/i })).not.toBeChecked();
|
||||
});
|
||||
|
||||
const messageInput = screen.getByPlaceholderText("Type your message... (Shift+Enter for new line)");
|
||||
await act(async () => {
|
||||
fireEvent.change(messageInput, { target: { value: "hello" } });
|
||||
});
|
||||
await act(async () => {
|
||||
fireEvent.keyDown(messageInput, { key: "Enter", code: "Enter" });
|
||||
});
|
||||
|
||||
await waitFor(() => {
|
||||
expect(makeOpenAIChatCompletionRequest).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
const requestArgs = vi.mocked(makeOpenAIChatCompletionRequest).mock.calls[0];
|
||||
expect(requestArgs).toHaveLength(CHAT_REQUEST_ARG_COUNT);
|
||||
expect(requestArgs[STREAMING_ENABLED_ARG_INDEX]).toBe(false);
|
||||
});
|
||||
|
||||
it("should force streaming in simplified mode even when the playground setting is off", async () => {
|
||||
sessionStorage.setItem("streamingEnabled", "false");
|
||||
|
||||
render(
|
||||
<ChatUI
|
||||
accessToken="1234567890"
|
||||
token="1234567890"
|
||||
userRole="user"
|
||||
userID="1234567890"
|
||||
disabledPersonalKeyCreation={false}
|
||||
simplified
|
||||
fixedModel="Model 1"
|
||||
/>,
|
||||
);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText("Chat")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
const messageInput = screen.getByPlaceholderText("Type your message... (Shift+Enter for new line)");
|
||||
await act(async () => {
|
||||
fireEvent.change(messageInput, { target: { value: "hello" } });
|
||||
});
|
||||
await act(async () => {
|
||||
fireEvent.keyDown(messageInput, { key: "Enter", code: "Enter" });
|
||||
});
|
||||
|
||||
await waitFor(() => {
|
||||
expect(makeOpenAIChatCompletionRequest).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
const requestArgs = vi.mocked(makeOpenAIChatCompletionRequest).mock.calls[0];
|
||||
expect(requestArgs).toHaveLength(CHAT_REQUEST_ARG_COUNT);
|
||||
expect(requestArgs[STREAMING_ENABLED_ARG_INDEX]).toBe(true);
|
||||
expect(sessionStorage.getItem("streamingEnabled")).toBe("false");
|
||||
});
|
||||
|
||||
it("should offer the streaming toggle for a responses-only model without advanced params", async () => {
|
||||
(fetchModelsModule.fetchAvailableModels as any).mockResolvedValue([
|
||||
{ model_group: "ResponsesModel", mode: "responses" },
|
||||
]);
|
||||
|
||||
render(
|
||||
<ChatUI
|
||||
accessToken="1234567890"
|
||||
token="1234567890"
|
||||
userRole="user"
|
||||
userID="1234567890"
|
||||
disabledPersonalKeyCreation={false}
|
||||
/>,
|
||||
);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText("Test Key")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
const endpointTypeText = screen.getByText("Endpoint Type");
|
||||
const endpointSelect = endpointTypeText.parentElement?.querySelector(".ant-select-selector");
|
||||
await act(async () => {
|
||||
fireEvent.mouseDown(endpointSelect!);
|
||||
});
|
||||
await act(async () => {
|
||||
fireEvent.click(screen.getByText("/v1/responses"));
|
||||
});
|
||||
|
||||
const selectModelLabel = screen.getByText("Select Model");
|
||||
const modelSelect = selectModelLabel.closest("div")?.querySelector(".ant-select-selector");
|
||||
await act(async () => {
|
||||
fireEvent.mouseDown(modelSelect!);
|
||||
});
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getAllByText("ResponsesModel").length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
const modelOptions = screen.getAllByText("ResponsesModel");
|
||||
await act(async () => {
|
||||
fireEvent.click(modelOptions[modelOptions.length - 1]);
|
||||
});
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByTestId("model-settings-button")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
await act(async () => {
|
||||
fireEvent.click(screen.getByTestId("model-settings-button"));
|
||||
});
|
||||
|
||||
expect(await screen.findByRole("checkbox", { name: /Stream responses/i })).toBeChecked();
|
||||
expect(screen.queryByText("Temperature")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("Use Advanced Parameters")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("should show Fill button and populate customProxyBaseUrl when proxySettings.LITELLM_UI_API_DOC_BASE_URL is provided", async () => {
|
||||
const testProxyUrl = "http://localhost:5000";
|
||||
|
||||
|
|
|
|||
|
|
@ -261,6 +261,11 @@ const ChatUI: React.FC<ChatUIProps> = ({
|
|||
const [maxTokens, setMaxTokens] = useState<number>(2048);
|
||||
const [useAdvancedParams, setUseAdvancedParams] = useState<boolean>(false);
|
||||
const [mockTestFallbacks, setMockTestFallbacks] = useState<boolean>(false);
|
||||
const [streamingEnabled, setStreamingEnabled] = useState<boolean>(() => {
|
||||
if (simplified) return true;
|
||||
const saved = sessionStorage.getItem("streamingEnabled");
|
||||
return saved === null ? true : saved === "true";
|
||||
});
|
||||
|
||||
// Code Interpreter state (using custom hook)
|
||||
const codeInterpreter = useCodeInterpreter();
|
||||
|
|
@ -372,6 +377,7 @@ const ChatUI: React.FC<ChatUIProps> = ({
|
|||
sessionStorage.removeItem("selectedMCPTools"); // Clean up old key
|
||||
|
||||
if (!simplified) {
|
||||
sessionStorage.setItem("streamingEnabled", JSON.stringify(streamingEnabled));
|
||||
if (selectedModel) {
|
||||
sessionStorage.setItem("selectedModel", selectedModel);
|
||||
} else {
|
||||
|
|
@ -392,6 +398,7 @@ const ChatUI: React.FC<ChatUIProps> = ({
|
|||
selectedMCPServers,
|
||||
mcpServerToolRestrictions,
|
||||
selectedVoice,
|
||||
streamingEnabled,
|
||||
]);
|
||||
|
||||
useEffect(() => {
|
||||
|
|
@ -771,6 +778,7 @@ const ChatUI: React.FC<ChatUIProps> = ({
|
|||
handleMCPEvent,
|
||||
mockTestFallbacks,
|
||||
mcpToolsets,
|
||||
streamingEnabled,
|
||||
);
|
||||
} else if (endpointType === EndpointType.IMAGE) {
|
||||
// For image generation
|
||||
|
|
@ -852,6 +860,8 @@ const ChatUI: React.FC<ChatUIProps> = ({
|
|||
mcpServers,
|
||||
mcpServerToolRestrictions,
|
||||
mcpToolsets,
|
||||
streamingEnabled,
|
||||
updateTotalLatency,
|
||||
);
|
||||
} else if (endpointType === EndpointType.ANTHROPIC_MESSAGES) {
|
||||
const apiChatHistory = [
|
||||
|
|
@ -1035,6 +1045,8 @@ const ChatUI: React.FC<ChatUIProps> = ({
|
|||
return !model.mode || model.mode === "chat";
|
||||
};
|
||||
|
||||
const supportsStreamingToggle = endpointType === EndpointType.CHAT || endpointType === EndpointType.RESPONSES;
|
||||
|
||||
const antIcon = <LoadingOutlined style={{ fontSize: 24 }} spin />;
|
||||
|
||||
return (
|
||||
|
|
@ -1184,10 +1196,11 @@ const ChatUI: React.FC<ChatUIProps> = ({
|
|||
<span className="flex items-center">
|
||||
<RobotOutlined className="mr-2" /> Select Model
|
||||
</span>
|
||||
{isChatModel() ? (
|
||||
{isChatModel() || supportsStreamingToggle ? (
|
||||
<Popover
|
||||
content={
|
||||
<AdditionalModelSettings
|
||||
showAdvancedParams={isChatModel()}
|
||||
temperature={temperature}
|
||||
maxTokens={maxTokens}
|
||||
useAdvancedParams={useAdvancedParams}
|
||||
|
|
@ -1196,6 +1209,8 @@ const ChatUI: React.FC<ChatUIProps> = ({
|
|||
onUseAdvancedParamsChange={setUseAdvancedParams}
|
||||
mockTestFallbacks={mockTestFallbacks}
|
||||
onMockTestFallbacksChange={setMockTestFallbacks}
|
||||
streamingEnabled={streamingEnabled}
|
||||
onStreamingChange={supportsStreamingToggle ? setStreamingEnabled : undefined}
|
||||
/>
|
||||
}
|
||||
title="Model Settings"
|
||||
|
|
|
|||
|
|
@ -224,6 +224,143 @@ describe("chat_completion", () => {
|
|||
expect(callArgs.mock_testing_fallbacks).toBe(true);
|
||||
});
|
||||
|
||||
it("should send a non-streaming request and render the whole message at once when streaming is disabled", async () => {
|
||||
mockCreate.mockResolvedValueOnce({
|
||||
id: "chatcmpl-1",
|
||||
object: "chat.completion",
|
||||
created: 1,
|
||||
model: "gpt-4",
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
finish_reason: "stop",
|
||||
message: { role: "assistant", content: "Hello there" },
|
||||
},
|
||||
],
|
||||
usage: {
|
||||
completion_tokens: 2,
|
||||
prompt_tokens: 5,
|
||||
total_tokens: 7,
|
||||
cost: 0.25,
|
||||
},
|
||||
});
|
||||
|
||||
const onTimingData = vi.fn();
|
||||
const onUsageData = vi.fn();
|
||||
const onTotalLatency = vi.fn();
|
||||
|
||||
await makeOpenAIChatCompletionRequest(
|
||||
mockChatHistory,
|
||||
mockUpdateUI,
|
||||
"gpt-4",
|
||||
"test-token",
|
||||
undefined, // tags
|
||||
undefined, // signal
|
||||
undefined, // onReasoningContent
|
||||
onTimingData,
|
||||
onUsageData,
|
||||
undefined, // traceId
|
||||
undefined, // vector_store_ids
|
||||
undefined, // guardrails
|
||||
undefined, // policies
|
||||
undefined, // selectedMCPServers
|
||||
undefined, // onImageGenerated
|
||||
undefined, // onSearchResults
|
||||
undefined, // temperature
|
||||
undefined, // max_tokens
|
||||
onTotalLatency,
|
||||
undefined, // customBaseUrl
|
||||
undefined, // mcpServers
|
||||
undefined, // mcpServerToolRestrictions
|
||||
undefined, // onMCPEvent
|
||||
undefined, // mockTestFallbacks
|
||||
undefined, // mcpToolsets
|
||||
false, // streamingEnabled
|
||||
);
|
||||
|
||||
expect(mockCreate).toHaveBeenCalledTimes(1);
|
||||
const callArgs = mockCreate.mock.calls[0][0];
|
||||
expect(callArgs.stream).toBe(false);
|
||||
expect(callArgs).not.toHaveProperty("stream_options");
|
||||
|
||||
expect(mockUpdateUI).toHaveBeenCalledTimes(1);
|
||||
expect(mockUpdateUI).toHaveBeenCalledWith("Hello there", "gpt-4");
|
||||
|
||||
expect(onUsageData).toHaveBeenCalledWith({
|
||||
completionTokens: 2,
|
||||
promptTokens: 5,
|
||||
totalTokens: 7,
|
||||
cost: 0.25,
|
||||
});
|
||||
expect(onTimingData).not.toHaveBeenCalled();
|
||||
expect(onTotalLatency).toHaveBeenCalledWith(expect.any(Number));
|
||||
});
|
||||
|
||||
it("should surface reasoning content and MCP metadata from a non-streaming response", async () => {
|
||||
mockCreate.mockResolvedValueOnce({
|
||||
model: "gpt-4",
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
finish_reason: "stop",
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: "done",
|
||||
reasoning_content: "thinking",
|
||||
provider_specific_fields: {
|
||||
mcp_tool_calls: [{ id: "call_1", function: { name: "search_docs", arguments: "{}" } }],
|
||||
mcp_call_results: [{ tool_call_id: "call_1", result: "found it" }],
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const onReasoningContent = vi.fn();
|
||||
const onMCPEvent = vi.fn();
|
||||
|
||||
await makeOpenAIChatCompletionRequest(
|
||||
mockChatHistory,
|
||||
mockUpdateUI,
|
||||
"gpt-4",
|
||||
"test-token",
|
||||
undefined, // tags
|
||||
undefined, // signal
|
||||
onReasoningContent,
|
||||
undefined, // onTimingData
|
||||
undefined, // onUsageData
|
||||
undefined, // traceId
|
||||
undefined, // vector_store_ids
|
||||
undefined, // guardrails
|
||||
undefined, // policies
|
||||
undefined, // selectedMCPServers
|
||||
undefined, // onImageGenerated
|
||||
undefined, // onSearchResults
|
||||
undefined, // temperature
|
||||
undefined, // max_tokens
|
||||
undefined, // onTotalLatency
|
||||
undefined, // customBaseUrl
|
||||
undefined, // mcpServers
|
||||
undefined, // mcpServerToolRestrictions
|
||||
onMCPEvent,
|
||||
undefined, // mockTestFallbacks
|
||||
undefined, // mcpToolsets
|
||||
false, // streamingEnabled
|
||||
);
|
||||
|
||||
expect(onReasoningContent).toHaveBeenCalledWith("thinking");
|
||||
expect(onMCPEvent).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
type: "response.output_item.done",
|
||||
item: expect.objectContaining({
|
||||
type: "mcp_call",
|
||||
name: "search_docs",
|
||||
output: "found it",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("should not include mock_testing_fallbacks in request body when mockTestFallbacks is false or undefined", async () => {
|
||||
await makeOpenAIChatCompletionRequest(
|
||||
mockChatHistory,
|
||||
|
|
|
|||
|
|
@ -1,10 +1,26 @@
|
|||
import openai from "openai";
|
||||
import { ChatCompletionMessageParam } from "openai/resources/chat/completions";
|
||||
import { ChatCompletion, ChatCompletionChunk, ChatCompletionMessageParam } from "openai/resources/chat/completions";
|
||||
import { TokenUsage } from "../chat_ui/ResponseMetrics";
|
||||
import { VectorStoreSearchResponse } from "../chat_ui/types";
|
||||
import { getProxyBaseUrl } from "@/components/networking";
|
||||
import { MCPServer, MCPToolset, type MCPEvent } from "@/components/mcp_tools/types";
|
||||
|
||||
const completionAsSingleChunk = (completion: ChatCompletion): ChatCompletionChunk =>
|
||||
({
|
||||
id: completion.id,
|
||||
object: "chat.completion.chunk",
|
||||
created: completion.created,
|
||||
model: completion.model,
|
||||
usage: completion.usage,
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
finish_reason: completion.choices[0]?.finish_reason ?? null,
|
||||
delta: completion.choices[0]?.message ?? {},
|
||||
},
|
||||
],
|
||||
}) as unknown as ChatCompletionChunk;
|
||||
|
||||
export async function makeOpenAIChatCompletionRequest(
|
||||
chatHistory: { role: string; content: string | any[] }[],
|
||||
updateUI: (chunk: string, model?: string) => void,
|
||||
|
|
@ -31,6 +47,7 @@ export async function makeOpenAIChatCompletionRequest(
|
|||
onMCPEvent?: (event: MCPEvent) => void,
|
||||
mockTestFallbacks?: boolean,
|
||||
mcpToolsets?: MCPToolset[],
|
||||
streamingEnabled: boolean = true,
|
||||
) {
|
||||
// base url should be the current base_url
|
||||
const isLocal = process.env.NODE_ENV === "development";
|
||||
|
|
@ -111,26 +128,25 @@ export async function makeOpenAIChatCompletionRequest(
|
|||
}
|
||||
}
|
||||
|
||||
// @ts-ignore
|
||||
const response = await client.chat.completions.create(
|
||||
{
|
||||
model: selectedModel,
|
||||
stream: true,
|
||||
stream_options: {
|
||||
include_usage: true,
|
||||
},
|
||||
litellm_trace_id: traceId,
|
||||
messages: chatHistory as ChatCompletionMessageParam[],
|
||||
...(vector_store_ids ? { vector_store_ids } : {}),
|
||||
...(guardrails ? { guardrails } : {}),
|
||||
...(policies ? { policies } : {}),
|
||||
...(tools.length > 0 ? { tools, tool_choice: "auto" } : {}),
|
||||
...(temperature !== undefined ? { temperature } : {}),
|
||||
...(max_tokens !== undefined ? { max_tokens } : {}),
|
||||
...(mockTestFallbacks ? { mock_testing_fallbacks: true } : {}),
|
||||
},
|
||||
{ signal },
|
||||
);
|
||||
const requestBody = {
|
||||
model: selectedModel,
|
||||
litellm_trace_id: traceId,
|
||||
messages: chatHistory as ChatCompletionMessageParam[],
|
||||
...(vector_store_ids ? { vector_store_ids } : {}),
|
||||
...(guardrails ? { guardrails } : {}),
|
||||
...(policies ? { policies } : {}),
|
||||
...(tools.length > 0 ? { tools, tool_choice: "auto" as const } : {}),
|
||||
...(temperature !== undefined ? { temperature } : {}),
|
||||
...(max_tokens !== undefined ? { max_tokens } : {}),
|
||||
...(mockTestFallbacks ? { mock_testing_fallbacks: true } : {}),
|
||||
};
|
||||
|
||||
const response: AsyncIterable<ChatCompletionChunk> | ChatCompletionChunk[] = streamingEnabled
|
||||
? await client.chat.completions.create(
|
||||
{ ...requestBody, stream: true, stream_options: { include_usage: true } },
|
||||
{ signal },
|
||||
)
|
||||
: [completionAsSingleChunk(await client.chat.completions.create({ ...requestBody, stream: false }, { signal }))];
|
||||
|
||||
for await (const chunk of response) {
|
||||
// Process content and measure time to first token
|
||||
|
|
@ -142,7 +158,7 @@ export async function makeOpenAIChatCompletionRequest(
|
|||
if (!firstTokenReceived && (chunk.choices[0]?.delta?.content || (delta && delta.reasoning_content))) {
|
||||
firstTokenReceived = true;
|
||||
timeToFirstToken = Date.now() - startTime;
|
||||
if (onTimingData) {
|
||||
if (onTimingData && streamingEnabled) {
|
||||
onTimingData(timeToFirstToken);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -69,6 +69,158 @@ describe("responses_api", () => {
|
|||
expect(mockUpdateTextUI).toHaveBeenCalledWith("assistant", "Hi", "gpt-4");
|
||||
});
|
||||
|
||||
it("should send a non-streaming request and render the whole output at once when streaming is disabled", async () => {
|
||||
mockResponsesCreate.mockResolvedValueOnce({
|
||||
id: "resp_456",
|
||||
output: [
|
||||
{
|
||||
type: "message",
|
||||
content: [
|
||||
{ type: "output_text", text: "Full " },
|
||||
{ type: "output_text", text: "answer" },
|
||||
],
|
||||
},
|
||||
],
|
||||
usage: { output_tokens: 3, input_tokens: 4, total_tokens: 7 },
|
||||
});
|
||||
|
||||
const onTimingData = vi.fn();
|
||||
const onUsageData = vi.fn();
|
||||
const onResponseId = vi.fn();
|
||||
|
||||
await makeOpenAIResponsesRequest(
|
||||
messages,
|
||||
mockUpdateTextUI,
|
||||
"gpt-4",
|
||||
"test-token",
|
||||
undefined, // tags
|
||||
undefined, // signal
|
||||
undefined, // onReasoningContent
|
||||
onTimingData,
|
||||
onUsageData,
|
||||
undefined, // traceId
|
||||
undefined, // vector_store_ids
|
||||
undefined, // guardrails
|
||||
undefined, // policies
|
||||
undefined, // selectedMCPServers
|
||||
undefined, // previousResponseId
|
||||
onResponseId,
|
||||
undefined, // onMCPEvent
|
||||
undefined, // codeInterpreterEnabled
|
||||
undefined, // onCodeInterpreterResult
|
||||
undefined, // customBaseUrl
|
||||
undefined, // mcpServers
|
||||
undefined, // mcpServerToolRestrictions
|
||||
undefined, // mcpToolsets
|
||||
false, // streamingEnabled
|
||||
);
|
||||
|
||||
expect(mockResponsesCreate).toHaveBeenCalledTimes(1);
|
||||
expect(mockResponsesCreate.mock.calls[0][0].stream).toBe(false);
|
||||
|
||||
expect(mockUpdateTextUI).toHaveBeenCalledTimes(1);
|
||||
expect(mockUpdateTextUI).toHaveBeenCalledWith("assistant", "Full answer", "gpt-4");
|
||||
|
||||
expect(onUsageData).toHaveBeenCalledWith({ completionTokens: 3, promptTokens: 4, totalTokens: 7 }, "");
|
||||
expect(onResponseId).toHaveBeenCalledWith("resp_456");
|
||||
expect(onTimingData).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("should report total latency in both streaming and non-streaming modes", async () => {
|
||||
const onTotalLatency = vi.fn();
|
||||
const callWithStreaming = (streamingEnabled: boolean) =>
|
||||
makeOpenAIResponsesRequest(
|
||||
messages,
|
||||
mockUpdateTextUI,
|
||||
"gpt-4",
|
||||
"test-token",
|
||||
undefined, // tags
|
||||
undefined, // signal
|
||||
undefined, // onReasoningContent
|
||||
undefined, // onTimingData
|
||||
undefined, // onUsageData
|
||||
undefined, // traceId
|
||||
undefined, // vector_store_ids
|
||||
undefined, // guardrails
|
||||
undefined, // policies
|
||||
undefined, // selectedMCPServers
|
||||
undefined, // previousResponseId
|
||||
undefined, // onResponseId
|
||||
undefined, // onMCPEvent
|
||||
undefined, // codeInterpreterEnabled
|
||||
undefined, // onCodeInterpreterResult
|
||||
undefined, // customBaseUrl
|
||||
undefined, // mcpServers
|
||||
undefined, // mcpServerToolRestrictions
|
||||
undefined, // mcpToolsets
|
||||
streamingEnabled,
|
||||
onTotalLatency,
|
||||
);
|
||||
|
||||
await callWithStreaming(true);
|
||||
expect(onTotalLatency).toHaveBeenCalledTimes(1);
|
||||
expect(onTotalLatency).toHaveBeenLastCalledWith(expect.any(Number));
|
||||
|
||||
mockResponsesCreate.mockResolvedValueOnce({
|
||||
id: "resp_latency",
|
||||
output: [{ type: "message", content: [{ type: "output_text", text: "Answer" }] }],
|
||||
});
|
||||
|
||||
await callWithStreaming(false);
|
||||
expect(onTotalLatency).toHaveBeenCalledTimes(2);
|
||||
expect(onTotalLatency).toHaveBeenLastCalledWith(expect.any(Number));
|
||||
});
|
||||
|
||||
it("should replay MCP output items as events for a non-streaming response", async () => {
|
||||
mockResponsesCreate.mockResolvedValueOnce({
|
||||
id: "resp_789",
|
||||
output: [
|
||||
{ type: "mcp_call", id: "mcp_1", name: "search_docs", arguments: "{}", output: "found it" },
|
||||
{ type: "message", content: [{ type: "output_text", text: "Answer" }] },
|
||||
],
|
||||
usage: { output_tokens: 1, input_tokens: 1, total_tokens: 2 },
|
||||
});
|
||||
|
||||
const onMCPEvent = vi.fn();
|
||||
const onUsageData = vi.fn();
|
||||
|
||||
await makeOpenAIResponsesRequest(
|
||||
messages,
|
||||
mockUpdateTextUI,
|
||||
"gpt-4",
|
||||
"test-token",
|
||||
undefined, // tags
|
||||
undefined, // signal
|
||||
undefined, // onReasoningContent
|
||||
undefined, // onTimingData
|
||||
onUsageData,
|
||||
undefined, // traceId
|
||||
undefined, // vector_store_ids
|
||||
undefined, // guardrails
|
||||
undefined, // policies
|
||||
undefined, // selectedMCPServers
|
||||
undefined, // previousResponseId
|
||||
undefined, // onResponseId
|
||||
onMCPEvent,
|
||||
undefined, // codeInterpreterEnabled
|
||||
undefined, // onCodeInterpreterResult
|
||||
undefined, // customBaseUrl
|
||||
undefined, // mcpServers
|
||||
undefined, // mcpServerToolRestrictions
|
||||
undefined, // mcpToolsets
|
||||
false, // streamingEnabled
|
||||
);
|
||||
|
||||
expect(onMCPEvent).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
type: "response.output_item.done",
|
||||
item_id: "mcp_1",
|
||||
item: expect.objectContaining({ type: "mcp_call", name: "search_docs", output: "found it" }),
|
||||
}),
|
||||
);
|
||||
expect(onUsageData).toHaveBeenCalledWith(expect.anything(), "search_docs");
|
||||
});
|
||||
|
||||
it("should configure MCP tools per server with restrictions", async () => {
|
||||
const selectedMCPServers = ["server-1", "server-2"];
|
||||
const mcpServers = [
|
||||
|
|
|
|||
|
|
@ -14,6 +14,49 @@ import {
|
|||
|
||||
export type { CodeInterpreterResult } from "./code_interpreter_handler";
|
||||
|
||||
interface ResponseOutputPart {
|
||||
type?: string;
|
||||
text?: string;
|
||||
}
|
||||
|
||||
interface ResponseOutputItem {
|
||||
type?: string;
|
||||
content?: ResponseOutputPart[];
|
||||
summary?: ResponseOutputPart[];
|
||||
}
|
||||
|
||||
interface NonStreamedResponse {
|
||||
output?: ResponseOutputItem[];
|
||||
}
|
||||
|
||||
type SynthesizedResponseEvent =
|
||||
| { type: "response.output_item.done"; item: ResponseOutputItem }
|
||||
| { type: "response.reasoning.delta"; delta: string }
|
||||
| { type: "response.output_text.delta"; delta: string }
|
||||
| { type: "response.completed"; response: NonStreamedResponse };
|
||||
|
||||
const responseAsEvents = (response: NonStreamedResponse): SynthesizedResponseEvent[] => {
|
||||
const outputItems = response.output ?? [];
|
||||
const outputText = outputItems
|
||||
.filter((item) => item.type === "message")
|
||||
.flatMap((item) => item.content ?? [])
|
||||
.filter((part) => part.type === "output_text")
|
||||
.map((part) => part.text ?? "")
|
||||
.join("");
|
||||
const reasoningText = outputItems
|
||||
.filter((item) => item.type === "reasoning")
|
||||
.flatMap((item) => item.summary ?? [])
|
||||
.map((part) => part.text ?? "")
|
||||
.join("");
|
||||
|
||||
return [
|
||||
...outputItems.map((item) => ({ type: "response.output_item.done" as const, item })),
|
||||
...(reasoningText ? [{ type: "response.reasoning.delta" as const, delta: reasoningText }] : []),
|
||||
...(outputText ? [{ type: "response.output_text.delta" as const, delta: outputText }] : []),
|
||||
{ type: "response.completed" as const, response },
|
||||
];
|
||||
};
|
||||
|
||||
export async function makeOpenAIResponsesRequest(
|
||||
messages: MessageType[],
|
||||
updateTextUI: (role: string, delta: string, model?: string) => void,
|
||||
|
|
@ -38,6 +81,8 @@ export async function makeOpenAIResponsesRequest(
|
|||
mcpServers?: MCPServer[],
|
||||
mcpServerToolRestrictions?: Record<string, string[]>,
|
||||
mcpToolsets?: MCPToolset[],
|
||||
streamingEnabled: boolean = true,
|
||||
onTotalLatency?: (latency: number) => void,
|
||||
) {
|
||||
if (!accessToken) {
|
||||
throw new Error("Virtual Key is required");
|
||||
|
|
@ -143,27 +188,26 @@ export async function makeOpenAIResponsesRequest(
|
|||
});
|
||||
}
|
||||
|
||||
const requestBody = {
|
||||
model: selectedModel,
|
||||
input: formattedInput,
|
||||
litellm_trace_id: traceId,
|
||||
...(previousResponseId ? { previous_response_id: previousResponseId } : {}),
|
||||
...(vector_store_ids ? { vector_store_ids } : {}),
|
||||
...(guardrails ? { guardrails } : {}),
|
||||
...(policies ? { policies } : {}),
|
||||
...(tools.length > 0 ? { tools, tool_choice: "auto" } : {}),
|
||||
};
|
||||
|
||||
// Create request to OpenAI responses API
|
||||
// Use 'any' type to avoid TypeScript issues with the experimental API
|
||||
const response = await (client as any).responses.create(
|
||||
{
|
||||
model: selectedModel,
|
||||
input: formattedInput,
|
||||
stream: true,
|
||||
litellm_trace_id: traceId,
|
||||
...(previousResponseId ? { previous_response_id: previousResponseId } : {}),
|
||||
...(vector_store_ids ? { vector_store_ids } : {}),
|
||||
...(guardrails ? { guardrails } : {}),
|
||||
...(policies ? { policies } : {}),
|
||||
...(tools.length > 0 ? { tools, tool_choice: "auto" } : {}),
|
||||
},
|
||||
{ signal },
|
||||
);
|
||||
const response = await (client as any).responses.create({ ...requestBody, stream: streamingEnabled }, { signal });
|
||||
const events = streamingEnabled ? response : responseAsEvents(response);
|
||||
|
||||
let mcpToolUsed = "";
|
||||
let codeInterpreterState: CodeInterpreterState = { code: "", containerId: "" };
|
||||
|
||||
for await (const event of response) {
|
||||
for await (const event of events) {
|
||||
// Use a type-safe approach to handle events
|
||||
if (typeof event === "object" && event !== null) {
|
||||
// Handle MCP events first
|
||||
|
|
@ -215,7 +259,7 @@ export async function makeOpenAIResponsesRequest(
|
|||
firstTokenReceived = true;
|
||||
const timeToFirstToken = Date.now() - startTime;
|
||||
|
||||
if (onTimingData) {
|
||||
if (onTimingData && streamingEnabled) {
|
||||
onTimingData(timeToFirstToken);
|
||||
}
|
||||
}
|
||||
|
|
@ -259,6 +303,10 @@ export async function makeOpenAIResponsesRequest(
|
|||
}
|
||||
}
|
||||
|
||||
if (onTotalLatency) {
|
||||
onTotalLatency(Date.now() - startTime);
|
||||
}
|
||||
|
||||
return response;
|
||||
} catch (error) {
|
||||
if (signal?.aborted) {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue