From 9056ff27b74d9323a656a192a2d5e08812266cab Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Thu, 6 Aug 2026 15:06:44 -0700 Subject: [PATCH] feat(ui): restyle realtime playground with shared chat composer Move RealtimePlayground off Ant Design onto shadcn controls and the shared ChatComposer, use the realtime-safe voice list, and reuse the same composer for Compare message input --- .../components/chat_ui/RealtimePlayground.tsx | 236 +++++++++--------- .../components/chat_ui/chatConstants.ts | 33 +++ .../compareUI/components/MessageInput.tsx | 47 +--- 3 files changed, 166 insertions(+), 150 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.tsx index 2bf645fced2..8ed967c6959 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.tsx @@ -1,12 +1,14 @@ "use client"; -import { AudioMutedOutlined, AudioOutlined, CloseCircleOutlined, SendOutlined, SoundOutlined } from "@ant-design/icons"; -import { Button, Input, Select, Typography } from "antd"; +import { Loader2, Mic, MicOff, Phone, PhoneOff, Volume2 } from "lucide-react"; import React, { useCallback, useEffect, useRef, useState } from "react"; import { getProxyBaseUrl } from "@/components/networking"; -import { OPEN_AI_VOICE_SELECT_OPTIONS } from "./chatConstants"; - -const { Text } = Typography; +import { Button } from "@/components/ui/button"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { cn } from "@/lib/cva.config"; +import ChatComposer from "./ChatComposer"; +import { OPEN_AI_REALTIME_VOICE_SELECT_OPTIONS, type OpenAIRealtimeVoice } from "./chatConstants"; interface RealtimeMessage { role: "user" | "assistant" | "system" | "status"; @@ -32,13 +34,14 @@ const RealtimePlayground: React.FC = ({ const [isConnected, setIsConnected] = useState(false); const [isConnecting, setIsConnecting] = useState(false); const [isRecording, setIsRecording] = useState(false); - const [selectedVoice, setSelectedVoice] = useState("alloy"); + const [selectedVoice, setSelectedVoice] = useState("alloy"); const wsRef = useRef(null); const audioContextRef = useRef(null); const mediaStreamRef = useRef(null); const processorRef = useRef(null); const messagesEndRef = useRef(null); const nextPlayTimeRef = useRef(0); + const configureSessionRef = useRef(false); const scrollToBottom = useCallback(() => { messagesEndRef.current?.scrollIntoView({ behavior: "smooth" }); @@ -85,6 +88,14 @@ const RealtimePlayground: React.FC = ({ nextPlayTimeRef.current = startTime + buffer.duration; }, []); + const stopRecording = useCallback(() => { + processorRef.current?.disconnect(); + processorRef.current = null; + mediaStreamRef.current?.getTracks().forEach((t) => t.stop()); + mediaStreamRef.current = null; + setIsRecording(false); + }, []); + const connect = useCallback(async () => { if (wsRef.current) return; if (!selectedModel) { @@ -123,7 +134,6 @@ const RealtimePlayground: React.FC = ({ const type = data.type; if (type === "session.created") { - // GA: session.type is required ("realtime" | "transcription") ws.send( JSON.stringify({ type: "session.update", @@ -138,17 +148,9 @@ const RealtimePlayground: React.FC = ({ }, }), ); - } else if (type === "session.updated") { - // session configured - } else if ( - // GA: response.output_audio.delta | beta: response.audio.delta - type === "response.output_audio.delta" || - type === "response.audio.delta" - ) { + } else if (type === "response.output_audio.delta" || type === "response.audio.delta") { if (data.delta) playAudioChunk(data.delta); } else if ( - // GA: response.output_text.delta / response.output_audio_transcript.delta - // beta: response.text.delta / response.audio_transcript.delta type === "response.output_text.delta" || type === "response.output_audio_transcript.delta" || type === "response.audio_transcript.delta" || @@ -158,8 +160,6 @@ const RealtimePlayground: React.FC = ({ } else if (type === "conversation.item.input_audio_transcription.completed") { if (data.transcript) addMessage("user", data.transcript); } else if (type === "response.done") { - // Ensure we have the full text if deltas were missed. - // Accept both beta (type=text/audio) and GA (type=output_text/output_audio) content. setMessages((prev) => { const last = prev[prev.length - 1]; if (last && last.role === "assistant" && last.content) return prev; @@ -167,8 +167,6 @@ const RealtimePlayground: React.FC = ({ const texts: string[] = []; for (const item of output) { for (const c of item.content || []) { - // beta: c.text (type=text), c.transcript (type=audio) - // GA: c.text (type=output_text), c.transcript (type=output_audio) const t = c.text || c.transcript; if (t) texts.push(t); } @@ -200,8 +198,9 @@ const RealtimePlayground: React.FC = ({ }; wsRef.current = ws; - } catch (err: any) { - addMessage("status", `Connection failed: ${err.message}`); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : "Unknown error"; + addMessage("status", `Connection failed: ${message}`); setIsConnecting(false); } }, [ @@ -224,13 +223,11 @@ const RealtimePlayground: React.FC = ({ nextPlayTimeRef.current = 0; configureSessionRef.current = false; setIsConnected(false); - }, []); + }, [stopRecording]); const startRecording = useCallback(async () => { if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) return; - // Switch to server VAD mode for voice input - // GA: session.type is required wsRef.current.send( JSON.stringify({ type: "session.update", @@ -261,7 +258,6 @@ const RealtimePlayground: React.FC = ({ if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) return; const input = e.inputBuffer.getChannelData(0); - // Resample to 24kHz if needed const sampleRate = ctx.sampleRate; const targetRate = 24000; let samples: Float32Array; @@ -276,46 +272,34 @@ const RealtimePlayground: React.FC = ({ samples = input; } - // Convert to PCM16 const pcm16 = new Int16Array(samples.length); for (let i = 0; i < samples.length; i++) { const s = Math.max(-1, Math.min(1, samples[i])); pcm16[i] = s < 0 ? s * 0x8000 : s * 0x7fff; } - // Base64 encode and send const bytes = new Uint8Array(pcm16.buffer); let binary = ""; for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]); const b64 = btoa(binary); - wsRef.current!.send(JSON.stringify({ type: "input_audio_buffer.append", audio: b64 })); + wsRef.current.send(JSON.stringify({ type: "input_audio_buffer.append", audio: b64 })); }; source.connect(processor); processor.connect(ctx.destination); setIsRecording(true); - addMessage("status", "๐ŸŽ™๏ธ Listening..."); - } catch (err: any) { - addMessage("status", `Microphone error: ${err.message}`); + addMessage("status", "Listening..."); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : "Unknown error"; + addMessage("status", `Microphone error: ${message}`); } - }, [addMessage]); - - const stopRecording = useCallback(() => { - processorRef.current?.disconnect(); - processorRef.current = null; - mediaStreamRef.current?.getTracks().forEach((t) => t.stop()); - mediaStreamRef.current = null; - setIsRecording(false); - }, []); - - const configureSessionRef = useRef(false); + }, [addMessage, selectedVoice]); const ensureTextSession = useCallback(() => { if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) return; if (configureSessionRef.current) return; configureSessionRef.current = true; - // GA: session.type is required wsRef.current.send( JSON.stringify({ type: "session.update", @@ -334,6 +318,7 @@ const RealtimePlayground: React.FC = ({ const sendTextMessage = useCallback(() => { if (!inputText.trim() || !wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) return; + ensureTextSession(); const text = inputText.trim(); addMessage("user", text); setInputText(""); @@ -359,68 +344,89 @@ const RealtimePlayground: React.FC = ({ }; }, []); + const voiceLabel = + OPEN_AI_REALTIME_VOICE_SELECT_OPTIONS.find((voice) => voice.value === selectedVoice)?.label ?? selectedVoice; + return ( -
- {/* Header */} -
-
- - Realtime Voice Chat - - - {isConnected ? "Connected" : isConnecting ? "Connecting..." : "Disconnected"} - +
+
+
+
-
+
{!isConnected ? ( - ) : ( - )}
- {/* Messages */} -
+
{messages.length === 0 && !isConnected && ( -
- - Realtime Voice Playground - - Click Connect to start a realtime session. You can speak using your microphone or type messages. - The AI will respond with voice and text. - +
+
)} {messages.map((msg, i) => (
{msg.role === "status" ? ( -
{msg.content}
+
{msg.content}
) : (
-
{msg.role === "user" ? "You" : "AI"}
-
{msg.content}
+
{msg.role === "user" ? "You" : "AI"}
+
{msg.content}
)}
@@ -428,42 +434,46 @@ const RealtimePlayground: React.FC = ({
- {/* Input area */} {isConnected && ( -
-
-
+
{isRecording && ( -
- - Listening โ€” speak into your microphone. Server VAD will detect when you stop. +
+
)} + + { + if (isRecording) { + stopRecording(); + } else { + void startRecording(); + } + }} + /> + } + > + {isRecording ? : } + + {isRecording ? "Stop recording" : "Start recording"} + + } + />
)}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/chatConstants.ts b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/chatConstants.ts index 2b59fbad2ee..7cb6789a46d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/chatConstants.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/chatConstants.ts @@ -33,6 +33,39 @@ export const OPEN_AI_VOICE_SELECT_OPTIONS = Object.entries(OPEN_AI_VOICES).map(( label: OPEN_AI_VOICE_LABELS[key as keyof typeof OPEN_AI_VOICE_LABELS], })); +export const OPEN_AI_REALTIME_VOICES = { + ALLOY: "alloy", + ASH: "ash", + BALLAD: "ballad", + CORAL: "coral", + ECHO: "echo", + SAGE: "sage", + SHIMMER: "shimmer", + VERSE: "verse", + MARIN: "marin", + CEDAR: "cedar", +} as const; + +export type OpenAIRealtimeVoice = (typeof OPEN_AI_REALTIME_VOICES)[keyof typeof OPEN_AI_REALTIME_VOICES]; + +export const OPEN_AI_REALTIME_VOICE_LABELS: Record = { + ALLOY: "Alloy - Professional and confident", + ASH: "Ash - Casual and relaxed", + BALLAD: "Ballad - Smooth and melodic", + CORAL: "Coral - Warm and engaging", + ECHO: "Echo - Friendly and conversational", + SAGE: "Sage - Wise and measured", + SHIMMER: "Shimmer - Bright and cheerful", + VERSE: "Verse - Expressive and clear", + MARIN: "Marin - Calm and natural", + CEDAR: "Cedar - Warm and steady", +}; + +export const OPEN_AI_REALTIME_VOICE_SELECT_OPTIONS = Object.entries(OPEN_AI_REALTIME_VOICES).map(([key, voice]) => ({ + value: voice, + label: OPEN_AI_REALTIME_VOICE_LABELS[key as keyof typeof OPEN_AI_REALTIME_VOICE_LABELS], +})); + export const ENDPOINT_OPTIONS = [ { value: EndpointType.CHAT, label: "/v1/chat/completions" }, { value: EndpointType.RESPONSES, label: "/v1/responses" }, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/compareUI/components/MessageInput.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/compareUI/components/MessageInput.tsx index fae964b372a..8c8d1937261 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/compareUI/components/MessageInput.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/compareUI/components/MessageInput.tsx @@ -1,8 +1,5 @@ import React from "react"; -import { Input, Button } from "antd"; -import { ArrowUpOutlined } from "@ant-design/icons"; - -const { TextArea } = Input; +import ChatComposer from "../../chat_ui/ChatComposer"; interface MessageInputProps { value: string; @@ -16,39 +13,15 @@ interface MessageInputProps { export function MessageInput({ value, onChange, onSend, disabled, hasAttachment, uploadComponent }: MessageInputProps) { const canSend = !disabled && (value.trim().length > 0 || Boolean(hasAttachment)); - const handleKeyDown = (e: React.KeyboardEvent) => { - if (e.key === "Enter" && !e.shiftKey) { - e.preventDefault(); - if (canSend) { - onSend(); - } - } - }; - return ( -
-
- {uploadComponent &&
{uploadComponent}
} -