diff --git a/src/lib/components/chat/MessageInput/CallOverlay.svelte b/src/lib/components/chat/MessageInput/CallOverlay.svelte index 55e197b01c..4406c3c1a2 100644 --- a/src/lib/components/chat/MessageInput/CallOverlay.svelte +++ b/src/lib/components/chat/MessageInput/CallOverlay.svelte @@ -12,7 +12,12 @@ import Tooltip from '$lib/components/common/Tooltip.svelte'; import VideoInputMenu from './CallOverlay/VideoInputMenu.svelte'; - import { hasLiveAudioTrack, singleFlight, watchAudioTrackEnd } from './CallOverlay/audio-stream'; + import { + createAsyncQueue, + hasLiveAudioTrack, + singleFlight, + watchAudioTrackEnd + } from './CallOverlay/audio-stream'; import { KokoroWorker } from '$lib/workers/KokoroWorker'; import { WEBUI_API_BASE_URL } from '$lib/constants'; @@ -48,6 +53,9 @@ let audioStatus: 'listening' | 'reconnecting' | 'unavailable' = 'reconnecting'; let stopWatchingAudioTrack = () => {}; let audioAnalysisGeneration = 0; + let callOverlayDestroyed = false; + let watchingAudioDevices = false; + const enqueueAudioTransition = createAsyncQueue(); let videoInputDevices = []; let selectedVideoInputDeviceId = null; @@ -234,8 +242,13 @@ } }; - const startRecording = async () => { - if ($showCallOverlay) { + const startRecordingUnlocked = async () => { + if ($showCallOverlay && !callOverlayDestroyed) { + if (mediaRecorder && hasLiveAudioTrack(audioStream)) { + audioStatus = 'listening'; + return true; + } + if (!hasLiveAudioTrack(audioStream)) { audioStatus = 'reconnecting'; @@ -248,14 +261,10 @@ } }); } catch (error) { - console.error('Error accessing media devices.', error); - audioStream = null; - audioStatus = 'unavailable'; - toast.error($i18n.t('Error accessing media devices.')); - return false; + return await handleAudioSetupError(error); } - if (!$showCallOverlay) { + if (!$showCallOverlay || callOverlayDestroyed) { audioStream.getAudioTracks().forEach((track) => track.stop()); audioStream = null; return false; @@ -268,36 +277,41 @@ } if (!hasLiveAudioTrack(audioStream)) { - audioStatus = 'unavailable'; - return false; + return await handleAudioSetupError(new Error('Acquired audio stream has no live track.')); } - mediaRecorder = new MediaRecorder(audioStream); - audioStatus = 'listening'; + try { + mediaRecorder = new MediaRecorder(audioStream); - mediaRecorder.onstart = () => { - console.log('Recording started'); - audioChunks = []; - }; + mediaRecorder.onstart = () => { + console.log('Recording started'); + audioChunks = []; + }; - mediaRecorder.ondataavailable = (event) => { - if (hasStartedSpeaking) { - audioChunks.push(event.data); - } - }; + mediaRecorder.ondataavailable = (event) => { + if (hasStartedSpeaking) { + audioChunks.push(event.data); + } + }; - mediaRecorder.onstop = (e) => { - console.log('Recording stopped', audioStream, e); - stopRecordingCallback(); - }; + mediaRecorder.onstop = (e) => { + console.log('Recording stopped', audioStream, e); + void stopRecordingCallback(); + }; - analyseAudio(audioStream, ++audioAnalysisGeneration); - return true; + analyseAudio(audioStream, ++audioAnalysisGeneration); + audioStatus = 'listening'; + return true; + } catch (error) { + return await handleAudioSetupError(error); + } } return false; }; + const startRecording = () => enqueueAudioTransition(startRecordingUnlocked); + const stopAudioStream = async () => { const recorder = mediaRecorder; mediaRecorder = false; @@ -324,19 +338,34 @@ audioStream = null; }; - const recoverAudioStream = singleFlight(async () => { - if (!$showCallOverlay) { + const handleAudioSetupError = async (error: unknown) => { + console.error('Error setting up media devices.', error); + await stopAudioStream(); + + if (!$showCallOverlay || callOverlayDestroyed) { return false; } - audioStatus = 'reconnecting'; - hasStartedSpeaking = false; - confirmed = false; - audioChunks = []; + audioStatus = 'unavailable'; + toast.error($i18n.t('Error accessing media devices.')); + return false; + }; - await stopAudioStream(); - return await startRecording(); - }); + const recoverAudioStream = singleFlight(() => + enqueueAudioTransition(async () => { + if (!$showCallOverlay || callOverlayDestroyed) { + return false; + } + + audioStatus = 'reconnecting'; + hasStartedSpeaking = false; + confirmed = false; + audioChunks = []; + + await stopAudioStream(); + return await startRecordingUnlocked(); + }) + ); const handleAudioDeviceChange = () => { if ($showCallOverlay && !hasLiveAudioTrack(audioStream)) { @@ -344,6 +373,20 @@ } }; + const startWatchingAudioDevices = () => { + if (!watchingAudioDevices && !callOverlayDestroyed) { + navigator.mediaDevices.addEventListener('devicechange', handleAudioDeviceChange); + watchingAudioDevices = true; + } + }; + + const stopWatchingAudioDevices = () => { + if (watchingAudioDevices) { + navigator.mediaDevices.removeEventListener('devicechange', handleAudioDeviceChange); + watchingAudioDevices = false; + } + }; + // Function to calculate the RMS level from time domain data const calculateRMS = (data: Uint8Array) => { let sumSquares = 0; @@ -354,13 +397,23 @@ return Math.sqrt(sumSquares / data.length); }; - const analyseAudio = (stream, generation) => { + const analyseAudio = (stream: MediaStream, generation: number) => { const audioContext = new AudioContext(); - const audioStreamSource = audioContext.createMediaStreamSource(stream); + let audioStreamSource; + let analyser; + const closeAudioContext = () => { + void audioContext.close().catch(() => {}); + }; - const analyser = audioContext.createAnalyser(); - analyser.minDecibels = MIN_DECIBELS; - audioStreamSource.connect(analyser); + try { + audioStreamSource = audioContext.createMediaStreamSource(stream); + analyser = audioContext.createAnalyser(); + analyser.minDecibels = MIN_DECIBELS; + audioStreamSource.connect(analyser); + } catch (error) { + closeAudioContext(); + throw error; + } const bufferLength = analyser.frequencyBinCount; @@ -371,10 +424,20 @@ hasStartedSpeaking = false; console.log('🔊 Sound detection started', lastSoundTime, hasStartedSpeaking); + let analysisStopped = false; + const stopAnalysis = () => { + if (analysisStopped) { + return; + } + + analysisStopped = true; + closeAudioContext(); + }; const detectSound = () => { const processFrame = () => { if (!mediaRecorder || !$showCallOverlay || generation !== audioAnalysisGeneration) { + stopAnalysis(); return; } @@ -422,6 +485,7 @@ if (mediaRecorder) { console.log('%c%s', 'color: red; font-size: 20px;', '🔇 Silence detected'); mediaRecorder.stop(); + stopAnalysis(); return; } } @@ -740,7 +804,7 @@ } }; - onMount(async () => { + onMount(() => { const setWakeLock = async () => { try { wakeLock = await navigator.wakeLock.request('screen'); @@ -758,51 +822,40 @@ } }; - if ('wakeLock' in navigator) { - await setWakeLock(); + const handleVisibilityChange = () => { + if (wakeLock !== null && document.visibilityState === 'visible') { + void setWakeLock(); + } + }; - document.addEventListener('visibilitychange', async () => { - // Re-request the wake lock if the document becomes visible - if (wakeLock !== null && document.visibilityState === 'visible') { - await setWakeLock(); - } - }); + if ('wakeLock' in navigator) { + void setWakeLock(); + document.addEventListener('visibilitychange', handleVisibilityChange); } model = $models.find((m) => m.id === modelId); - await startRecording(); - navigator.mediaDevices.addEventListener('devicechange', handleAudioDeviceChange); - eventTarget.addEventListener('chat:start', chatStartHandler); eventTarget.addEventListener('chat', chatEventHandler); eventTarget.addEventListener('chat:finish', chatFinishHandler); document.addEventListener('keydown', handleKeydown); + void startRecording().finally(startWatchingAudioDevices); - return async () => { - await stopAllAudio(); - - stopAudioStream(); - + return () => { + callOverlayDestroyed = true; eventTarget.removeEventListener('chat:start', chatStartHandler); eventTarget.removeEventListener('chat', chatEventHandler); eventTarget.removeEventListener('chat:finish', chatFinishHandler); document.removeEventListener('keydown', handleKeydown); - navigator.mediaDevices.removeEventListener('devicechange', handleAudioDeviceChange); - - audioAbortController.abort(); - await tick(); - - await stopAllAudio(); - - await stopRecordingCallback(false); - await stopCamera(); + document.removeEventListener('visibilitychange', handleVisibilityChange); + stopWatchingAudioDevices(); }; }); onDestroy(async () => { + callOverlayDestroyed = true; await stopAllAudio(); await stopRecordingCallback(false); await stopCamera(); @@ -813,7 +866,7 @@ eventTarget.removeEventListener('chat:finish', chatFinishHandler); document.removeEventListener('keydown', handleKeydown); - navigator.mediaDevices.removeEventListener('devicechange', handleAudioDeviceChange); + stopWatchingAudioDevices(); audioAbortController.abort(); diff --git a/src/lib/components/chat/MessageInput/CallOverlay/audio-stream.test.ts b/src/lib/components/chat/MessageInput/CallOverlay/audio-stream.test.ts index 9d2db39005..8a3a6b6add 100644 --- a/src/lib/components/chat/MessageInput/CallOverlay/audio-stream.test.ts +++ b/src/lib/components/chat/MessageInput/CallOverlay/audio-stream.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it, vi } from 'vitest'; -import { hasLiveAudioTrack, singleFlight, watchAudioTrackEnd } from './audio-stream'; +import { + createAsyncQueue, + hasLiveAudioTrack, + singleFlight, + watchAudioTrackEnd +} from './audio-stream'; class FakeAudioTrack extends EventTarget { readyState: MediaStreamTrackState; @@ -75,4 +80,41 @@ describe('call audio stream lifecycle', () => { finishRecovery?.(); await laterRecovery; }); + + it('serializes recording lifecycle transitions', async () => { + const enqueue = createAsyncQueue(); + let finishStart: (() => void) | undefined; + const events: string[] = []; + + const start = enqueue( + () => + new Promise((resolve) => { + events.push('start'); + finishStart = resolve; + }) + ); + const recover = enqueue(async () => { + events.push('recover'); + }); + + await Promise.resolve(); + expect(events).toEqual(['start']); + + finishStart?.(); + await start; + await recover; + + expect(events).toEqual(['start', 'recover']); + }); + + it('continues serialized transitions after a failure', async () => { + const enqueue = createAsyncQueue(); + const failed = enqueue(async () => { + throw new Error('setup failed'); + }); + const recovered = enqueue(async () => 'recovered'); + + await expect(failed).rejects.toThrow('setup failed'); + await expect(recovered).resolves.toBe('recovered'); + }); }); diff --git a/src/lib/components/chat/MessageInput/CallOverlay/audio-stream.ts b/src/lib/components/chat/MessageInput/CallOverlay/audio-stream.ts index 2247ae6770..46267e8587 100644 --- a/src/lib/components/chat/MessageInput/CallOverlay/audio-stream.ts +++ b/src/lib/components/chat/MessageInput/CallOverlay/audio-stream.ts @@ -41,3 +41,17 @@ export const singleFlight = (task: () => Promise) => { return active; }; }; + +export const createAsyncQueue = () => { + let tail = Promise.resolve(); + + return (task: () => Promise) => { + const result = tail.then(task, task); + tail = result.then( + () => undefined, + () => undefined + ); + + return result; + }; +};