From 51056f9b809200892c3f5fbcdb2b3f3b06e2f961 Mon Sep 17 00:00:00 2001 From: JP Ratliff Date: Mon, 10 Aug 2026 07:57:22 -0700 Subject: [PATCH] fix call microphone disconnect recovery --- .../chat/MessageInput/CallOverlay.svelte | 99 ++++++++++++++++--- .../CallOverlay/audio-stream.test.ts | 78 +++++++++++++++ .../MessageInput/CallOverlay/audio-stream.ts | 43 ++++++++ 3 files changed, 204 insertions(+), 16 deletions(-) create mode 100644 src/lib/components/chat/MessageInput/CallOverlay/audio-stream.test.ts create mode 100644 src/lib/components/chat/MessageInput/CallOverlay/audio-stream.ts diff --git a/src/lib/components/chat/MessageInput/CallOverlay.svelte b/src/lib/components/chat/MessageInput/CallOverlay.svelte index 17003e6ad4..55e197b01c 100644 --- a/src/lib/components/chat/MessageInput/CallOverlay.svelte +++ b/src/lib/components/chat/MessageInput/CallOverlay.svelte @@ -12,6 +12,7 @@ import Tooltip from '$lib/components/common/Tooltip.svelte'; import VideoInputMenu from './CallOverlay/VideoInputMenu.svelte'; + import { hasLiveAudioTrack, singleFlight, watchAudioTrackEnd } from './CallOverlay/audio-stream'; import { KokoroWorker } from '$lib/workers/KokoroWorker'; import { WEBUI_API_BASE_URL } from '$lib/constants'; @@ -44,6 +45,9 @@ let mediaRecorder; let audioStream = null; let audioChunks = []; + let audioStatus: 'listening' | 'reconnecting' | 'unavailable' = 'reconnecting'; + let stopWatchingAudioTrack = () => {}; + let audioAnalysisGeneration = 0; let videoInputDevices = []; let selectedVideoInputDeviceId = null; @@ -194,7 +198,7 @@ mediaRecorder = false; if (_continue) { - startRecording(); + await startRecording(); } if (confirmed) { @@ -232,21 +236,44 @@ const startRecording = async () => { if ($showCallOverlay) { - if (!audioStream) { - audioStream = await navigator.mediaDevices.getUserMedia({ - audio: { - echoCancellation: true, - noiseSuppression: true, - autoGainControl: true - } + if (!hasLiveAudioTrack(audioStream)) { + audioStatus = 'reconnecting'; + + try { + audioStream = await navigator.mediaDevices.getUserMedia({ + audio: { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true + } + }); + } catch (error) { + console.error('Error accessing media devices.', error); + audioStream = null; + audioStatus = 'unavailable'; + toast.error($i18n.t('Error accessing media devices.')); + return false; + } + + if (!$showCallOverlay) { + audioStream.getAudioTracks().forEach((track) => track.stop()); + audioStream = null; + return false; + } + + stopWatchingAudioTrack(); + stopWatchingAudioTrack = watchAudioTrackEnd(audioStream, () => { + void recoverAudioStream(); }); } - if (audioStream) { - // hardware track muting disabled to prevent backend translation errors with malformed WebM files + if (!hasLiveAudioTrack(audioStream)) { + audioStatus = 'unavailable'; + return false; } mediaRecorder = new MediaRecorder(audioStream); + audioStatus = 'listening'; mediaRecorder.onstart = () => { console.log('Recording started'); @@ -264,19 +291,30 @@ stopRecordingCallback(); }; - analyseAudio(audioStream); + analyseAudio(audioStream, ++audioAnalysisGeneration); + return true; } + + return false; }; const stopAudioStream = async () => { + const recorder = mediaRecorder; + mediaRecorder = false; + audioAnalysisGeneration += 1; + try { - if (mediaRecorder) { - mediaRecorder.stop(); + if (recorder && recorder.state !== 'inactive') { + recorder.onstop = null; + recorder.stop(); } } catch (error) { console.log('Error stopping audio stream:', error); } + stopWatchingAudioTrack(); + stopWatchingAudioTrack = () => {}; + if (!audioStream) return; audioStream.getAudioTracks().forEach(function (track) { @@ -286,6 +324,26 @@ audioStream = null; }; + const recoverAudioStream = singleFlight(async () => { + if (!$showCallOverlay) { + return false; + } + + audioStatus = 'reconnecting'; + hasStartedSpeaking = false; + confirmed = false; + audioChunks = []; + + await stopAudioStream(); + return await startRecording(); + }); + + const handleAudioDeviceChange = () => { + if ($showCallOverlay && !hasLiveAudioTrack(audioStream)) { + void recoverAudioStream(); + } + }; + // Function to calculate the RMS level from time domain data const calculateRMS = (data: Uint8Array) => { let sumSquares = 0; @@ -296,7 +354,7 @@ return Math.sqrt(sumSquares / data.length); }; - const analyseAudio = (stream) => { + const analyseAudio = (stream, generation) => { const audioContext = new AudioContext(); const audioStreamSource = audioContext.createMediaStreamSource(stream); @@ -316,7 +374,7 @@ const detectSound = () => { const processFrame = () => { - if (!mediaRecorder || !$showCallOverlay) { + if (!mediaRecorder || !$showCallOverlay || generation !== audioAnalysisGeneration) { return; } @@ -713,7 +771,8 @@ model = $models.find((m) => m.id === modelId); - startRecording(); + await startRecording(); + navigator.mediaDevices.addEventListener('devicechange', handleAudioDeviceChange); eventTarget.addEventListener('chat:start', chatStartHandler); eventTarget.addEventListener('chat', chatEventHandler); @@ -731,6 +790,7 @@ eventTarget.removeEventListener('chat:finish', chatFinishHandler); document.removeEventListener('keydown', handleKeydown); + navigator.mediaDevices.removeEventListener('devicechange', handleAudioDeviceChange); audioAbortController.abort(); await tick(); @@ -753,6 +813,7 @@ eventTarget.removeEventListener('chat:finish', chatFinishHandler); document.removeEventListener('keydown', handleKeydown); + navigator.mediaDevices.removeEventListener('devicechange', handleAudioDeviceChange); audioAbortController.abort(); @@ -957,6 +1018,8 @@ on:click={() => { if (assistantSpeaking) { stopAllAudio(); + } else if (audioStatus === 'unavailable') { + void recoverAudioStream(); } }} > @@ -967,6 +1030,10 @@ {$i18n.t('Muted')} {:else if assistantSpeaking} {$i18n.t('Tap to interrupt')} + {:else if audioStatus === 'reconnecting'} + {$i18n.t('Connection lost. Reconnecting...')} + {:else if audioStatus === 'unavailable'} + {$i18n.t('Error accessing media devices.')} {$i18n.t('Retry')} {:else} {$i18n.t('Listening...')} {/if} diff --git a/src/lib/components/chat/MessageInput/CallOverlay/audio-stream.test.ts b/src/lib/components/chat/MessageInput/CallOverlay/audio-stream.test.ts new file mode 100644 index 0000000000..9d2db39005 --- /dev/null +++ b/src/lib/components/chat/MessageInput/CallOverlay/audio-stream.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it, vi } from 'vitest'; +import { hasLiveAudioTrack, singleFlight, watchAudioTrackEnd } from './audio-stream'; + +class FakeAudioTrack extends EventTarget { + readyState: MediaStreamTrackState; + + constructor(readyState: MediaStreamTrackState) { + super(); + this.readyState = readyState; + } + + end() { + this.readyState = 'ended'; + this.dispatchEvent(new Event('ended')); + } +} + +const stream = (...tracks: FakeAudioTrack[]) => ({ + getAudioTracks: () => tracks +}); + +describe('call audio stream lifecycle', () => { + it('requires at least one live audio track', () => { + expect(hasLiveAudioTrack(null)).toBe(false); + expect(hasLiveAudioTrack(stream(new FakeAudioTrack('ended')))).toBe(false); + expect(hasLiveAudioTrack(stream(new FakeAudioTrack('ended'), new FakeAudioTrack('live')))).toBe( + true + ); + }); + + it('reports a disconnected stream only once', () => { + const first = new FakeAudioTrack('live'); + const second = new FakeAudioTrack('live'); + const onEnded = vi.fn(); + + watchAudioTrackEnd(stream(first, second), onEnded); + first.end(); + second.end(); + + expect(onEnded).toHaveBeenCalledOnce(); + }); + + it('does not report intentional shutdown after cleanup', () => { + const track = new FakeAudioTrack('live'); + const onEnded = vi.fn(); + const cleanup = watchAudioTrackEnd(stream(track), onEnded); + + cleanup(); + track.end(); + + expect(onEnded).not.toHaveBeenCalled(); + }); + + it('coalesces simultaneous recovery signals and permits a later retry', async () => { + let finishRecovery: (() => void) | undefined; + const recover = vi.fn( + () => + new Promise((resolve) => { + finishRecovery = resolve; + }) + ); + const runRecovery = singleFlight(recover); + + const trackEndedRecovery = runRecovery(); + const deviceChangeRecovery = runRecovery(); + + expect(recover).toHaveBeenCalledOnce(); + expect(deviceChangeRecovery).toBe(trackEndedRecovery); + + finishRecovery?.(); + await trackEndedRecovery; + + const laterRecovery = runRecovery(); + expect(recover).toHaveBeenCalledTimes(2); + finishRecovery?.(); + await laterRecovery; + }); +}); diff --git a/src/lib/components/chat/MessageInput/CallOverlay/audio-stream.ts b/src/lib/components/chat/MessageInput/CallOverlay/audio-stream.ts new file mode 100644 index 0000000000..2247ae6770 --- /dev/null +++ b/src/lib/components/chat/MessageInput/CallOverlay/audio-stream.ts @@ -0,0 +1,43 @@ +type AudioTrack = Pick; + +type AudioStream = { + getAudioTracks: () => AudioTrack[]; +}; + +export const hasLiveAudioTrack = (stream: AudioStream | null) => + stream?.getAudioTracks().some((track) => track.readyState === 'live') ?? false; + +export const watchAudioTrackEnd = (stream: AudioStream, onEnded: () => void) => { + const tracks = stream.getAudioTracks(); + let handled = false; + + const handleEnded = () => { + if (handled) { + return; + } + + handled = true; + onEnded(); + }; + + tracks.forEach((track) => track.addEventListener('ended', handleEnded)); + + return () => { + handled = true; + tracks.forEach((track) => track.removeEventListener('ended', handleEnded)); + }; +}; + +export const singleFlight = (task: () => Promise) => { + let active: Promise | null = null; + + return () => { + if (active === null) { + active = task().finally(() => { + active = null; + }); + } + + return active; + }; +};