harden call audio recovery lifecycle

src/lib/components/chat/MessageInput/CallOverlay.svelte

@@ -12,7 +12,12 @@
 
 	import Tooltip from '$lib/components/common/Tooltip.svelte';
 	import VideoInputMenu from './CallOverlay/VideoInputMenu.svelte';
-	import { hasLiveAudioTrack, singleFlight, watchAudioTrackEnd } from './CallOverlay/audio-stream';
+	import {
+		createAsyncQueue,
+		hasLiveAudioTrack,
+		singleFlight,
+		watchAudioTrackEnd
+	} from './CallOverlay/audio-stream';
 	import { KokoroWorker } from '$lib/workers/KokoroWorker';
 	import { WEBUI_API_BASE_URL } from '$lib/constants';
 
@@ -48,6 +53,9 @@
 	let audioStatus: 'listening' | 'reconnecting' | 'unavailable' = 'reconnecting';
 	let stopWatchingAudioTrack = () => {};
 	let audioAnalysisGeneration = 0;
+	let callOverlayDestroyed = false;
+	let watchingAudioDevices = false;
+	const enqueueAudioTransition = createAsyncQueue();
 
 	let videoInputDevices = [];
 	let selectedVideoInputDeviceId = null;
@@ -234,8 +242,13 @@
 		}
 	};
 
-	const startRecording = async () => {
-		if ($showCallOverlay) {
+	const startRecordingUnlocked = async () => {
+		if ($showCallOverlay && !callOverlayDestroyed) {
+			if (mediaRecorder && hasLiveAudioTrack(audioStream)) {
+				audioStatus = 'listening';
+				return true;
+			}
+
 			if (!hasLiveAudioTrack(audioStream)) {
 				audioStatus = 'reconnecting';
 
@@ -248,14 +261,10 @@
 						}
 					});
 				} catch (error) {
-					console.error('Error accessing media devices.', error);
-					audioStream = null;
-					audioStatus = 'unavailable';
-					toast.error($i18n.t('Error accessing media devices.'));
-					return false;
+					return await handleAudioSetupError(error);
 				}
 
-				if (!$showCallOverlay) {
+				if (!$showCallOverlay || callOverlayDestroyed) {
 					audioStream.getAudioTracks().forEach((track) => track.stop());
 					audioStream = null;
 					return false;
@@ -268,36 +277,41 @@
 			}
 
 			if (!hasLiveAudioTrack(audioStream)) {
-				audioStatus = 'unavailable';
-				return false;
+				return await handleAudioSetupError(new Error('Acquired audio stream has no live track.'));
 			}
 
-			mediaRecorder = new MediaRecorder(audioStream);
-			audioStatus = 'listening';
+			try {
+				mediaRecorder = new MediaRecorder(audioStream);
 
-			mediaRecorder.onstart = () => {
-				console.log('Recording started');
-				audioChunks = [];
-			};
+				mediaRecorder.onstart = () => {
+					console.log('Recording started');
+					audioChunks = [];
+				};
 
-			mediaRecorder.ondataavailable = (event) => {
-				if (hasStartedSpeaking) {
-					audioChunks.push(event.data);
-				}
-			};
+				mediaRecorder.ondataavailable = (event) => {
+					if (hasStartedSpeaking) {
+						audioChunks.push(event.data);
+					}
+				};
 
-			mediaRecorder.onstop = (e) => {
-				console.log('Recording stopped', audioStream, e);
-				stopRecordingCallback();
-			};
+				mediaRecorder.onstop = (e) => {
+					console.log('Recording stopped', audioStream, e);
+					void stopRecordingCallback();
+				};
 
-			analyseAudio(audioStream, ++audioAnalysisGeneration);
-			return true;
+				analyseAudio(audioStream, ++audioAnalysisGeneration);
+				audioStatus = 'listening';
+				return true;
+			} catch (error) {
+				return await handleAudioSetupError(error);
+			}
 		}
 
 		return false;
 	};
 
+	const startRecording = () => enqueueAudioTransition(startRecordingUnlocked);
+
 	const stopAudioStream = async () => {
 		const recorder = mediaRecorder;
 		mediaRecorder = false;
@@ -324,19 +338,34 @@
 		audioStream = null;
 	};
 
-	const recoverAudioStream = singleFlight(async () => {
-		if (!$showCallOverlay) {
+	const handleAudioSetupError = async (error: unknown) => {
+		console.error('Error setting up media devices.', error);
+		await stopAudioStream();
+
+		if (!$showCallOverlay || callOverlayDestroyed) {
 			return false;
 		}
 
-		audioStatus = 'reconnecting';
-		hasStartedSpeaking = false;
-		confirmed = false;
-		audioChunks = [];
+		audioStatus = 'unavailable';
+		toast.error($i18n.t('Error accessing media devices.'));
+		return false;
+	};
 
-		await stopAudioStream();
-		return await startRecording();
-	});
+	const recoverAudioStream = singleFlight(() =>
+		enqueueAudioTransition(async () => {
+			if (!$showCallOverlay || callOverlayDestroyed) {
+				return false;
+			}
+
+			audioStatus = 'reconnecting';
+			hasStartedSpeaking = false;
+			confirmed = false;
+			audioChunks = [];
+
+			await stopAudioStream();
+			return await startRecordingUnlocked();
+		})
+	);
 
 	const handleAudioDeviceChange = () => {
 		if ($showCallOverlay && !hasLiveAudioTrack(audioStream)) {
@@ -344,6 +373,20 @@
 		}
 	};
 
+	const startWatchingAudioDevices = () => {
+		if (!watchingAudioDevices && !callOverlayDestroyed) {
+			navigator.mediaDevices.addEventListener('devicechange', handleAudioDeviceChange);
+			watchingAudioDevices = true;
+		}
+	};
+
+	const stopWatchingAudioDevices = () => {
+		if (watchingAudioDevices) {
+			navigator.mediaDevices.removeEventListener('devicechange', handleAudioDeviceChange);
+			watchingAudioDevices = false;
+		}
+	};
+
 	// Function to calculate the RMS level from time domain data
 	const calculateRMS = (data: Uint8Array) => {
 		let sumSquares = 0;
@@ -354,13 +397,23 @@
 		return Math.sqrt(sumSquares / data.length);
 	};
 
-	const analyseAudio = (stream, generation) => {
+	const analyseAudio = (stream: MediaStream, generation: number) => {
 		const audioContext = new AudioContext();
-		const audioStreamSource = audioContext.createMediaStreamSource(stream);
+		let audioStreamSource;
+		let analyser;
+		const closeAudioContext = () => {
+			void audioContext.close().catch(() => {});
+		};
 
-		const analyser = audioContext.createAnalyser();
-		analyser.minDecibels = MIN_DECIBELS;
-		audioStreamSource.connect(analyser);
+		try {
+			audioStreamSource = audioContext.createMediaStreamSource(stream);
+			analyser = audioContext.createAnalyser();
+			analyser.minDecibels = MIN_DECIBELS;
+			audioStreamSource.connect(analyser);
+		} catch (error) {
+			closeAudioContext();
+			throw error;
+		}
 
 		const bufferLength = analyser.frequencyBinCount;
 
@@ -371,10 +424,20 @@
 		hasStartedSpeaking = false;
 
 		console.log('🔊 Sound detection started', lastSoundTime, hasStartedSpeaking);
+		let analysisStopped = false;
+		const stopAnalysis = () => {
+			if (analysisStopped) {
+				return;
+			}
+
+			analysisStopped = true;
+			closeAudioContext();
+		};
 
 		const detectSound = () => {
 			const processFrame = () => {
 				if (!mediaRecorder || !$showCallOverlay || generation !== audioAnalysisGeneration) {
+					stopAnalysis();
 					return;
 				}
 
@@ -422,6 +485,7 @@
 						if (mediaRecorder) {
 							console.log('%c%s', 'color: red; font-size: 20px;', '🔇 Silence detected');
 							mediaRecorder.stop();
+							stopAnalysis();
 							return;
 						}
 					}
@@ -740,7 +804,7 @@
 		}
 	};
 
-	onMount(async () => {
+	onMount(() => {
 		const setWakeLock = async () => {
 			try {
 				wakeLock = await navigator.wakeLock.request('screen');
@@ -758,51 +822,40 @@
 			}
 		};
 
-		if ('wakeLock' in navigator) {
-			await setWakeLock();
+		const handleVisibilityChange = () => {
+			if (wakeLock !== null && document.visibilityState === 'visible') {
+				void setWakeLock();
+			}
+		};
 
-			document.addEventListener('visibilitychange', async () => {
-				// Re-request the wake lock if the document becomes visible
-				if (wakeLock !== null && document.visibilityState === 'visible') {
-					await setWakeLock();
-				}
-			});
+		if ('wakeLock' in navigator) {
+			void setWakeLock();
+			document.addEventListener('visibilitychange', handleVisibilityChange);
 		}
 
 		model = $models.find((m) => m.id === modelId);
 
-		await startRecording();
-		navigator.mediaDevices.addEventListener('devicechange', handleAudioDeviceChange);
-
 		eventTarget.addEventListener('chat:start', chatStartHandler);
 		eventTarget.addEventListener('chat', chatEventHandler);
 		eventTarget.addEventListener('chat:finish', chatFinishHandler);
 
 		document.addEventListener('keydown', handleKeydown);
+		void startRecording().finally(startWatchingAudioDevices);
 
-		return async () => {
-			await stopAllAudio();
-
-			stopAudioStream();
-
+		return () => {
+			callOverlayDestroyed = true;
 			eventTarget.removeEventListener('chat:start', chatStartHandler);
 			eventTarget.removeEventListener('chat', chatEventHandler);
 			eventTarget.removeEventListener('chat:finish', chatFinishHandler);
 
 			document.removeEventListener('keydown', handleKeydown);
-			navigator.mediaDevices.removeEventListener('devicechange', handleAudioDeviceChange);
-
-			audioAbortController.abort();
-			await tick();
-
-			await stopAllAudio();
-
-			await stopRecordingCallback(false);
-			await stopCamera();
+			document.removeEventListener('visibilitychange', handleVisibilityChange);
+			stopWatchingAudioDevices();
 		};
 	});
 
 	onDestroy(async () => {
+		callOverlayDestroyed = true;
 		await stopAllAudio();
 		await stopRecordingCallback(false);
 		await stopCamera();
@@ -813,7 +866,7 @@
 		eventTarget.removeEventListener('chat:finish', chatFinishHandler);
 
 		document.removeEventListener('keydown', handleKeydown);
-		navigator.mediaDevices.removeEventListener('devicechange', handleAudioDeviceChange);
+		stopWatchingAudioDevices();
 
 		audioAbortController.abort();
 

src/lib/components/chat/MessageInput/CallOverlay/audio-stream.test.ts

@@ -1,5 +1,10 @@
 import { describe, expect, it, vi } from 'vitest';
-import { hasLiveAudioTrack, singleFlight, watchAudioTrackEnd } from './audio-stream';
+import {
+	createAsyncQueue,
+	hasLiveAudioTrack,
+	singleFlight,
+	watchAudioTrackEnd
+} from './audio-stream';
 
 class FakeAudioTrack extends EventTarget {
 	readyState: MediaStreamTrackState;
@@ -75,4 +80,41 @@ describe('call audio stream lifecycle', () => {
 		finishRecovery?.();
 		await laterRecovery;
 	});
+
+	it('serializes recording lifecycle transitions', async () => {
+		const enqueue = createAsyncQueue();
+		let finishStart: (() => void) | undefined;
+		const events: string[] = [];
+
+		const start = enqueue(
+			() =>
+				new Promise<void>((resolve) => {
+					events.push('start');
+					finishStart = resolve;
+				})
+		);
+		const recover = enqueue(async () => {
+			events.push('recover');
+		});
+
+		await Promise.resolve();
+		expect(events).toEqual(['start']);
+
+		finishStart?.();
+		await start;
+		await recover;
+
+		expect(events).toEqual(['start', 'recover']);
+	});
+
+	it('continues serialized transitions after a failure', async () => {
+		const enqueue = createAsyncQueue();
+		const failed = enqueue(async () => {
+			throw new Error('setup failed');
+		});
+		const recovered = enqueue(async () => 'recovered');
+
+		await expect(failed).rejects.toThrow('setup failed');
+		await expect(recovered).resolves.toBe('recovered');
+	});
 });

src/lib/components/chat/MessageInput/CallOverlay/audio-stream.ts

@@ -41,3 +41,17 @@ export const singleFlight = <T>(task: () => Promise<T>) => {
 		return active;
 	};
 };
+
+export const createAsyncQueue = () => {
+	let tail = Promise.resolve();
+
+	return <T>(task: () => Promise<T>) => {
+		const result = tail.then(task, task);
+		tail = result.then(
+			() => undefined,
+			() => undefined
+		);
+
+		return result;
+	};
+};