diff --git a/src/lib/components/chat/MessageInput/CallOverlay/BridgeCallOverlay.svelte b/src/lib/components/chat/MessageInput/CallOverlay/BridgeCallOverlay.svelte new file mode 100644 index 0000000000..87742bc34e --- /dev/null +++ b/src/lib/components/chat/MessageInput/CallOverlay/BridgeCallOverlay.svelte @@ -0,0 +1,316 @@ + + +
+
+ + +
+

+ {#if bridge.connecting}{/if} + {status} +

+

+ {#if bridge.speaking} + {bridge.muted ? $i18n.t('Microphone off') : $i18n.t('Tap to interrupt')} + {:else if bridge.muted && bridge.connected} + {$i18n.t('Unmute to continue')} + {/if} +

+
+ + {#if bridge.error} +
+

{bridge.error}

+ +
+ {/if} +
+ + +
+ + diff --git a/src/lib/components/chat/MessageInput/CallOverlay/VoiceOrb.svelte b/src/lib/components/chat/MessageInput/CallOverlay/VoiceOrb.svelte new file mode 100644 index 0000000000..7194b1f7b1 --- /dev/null +++ b/src/lib/components/chat/MessageInput/CallOverlay/VoiceOrb.svelte @@ -0,0 +1,247 @@ + + + + + diff --git a/src/lib/components/chat/MessageInput/CallPanel.svelte b/src/lib/components/chat/MessageInput/CallPanel.svelte index 0d10ce5181..857e6d931a 100644 --- a/src/lib/components/chat/MessageInput/CallPanel.svelte +++ b/src/lib/components/chat/MessageInput/CallPanel.svelte @@ -3,6 +3,7 @@ import { config, showCallOverlay, showControls } from '$lib/stores'; import type { RealtimeCall } from '$lib/utils/realtime'; import CallOverlay from './CallOverlay.svelte'; + import BridgeCallOverlay from './CallOverlay/BridgeCallOverlay.svelte'; import { toast } from 'svelte-sonner'; const i18n = getContext('i18n'); @@ -36,18 +37,28 @@ started = true; }); - $: if (started && callMode === 'bridge' && !bridge?.connected && !bridge?.connecting) close(); + $: if ( + started && + callMode === 'bridge' && + !bridge?.connected && + !bridge?.connecting && + !bridge?.error + ) + close(); {#if started} - + {#if callMode === 'bridge'} + + {:else} + + {/if} {/if} diff --git a/src/lib/utils/realtime-audio.js b/src/lib/utils/realtime-audio.js index 3c02690bf0..ee44128810 100644 --- a/src/lib/utils/realtime-audio.js +++ b/src/lib/utils/realtime-audio.js @@ -13,6 +13,9 @@ class RealtimeAudioProcessor extends AudioWorkletProcessor { this.rendered = new Map(); this.ended = new Set(); this.ticks = 0; + this.inputEnergy = 0; + this.outputEnergy = 0; + this.levelSamples = 0; this.port.onmessage = ({ data }) => { if (data.type === 'capture') { this.enabled = data.enabled; @@ -86,8 +89,21 @@ class RealtimeAudioProcessor extends AudioWorkletProcessor { } if (!this.queued) this.playing = false; } + for (let i = 0; i < output.length; i++) { + const sample = this.enabled ? (input?.[i] ?? 0) : 0; + this.inputEnergy += sample * sample; + this.outputEnergy += output[i] * output[i]; + } + this.levelSamples += output.length; if (++this.ticks % 8 === 0) { - this.port.postMessage({ type: 'playback', queued: this.queued, received: this.received }); + this.port.postMessage({ + type: 'playback', + queued: this.queued, + received: this.received, + inputLevel: Math.sqrt(this.inputEnergy / this.levelSamples), + outputLevel: Math.sqrt(this.outputEnergy / this.levelSamples) + }); + this.inputEnergy = this.outputEnergy = this.levelSamples = 0; } return true; } diff --git a/src/lib/utils/realtime.ts b/src/lib/utils/realtime.ts index 8bfc0541da..bb67d2889a 100644 --- a/src/lib/utils/realtime.ts +++ b/src/lib/utils/realtime.ts @@ -75,6 +75,9 @@ export class RealtimeCall { approval = false; model = ''; voice = ''; + inputLevel = 0; + outputLevel = 0; + error = ''; private ws?: WebSocket; private context?: AudioContext; private stream?: MediaStream; @@ -109,6 +112,7 @@ export class RealtimeCall { async connect(token: string) { if (this.connected || this.connecting) return; + this.error = ''; this.connecting = true; this.options.change(); const session = ++this.session; @@ -192,12 +196,21 @@ export class RealtimeCall { audio: btoa(String.fromCharCode(...new Uint8Array(data.pcm))) }); } else if (data.type === 'playback') { + const inputLevel = this.muted ? 0 : (data.inputLevel ?? 0); + const outputLevel = data.outputLevel ?? 0; + const levelsChanged = + Math.abs(inputLevel - this.inputLevel) > 0.002 || + Math.abs(outputLevel - this.outputLevel) > 0.002; const nextSpeaking = data.queued > 0 || this.sentSamples > data.received; const speakingChanged = this.speaking !== nextSpeaking; this.speaking = nextSpeaking; if (!this.speaking && !this.activeResponse && !this.responseRequested) this.speakingResponses.clear(); - if (speakingChanged) this.options.change(); + if (levelsChanged || speakingChanged) { + this.inputLevel = inputLevel; + this.outputLevel = outputLevel; + this.options.change(); + } this.flush(); } else if (data.type === 'overflow') { this.stopSpeaking(); @@ -591,6 +604,7 @@ export class RealtimeCall { } this.speakingResponses.clear(); this.speaking = false; + this.outputLevel = 0; const id = ++this.clearId; this.clears.set(id, responses); this.audio?.port.postMessage({ type: 'clear', id }); @@ -617,6 +631,7 @@ export class RealtimeCall { mute() { this.muted = !this.muted; + if (this.muted) this.inputLevel = 0; this.stream?.getAudioTracks().forEach((track) => { track.enabled = !this.muted; }); @@ -626,6 +641,7 @@ export class RealtimeCall { } private fail(message: string) { + this.error = message; this.options.error(message); this.end(); } @@ -657,6 +673,7 @@ export class RealtimeCall { this.context = undefined; this.audio = undefined; this.connected = this.connecting = this.speaking = this.working = this.approval = false; + this.inputLevel = this.outputLevel = 0; this.responseRequested = false; this.cancelRequested = false; this.receivingSpeech = false;