mirror of
https://github.com/open-webui/open-webui.git
synced 2026-10-08 03:08:02 +00:00
refac
This commit is contained in:
Timothy Jaeryang Baek
parent
71167131d2
commit
4f1f385419
5 changed files with 620 additions and 13 deletions
|
|
@ -0,0 +1,316 @@
|
|||
<script lang="ts">
|
||||
import { createEventDispatcher, getContext, onMount } from 'svelte';
|
||||
import { showCallOverlay } from '$lib/stores';
|
||||
import type { RealtimeCall } from '$lib/utils/realtime';
|
||||
import VoiceOrb from './VoiceOrb.svelte';
|
||||
|
||||
export let bridge: RealtimeCall;
|
||||
|
||||
const i18n = getContext<any>('i18n');
|
||||
const dispatch = createEventDispatcher();
|
||||
$: unavailable = !bridge.connected;
|
||||
$: status = bridge.error
|
||||
? $i18n.t('Call disconnected')
|
||||
: bridge.connecting
|
||||
? $i18n.t('Connecting...')
|
||||
: bridge.speaking
|
||||
? $i18n.t('Speaking')
|
||||
: bridge.muted
|
||||
? $i18n.t('Microphone off')
|
||||
: $i18n.t('Listening');
|
||||
|
||||
const endCall = () => {
|
||||
bridge.end();
|
||||
showCallOverlay.set(false);
|
||||
dispatch('close');
|
||||
};
|
||||
|
||||
onMount(() => {
|
||||
const handleKeydown = (event: KeyboardEvent) => {
|
||||
const target = event.target as HTMLElement;
|
||||
if (
|
||||
event.key.toLowerCase() === 'm' &&
|
||||
!event.repeat &&
|
||||
!event.metaKey &&
|
||||
!event.ctrlKey &&
|
||||
!event.altKey &&
|
||||
!target.closest('input, textarea, select, [contenteditable="true"], [role="textbox"]') &&
|
||||
bridge.connected
|
||||
) {
|
||||
event.preventDefault();
|
||||
bridge.mute();
|
||||
}
|
||||
};
|
||||
document.addEventListener('keydown', handleKeydown);
|
||||
return () => document.removeEventListener('keydown', handleKeydown);
|
||||
});
|
||||
</script>
|
||||
|
||||
<section class="bridge-call" aria-label={$i18n.t('Voice call')}>
|
||||
<div class="call-stage">
|
||||
<button
|
||||
type="button"
|
||||
class="orb-button"
|
||||
disabled={!bridge.speaking}
|
||||
aria-label={$i18n.t('Stop speaking')}
|
||||
on:click={() => bridge.stopSpeaking()}
|
||||
>
|
||||
<VoiceOrb
|
||||
speaking={bridge.speaking && !unavailable}
|
||||
level={unavailable
|
||||
? 0
|
||||
: bridge.speaking
|
||||
? bridge.outputLevel
|
||||
: bridge.muted
|
||||
? 0
|
||||
: bridge.inputLevel}
|
||||
muted={unavailable || (bridge.muted && !bridge.speaking)}
|
||||
/>
|
||||
</button>
|
||||
|
||||
<div class="call-status" role="status" aria-live="polite" aria-atomic="true">
|
||||
<p class="status-label">
|
||||
{#if bridge.connecting}<span class="connecting-dot" aria-hidden="true"></span>{/if}
|
||||
{status}
|
||||
</p>
|
||||
<p class="status-hint">
|
||||
{#if bridge.speaking}
|
||||
{bridge.muted ? $i18n.t('Microphone off') : $i18n.t('Tap to interrupt')}
|
||||
{:else if bridge.muted && bridge.connected}
|
||||
{$i18n.t('Unmute to continue')}
|
||||
{/if}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{#if bridge.error}
|
||||
<div class="call-error">
|
||||
<p role="alert">{bridge.error}</p>
|
||||
<button
|
||||
type="button"
|
||||
class="text-action"
|
||||
on:click={() => bridge.connect(localStorage.token)}
|
||||
>
|
||||
{$i18n.t('Retry')}
|
||||
</button>
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<footer>
|
||||
<div class="call-controls">
|
||||
<button
|
||||
type="button"
|
||||
class="call-control mute-control"
|
||||
class:is-muted={bridge.muted}
|
||||
aria-label={bridge.muted ? $i18n.t('Unmute') : $i18n.t('Mute')}
|
||||
aria-pressed={bridge.muted}
|
||||
title={`${bridge.muted ? $i18n.t('Unmute') : $i18n.t('Mute')} (M)`}
|
||||
disabled={unavailable}
|
||||
on:click={() => bridge.mute()}
|
||||
>
|
||||
<svg
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
stroke-width="1.5"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<path
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
d="M12 15.75a3 3 0 0 0 3-3V4.5a3 3 0 0 0-6 0v8.25a3 3 0 0 0 3 3Zm-6-4.5v1.5a6 6 0 0 0 12 0v-1.5M12 18.75v3M9 21.75h6"
|
||||
/>
|
||||
{#if bridge.muted}<path stroke-linecap="round" d="m3 3 18 18" />{/if}
|
||||
</svg>
|
||||
<span>{bridge.muted ? $i18n.t('Unmute') : $i18n.t('Mute')}</span>
|
||||
</button>
|
||||
<button type="button" class="call-control end-control" on:click={endCall}>
|
||||
<svg
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
stroke-width="1.5"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<path
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
d="M3.3 15.4c-.8.3-1.7-.3-1.7-1.2v-2.1c0-.8.3-1.5 1-1.9 5.5-3.5 13.3-3.5 18.8 0 .7.4 1 1.1 1 1.9v2.1c0 .9-.9 1.5-1.7 1.2l-4-1.4a1.3 1.3 0 0 1-.9-1.2v-1.3a15 15 0 0 0-7.6 0v1.3c0 .5-.4 1-.9 1.2Z"
|
||||
/>
|
||||
</svg>
|
||||
<span>{$i18n.t('End call')}</span>
|
||||
</button>
|
||||
</div>
|
||||
</footer>
|
||||
</section>
|
||||
|
||||
<style>
|
||||
.bridge-call {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
width: 100%;
|
||||
min-height: 100%;
|
||||
padding: 24px 16px max(24px, env(safe-area-inset-bottom));
|
||||
color: var(--color-gray-900);
|
||||
}
|
||||
:global(.dark) .bridge-call {
|
||||
color: var(--color-gray-100);
|
||||
}
|
||||
.call-stage {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
min-height: 260px;
|
||||
padding: 40px 16px;
|
||||
}
|
||||
.orb-button {
|
||||
flex-shrink: 0;
|
||||
padding: 12px;
|
||||
border-radius: 50%;
|
||||
}
|
||||
.orb-button:disabled {
|
||||
cursor: default;
|
||||
}
|
||||
.call-status {
|
||||
margin-top: 16px;
|
||||
text-align: center;
|
||||
}
|
||||
.status-label {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 8px;
|
||||
font-size: 14px;
|
||||
line-height: 20px;
|
||||
font-weight: 400;
|
||||
}
|
||||
.status-hint {
|
||||
min-height: 18px;
|
||||
margin-top: 8px;
|
||||
color: var(--color-gray-600);
|
||||
font-size: 12px;
|
||||
line-height: 18px;
|
||||
}
|
||||
:global(.dark) .status-hint {
|
||||
color: var(--color-gray-400);
|
||||
}
|
||||
.connecting-dot {
|
||||
flex-shrink: 0;
|
||||
width: 5px;
|
||||
height: 5px;
|
||||
border-radius: 50%;
|
||||
background: currentColor;
|
||||
}
|
||||
.connecting-dot {
|
||||
animation: pulse 1.4s ease-in-out infinite;
|
||||
}
|
||||
.call-error {
|
||||
max-width: 260px;
|
||||
margin-top: 12px;
|
||||
text-align: center;
|
||||
font-size: 12px;
|
||||
line-height: 18px;
|
||||
}
|
||||
.call-error .text-action {
|
||||
margin-top: 8px;
|
||||
}
|
||||
footer {
|
||||
flex-shrink: 0;
|
||||
}
|
||||
.text-action {
|
||||
min-height: 44px;
|
||||
text-align: start;
|
||||
font-size: 12px;
|
||||
font-weight: 500;
|
||||
text-underline-offset: 3px;
|
||||
}
|
||||
.text-action:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
.call-controls {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
flex-wrap: wrap;
|
||||
gap: 12px;
|
||||
}
|
||||
.call-control {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 8px;
|
||||
min-width: 104px;
|
||||
min-height: 44px;
|
||||
padding: 10px 16px;
|
||||
border-radius: 999px;
|
||||
font-size: 12px;
|
||||
line-height: 20px;
|
||||
font-weight: 500;
|
||||
transition:
|
||||
background 160ms ease,
|
||||
color 160ms ease;
|
||||
}
|
||||
.call-control svg {
|
||||
width: 18px;
|
||||
height: 18px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
.mute-control {
|
||||
background: var(--color-gray-100);
|
||||
}
|
||||
.mute-control:hover {
|
||||
background: var(--color-gray-200);
|
||||
}
|
||||
:global(.dark) .mute-control {
|
||||
background: var(--color-gray-800);
|
||||
}
|
||||
:global(.dark) .mute-control:hover {
|
||||
background: var(--color-gray-700);
|
||||
}
|
||||
.mute-control.is-muted {
|
||||
background: var(--color-gray-900);
|
||||
color: white;
|
||||
}
|
||||
:global(.dark) .mute-control.is-muted {
|
||||
background: var(--color-gray-100);
|
||||
color: var(--color-gray-900);
|
||||
}
|
||||
.end-control {
|
||||
background: #fff0ef;
|
||||
color: #b42324;
|
||||
}
|
||||
.end-control:hover {
|
||||
background: #ffe1df;
|
||||
}
|
||||
:global(.dark) .end-control {
|
||||
background: #392526;
|
||||
color: #f2a3a3;
|
||||
}
|
||||
:global(.dark) .end-control:hover {
|
||||
background: #482a2c;
|
||||
}
|
||||
button:focus-visible {
|
||||
outline: 2px solid currentColor;
|
||||
outline-offset: 4px;
|
||||
}
|
||||
.call-control:disabled,
|
||||
.text-action:disabled {
|
||||
opacity: 0.45;
|
||||
cursor: default;
|
||||
}
|
||||
@keyframes pulse {
|
||||
50% {
|
||||
opacity: 0.3;
|
||||
}
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.connecting-dot {
|
||||
animation: none;
|
||||
}
|
||||
.call-control {
|
||||
transition: none;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
247
src/lib/components/chat/MessageInput/CallOverlay/VoiceOrb.svelte
Normal file
247
src/lib/components/chat/MessageInput/CallOverlay/VoiceOrb.svelte
Normal file
|
|
@ -0,0 +1,247 @@
|
|||
<script lang="ts">
|
||||
import { onMount } from 'svelte';
|
||||
|
||||
let {
|
||||
level = 0,
|
||||
muted = false,
|
||||
speaking = false
|
||||
}: { level?: number; muted?: boolean; speaking?: boolean } = $props();
|
||||
let canvas: HTMLCanvasElement;
|
||||
let refresh = () => {};
|
||||
|
||||
$effect(() => {
|
||||
level;
|
||||
muted;
|
||||
speaking;
|
||||
refresh();
|
||||
});
|
||||
|
||||
onMount(() => {
|
||||
const gl = canvas.getContext('webgl', {
|
||||
alpha: true,
|
||||
antialias: false,
|
||||
premultipliedAlpha: false
|
||||
});
|
||||
if (!gl) return;
|
||||
|
||||
const vertex = gl.createShader(gl.VERTEX_SHADER)!;
|
||||
const fragment = gl.createShader(gl.FRAGMENT_SHADER)!;
|
||||
const program = gl.createProgram()!;
|
||||
const buffer = gl.createBuffer()!;
|
||||
const dispose = () => {
|
||||
gl.deleteBuffer(buffer);
|
||||
gl.deleteProgram(program);
|
||||
gl.deleteShader(vertex);
|
||||
gl.deleteShader(fragment);
|
||||
};
|
||||
|
||||
gl.shaderSource(
|
||||
vertex,
|
||||
`
|
||||
attribute vec2 position;
|
||||
varying vec2 uv;
|
||||
void main() { uv = position; gl_Position = vec4(position, 0.0, 1.0); }
|
||||
`
|
||||
);
|
||||
gl.shaderSource(
|
||||
fragment,
|
||||
`
|
||||
precision highp float;
|
||||
varying vec2 uv;
|
||||
uniform float time;
|
||||
uniform float energy;
|
||||
uniform float speech;
|
||||
uniform float dark;
|
||||
|
||||
uniform float resolution;
|
||||
|
||||
// Smooth, evolving material coordinates on a sphere. No image textures.
|
||||
float material(vec3 p) {
|
||||
float t = time * 0.34;
|
||||
vec3 warp = vec3(
|
||||
sin(p.y * 2.1 + p.z * 1.3 + t),
|
||||
sin(p.z * 2.3 - p.x * 1.2 - t * 0.8),
|
||||
sin(p.x * 1.8 + p.y * 1.4 + t * 0.65)
|
||||
);
|
||||
vec3 q = p + (0.38 + speech * 0.05 + energy * (0.12 + speech * 0.08)) * warp;
|
||||
return sin(q.y * 2.2 + q.x * 1.5 + t * 0.3)
|
||||
+ 0.28 * sin(q.x * 3.2 - q.z * 2.4 - t * 0.7);
|
||||
}
|
||||
|
||||
void main() {
|
||||
float r = length(uv);
|
||||
float edge = 0.94;
|
||||
float alpha = 1.0 - smoothstep(edge - 2.0 / resolution, edge, r);
|
||||
if (alpha <= 0.0) { gl_FragColor = vec4(0.0); return; }
|
||||
|
||||
vec2 xy = uv / edge;
|
||||
vec3 p = vec3(xy, sqrt(max(0.0, 1.0 - dot(xy, xy))));
|
||||
float field = material(p);
|
||||
float e = 0.004;
|
||||
vec3 gradient = vec3(
|
||||
material(p + vec3(e, 0.0, 0.0)),
|
||||
material(p + vec3(0.0, e, 0.0)),
|
||||
material(p + vec3(0.0, 0.0, e))
|
||||
) - field;
|
||||
gradient /= e;
|
||||
// Tangential displacement changes the lighting without distorting the silhouette.
|
||||
vec3 tangent = gradient - p * dot(gradient, p);
|
||||
float crease = exp(-field * field * 2.0);
|
||||
vec3 n = normalize(p - tangent * crease * (0.10 + energy * 0.03));
|
||||
vec3 light = normalize(vec3(0.55, 0.9, 1.4));
|
||||
float diffuse = max(dot(n, light), 0.0);
|
||||
float fill = max(dot(n, normalize(vec3(-1.0, 0.15, 0.8))), 0.0);
|
||||
float highlight = pow(max(dot(n, normalize(light + vec3(0.0, 0.0, 1.0))), 0.0), 8.0);
|
||||
float flow = smoothstep(-0.95, 0.95, field);
|
||||
vec3 low = mix(vec3(0.145, 0.15, 0.16), vec3(0.22, 0.225, 0.235), dark);
|
||||
vec3 high = mix(vec3(0.33, 0.335, 0.345), vec3(0.50, 0.505, 0.515), dark);
|
||||
float emphasis = speech * (0.35 + 0.20 * energy);
|
||||
low *= 1.0 - 0.30 * emphasis;
|
||||
high += vec3(0.10) * emphasis;
|
||||
vec3 base = mix(low, high, flow);
|
||||
vec3 color = base * (0.64 + 0.30 * diffuse + 0.06 * fill);
|
||||
color += vec3(0.04 + 0.02 * emphasis) * highlight;
|
||||
color = pow(color, vec3(1.0 / 2.2));
|
||||
gl_FragColor = vec4(color, alpha);
|
||||
}
|
||||
|
||||
`
|
||||
);
|
||||
gl.compileShader(vertex);
|
||||
gl.compileShader(fragment);
|
||||
gl.attachShader(program, vertex);
|
||||
gl.attachShader(program, fragment);
|
||||
gl.linkProgram(program);
|
||||
if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
|
||||
dispose();
|
||||
return;
|
||||
}
|
||||
|
||||
gl.useProgram(program);
|
||||
gl.bindBuffer(gl.ARRAY_BUFFER, buffer);
|
||||
gl.bufferData(
|
||||
gl.ARRAY_BUFFER,
|
||||
new Float32Array([-1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, 1]),
|
||||
gl.STATIC_DRAW
|
||||
);
|
||||
const position = gl.getAttribLocation(program, 'position');
|
||||
gl.enableVertexAttribArray(position);
|
||||
gl.vertexAttribPointer(position, 2, gl.FLOAT, false, 0, 0);
|
||||
const timeUniform = gl.getUniformLocation(program, 'time');
|
||||
const energyUniform = gl.getUniformLocation(program, 'energy');
|
||||
const speechUniform = gl.getUniformLocation(program, 'speech');
|
||||
const darkUniform = gl.getUniformLocation(program, 'dark');
|
||||
const resolutionUniform = gl.getUniformLocation(program, 'resolution');
|
||||
const reducedMotion = matchMedia('(prefers-reduced-motion: reduce)');
|
||||
let frame = 0;
|
||||
let lastTime = 0;
|
||||
let elapsed = 0;
|
||||
let amplitude = 0;
|
||||
let speech = 0;
|
||||
let disposed = false;
|
||||
|
||||
const draw = (now: number) => {
|
||||
frame = 0;
|
||||
if (disposed || gl.isContextLost() || document.hidden) return;
|
||||
const delta = lastTime ? Math.min((now - lastTime) / 1000, 0.05) : 0;
|
||||
lastTime = now;
|
||||
const active = !muted && !reducedMotion.matches;
|
||||
const target = active
|
||||
? Math.min(1, Math.pow(Math.max(0, level * (speaking ? 12 : 6)), speaking ? 0.7 : 1))
|
||||
: 0;
|
||||
amplitude += (target - amplitude) * (1 - Math.exp(-delta * 6));
|
||||
speech += ((active && speaking ? 1 : 0) - speech) * (1 - Math.exp(-delta * 4));
|
||||
if (active) elapsed += delta * (0.8 + amplitude * 0.3 + speech * (0.6 + amplitude * 0.4));
|
||||
gl.uniform1f(timeUniform, reducedMotion.matches ? 0 : elapsed);
|
||||
gl.uniform1f(energyUniform, active ? amplitude : 0);
|
||||
gl.uniform1f(speechUniform, active ? speech : 0);
|
||||
gl.uniform1f(darkUniform, canvas.closest('.dark') ? 1 : 0);
|
||||
gl.uniform1f(resolutionUniform, canvas.width);
|
||||
gl.drawArrays(gl.TRIANGLES, 0, 6);
|
||||
if (!canvas.classList.contains('rendered')) canvas.classList.add('rendered');
|
||||
if (active) frame = requestAnimationFrame(draw);
|
||||
};
|
||||
|
||||
refresh = () => {
|
||||
if (!frame && !disposed) {
|
||||
lastTime = 0;
|
||||
frame = requestAnimationFrame(draw);
|
||||
}
|
||||
};
|
||||
const resize = () => {
|
||||
const size = Math.max(1, Math.round(canvas.clientWidth * 2));
|
||||
canvas.width = canvas.height = size;
|
||||
gl.viewport(0, 0, size, size);
|
||||
refresh();
|
||||
};
|
||||
const onVisibility = () => {
|
||||
cancelAnimationFrame(frame);
|
||||
frame = 0;
|
||||
if (!document.hidden) refresh();
|
||||
};
|
||||
const onContextLost = () => {
|
||||
cancelAnimationFrame(frame);
|
||||
frame = 0;
|
||||
canvas.classList.remove('rendered');
|
||||
};
|
||||
const observer = new ResizeObserver(resize);
|
||||
observer.observe(canvas);
|
||||
const themeObserver = new MutationObserver(refresh);
|
||||
themeObserver.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
subtree: true,
|
||||
attributeFilter: ['class']
|
||||
});
|
||||
reducedMotion.addEventListener('change', refresh);
|
||||
document.addEventListener('visibilitychange', onVisibility);
|
||||
canvas.addEventListener('webglcontextlost', onContextLost);
|
||||
resize();
|
||||
|
||||
return () => {
|
||||
disposed = true;
|
||||
cancelAnimationFrame(frame);
|
||||
observer.disconnect();
|
||||
themeObserver.disconnect();
|
||||
reducedMotion.removeEventListener('change', refresh);
|
||||
refresh = () => {};
|
||||
document.removeEventListener('visibilitychange', onVisibility);
|
||||
canvas.removeEventListener('webglcontextlost', onContextLost);
|
||||
dispose();
|
||||
};
|
||||
});
|
||||
</script>
|
||||
|
||||
<span class="voice-orb" class:muted aria-hidden="true">
|
||||
<canvas bind:this={canvas}></canvas>
|
||||
</span>
|
||||
|
||||
<style>
|
||||
.voice-orb {
|
||||
display: block;
|
||||
position: relative;
|
||||
width: 112px;
|
||||
aspect-ratio: 1;
|
||||
transition: opacity 240ms ease;
|
||||
}
|
||||
canvas {
|
||||
display: block;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
border-radius: 50%;
|
||||
background: radial-gradient(circle at 68% 22%, #9a9ca1, #7b7d82 49%, #5d6065);
|
||||
}
|
||||
:global(.dark) canvas {
|
||||
background: radial-gradient(circle at 68% 22%, #b9bbc0, #93959a 49%, #6d7076);
|
||||
}
|
||||
canvas:global(.rendered) {
|
||||
background: transparent;
|
||||
}
|
||||
.muted {
|
||||
opacity: 0.55;
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.voice-orb {
|
||||
transition: none;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
|
|
@ -3,6 +3,7 @@
|
|||
import { config, showCallOverlay, showControls } from '$lib/stores';
|
||||
import type { RealtimeCall } from '$lib/utils/realtime';
|
||||
import CallOverlay from './CallOverlay.svelte';
|
||||
import BridgeCallOverlay from './CallOverlay/BridgeCallOverlay.svelte';
|
||||
import { toast } from 'svelte-sonner';
|
||||
|
||||
const i18n = getContext<any>('i18n');
|
||||
|
|
@ -36,18 +37,28 @@
|
|||
started = true;
|
||||
});
|
||||
|
||||
$: if (started && callMode === 'bridge' && !bridge?.connected && !bridge?.connecting) close();
|
||||
$: if (
|
||||
started &&
|
||||
callMode === 'bridge' &&
|
||||
!bridge?.connected &&
|
||||
!bridge?.connecting &&
|
||||
!bridge?.error
|
||||
)
|
||||
close();
|
||||
</script>
|
||||
|
||||
{#if started}
|
||||
<CallOverlay
|
||||
bind:files
|
||||
{submitPrompt}
|
||||
{stopResponse}
|
||||
{modelId}
|
||||
{chatId}
|
||||
{eventTarget}
|
||||
bridge={callMode === 'bridge' ? bridge : undefined}
|
||||
on:close
|
||||
/>
|
||||
{#if callMode === 'bridge'}
|
||||
<BridgeCallOverlay {bridge} on:close />
|
||||
{:else}
|
||||
<CallOverlay
|
||||
bind:files
|
||||
{submitPrompt}
|
||||
{stopResponse}
|
||||
{modelId}
|
||||
{chatId}
|
||||
{eventTarget}
|
||||
on:close
|
||||
/>
|
||||
{/if}
|
||||
{/if}
|
||||
|
|
|
|||
|
|
@ -13,6 +13,9 @@ class RealtimeAudioProcessor extends AudioWorkletProcessor {
|
|||
this.rendered = new Map();
|
||||
this.ended = new Set();
|
||||
this.ticks = 0;
|
||||
this.inputEnergy = 0;
|
||||
this.outputEnergy = 0;
|
||||
this.levelSamples = 0;
|
||||
this.port.onmessage = ({ data }) => {
|
||||
if (data.type === 'capture') {
|
||||
this.enabled = data.enabled;
|
||||
|
|
@ -86,8 +89,21 @@ class RealtimeAudioProcessor extends AudioWorkletProcessor {
|
|||
}
|
||||
if (!this.queued) this.playing = false;
|
||||
}
|
||||
for (let i = 0; i < output.length; i++) {
|
||||
const sample = this.enabled ? (input?.[i] ?? 0) : 0;
|
||||
this.inputEnergy += sample * sample;
|
||||
this.outputEnergy += output[i] * output[i];
|
||||
}
|
||||
this.levelSamples += output.length;
|
||||
if (++this.ticks % 8 === 0) {
|
||||
this.port.postMessage({ type: 'playback', queued: this.queued, received: this.received });
|
||||
this.port.postMessage({
|
||||
type: 'playback',
|
||||
queued: this.queued,
|
||||
received: this.received,
|
||||
inputLevel: Math.sqrt(this.inputEnergy / this.levelSamples),
|
||||
outputLevel: Math.sqrt(this.outputEnergy / this.levelSamples)
|
||||
});
|
||||
this.inputEnergy = this.outputEnergy = this.levelSamples = 0;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -75,6 +75,9 @@ export class RealtimeCall {
|
|||
approval = false;
|
||||
model = '';
|
||||
voice = '';
|
||||
inputLevel = 0;
|
||||
outputLevel = 0;
|
||||
error = '';
|
||||
private ws?: WebSocket;
|
||||
private context?: AudioContext;
|
||||
private stream?: MediaStream;
|
||||
|
|
@ -109,6 +112,7 @@ export class RealtimeCall {
|
|||
|
||||
async connect(token: string) {
|
||||
if (this.connected || this.connecting) return;
|
||||
this.error = '';
|
||||
this.connecting = true;
|
||||
this.options.change();
|
||||
const session = ++this.session;
|
||||
|
|
@ -192,12 +196,21 @@ export class RealtimeCall {
|
|||
audio: btoa(String.fromCharCode(...new Uint8Array(data.pcm)))
|
||||
});
|
||||
} else if (data.type === 'playback') {
|
||||
const inputLevel = this.muted ? 0 : (data.inputLevel ?? 0);
|
||||
const outputLevel = data.outputLevel ?? 0;
|
||||
const levelsChanged =
|
||||
Math.abs(inputLevel - this.inputLevel) > 0.002 ||
|
||||
Math.abs(outputLevel - this.outputLevel) > 0.002;
|
||||
const nextSpeaking = data.queued > 0 || this.sentSamples > data.received;
|
||||
const speakingChanged = this.speaking !== nextSpeaking;
|
||||
this.speaking = nextSpeaking;
|
||||
if (!this.speaking && !this.activeResponse && !this.responseRequested)
|
||||
this.speakingResponses.clear();
|
||||
if (speakingChanged) this.options.change();
|
||||
if (levelsChanged || speakingChanged) {
|
||||
this.inputLevel = inputLevel;
|
||||
this.outputLevel = outputLevel;
|
||||
this.options.change();
|
||||
}
|
||||
this.flush();
|
||||
} else if (data.type === 'overflow') {
|
||||
this.stopSpeaking();
|
||||
|
|
@ -591,6 +604,7 @@ export class RealtimeCall {
|
|||
}
|
||||
this.speakingResponses.clear();
|
||||
this.speaking = false;
|
||||
this.outputLevel = 0;
|
||||
const id = ++this.clearId;
|
||||
this.clears.set(id, responses);
|
||||
this.audio?.port.postMessage({ type: 'clear', id });
|
||||
|
|
@ -617,6 +631,7 @@ export class RealtimeCall {
|
|||
|
||||
mute() {
|
||||
this.muted = !this.muted;
|
||||
if (this.muted) this.inputLevel = 0;
|
||||
this.stream?.getAudioTracks().forEach((track) => {
|
||||
track.enabled = !this.muted;
|
||||
});
|
||||
|
|
@ -626,6 +641,7 @@ export class RealtimeCall {
|
|||
}
|
||||
|
||||
private fail(message: string) {
|
||||
this.error = message;
|
||||
this.options.error(message);
|
||||
this.end();
|
||||
}
|
||||
|
|
@ -657,6 +673,7 @@ export class RealtimeCall {
|
|||
this.context = undefined;
|
||||
this.audio = undefined;
|
||||
this.connected = this.connecting = this.speaking = this.working = this.approval = false;
|
||||
this.inputLevel = this.outputLevel = 0;
|
||||
this.responseRequested = false;
|
||||
this.cancelRequested = false;
|
||||
this.receivingSpeech = false;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue