diff --git a/src/lib/components/workspace/Models/VoiceAvatarEditor.svelte b/src/lib/components/workspace/Models/VoiceAvatarEditor.svelte
new file mode 100644
index 0000000000..e10ed77036
--- /dev/null
+++ b/src/lib/components/workspace/Models/VoiceAvatarEditor.svelte
@@ -0,0 +1,389 @@
+
+
+
+
+
diff --git a/src/lib/components/workspace/Models/VoiceAvatarSettings.svelte b/src/lib/components/workspace/Models/VoiceAvatarSettings.svelte
new file mode 100644
index 0000000000..e722f90c95
--- /dev/null
+++ b/src/lib/components/workspace/Models/VoiceAvatarSettings.svelte
@@ -0,0 +1,90 @@
+
+
+
+
+
{$i18n.t('Voice avatar')}
+
{$i18n.t(value ? 'Custom avatar' : 'Default orb')}
+
+
+
+
+
+
+
+
{$i18n.t('Avatar setup')}
+
+
+ {#if show}
+
+
+
+ {/if}
+
+
+
+
+
diff --git a/src/lib/utils/realtime-audio.js b/src/lib/utils/realtime-audio.js
index ee44128810..7337472487 100644
--- a/src/lib/utils/realtime-audio.js
+++ b/src/lib/utils/realtime-audio.js
@@ -16,6 +16,8 @@ class RealtimeAudioProcessor extends AudioWorkletProcessor {
this.inputEnergy = 0;
this.outputEnergy = 0;
this.levelSamples = 0;
+ this.playbackSamples = 0;
+ this.clearId = 0;
this.port.onmessage = ({ data }) => {
if (data.type === 'capture') {
this.enabled = data.enabled;
@@ -31,6 +33,8 @@ class RealtimeAudioProcessor extends AudioWorkletProcessor {
} else if (data.type === 'done') {
this.ended.add(data.response_id);
} else if (data.type === 'clear') {
+ this.clearId = data.id;
+ this.playbackSamples = this.outputEnergy = 0;
this.port.postMessage({
type: 'cleared',
id: data.id,
@@ -76,6 +80,7 @@ class RealtimeAudioProcessor extends AudioWorkletProcessor {
offset += count;
chunk.offset += count;
this.queued -= count;
+ this.playbackSamples += count;
const key = `${chunk.item_id}:${chunk.content_index}`;
const position = this.rendered.get(key) ?? {
response_id: chunk.response_id,
@@ -98,12 +103,14 @@ class RealtimeAudioProcessor extends AudioWorkletProcessor {
if (++this.ticks % 8 === 0) {
this.port.postMessage({
type: 'playback',
+ clearId: this.clearId,
+ playbackActive: this.playbackSamples > 0,
queued: this.queued,
received: this.received,
inputLevel: Math.sqrt(this.inputEnergy / this.levelSamples),
outputLevel: Math.sqrt(this.outputEnergy / this.levelSamples)
});
- this.inputEnergy = this.outputEnergy = this.levelSamples = 0;
+ this.inputEnergy = this.outputEnergy = this.levelSamples = this.playbackSamples = 0;
}
return true;
}
diff --git a/src/lib/utils/realtime.ts b/src/lib/utils/realtime.ts
index aed2a09c54..e220a15321 100644
--- a/src/lib/utils/realtime.ts
+++ b/src/lib/utils/realtime.ts
@@ -80,6 +80,9 @@ export class RealtimeCall {
inputLevel = 0;
outputLevel = 0;
error = '';
+ animationPlayer?: (name: string) => string;
+ animationInterruption = 0;
+ private animationCalls = new Set
();
private ws?: WebSocket;
private context?: AudioContext;
private stream?: MediaStream;
@@ -441,6 +444,29 @@ export class RealtimeCall {
event.item.status === 'completed'
) {
const response = this.responses.get(event.response_id);
+ if (event.item.name === 'play_animation') {
+ if (this.animationCalls.has(event.item.call_id)) return;
+ this.animationCalls.add(event.item.call_id);
+ let status = 'unavailable';
+ if (!response || this.interrupted.has(event.response_id) || this.receivingSpeech)
+ status = 'cancelled';
+ else {
+ try {
+ const args = JSON.parse(event.item.arguments);
+ if (
+ event.animation_valid !== false &&
+ Object.keys(args).length === 1 &&
+ typeof args.name === 'string'
+ )
+ status = this.animationPlayer?.(args.name) ?? 'unavailable';
+ } catch {
+ /* Bad or unavailable gestures never interrupt a voice call. */
+ }
+ response.animation = true;
+ }
+ this.send({ type: 'bridge.animation.result', call_id: event.item.call_id, status });
+ return;
+ }
if (
!response ||
event.item.name !== 'generate_chat_completion' ||
@@ -521,6 +547,14 @@ export class RealtimeCall {
if (response) {
response.done = true;
this.saveSpeech(response);
+ if (
+ response.animation &&
+ !response.delegated &&
+ event.response.status === 'completed' &&
+ !this.interrupted.has(response.id)
+ ) {
+ this.enqueue({ type: 'bridge.animation.respond', response_id: response.id });
+ }
}
if (['failed', 'incomplete'].includes(event.response.status))
this.options.error('The voice response did not complete.');
@@ -619,6 +653,7 @@ export class RealtimeCall {
}
stopSpeaking() {
+ this.animationInterruption++;
if (this.responseRequested) this.cancelRequested = true;
const responses = new Set(
[...this.speakingResponses].filter((id) => !this.interrupted.has(id))
@@ -639,7 +674,9 @@ export class RealtimeCall {
const id = ++this.clearId;
this.clears.set(id, responses);
this.audio?.port.postMessage({ type: 'clear', id });
- this.commands = this.commands.filter((command) => command.type !== 'bridge.status');
+ this.commands = this.commands.filter(
+ (command) => command.type !== 'bridge.status' && command.type !== 'bridge.animation.respond'
+ );
this.options.change();
}
@@ -723,6 +760,7 @@ export class RealtimeCall {
this.responses.clear();
this.inputs.clear();
this.calls.clear();
+ this.animationCalls.clear();
this.clears.clear();
this.clearId = 0;
this.options.change();
diff --git a/src/lib/utils/voice-avatar-animation.ts b/src/lib/utils/voice-avatar-animation.ts
new file mode 100644
index 0000000000..f849ab7006
--- /dev/null
+++ b/src/lib/utils/voice-avatar-animation.ts
@@ -0,0 +1,194 @@
+import * as THREE from 'three';
+import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
+import {
+ VRMAnimationLoaderPlugin,
+ createVRMAnimationHumanoidTracks
+} from '@pixiv/three-vrm-animation';
+import type { VRM } from '@pixiv/three-vrm';
+
+export const ANIMATION_MAX_BYTES = 10 * 1024 * 1024;
+
+export function validateAnimation(data: ArrayBuffer) {
+ if (data.byteLength > ANIMATION_MAX_BYTES) throw new Error('Animation must be at most 10 MiB.');
+ if (data.byteLength < 28) throw new Error('Upload a binary VRMA file.');
+ const view = new DataView(data);
+ const length = view.getUint32(12, true);
+ if (
+ view.getUint32(0, true) !== 0x46546c67 ||
+ view.getUint32(4, true) !== 2 ||
+ view.getUint32(8, true) !== data.byteLength ||
+ view.getUint32(16, true) !== 0x4e4f534a ||
+ length > 2 * 1024 * 1024 ||
+ length + 28 > data.byteLength
+ )
+ throw new Error('Invalid VRMA container.');
+ const start = 28 + length;
+ const bytes = view.getUint32(20 + length, true);
+ if (view.getUint32(24 + length, true) !== 0x004e4942 || start + bytes !== data.byteLength)
+ throw new Error('Invalid animation binary data.');
+ const json = JSON.parse(new TextDecoder().decode(new Uint8Array(data, 20, length)));
+ const ext = json.extensions?.VRMC_vrm_animation;
+ const bones = Object.values(ext?.humanoid?.humanBones ?? {}) as { node: number }[];
+ if (
+ ext?.specVersion !== '1.0' ||
+ !bones.length ||
+ !json.nodes?.length ||
+ json.nodes.length > 512 ||
+ bones.some((b) => !Number.isInteger(b.node) || !json.nodes[b.node])
+ )
+ throw new Error('Use a VRMA 1.0 humanoid animation.');
+ const parents = new Set(),
+ visiting = new Set(),
+ visited = new Set();
+ const visit = (index: number) => {
+ if (!Number.isInteger(index) || !json.nodes[index] || visiting.has(index))
+ throw new Error('Invalid animation node hierarchy.');
+ if (visited.has(index)) return;
+ visiting.add(index);
+ const node = json.nodes[index];
+ for (const [key, length] of Object.entries({
+ translation: 3,
+ rotation: 4,
+ scale: 3,
+ matrix: 16
+ })) {
+ if (
+ node[key] &&
+ (node[key].length !== length || node[key].some((v: number) => !Number.isFinite(v)))
+ )
+ throw new Error('Invalid animation transform.');
+ }
+ for (const child of node.children ?? []) {
+ if (parents.has(child)) throw new Error('Animation nodes must have a single parent.');
+ parents.add(child);
+ visit(child);
+ }
+ visiting.delete(index);
+ visited.add(index);
+ };
+ json.nodes.forEach((_: unknown, index: number) => visit(index));
+ for (const scene of json.scenes ?? [])
+ if ((scene.nodes ?? []).some((n: number) => !Number.isInteger(n) || !json.nodes[n]))
+ throw new Error('Invalid animation scene.');
+ if (
+ json.buffers?.length !== 1 ||
+ 'uri' in json.buffers[0] ||
+ json.buffers[0].byteLength > bytes ||
+ json.buffers[0].byteLength < bytes - 3 ||
+ ['images', 'textures', 'meshes', 'skins'].some((k) => json[k]?.length) ||
+ (json.extensionsRequired ?? []).some((e: string) => e !== 'VRMC_vrm_animation')
+ )
+ throw new Error('Export embedded animation only, without meshes or textures.');
+ for (const buffer of json.bufferViews ?? []) {
+ if (
+ (buffer.buffer ?? 0) !== 0 ||
+ (buffer.byteOffset ?? 0) < 0 ||
+ !(buffer.byteLength > 0) ||
+ (buffer.byteOffset ?? 0) + buffer.byteLength > bytes
+ )
+ throw new Error('Invalid animation buffer.');
+ }
+ let count = 0;
+ const values = (json.accessors ?? []).map((a: any) => {
+ count += a.count;
+ const components = ({ SCALAR: 1, VEC3: 3, VEC4: 4 } as Record)[
+ a.type
+ ] as number;
+ const buffer = json.bufferViews?.[a.bufferView];
+ const stride = buffer?.byteStride ?? components * 4;
+ const offset = a.byteOffset ?? 0;
+ if (
+ !Number.isInteger(a.count) ||
+ a.count <= 0 ||
+ count > 500000 ||
+ a.sparse ||
+ a.componentType !== 5126 ||
+ !components ||
+ !buffer ||
+ stride < components * 4 ||
+ stride % 4 ||
+ offset < 0 ||
+ offset + (a.count - 1) * stride + components * 4 > buffer.byteLength
+ )
+ throw new Error('Invalid animation keyframes.');
+ return Array.from({ length: a.count }, (_, i) =>
+ Array.from({ length: components }, (_, c) => {
+ const value = view.getFloat32(
+ start + (buffer.byteOffset ?? 0) + offset + i * stride + c * 4,
+ true
+ );
+ if (!Number.isFinite(value)) throw new Error('Animation keyframes must be finite.');
+ return value;
+ })
+ );
+ });
+ const animation = json.animations?.[0];
+ if (
+ json.animations?.length !== 1 ||
+ !animation?.channels?.length ||
+ animation.channels.length > 256
+ )
+ throw new Error('Export exactly one animation per VRMA file.');
+ let duration = 0,
+ body = false;
+ for (const channel of animation.channels) {
+ const target = channel.target;
+ const sampler = animation.samplers?.[channel.sampler];
+ const times = values[sampler?.input],
+ output = values[sampler?.output];
+ const interpolation = sampler?.interpolation ?? 'LINEAR';
+ if (!['LINEAR', 'STEP'].includes(interpolation))
+ throw new Error('Export baked animation with linear or stepped keyframes.');
+ if (
+ !json.nodes[target?.node] ||
+ !['rotation', 'translation'].includes(target.path) ||
+ !times ||
+ !output ||
+ json.accessors[sampler.input].type !== 'SCALAR' ||
+ !['LINEAR', 'STEP'].includes(interpolation) ||
+ output.length !== times.length ||
+ output[0].length !== (target.path === 'rotation' ? 4 : 3) ||
+ times.some((t: number[], i: number) => t[0] < 0 || (i > 0 && t[0] <= times[i - 1][0]))
+ )
+ throw new Error('Invalid animation channels or timing.');
+ duration = Math.max(duration, times[times.length - 1][0]);
+ body ||= bones.some((b) => b.node === target.node);
+ }
+ if (!body || duration <= 0 || duration > 60)
+ throw new Error('Use a body animation between 0 and 60 seconds.');
+}
+
+export async function loadBodyAnimation(data: ArrayBuffer, vrm: VRM) {
+ validateAnimation(data);
+ const manager = new THREE.LoadingManager();
+ manager.setURLModifier(() => {
+ throw new Error('External animation resources are not supported.');
+ });
+ const loader = new GLTFLoader(manager);
+ loader.register((parser) => new VRMAnimationLoaderPlugin(parser));
+ const gltf = await loader.parseAsync(data, '');
+ const animation = gltf.userData.vrmAnimations?.[0];
+ if (
+ !animation ||
+ !Number.isFinite(animation.restHipsPosition.y) ||
+ animation.restHipsPosition.y <= 0
+ )
+ throw new Error('Animation needs a valid humanoid rest pose.');
+ const humanoid = createVRMAnimationHumanoidTracks(animation, vrm.humanoid, vrm.meta.metaVersion);
+ for (const [name, track] of humanoid.rotation)
+ track.setInterpolation(animation.humanoidTracks.rotation.get(name).getInterpolation());
+ const tracks: THREE.KeyframeTrack[] = [
+ ...humanoid.rotation.values(),
+ ...humanoid.translation.values()
+ ];
+ if (!tracks.length) throw new Error('Animation has no compatible body tracks.');
+ const hips = vrm.humanoid.getNormalizedBoneNode('hips')!;
+ for (const track of tracks) {
+ if (!track.name.endsWith('.position')) continue;
+ for (let i = 0; i < track.values.length; i += 3) {
+ track.values[i] = hips.position.x;
+ track.values[i + 2] = hips.position.z;
+ }
+ }
+ return new THREE.AnimationClip('body', animation.duration, tracks);
+}
diff --git a/src/lib/utils/voice-avatar-renderer.ts b/src/lib/utils/voice-avatar-renderer.ts
new file mode 100644
index 0000000000..dd08b5730e
--- /dev/null
+++ b/src/lib/utils/voice-avatar-renderer.ts
@@ -0,0 +1,308 @@
+import * as THREE from 'three';
+import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
+import { VRMLoaderPlugin, VRMUtils, type VRM } from '@pixiv/three-vrm';
+import { loadBodyAnimation } from './voice-avatar-animation';
+import { validateAvatar, type VoiceAvatarConfig } from './voice-avatar';
+
+export type AvatarSignals = {
+ speaking: boolean;
+ listening: boolean;
+ level: number;
+ active: boolean;
+};
+
+export async function createAvatarRenderer(
+ canvas: HTMLCanvasElement,
+ data: ArrayBuffer,
+ signal: AbortSignal
+) {
+ await validateAvatar(data);
+ signal.throwIfAborted();
+ const manager = new THREE.LoadingManager();
+ manager.setURLModifier((url) => {
+ if (!url.startsWith('blob:')) throw new Error('External avatar resources are not supported.');
+ return url;
+ });
+ const loader = new GLTFLoader(manager);
+ loader.register((parser) => new VRMLoaderPlugin(parser));
+ const gltf = await loader.parseAsync(data, '');
+ const vrm: VRM | undefined = gltf.userData.vrm;
+ if (!vrm || signal.aborted) {
+ VRMUtils.deepDispose(gltf.scene);
+ signal.throwIfAborted();
+ throw new Error('This VRM could not be loaded.');
+ }
+ VRMUtils.rotateVRM0(vrm);
+ const scene = new THREE.Scene();
+ scene.add(vrm.scene);
+ scene.add(new THREE.HemisphereLight(0xffffff, 0x8b929f, 2));
+ const light = new THREE.DirectionalLight(0xffffff, 2.2);
+ light.position.set(-1, 2, 3);
+ scene.add(light);
+ let renderer: THREE.WebGLRenderer;
+ try {
+ renderer = new THREE.WebGLRenderer({
+ canvas,
+ alpha: true,
+ antialias: true,
+ powerPreference: 'low-power'
+ });
+ } catch (error) {
+ VRMUtils.deepDispose(vrm.scene);
+ throw error;
+ }
+ renderer.setPixelRatio(Math.min(window.devicePixelRatio || 1, 2));
+ renderer.setClearColor(0x000000, 0);
+ renderer.outputColorSpace = THREE.SRGBColorSpace;
+ const camera = new THREE.PerspectiveCamera(30, 1, 0.01, 100);
+ const bone = (name: Parameters[0]) =>
+ vrm.humanoid.getNormalizedBoneNode(name);
+ const head = bone('head');
+ const spine = bone('spine');
+ const leftArm = bone('leftUpperArm');
+ const rightArm = bone('rightUpperArm');
+ // VRM's normalized skeleton gives both VRM 0 and 1 a predictable T-pose.
+ if (leftArm) leftArm.rotation.z = -1.22;
+ if (rightArm) rightArm.rotation.z = 1.22;
+ const leftElbow = bone('leftLowerArm');
+ const rightElbow = bone('rightLowerArm');
+ if (leftElbow) leftElbow.rotation.y = -0.12;
+ if (rightElbow) rightElbow.rotation.y = 0.12;
+ vrm.update(0);
+ vrm.scene.updateMatrixWorld(true);
+ const bounds = new THREE.Box3().setFromObject(vrm.scene);
+ const size = bounds.getSize(new THREE.Vector3());
+ const center = bounds.getCenter(new THREE.Vector3());
+ const headPosition = head?.getWorldPosition(new THREE.Vector3()) ?? center.clone();
+ const bodyHeight = Math.max(size.y, 0.5);
+ const target = new THREE.Object3D();
+ scene.add(target);
+ if (vrm.lookAt) vrm.lookAt.target = target;
+ const expressions = vrm.expressionManager;
+ const mouth = !!expressions?.getExpression('aa');
+ const blink = !!expressions?.getExpression('blink');
+ const separateBlink =
+ !!expressions?.getExpression('blinkLeft') && !!expressions?.getExpression('blinkRight');
+ let speaking = 0,
+ listening = 0,
+ speechTime = 0,
+ listenTime = 0,
+ mouthWeight = 0,
+ elapsed = 0,
+ nextBlink = 2.8,
+ blinkStart = -10;
+ let width = 0,
+ height = 0;
+
+ const mixer = new THREE.AnimationMixer(vrm.scene);
+ const poses = Object.values(vrm.humanoid.normalizedHumanBones).map(({ node }) => ({
+ node,
+ restQ: node.quaternion.clone(),
+ restP: node.position.clone(),
+ q: node.quaternion.clone(),
+ p: node.position.clone()
+ }));
+ const clips = new Map();
+ let current: THREE.AnimationAction | null = null;
+ let outgoing: THREE.AnimationAction | null = null;
+ let fadeTime = 0,
+ bodyWeight = 0,
+ speechHold = 0;
+ let gesture = false,
+ gestureElapsed = 0;
+ let enabled = true;
+ let cameraDistance = 0;
+ const switchClip = (clip: THREE.AnimationClip, once: boolean) => {
+ outgoing?.stop();
+ outgoing = current;
+ current = mixer.clipAction(clip);
+ current.reset().setLoop(once ? THREE.LoopOnce : THREE.LoopRepeat, once ? 1 : Infinity);
+ current.clampWhenFinished = true;
+ current.enabled = true;
+ current.setEffectiveWeight(1).play();
+ if (outgoing && outgoing !== current) current.crossFadeFrom(outgoing, 0.3, false);
+ fadeTime = 0;
+ };
+ const cancelGesture = () => {
+ gesture = false;
+ gestureElapsed = 0;
+ speechHold = 0;
+ };
+
+ return {
+ capabilities: { mouth, blink: blink || separateBlink },
+ async loadAnimation(id: string, data: ArrayBuffer) {
+ const clip = await loadBodyAnimation(data, vrm);
+ signal.throwIfAborted();
+ // Every body clip owns the full pose, including bones omitted by its author.
+ const names = new Set(clip.tracks.map((track) => track.name));
+ for (const { node, restQ, restP } of poses) {
+ if (!names.has(`${node.name}.quaternion`))
+ clip.tracks.push(
+ new THREE.QuaternionKeyframeTrack(`${node.name}.quaternion`, [0], restQ.toArray())
+ );
+ if (!names.has(`${node.name}.position`))
+ clip.tracks.push(
+ new THREE.VectorKeyframeTrack(`${node.name}.position`, [0], restP.toArray())
+ );
+ }
+ clips.set(id, clip);
+ },
+ playGesture(id: string) {
+ if (!enabled) return 'unavailable' as const;
+ if (gesture) return 'busy' as const;
+ const clip = clips.get(id);
+ if (!clip) return 'unavailable' as const;
+ switchClip(clip, true);
+ gesture = true;
+ gestureElapsed = 0;
+ return 'started' as const;
+ },
+ cancelGesture,
+ render(delta: number, config: VoiceAvatarConfig, input: AvatarSignals, reducedMotion: boolean) {
+ const dt = Math.min(delta, 0.05);
+ elapsed += dt;
+ enabled = input.active && !reducedMotion;
+ if (!enabled) cancelGesture();
+ if (input.speaking && input.active) speechHold = 0.2;
+ else speechHold = Math.max(0, speechHold - dt);
+ if (input.listening) speechHold = 0;
+ const state =
+ input.speaking || speechHold > 0 ? 'speaking' : input.listening ? 'listening' : 'idle';
+ if (gesture) {
+ gestureElapsed += dt;
+ if (gestureElapsed >= current!.getClip().duration) cancelGesture();
+ }
+ const stateClip = enabled ? clips.get(config.states?.[state]?.file_id ?? '') : undefined;
+ if (
+ !gesture &&
+ stateClip &&
+ (current?.getClip() !== stateClip || current.loop === THREE.LoopOnce)
+ )
+ switchClip(stateClip, false);
+ const authored = enabled && (gesture || !!stateClip);
+ bodyWeight = THREE.MathUtils.clamp(bodyWeight + (authored ? dt : -dt) / 0.3, 0, 1);
+ if (!enabled) bodyWeight = 0;
+ // Start the procedural pose from rest, never from last frame's authored pose.
+ for (const pose of poses) {
+ pose.node.quaternion.copy(pose.restQ);
+ pose.node.position.copy(pose.restP);
+ }
+ const rect = canvas.getBoundingClientRect();
+ if (!rect.width || !rect.height) return;
+ if (rect.width !== width || rect.height !== height) {
+ width = rect.width;
+ height = rect.height;
+ renderer.setSize(width, height, false);
+ camera.aspect = width / height;
+ camera.updateProjectionMatrix();
+ }
+
+ const smooth = 1 - Math.exp(-dt * 5);
+ speaking += ((input.active && input.speaking ? 1 : 0) - speaking) * smooth;
+ listening +=
+ ((input.active && input.listening && !input.speaking ? 1 : 0) - listening) * smooth;
+ speechTime = input.active && input.speaking ? speechTime + dt : 0;
+ listenTime = input.active && input.listening ? listenTime + dt : 0;
+ // Full-body framing needs readable poses, not faster oscillation.
+ const amount = reducedMotion ? 0 : Math.sqrt(0.35);
+ const breath = Math.sin(elapsed * 1.5);
+ const phrase = 0.5 + 0.5 * Math.sin(speechTime * 1.1);
+ const leftGesture = speaking * (0.45 + 0.55 * phrase);
+ const rightGesture = speaking * (0.15 + 0.55 * (1 - phrase));
+ // One small acknowledgment every few seconds, with stillness between nods.
+ const nodPhase = listenTime % 5;
+ const nod = nodPhase < 1.2 ? Math.sin((nodPhase / 1.2) * Math.PI) : 0;
+ if (spine) {
+ spine.rotation.x = amount * (0.012 * breath + 0.065 * listening);
+ spine.rotation.y = amount * speaking * 0.045 * Math.sin(speechTime * 0.8);
+ }
+ if (head) {
+ head.rotation.x =
+ amount *
+ (0.018 * Math.sin(elapsed * 0.7) +
+ speaking * 0.085 * Math.sin(speechTime * 2.2) +
+ listening * (0.08 + 0.12 * nod));
+ head.rotation.y =
+ amount *
+ (0.025 * Math.sin(elapsed * 0.47) + speaking * 0.09 * Math.sin(speechTime * 0.9));
+ head.rotation.z = amount * (0.02 * Math.sin(elapsed * 0.63) + listening * 0.18);
+ }
+ if (leftArm) leftArm.rotation.z = -1.22 + amount * (0.015 * breath + 0.22 * leftGesture);
+ if (rightArm) rightArm.rotation.z = 1.22 - amount * (0.015 * breath + 0.22 * rightGesture);
+ if (leftElbow) leftElbow.rotation.z = amount * 1.25 * leftGesture;
+ if (rightElbow) rightElbow.rotation.z = -amount * 1.25 * rightGesture;
+ if (current && bodyWeight > 0) {
+ for (const pose of poses) {
+ pose.q.copy(pose.node.quaternion);
+ pose.p.copy(pose.node.position);
+ }
+ mixer.update(dt);
+ fadeTime += dt;
+ if (outgoing && fadeTime >= 0.3) {
+ if (outgoing !== current) outgoing.stop();
+ outgoing = null;
+ }
+ for (const pose of poses) {
+ pose.node.quaternion.slerpQuaternions(pose.q, pose.node.quaternion, bodyWeight);
+ pose.node.position.lerpVectors(pose.p, pose.node.position, bodyWeight);
+ }
+ } else if (current) {
+ mixer.stopAllAction();
+ current = outgoing = null;
+ }
+ // Only assistant PCM drives the mouth. Silence and interruptions close it immediately.
+ const desiredMouth =
+ input.active && input.speaking ? Math.min(1, Math.max(0, input.level) * 7) * 0.7 : 0;
+ mouthWeight =
+ desiredMouth === 0
+ ? 0
+ : mouthWeight + (desiredMouth - mouthWeight) * (1 - Math.exp(-dt * 24));
+ if (mouth) expressions!.setValue('aa', mouthWeight);
+ if (elapsed >= nextBlink) {
+ blinkStart = elapsed;
+ nextBlink = elapsed + 3.2 + Math.random() * 2.8;
+ }
+ const blinkPhase = (elapsed - blinkStart) / 0.17;
+ const blinkWeight = blinkPhase >= 0 && blinkPhase <= 1 ? Math.sin(blinkPhase * Math.PI) : 0;
+ if (blink) expressions!.setValue('blink', blinkWeight);
+ else if (separateBlink) {
+ expressions!.setValue('blinkLeft', blinkWeight);
+ expressions!.setValue('blinkRight', blinkWeight);
+ }
+ if (vrm.lookAt) vrm.lookAt.autoUpdate = true;
+ target.position.set(
+ camera.position.x + amount * 0.1 * Math.sin(elapsed * 0.4),
+ headPosition.y,
+ camera.position.z
+ );
+ vrm.update(reducedMotion ? 0 : dt);
+ vrm.scene.updateMatrixWorld(true);
+ const animatedBounds = bounds.clone();
+ const bonePosition = new THREE.Vector3();
+ for (const { node } of poses)
+ animatedBounds.expandByPoint(node.getWorldPosition(bonePosition));
+ animatedBounds.expandByScalar(bodyHeight * 0.06);
+ const extent = animatedBounds.getSize(new THREE.Vector3());
+ const animatedCenter = animatedBounds.getCenter(new THREE.Vector3());
+ const fitHeight = Math.max(bodyHeight, extent.y + 2 * Math.abs(animatedCenter.y - center.y));
+ const fitWidth = Math.max(size.x, extent.x + 2 * Math.abs(animatedCenter.x - center.x));
+ const distance =
+ Math.max(size.z, extent.z) / 2 +
+ (Math.max(fitHeight, fitWidth / camera.aspect) * 1.2) / (2 * Math.tan(Math.PI / 12));
+ // Widen before a limb reaches the edge; never zoom in/out on each gesture.
+ cameraDistance = Math.max(distance, cameraDistance);
+ camera.position.set(center.x, center.y, center.z + cameraDistance);
+ camera.lookAt(center);
+ renderer.render(scene, camera);
+ },
+ dispose() {
+ mixer.stopAllAction();
+ mixer.uncacheRoot(vrm.scene);
+ clips.clear();
+ VRMUtils.deepDispose(vrm.scene);
+ renderer.dispose();
+ renderer.forceContextLoss();
+ }
+ };
+}
diff --git a/src/lib/utils/voice-avatar.ts b/src/lib/utils/voice-avatar.ts
new file mode 100644
index 0000000000..9caeb4fcc3
--- /dev/null
+++ b/src/lib/utils/voice-avatar.ts
@@ -0,0 +1,112 @@
+export const AVATAR_MAX_BYTES = 25 * 1024 * 1024;
+
+export type AvatarState = 'idle' | 'listening' | 'speaking';
+export type AvatarAnimation = { file_id: string };
+export type AvatarGesture = AvatarAnimation & { name: string; description: string };
+export type AnimationFiles = Record;
+export function avatarAssetIds(config?: VoiceAvatarConfig | null): string[] {
+ return config
+ ? [
+ config.file_id,
+ ...Object.values(config.states ?? {}).map((a) => a.file_id),
+ ...(config.gestures ?? []).map((a) => a.file_id)
+ ].filter(Boolean)
+ : [];
+}
+
+export type VoiceAvatarConfig = {
+ file_id: string;
+ states?: Partial>;
+ gestures?: AvatarGesture[];
+};
+
+// Validate before GLTFLoader can fetch resources or allocate GPU buffers.
+export function readAvatar(data: ArrayBuffer) {
+ if (data.byteLength > AVATAR_MAX_BYTES) throw new Error('Avatar must be at most 25 MiB.');
+ if (data.byteLength < 20) throw new Error('Upload a VRM 0.x or VRM 1.0 file.');
+ const view = new DataView(data);
+ if (
+ view.getUint32(0, true) !== 0x46546c67 ||
+ view.getUint32(4, true) !== 2 ||
+ view.getUint32(8, true) !== data.byteLength ||
+ view.getUint32(16, true) !== 0x4e4f534a
+ ) {
+ throw new Error('Upload a binary VRM file.');
+ }
+ const length = view.getUint32(12, true);
+ if (length > 2 * 1024 * 1024 || 20 + length + 8 > data.byteLength)
+ throw new Error('Invalid avatar container.');
+ const json = JSON.parse(new TextDecoder().decode(new Uint8Array(data, 20, length)));
+ const binaryStart = 28 + length;
+ if (
+ view.getUint32(24 + length, true) !== 0x004e4942 ||
+ binaryStart + view.getUint32(20 + length, true) !== data.byteLength
+ )
+ throw new Error('Invalid avatar binary data.');
+ const vrm = json.extensions?.VRMC_vrm ?? json.extensions?.VRM;
+ if (!vrm?.humanoid?.humanBones)
+ throw new Error('This model needs a VRM humanoid rig. A plain GLB is not enough.');
+ const bones = Array.isArray(vrm.humanoid.humanBones)
+ ? Object.fromEntries(vrm.humanoid.humanBones.map((bone: any) => [bone.bone, bone]))
+ : vrm.humanoid.humanBones;
+ for (const name of [
+ 'hips',
+ 'spine',
+ 'head',
+ 'leftUpperArm',
+ 'rightUpperArm',
+ 'leftLowerArm',
+ 'rightLowerArm'
+ ]) {
+ if (!Number.isInteger(bones[name]?.node) || !json.nodes?.[bones[name].node])
+ throw new Error(`Avatar is missing its ${name} bone.`);
+ }
+ if (
+ json.buffers?.length !== 1 ||
+ json.buffers[0].uri ||
+ json.buffers[0].byteLength > data.byteLength - binaryStart ||
+ (json.images ?? []).some((image: any) => image.uri || !Number.isInteger(image.bufferView))
+ ) {
+ throw new Error(
+ 'Embed all textures and buffers in the VRM file. External resources are not supported.'
+ );
+ }
+ const vertexCount = (json.accessors ?? []).reduce(
+ (sum: number, a: any) => sum + (a.count ?? 0),
+ 0
+ );
+ if (
+ (json.nodes?.length ?? 0) > 512 ||
+ (json.images?.length ?? 0) > 32 ||
+ vertexCount > 5_000_000
+ ) {
+ throw new Error('This avatar is too complex for a voice call. Use a lighter export.');
+ }
+ return { json, binaryStart };
+}
+
+export async function validateAvatar(data: ArrayBuffer) {
+ const { json, binaryStart } = readAvatar(data);
+ let pixels = 0;
+ for (const image of json.images ?? []) {
+ const buffer = json.bufferViews?.[image.bufferView];
+ const start = binaryStart + (buffer?.byteOffset ?? 0);
+ if (
+ !buffer ||
+ (buffer.buffer ?? 0) !== 0 ||
+ !Number.isInteger(buffer.byteLength) ||
+ start < binaryStart ||
+ start + buffer.byteLength > data.byteLength ||
+ !['image/png', 'image/jpeg', 'image/webp'].includes(image.mimeType)
+ )
+ throw new Error('Invalid embedded avatar texture.');
+ const bitmap = await createImageBitmap(
+ new Blob([data.slice(start, start + buffer.byteLength)], { type: image.mimeType })
+ );
+ const tooLarge = bitmap.width > 4096 || bitmap.height > 4096;
+ pixels += bitmap.width * bitmap.height;
+ bitmap.close();
+ if (tooLarge || pixels > 32 * 1024 * 1024)
+ throw new Error('Avatar textures are too large. Export at 2048px or below.');
+ }
+}