mirror of
https://github.com/open-webui/open-webui.git
synced 2026-10-08 03:08:02 +00:00
refac
This commit is contained in:
Timothy Jaeryang Baek
parent
6a8f1e2cb5
commit
7401f41630
16 changed files with 1880 additions and 36 deletions
|
|
@ -106,6 +106,38 @@ class ModelVoice(BaseModel):
|
|||
voice: str | None = Field(default=None, min_length=1, max_length=200, pattern=r'^\S+$')
|
||||
|
||||
|
||||
class ModelAvatarAnimation(BaseModel):
|
||||
model_config = ConfigDict(extra='forbid')
|
||||
file_id: str = Field(pattern=r'^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$')
|
||||
|
||||
|
||||
class ModelAvatarGesture(ModelAvatarAnimation):
|
||||
name: str = Field(pattern=r'^[a-z][a-z0-9_]{0,47}$')
|
||||
description: str = Field(min_length=1, max_length=500)
|
||||
|
||||
|
||||
class ModelVoiceAvatar(BaseModel):
|
||||
model_config = ConfigDict(extra='forbid', allow_inf_nan=False)
|
||||
|
||||
file_id: str = Field(pattern=r'^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$')
|
||||
states: dict[Literal['idle', 'listening', 'speaking'], ModelAvatarAnimation] = Field(default_factory=dict)
|
||||
gestures: list[ModelAvatarGesture] = Field(default_factory=list, max_length=16)
|
||||
|
||||
@model_validator(mode='before')
|
||||
@classmethod
|
||||
def discard_legacy_movement_settings(cls, value):
|
||||
if isinstance(value, dict):
|
||||
return {key: item for key, item in value.items() if key not in {'preset', 'movement', 'mouth', 'gaze'}}
|
||||
return value
|
||||
|
||||
@model_validator(mode='after')
|
||||
def unique_gestures(self):
|
||||
names = [gesture.name for gesture in self.gestures]
|
||||
if len(set(names)) != len(names) or any(not gesture.description.strip() for gesture in self.gestures):
|
||||
raise ValueError('Gestures need unique names and a description.')
|
||||
return self
|
||||
|
||||
|
||||
class ModelMeta(BaseModel):
|
||||
"""Metadata for a workspace model entry (profile, description, tags, capabilities)."""
|
||||
|
||||
|
|
@ -116,6 +148,7 @@ class ModelMeta(BaseModel):
|
|||
capabilities: dict | None = None
|
||||
knowledge: list[Any] | None = None
|
||||
voice: ModelVoice | None = None
|
||||
voice_avatar: ModelVoiceAvatar | None = None
|
||||
|
||||
model_config = ConfigDict(extra='allow')
|
||||
|
||||
|
|
@ -329,12 +362,13 @@ class ModelsTable:
|
|||
async def get_model_owner_ids_by_file_id(
|
||||
self, file_id: str, db: AsyncSession | None = None, include_background: bool = False
|
||||
) -> dict[str, str]:
|
||||
"""Return model IDs mapped to owner IDs for models referencing the file."""
|
||||
"""Find file references; include_background adds read-only background/avatar assets."""
|
||||
async with get_async_db_context(db) as db:
|
||||
# File ids are server-generated uuids, so the text match can only over-match.
|
||||
result = await db.execute(
|
||||
select(Model.id, Model.user_id, Model.meta).filter(
|
||||
Model.base_model_id.is_not(None), cast(Model.meta, String).like(f'%{file_id}%')
|
||||
(Model.base_model_id.is_not(None) if not include_background else True),
|
||||
cast(Model.meta, String).like(f'%{file_id}%'),
|
||||
)
|
||||
)
|
||||
return {
|
||||
|
|
@ -344,7 +378,17 @@ class ModelsTable:
|
|||
isinstance(item, dict) and item.get('type') == 'file' and item.get('id') == file_id
|
||||
for item in meta.get('knowledge') or []
|
||||
)
|
||||
or (include_background and meta.get('background_image_url') == f'/api/v1/files/{file_id}/content')
|
||||
or (
|
||||
include_background
|
||||
and (
|
||||
meta.get('background_image_url') == f'/api/v1/files/{file_id}/content'
|
||||
or (meta.get('voice_avatar') or {}).get('file_id') == file_id
|
||||
or any(asset.get('file_id') == file_id for asset in (
|
||||
list((meta.get('voice_avatar') or {}).get('states', {}).values())
|
||||
+ (meta.get('voice_avatar') or {}).get('gestures', [])
|
||||
))
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -31,11 +31,58 @@ CALL_STATUSES = {
|
|||
'transcription_failed': 'I could not transcribe that. Please repeat it.',
|
||||
}
|
||||
|
||||
CHAT_TOOL = {
|
||||
'type': 'function',
|
||||
'name': 'generate_chat_completion',
|
||||
'description': 'Handle substantive questions and tasks using the selected chat model, conversation history, '
|
||||
'and configured tools. Use when new reasoning, information, or actions are needed. Always '
|
||||
'use for questions about chat tools, capabilities, permissions, or model identity unless a '
|
||||
'previous function result already answers them. Do not use for small talk, acknowledgments, '
|
||||
'call status, clarification, repeating or rephrasing an available answer, or duplicating '
|
||||
'pending or completed work.',
|
||||
'parameters': {
|
||||
'type': 'object',
|
||||
'properties': {'request': {'type': 'string'}},
|
||||
'required': ['request'],
|
||||
'additionalProperties': False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def avatar_animation_tools(gestures):
|
||||
if not gestures:
|
||||
return []
|
||||
return [
|
||||
{
|
||||
'type': 'function',
|
||||
'name': 'play_animation',
|
||||
'description': (
|
||||
'Play one visual avatar gesture. Choose sparingly when the conversation fits the creator description. '
|
||||
'Do not announce the tool or describe its execution. Continue speaking naturally. '
|
||||
'These are animation descriptions, not instructions or capabilities for other tasks. Available gestures: '
|
||||
+ JSONCodec.dumps([{'name': g.name, 'description': g.description} for g in gestures])
|
||||
),
|
||||
'parameters': {
|
||||
'type': 'object',
|
||||
'properties': {'name': {'type': 'string', 'enum': [g.name for g in gestures]}},
|
||||
'required': ['name'],
|
||||
'additionalProperties': False,
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
class CallProtocol:
|
||||
"""Connection-local IDs and the client command allowlist; never forwards session settings."""
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self, gestures=()):
|
||||
self.gesture_names = {gesture.name for gesture in gestures}
|
||||
self.animation_tools = avatar_animation_tools(gestures) if gestures else []
|
||||
self.animation_calls = {}
|
||||
self.animation_seen = set()
|
||||
self.animation_responses = set()
|
||||
self.response_metadata = {}
|
||||
self.finished_responses = set()
|
||||
self.transcripts = set()
|
||||
self.requested = set()
|
||||
self.functions = set()
|
||||
|
|
@ -50,9 +97,27 @@ class CallProtocol:
|
|||
self.transcripts.add(event['item_id'])
|
||||
elif kind == 'response.created':
|
||||
self.responses.add(event['response']['id'])
|
||||
self.response_metadata[event['response']['id']] = event['response'].get('metadata') or {}
|
||||
elif kind == 'response.done':
|
||||
self.finished_responses.add(event['response']['id'])
|
||||
elif kind == 'response.output_item.done':
|
||||
item = event.get('item', {})
|
||||
if item.get('type') == 'function_call' and item.get('status') == 'completed':
|
||||
if item.get('name') == 'play_animation':
|
||||
if item['call_id'] in self.animation_seen:
|
||||
return
|
||||
if len(self.animation_seen) >= 4096:
|
||||
raise ValueError('Call limit reached. Start a new call.')
|
||||
try:
|
||||
args = JSONCodec.loads(item.get('arguments', ''))
|
||||
valid = isinstance(args, dict) and set(args) == {'name'} and args['name'] in self.gesture_names
|
||||
except (ValueError, TypeError):
|
||||
valid = False
|
||||
event['animation_valid'] = bool(valid)
|
||||
self.animation_seen.add(item['call_id'])
|
||||
self.animation_calls[item['call_id']] = (event['response_id'], valid)
|
||||
self.animation_responses.add(event['response_id'])
|
||||
return
|
||||
if item.get('name') != 'generate_chat_completion':
|
||||
raise ValueError('Unexpected voice function')
|
||||
args = JSONCodec.loads(item.get('arguments', ''))
|
||||
|
|
@ -134,6 +199,42 @@ class CallProtocol:
|
|||
}
|
||||
)
|
||||
return items
|
||||
if kind == 'bridge.animation.result' and set(event) == {'type', 'call_id', 'status'}:
|
||||
pending = self.animation_calls.pop(event['call_id'], None)
|
||||
if pending is None or event['status'] not in {'started', 'busy', 'unavailable', 'cancelled'}:
|
||||
raise ValueError('Invalid animation result')
|
||||
return {
|
||||
'type': 'conversation.item.create',
|
||||
'item': {
|
||||
'type': 'function_call_output',
|
||||
'call_id': event['call_id'],
|
||||
'output': JSONCodec.dumps({'status': event['status'] if pending[1] else 'unavailable'}),
|
||||
},
|
||||
}
|
||||
if kind == 'bridge.animation.respond' and set(event) == {'type', 'response_id'}:
|
||||
response_id = event['response_id']
|
||||
if (
|
||||
response_id not in self.animation_responses
|
||||
or response_id not in self.finished_responses
|
||||
or any(p[0] == response_id for p in self.animation_calls.values())
|
||||
):
|
||||
raise ValueError('Animation response is not ready')
|
||||
self.animation_responses.remove(response_id)
|
||||
metadata = {
|
||||
k: v
|
||||
for k, v in self.response_metadata.get(response_id, {}).items()
|
||||
if k in {'input_item_id', 'call_id'}
|
||||
}
|
||||
# A gesture cannot cause a chain of gesture-only replies. Chat delegation remains available.
|
||||
tools = [] if 'call_id' in metadata else [CHAT_TOOL]
|
||||
return {
|
||||
'type': 'response.create',
|
||||
'response': {
|
||||
'metadata': metadata,
|
||||
'tools': tools,
|
||||
'tool_choice': 'auto' if tools else 'none',
|
||||
},
|
||||
}
|
||||
if kind == 'bridge.result' and set(event) == {'type', 'call_id', 'status', 'answer'}:
|
||||
if event['call_id'] not in self.functions:
|
||||
raise ValueError('Unknown or resolved function call')
|
||||
|
|
@ -166,8 +267,8 @@ class CallProtocol:
|
|||
return {
|
||||
'type': 'response.create',
|
||||
'response': {
|
||||
'tools': [],
|
||||
'tool_choice': 'none',
|
||||
'tools': self.animation_tools,
|
||||
'tool_choice': 'auto' if self.animation_tools else 'none',
|
||||
'metadata': {'call_id': call_id},
|
||||
},
|
||||
}
|
||||
|
|
@ -240,6 +341,9 @@ async def realtime_call(ws: WebSocket):
|
|||
await check_model_access(user, model, model_info=model_info)
|
||||
except Exception:
|
||||
raise ValueError('Chat model access denied') from None
|
||||
protocol = CallProtocol(
|
||||
model_info.meta.voice_avatar.gestures if model_info and model_info.meta.voice_avatar else ()
|
||||
)
|
||||
override = ModelVoice.model_validate((model_info.meta.model_dump().get('voice') if model_info else None) or {})
|
||||
voice_model = config.get('audio.realtime.model')
|
||||
voice = override.voice or config.get('audio.realtime.voice')
|
||||
|
|
@ -300,19 +404,7 @@ async def realtime_call(ws: WebSocket):
|
|||
},
|
||||
'output': {'format': {'type': 'audio/pcm', 'rate': 24000}, 'voice': voice},
|
||||
},
|
||||
'tools': [
|
||||
{
|
||||
'type': 'function',
|
||||
'name': 'generate_chat_completion',
|
||||
'description': 'Handle substantive questions and tasks using the selected chat model, conversation history, and configured tools. Use when new reasoning, information, or actions are needed. Always use for questions about chat tools, capabilities, permissions, or model identity unless a previous function result already answers them. Do not use for small talk, acknowledgments, call status, clarification, repeating or rephrasing an available answer, or duplicating pending or completed work.',
|
||||
'parameters': {
|
||||
'type': 'object',
|
||||
'properties': {'request': {'type': 'string'}},
|
||||
'required': ['request'],
|
||||
'additionalProperties': False,
|
||||
},
|
||||
}
|
||||
],
|
||||
'tools': [CHAT_TOOL, *protocol.animation_tools],
|
||||
'tool_choice': 'auto',
|
||||
},
|
||||
}
|
||||
|
|
@ -321,7 +413,6 @@ async def realtime_call(ws: WebSocket):
|
|||
if event.get('type') != 'session.updated':
|
||||
raise ValueError('Provider rejected voice configuration. Check model, voice, and transcription model.')
|
||||
await ws.send_json({'type': 'bridge.ready', 'model': voice_model, 'voice': voice, 'sample_rate': 24000})
|
||||
protocol = CallProtocol()
|
||||
|
||||
async def client_events():
|
||||
while True:
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@ from open_webui.utils.chat_variables import get_chat_variables_schema
|
|||
from open_webui.utils.json_codec import JSONCodec
|
||||
from open_webui.utils.models import get_all_models
|
||||
from open_webui.utils.validate import BACKGROUND_IMAGE_MAX_BYTES, validate_background_image
|
||||
from open_webui.utils.voice_avatar import AVATAR_MAX_BYTES, ANIMATION_MAX_BYTES, validate_voice_avatar, validate_voice_animation
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
|
|
@ -150,6 +151,34 @@ async def _verify_background_image(url: str | None, user, db, previous_url: str
|
|||
raise HTTPException(status_code=500, detail='Could not validate background image.')
|
||||
|
||||
|
||||
async def _verify_voice_avatar(avatar, user, db, previous=None) -> None:
|
||||
if not avatar:
|
||||
return
|
||||
assets = {avatar.file_id: (AVATAR_MAX_BYTES, validate_voice_avatar)}
|
||||
for asset in [*avatar.states.values(), *avatar.gestures]:
|
||||
if asset.file_id == avatar.file_id:
|
||||
raise HTTPException(status_code=400, detail='An animation must be a VRMA file, not the avatar.')
|
||||
assets[asset.file_id] = (ANIMATION_MAX_BYTES, validate_voice_animation)
|
||||
previous_assets = ({previous.file_id: validate_voice_avatar} | {
|
||||
asset.file_id: validate_voice_animation for asset in [*previous.states.values(), *previous.gestures]
|
||||
}) if previous else {}
|
||||
for file_id, (limit, validate) in assets.items():
|
||||
if previous_assets.get(file_id) is validate:
|
||||
continue
|
||||
file = await Files.get_file_by_id(file_id, db=db)
|
||||
if not file or not (
|
||||
user.role == 'admin' or file.user_id == user.id or await has_access_to_file(file_id, 'read', user, db=db)
|
||||
):
|
||||
raise HTTPException(status_code=403, detail='Avatar or animation file is not accessible. Upload it again.')
|
||||
try:
|
||||
path = await asyncio.to_thread(Storage.get_file, file.path)
|
||||
with open(path, 'rb') as source:
|
||||
data = await asyncio.to_thread(source.read, limit + 1)
|
||||
await asyncio.to_thread(validate, data)
|
||||
except (ValueError, OSError) as error:
|
||||
raise HTTPException(status_code=400, detail=str(error)) from error
|
||||
|
||||
|
||||
async def _verify_knowledge_file_access(
|
||||
knowledge_items: list | None,
|
||||
user,
|
||||
|
|
@ -370,6 +399,7 @@ async def create_new_model(
|
|||
)
|
||||
|
||||
await _verify_background_image(form_data.meta.background_image_url, user, db)
|
||||
await _verify_voice_avatar(form_data.meta.voice_avatar, user, db)
|
||||
|
||||
form_data.access_grants = await filter_allowed_access_grants(
|
||||
await Config.get('user.permissions'),
|
||||
|
|
@ -666,6 +696,14 @@ async def import_models(
|
|||
db,
|
||||
existing_model.meta.background_image_url if existing_model else None,
|
||||
)
|
||||
if existing_model and 'voice_avatar' not in imported_model.meta.model_fields_set:
|
||||
imported_model.meta.voice_avatar = existing_model.meta.voice_avatar
|
||||
await _verify_voice_avatar(
|
||||
imported_model.meta.voice_avatar,
|
||||
user,
|
||||
db,
|
||||
existing_model.meta.voice_avatar if existing_model else None,
|
||||
)
|
||||
saved = (
|
||||
await Models.update_model_by_id(model_id, imported_model, db=db)
|
||||
if existing_model
|
||||
|
|
@ -722,6 +760,9 @@ async def sync_models(
|
|||
for model in form_data.models:
|
||||
previous = existing.get(model.id)
|
||||
await _check_model_controls(model, previous, user, request)
|
||||
if previous and 'voice_avatar' not in model.meta.model_fields_set:
|
||||
model.meta.voice_avatar = previous.meta.voice_avatar
|
||||
await _verify_voice_avatar(model.meta.voice_avatar, user, db, previous.meta.voice_avatar if previous else None)
|
||||
if previous and 'background_image_url' not in model.meta.model_fields_set:
|
||||
model.meta.background_image_url = previous.meta.background_image_url
|
||||
await _verify_background_image(
|
||||
|
|
@ -1014,6 +1055,9 @@ async def update_model_by_id(
|
|||
if 'profile_image_url' not in form_data.meta.model_fields_set:
|
||||
form_data.meta.profile_image_url = model.meta.profile_image_url
|
||||
|
||||
if 'voice_avatar' not in form_data.meta.model_fields_set:
|
||||
form_data.meta.voice_avatar = model.meta.voice_avatar
|
||||
await _verify_voice_avatar(form_data.meta.voice_avatar, user, db, model.meta.voice_avatar)
|
||||
if 'background_image_url' not in form_data.meta.model_fields_set:
|
||||
form_data.meta.background_image_url = model.meta.background_image_url
|
||||
await _verify_background_image(form_data.meta.background_image_url, user, db, model.meta.background_image_url)
|
||||
|
|
|
|||
223
backend/open_webui/utils/voice_avatar.py
Normal file
223
backend/open_webui/utils/voice_avatar.py
Normal file
|
|
@ -0,0 +1,223 @@
|
|||
"""Validation for self-contained VRM uploads used by bridge voice calls."""
|
||||
|
||||
import io
|
||||
import json
|
||||
import struct
|
||||
|
||||
from PIL import Image
|
||||
|
||||
AVATAR_MAX_BYTES = 25 * 1024 * 1024
|
||||
|
||||
|
||||
def validate_voice_avatar(data: bytes) -> None:
|
||||
try:
|
||||
_validate(data)
|
||||
except (
|
||||
KeyError,
|
||||
TypeError,
|
||||
IndexError,
|
||||
AttributeError,
|
||||
struct.error,
|
||||
UnicodeError,
|
||||
json.JSONDecodeError,
|
||||
Image.DecompressionBombError,
|
||||
) as error:
|
||||
raise ValueError('Invalid VRM file.') from error
|
||||
|
||||
|
||||
def _validate(data: bytes) -> None:
|
||||
if len(data) > AVATAR_MAX_BYTES:
|
||||
raise ValueError('Avatar must be at most 25 MiB.')
|
||||
magic, version, size, json_size, chunk = struct.unpack_from('<5I', data)
|
||||
if magic != 0x46546C67 or version != 2 or size != len(data) or chunk != 0x4E4F534A:
|
||||
raise ValueError('Upload a binary VRM file.')
|
||||
if json_size > 2 * 1024 * 1024 or 28 + json_size > len(data):
|
||||
raise ValueError('Invalid avatar container.')
|
||||
model = json.loads(data[20 : 20 + json_size])
|
||||
binary_size, binary_type = struct.unpack_from('<2I', data, 20 + json_size)
|
||||
start = 28 + json_size
|
||||
if binary_type != 0x004E4942 or start + binary_size != len(data):
|
||||
raise ValueError('Invalid avatar binary data.')
|
||||
extension = model.get('extensions', {})
|
||||
vrm = extension.get('VRMC_vrm') or extension.get('VRM') or {}
|
||||
bones = vrm.get('humanoid', {}).get('humanBones', {})
|
||||
if isinstance(bones, list):
|
||||
bones = {bone['bone']: bone for bone in bones}
|
||||
nodes = model.get('nodes', [])
|
||||
for name in ('hips', 'spine', 'head', 'leftUpperArm', 'rightUpperArm', 'leftLowerArm', 'rightLowerArm'):
|
||||
node = bones.get(name, {}).get('node')
|
||||
if not isinstance(node, int) or not 0 <= node < len(nodes):
|
||||
raise ValueError(f'Avatar is missing its {name} bone. Upload a rigged VRM file.')
|
||||
buffers = model.get('buffers', [])
|
||||
images = model.get('images', [])
|
||||
if len(buffers) != 1 or buffers[0].get('uri') or not 0 <= buffers[0]['byteLength'] <= binary_size:
|
||||
raise ValueError('Embed all buffers in the VRM file.')
|
||||
if len(nodes) > 512 or len(images) > 32 or sum(a.get('count', 0) for a in model.get('accessors', [])) > 5_000_000:
|
||||
raise ValueError('This avatar is too complex for a voice call. Use a lighter export.')
|
||||
_validate_textures(model, data, start, binary_size)
|
||||
|
||||
|
||||
def _validate_textures(model: dict, data: bytes, start: int, binary_size: int) -> None:
|
||||
pixels = 0
|
||||
for image in model.get('images', []):
|
||||
if image.get('uri') or image.get('mimeType') not in ('image/png', 'image/jpeg', 'image/webp'):
|
||||
raise ValueError('Embed PNG, JPEG or WebP textures in the VRM file.')
|
||||
view = model['bufferViews'][image['bufferView']]
|
||||
offset, length = view.get('byteOffset', 0), view['byteLength']
|
||||
if view.get('buffer', 0) != 0 or offset < 0 or length <= 0 or offset + length > binary_size:
|
||||
raise ValueError('Invalid embedded avatar texture.')
|
||||
with Image.open(io.BytesIO(data[start + offset : start + offset + length])) as texture:
|
||||
pixels += texture.width * texture.height
|
||||
if max(texture.size) > 4096 or pixels > 32 * 1024 * 1024:
|
||||
raise ValueError('Avatar textures are too large. Export at 2048px or below.')
|
||||
texture.verify()
|
||||
|
||||
|
||||
ANIMATION_MAX_BYTES = 10 * 1024 * 1024
|
||||
|
||||
|
||||
def validate_voice_animation(data: bytes) -> None:
|
||||
"""Bound the binary data before any browser loader processes an authored clip."""
|
||||
import math
|
||||
|
||||
try:
|
||||
if len(data) > ANIMATION_MAX_BYTES:
|
||||
raise ValueError('Animation must be at most 10 MiB.')
|
||||
magic, version, size, json_size, chunk = struct.unpack_from('<5I', data)
|
||||
if (magic, version, size, chunk) != (0x46546C67, 2, len(data), 0x4E4F534A):
|
||||
raise ValueError('Upload a binary VRMA file.')
|
||||
if json_size > 2 * 1024 * 1024 or 28 + json_size > len(data):
|
||||
raise ValueError('Invalid animation container.')
|
||||
model = json.loads(data[20 : 20 + json_size])
|
||||
binary_size, binary_type = struct.unpack_from('<2I', data, 20 + json_size)
|
||||
start = 28 + json_size
|
||||
if binary_type != 0x004E4942 or start + binary_size != len(data):
|
||||
raise ValueError('Invalid animation binary data.')
|
||||
ext = model.get('extensions', {}).get('VRMC_vrm_animation', {})
|
||||
bones = ext.get('humanoid', {}).get('humanBones', {})
|
||||
nodes = model.get('nodes', [])
|
||||
if ext.get('specVersion') != '1.0' or not bones or not 0 < len(nodes) <= 512:
|
||||
raise ValueError('Use a VRMA 1.0 humanoid animation.')
|
||||
for bone in bones.values():
|
||||
if type(bone.get('node')) is not int or not 0 <= bone['node'] < len(nodes):
|
||||
raise ValueError('Invalid animation bone.')
|
||||
parents = set()
|
||||
visiting, visited = set(), set()
|
||||
|
||||
def visit(index):
|
||||
if type(index) is not int or not 0 <= index < len(nodes) or index in visiting:
|
||||
raise ValueError('Invalid animation node hierarchy.')
|
||||
if index in visited:
|
||||
return
|
||||
visiting.add(index)
|
||||
node = nodes[index]
|
||||
for key, length in (('translation', 3), ('rotation', 4), ('scale', 3), ('matrix', 16)):
|
||||
if key in node and (
|
||||
len(node[key]) != length
|
||||
or any(not isinstance(v, (int, float)) or not math.isfinite(v) for v in node[key])
|
||||
):
|
||||
raise ValueError('Invalid animation transform.')
|
||||
for child in node.get('children', []):
|
||||
if child in parents:
|
||||
raise ValueError('Animation nodes must have a single parent.')
|
||||
parents.add(child)
|
||||
visit(child)
|
||||
visiting.remove(index)
|
||||
visited.add(index)
|
||||
|
||||
for index in range(len(nodes)):
|
||||
visit(index)
|
||||
for scene in model.get('scenes', []):
|
||||
if any(type(n) is not int or not 0 <= n < len(nodes) for n in scene.get('nodes', [])):
|
||||
raise ValueError('Invalid animation scene.')
|
||||
|
||||
def at(items, index):
|
||||
if type(index) is not int or not 0 <= index < len(items):
|
||||
raise ValueError('Invalid animation reference.')
|
||||
return items[index]
|
||||
|
||||
buffers = model.get('buffers', [])
|
||||
if len(buffers) != 1 or 'uri' in buffers[0] or not binary_size - 3 <= buffers[0]['byteLength'] <= binary_size:
|
||||
raise ValueError('Embed all animation data in the VRMA file.')
|
||||
if any(model.get(key) for key in ('images', 'textures', 'meshes', 'skins')):
|
||||
raise ValueError('Export animation only, without meshes or textures.')
|
||||
if set(model.get('extensionsRequired', [])) - {'VRMC_vrm_animation'}:
|
||||
raise ValueError('Unsupported animation extension.')
|
||||
views = model.get('bufferViews', [])
|
||||
for view in views:
|
||||
offset, length = view.get('byteOffset', 0), view['byteLength']
|
||||
if view.get('buffer', 0) != 0 or offset < 0 or length <= 0 or offset + length > binary_size:
|
||||
raise ValueError('Invalid animation buffer.')
|
||||
accessors = model.get('accessors', [])
|
||||
if sum(a.get('count', 0) for a in accessors) > 500_000:
|
||||
raise ValueError('Animation has too many keyframes.')
|
||||
values = []
|
||||
for accessor in accessors:
|
||||
count = accessor['count']
|
||||
components = {'SCALAR': 1, 'VEC3': 3, 'VEC4': 4}.get(accessor['type'])
|
||||
if (
|
||||
accessor.get('sparse')
|
||||
or accessor['componentType'] != 5126
|
||||
or not components
|
||||
or not 0 < count <= 500_000
|
||||
):
|
||||
raise ValueError('Use float animation keyframes without sparse accessors.')
|
||||
view = at(views, accessor['bufferView'])
|
||||
stride = view.get('byteStride', components * 4)
|
||||
offset = accessor.get('byteOffset', 0)
|
||||
if (
|
||||
stride < components * 4
|
||||
or stride % 4
|
||||
or offset < 0
|
||||
or offset + (count - 1) * stride + components * 4 > view['byteLength']
|
||||
):
|
||||
raise ValueError('Invalid animation accessor.')
|
||||
rows = [
|
||||
struct.unpack_from(
|
||||
'<' + 'f' * components, data, start + view.get('byteOffset', 0) + offset + i * stride
|
||||
)
|
||||
for i in range(count)
|
||||
]
|
||||
if any(not math.isfinite(v) for row in rows for v in row):
|
||||
raise ValueError('Animation keyframes must be finite.')
|
||||
values.append(rows)
|
||||
animations = model.get('animations', [])
|
||||
if len(animations) != 1 or not animations[0].get('channels') or len(animations[0]['channels']) > 256:
|
||||
raise ValueError('Export exactly one animation per VRMA file.')
|
||||
animation = animations[0]
|
||||
duration = 0
|
||||
body_nodes = {bone['node'] for bone in bones.values()}
|
||||
body = False
|
||||
for channel in animation['channels']:
|
||||
target = channel['target']
|
||||
if not 0 <= target['node'] < len(nodes) or target['path'] not in ('rotation', 'translation'):
|
||||
raise ValueError('Unsupported animation channel.')
|
||||
body |= target['node'] in body_nodes
|
||||
sampler = at(animation['samplers'], channel['sampler'])
|
||||
times, outputs = at(values, sampler['input']), at(values, sampler['output'])
|
||||
if accessors[sampler['input']]['type'] != 'SCALAR' or any(
|
||||
t[0] < 0 or (i and t[0] <= times[i - 1][0]) for i, t in enumerate(times)
|
||||
):
|
||||
raise ValueError('Invalid animation timing.')
|
||||
interpolation = sampler.get('interpolation', 'LINEAR')
|
||||
if interpolation not in ('LINEAR', 'STEP'):
|
||||
raise ValueError('Export baked animation with linear or stepped keyframes.')
|
||||
if len(outputs) != len(times) or len(outputs[0]) != (
|
||||
4 if target['path'] == 'rotation' else 3
|
||||
):
|
||||
raise ValueError('Invalid animation output.')
|
||||
duration = max(duration, times[-1][0])
|
||||
if not body or not 0 < duration <= 60:
|
||||
raise ValueError('Use a body animation between 0 and 60 seconds.')
|
||||
except (
|
||||
KeyError,
|
||||
TypeError,
|
||||
IndexError,
|
||||
AttributeError,
|
||||
struct.error,
|
||||
UnicodeError,
|
||||
json.JSONDecodeError,
|
||||
OverflowError,
|
||||
RecursionError,
|
||||
) as error:
|
||||
raise ValueError('Invalid VRMA file.') from error
|
||||
|
|
@ -1781,6 +1781,7 @@ export interface ModelConfig {
|
|||
}
|
||||
|
||||
export interface ModelMeta {
|
||||
voice_avatar?: import('$lib/utils/voice-avatar').VoiceAvatarConfig | null;
|
||||
voice?: { voice?: string };
|
||||
toolIds: never[];
|
||||
description?: string;
|
||||
|
|
|
|||
|
|
@ -1,10 +1,19 @@
|
|||
<script lang="ts">
|
||||
import { createEventDispatcher, getContext, onMount } from 'svelte';
|
||||
import { showCallOverlay } from '$lib/stores';
|
||||
import { models, showCallOverlay } from '$lib/stores';
|
||||
import type { RealtimeCall } from '$lib/utils/realtime';
|
||||
import VoiceOrb from './VoiceOrb.svelte';
|
||||
import VoiceAvatar from './VoiceAvatar.svelte';
|
||||
|
||||
export let bridge: RealtimeCall;
|
||||
export let modelId = '';
|
||||
let failedAvatar = '';
|
||||
let avatarView: VoiceAvatar;
|
||||
$: bridge.animationPlayer = (name) =>
|
||||
showAvatar ? (avatarView?.playAnimation(name) ?? 'unavailable') : 'unavailable';
|
||||
let readyAvatar = '';
|
||||
$: avatar = $models.find((model) => model.id === modelId)?.info?.meta?.voice_avatar;
|
||||
$: showAvatar = avatar?.file_id && failedAvatar !== avatar.file_id;
|
||||
|
||||
const i18n = getContext<any>('i18n');
|
||||
const dispatch = createEventDispatcher();
|
||||
|
|
@ -42,7 +51,10 @@
|
|||
}
|
||||
};
|
||||
document.addEventListener('keydown', handleKeydown);
|
||||
return () => document.removeEventListener('keydown', handleKeydown);
|
||||
return () => {
|
||||
document.removeEventListener('keydown', handleKeydown);
|
||||
bridge.animationPlayer = undefined;
|
||||
};
|
||||
});
|
||||
</script>
|
||||
|
||||
|
|
@ -51,21 +63,48 @@
|
|||
<button
|
||||
type="button"
|
||||
class="orb-button"
|
||||
class:avatar-button={showAvatar}
|
||||
disabled={!bridge.speaking}
|
||||
aria-label={$i18n.t('Stop speaking')}
|
||||
on:click={() => bridge.stopSpeaking()}
|
||||
>
|
||||
<VoiceOrb
|
||||
speaking={bridge.speaking && !unavailable}
|
||||
level={unavailable
|
||||
? 0
|
||||
: bridge.speaking
|
||||
? bridge.outputLevel
|
||||
: bridge.muted
|
||||
? 0
|
||||
: bridge.inputLevel}
|
||||
muted={unavailable || (bridge.muted && !bridge.speaking)}
|
||||
/>
|
||||
{#if showAvatar && avatar}
|
||||
{@const selectedAvatar = avatar}
|
||||
{#key selectedAvatar.file_id}
|
||||
<div class="avatar-content" class:loading-avatar={readyAvatar !== selectedAvatar.file_id}>
|
||||
<VoiceAvatar
|
||||
bind:this={avatarView}
|
||||
interruption={bridge.animationInterruption}
|
||||
config={selectedAvatar}
|
||||
speaking={bridge.playbackActive && !unavailable}
|
||||
listening={bridge.userSpeaking && !bridge.muted && !unavailable}
|
||||
level={bridge.outputLevel}
|
||||
active={!unavailable}
|
||||
on:ready={() => {
|
||||
readyAvatar = selectedAvatar.file_id;
|
||||
}}
|
||||
on:error={() => {
|
||||
failedAvatar = selectedAvatar.file_id;
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
{/key}
|
||||
{#if readyAvatar !== selectedAvatar.file_id}<div class="avatar-loading">
|
||||
<VoiceOrb speaking={false} level={0} muted={true} />
|
||||
</div>{/if}
|
||||
{:else}
|
||||
<VoiceOrb
|
||||
speaking={bridge.speaking && !unavailable}
|
||||
level={unavailable
|
||||
? 0
|
||||
: bridge.speaking
|
||||
? bridge.outputLevel
|
||||
: bridge.muted
|
||||
? 0
|
||||
: bridge.inputLevel}
|
||||
muted={unavailable || (bridge.muted && !bridge.speaking)}
|
||||
/>
|
||||
{/if}
|
||||
</button>
|
||||
|
||||
<div class="call-status" role="status" aria-live="polite" aria-atomic="true">
|
||||
|
|
@ -170,6 +209,27 @@
|
|||
padding: 12px;
|
||||
border-radius: 50%;
|
||||
}
|
||||
.avatar-button {
|
||||
width: 100%;
|
||||
height: clamp(240px, 42vh, 360px);
|
||||
position: relative;
|
||||
padding: 0;
|
||||
border-radius: 16px;
|
||||
overflow: hidden;
|
||||
}
|
||||
.avatar-content {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
}
|
||||
.loading-avatar {
|
||||
opacity: 0;
|
||||
}
|
||||
.avatar-loading {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
display: grid;
|
||||
place-items: center;
|
||||
}
|
||||
.orb-button:disabled {
|
||||
cursor: default;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,174 @@
|
|||
<script lang="ts">
|
||||
import { createEventDispatcher, onMount } from 'svelte';
|
||||
import { WEBUI_API_BASE_URL } from '$lib/constants';
|
||||
import {
|
||||
avatarAssetIds,
|
||||
type AnimationFiles,
|
||||
type VoiceAvatarConfig
|
||||
} from '$lib/utils/voice-avatar';
|
||||
import type { createAvatarRenderer } from '$lib/utils/voice-avatar-renderer';
|
||||
|
||||
export let config: VoiceAvatarConfig;
|
||||
export let file: File | null = null;
|
||||
export let speaking = false;
|
||||
export let listening = false;
|
||||
export let level = 0;
|
||||
export let active = true;
|
||||
export let animationFiles: AnimationFiles = {};
|
||||
export let interruption = 0;
|
||||
let lastInterruption = interruption;
|
||||
let renderer: Awaited<ReturnType<typeof createAvatarRenderer>> | undefined;
|
||||
let syncAnimations: ((config: VoiceAvatarConfig, files: AnimationFiles) => void) | undefined;
|
||||
$: if (syncAnimations) syncAnimations(config, animationFiles);
|
||||
$: if (interruption !== lastInterruption) {
|
||||
renderer?.cancelGesture();
|
||||
lastInterruption = interruption;
|
||||
}
|
||||
export function playAnimation(name: string) {
|
||||
if (!active || document.hidden || window.matchMedia('(prefers-reduced-motion: reduce)').matches)
|
||||
return 'unavailable';
|
||||
const asset = config.gestures?.find((gesture) => gesture.name === name);
|
||||
return asset ? (renderer?.playGesture(asset.file_id) ?? 'unavailable') : 'unavailable';
|
||||
}
|
||||
export function cancelAnimation() {
|
||||
renderer?.cancelGesture();
|
||||
}
|
||||
const dispatch = createEventDispatcher();
|
||||
let canvas: HTMLCanvasElement;
|
||||
|
||||
// The parent keys this component by file, keeping settings changes inexpensive.
|
||||
onMount(() => {
|
||||
const abort = new AbortController();
|
||||
let frame = 0,
|
||||
previous = 0;
|
||||
let visible = true;
|
||||
const motion = window.matchMedia('(prefers-reduced-motion: reduce)');
|
||||
const animate = (now: number) => {
|
||||
if (abort.signal.aborted || document.hidden || !visible) {
|
||||
frame = 0;
|
||||
previous = 0;
|
||||
return;
|
||||
}
|
||||
try {
|
||||
renderer?.render(
|
||||
previous ? (now - previous) / 1000 : 0,
|
||||
config,
|
||||
{ speaking, listening, level, active },
|
||||
motion.matches
|
||||
);
|
||||
previous = now;
|
||||
frame = requestAnimationFrame(animate);
|
||||
} catch (error) {
|
||||
frame = 0;
|
||||
dispatch('error', error);
|
||||
}
|
||||
};
|
||||
const resume = () => {
|
||||
if (!frame && renderer && visible && !document.hidden) frame = requestAnimationFrame(animate);
|
||||
};
|
||||
const observer = new IntersectionObserver(([entry]) => {
|
||||
visible = entry.isIntersecting;
|
||||
resume();
|
||||
});
|
||||
observer.observe(canvas);
|
||||
document.addEventListener('visibilitychange', resume);
|
||||
const contextLost = (event: Event) => {
|
||||
event.preventDefault();
|
||||
dispatch('error', new Error('Avatar graphics context was lost.'));
|
||||
};
|
||||
canvas.addEventListener('webglcontextlost', contextLost);
|
||||
void (async () => {
|
||||
try {
|
||||
let data: ArrayBuffer;
|
||||
if (file) data = await file.arrayBuffer();
|
||||
else {
|
||||
if (!/^[a-f0-9-]{36}$/i.test(config.file_id)) throw new Error('Avatar file is missing.');
|
||||
const response = await fetch(`${WEBUI_API_BASE_URL}/files/${config.file_id}/content`, {
|
||||
headers: { Authorization: `Bearer ${localStorage.token}` },
|
||||
signal: abort.signal
|
||||
});
|
||||
if (!response.ok) throw new Error('Avatar file is unavailable. Upload it again.');
|
||||
data = await response.arrayBuffer();
|
||||
}
|
||||
const { createAvatarRenderer } = await import('$lib/utils/voice-avatar-renderer');
|
||||
abort.signal.throwIfAborted();
|
||||
renderer = await createAvatarRenderer(canvas, data, abort.signal);
|
||||
if (abort.signal.aborted) {
|
||||
renderer.dispose();
|
||||
return;
|
||||
}
|
||||
const pending = new Map<string, Promise<string | null>>();
|
||||
let revision = 0;
|
||||
let previousAssets = '';
|
||||
syncAnimations = (next, files) => {
|
||||
const ids = [...new Set(avatarAssetIds(next).filter((id) => id !== next.file_id))];
|
||||
const key = ids.join(',');
|
||||
if (key === previousAssets) return;
|
||||
previousAssets = key;
|
||||
const version = ++revision;
|
||||
dispatch('animations', { loading: ids.length > 0, errors: {} });
|
||||
void Promise.all(
|
||||
ids.map((id) => {
|
||||
if (!pending.has(id))
|
||||
pending.set(
|
||||
id,
|
||||
(async () => {
|
||||
try {
|
||||
let bytes: ArrayBuffer;
|
||||
if (files[id]) bytes = await files[id].arrayBuffer();
|
||||
else {
|
||||
const response = await fetch(`${WEBUI_API_BASE_URL}/files/${id}/content`, {
|
||||
headers: { Authorization: `Bearer ${localStorage.token}` },
|
||||
signal: abort.signal
|
||||
});
|
||||
if (!response.ok)
|
||||
throw new Error('Animation file is unavailable. Upload it again.');
|
||||
bytes = await response.arrayBuffer();
|
||||
}
|
||||
abort.signal.throwIfAborted();
|
||||
await renderer!.loadAnimation(id, bytes);
|
||||
return null;
|
||||
} catch (error) {
|
||||
return error instanceof Error ? error.message : 'Could not load animation.';
|
||||
}
|
||||
})()
|
||||
);
|
||||
return pending.get(id)!.then((error) => [id, error] as const);
|
||||
})
|
||||
).then((results) => {
|
||||
if (!abort.signal.aborted && version === revision)
|
||||
dispatch('animations', {
|
||||
loading: false,
|
||||
errors: Object.fromEntries(results.filter(([, error]) => error))
|
||||
});
|
||||
});
|
||||
};
|
||||
syncAnimations(config, animationFiles);
|
||||
dispatch('ready', renderer.capabilities);
|
||||
resume();
|
||||
} catch (error) {
|
||||
if (!abort.signal.aborted) dispatch('error', error);
|
||||
}
|
||||
})();
|
||||
return () => {
|
||||
abort.abort();
|
||||
syncAnimations = undefined;
|
||||
cancelAnimationFrame(frame);
|
||||
observer.disconnect();
|
||||
document.removeEventListener('visibilitychange', resume);
|
||||
canvas.removeEventListener('webglcontextlost', contextLost);
|
||||
renderer?.dispose();
|
||||
};
|
||||
});
|
||||
</script>
|
||||
|
||||
<canvas bind:this={canvas} aria-hidden="true" class="avatar-canvas"></canvas>
|
||||
|
||||
<style>
|
||||
.avatar-canvas {
|
||||
display: block;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
pointer-events: none;
|
||||
}
|
||||
</style>
|
||||
|
|
@ -49,7 +49,7 @@
|
|||
|
||||
{#if started}
|
||||
{#if callMode === 'bridge'}
|
||||
<BridgeCallOverlay {bridge} on:close />
|
||||
<BridgeCallOverlay {bridge} {modelId} on:close />
|
||||
{:else}
|
||||
<CallOverlay
|
||||
bind:files
|
||||
|
|
|
|||
|
|
@ -36,6 +36,12 @@
|
|||
import PromptSuggestions from './PromptSuggestions.svelte';
|
||||
import TerminalSelector from './TerminalSelector.svelte';
|
||||
import TTSVoiceInput from './TTSVoiceInput.svelte';
|
||||
import VoiceAvatarSettings from './VoiceAvatarSettings.svelte';
|
||||
import {
|
||||
avatarAssetIds,
|
||||
type AnimationFiles,
|
||||
type VoiceAvatarConfig
|
||||
} from '$lib/utils/voice-avatar';
|
||||
import AccessControlModal from '../common/AccessControlModal.svelte';
|
||||
import AccessButton from '$lib/components/common/AccessButton.svelte';
|
||||
import { copyToClipboard, extractInputVariables } from '$lib/utils';
|
||||
|
|
@ -53,6 +59,9 @@
|
|||
export let preset = true;
|
||||
|
||||
let loading = false;
|
||||
let voiceAvatar: VoiceAvatarConfig | null = null;
|
||||
let avatarFile: File | null = null;
|
||||
let animationFiles: AnimationFiles = {};
|
||||
let backgroundFile: File | null = null;
|
||||
let backgroundInput: HTMLInputElement;
|
||||
let backgroundPreview: string | null = null;
|
||||
|
|
@ -104,6 +113,7 @@
|
|||
profile_image_url: `${WEBUI_BASE_URL}/static/favicon.png`,
|
||||
background_image_url: null as string | null,
|
||||
voice: undefined as { voice?: string } | undefined,
|
||||
voice_avatar: null as VoiceAvatarConfig | null,
|
||||
description: '',
|
||||
i18n: {},
|
||||
suggestion_prompts: null,
|
||||
|
|
@ -288,6 +298,7 @@
|
|||
const modelInfo = structuredClone(info);
|
||||
|
||||
modelInfo.id = id;
|
||||
modelInfo.meta.voice_avatar = voiceAvatar;
|
||||
modelInfo.name = name;
|
||||
|
||||
modelInfo.params = { ...modelInfo.params, ...params };
|
||||
|
|
@ -452,6 +463,8 @@
|
|||
|
||||
let uploadedId: string | null = null;
|
||||
const previousBackground = info.meta.background_image_url;
|
||||
const previousAvatar = structuredClone(info.meta.voice_avatar);
|
||||
const uploadedAvatarIds: string[] = [];
|
||||
|
||||
try {
|
||||
if (backgroundFile) {
|
||||
|
|
@ -460,12 +473,59 @@
|
|||
uploadedId = uploaded.id;
|
||||
info.meta.background_image_url = `/api/v1/files/${uploaded.id}/content`;
|
||||
}
|
||||
if (avatarFile && info.meta.voice_avatar) {
|
||||
const uploaded = await uploadFile(localStorage.token, avatarFile, null, false, false);
|
||||
if (!uploaded?.id) throw new Error($i18n.t('Failed to upload avatar.'));
|
||||
uploadedAvatarIds.push(uploaded.id);
|
||||
info.meta.voice_avatar = { ...info.meta.voice_avatar, file_id: uploaded.id };
|
||||
}
|
||||
if (info.meta.voice_avatar) {
|
||||
const avatar = info.meta.voice_avatar;
|
||||
const replacements = new Map<string, string>();
|
||||
for (const id of new Set(avatarAssetIds(avatar))) {
|
||||
if (!animationFiles[id]) continue;
|
||||
const uploaded = await uploadFile(
|
||||
localStorage.token,
|
||||
animationFiles[id],
|
||||
null,
|
||||
false,
|
||||
false
|
||||
);
|
||||
if (!uploaded?.id) throw new Error($i18n.t('Failed to upload animation.'));
|
||||
uploadedAvatarIds.push(uploaded.id);
|
||||
replacements.set(id, uploaded.id);
|
||||
}
|
||||
for (const asset of [...Object.values(avatar.states ?? {}), ...(avatar.gestures ?? [])]) {
|
||||
asset.file_id = replacements.get(asset.file_id) ?? asset.file_id;
|
||||
}
|
||||
}
|
||||
const saved = await onSubmit(info);
|
||||
if (saved === false) throw new Error($i18n.t('Failed to save model'));
|
||||
backgroundFile = null;
|
||||
avatarFile = null;
|
||||
animationFiles = {};
|
||||
voiceAvatar = info.meta.voice_avatar;
|
||||
clearBackgroundPreview();
|
||||
} catch (error: any) {
|
||||
info.meta.background_image_url = previousBackground;
|
||||
info.meta.voice_avatar = previousAvatar;
|
||||
if (uploadedAvatarIds.length) {
|
||||
try {
|
||||
const response = await fetch(
|
||||
`${WEBUI_API_BASE_URL}/models/model?${new URLSearchParams({ id: info.id })}`,
|
||||
{ headers: { authorization: `Bearer ${localStorage.token}` } }
|
||||
);
|
||||
if (response.status === 404 || response.ok) {
|
||||
const referenced = response.ok
|
||||
? avatarAssetIds((await response.json())?.meta?.voice_avatar)
|
||||
: [];
|
||||
for (const id of uploadedAvatarIds)
|
||||
if (!referenced.includes(id)) await deleteFileById(localStorage.token, id);
|
||||
}
|
||||
} catch {
|
||||
/* Keep uploads when the save result is uncertain. */
|
||||
}
|
||||
}
|
||||
if (uploadedId) {
|
||||
// A failed response can follow a committed save; only delete an unused upload.
|
||||
try {
|
||||
|
|
@ -524,6 +584,7 @@
|
|||
|
||||
if (model) {
|
||||
name = model.name;
|
||||
voiceAvatar = model.meta?.voice_avatar ? structuredClone(model.meta.voice_avatar) : null;
|
||||
await tick();
|
||||
|
||||
id = model.id;
|
||||
|
|
@ -1310,6 +1371,14 @@
|
|||
/>
|
||||
</div>
|
||||
{/if}
|
||||
{#if $config?.audio?.realtime?.enabled || voiceAvatar}
|
||||
<VoiceAvatarSettings
|
||||
bind:value={voiceAvatar}
|
||||
bind:file={avatarFile}
|
||||
bind:animationFiles
|
||||
disabled={loading}
|
||||
/>
|
||||
{/if}
|
||||
<div class="my-3">
|
||||
<div class="flex w-full justify-between mb-1">
|
||||
<div class="self-center text-xs font-normal text-gray-500">
|
||||
|
|
|
|||
389
src/lib/components/workspace/Models/VoiceAvatarEditor.svelte
Normal file
389
src/lib/components/workspace/Models/VoiceAvatarEditor.svelte
Normal file
|
|
@ -0,0 +1,389 @@
|
|||
<script lang="ts">
|
||||
import { getContext, onDestroy } from 'svelte';
|
||||
import {
|
||||
AVATAR_MAX_BYTES,
|
||||
type AnimationFiles,
|
||||
type AvatarState,
|
||||
type VoiceAvatarConfig
|
||||
} from '$lib/utils/voice-avatar';
|
||||
import VoiceAvatar from '$lib/components/chat/MessageInput/CallOverlay/VoiceAvatar.svelte';
|
||||
|
||||
export let value: VoiceAvatarConfig | null = null;
|
||||
export let file: File | null = null;
|
||||
export let valid = true;
|
||||
export let animationFiles: AnimationFiles = {};
|
||||
let avatarValid = true;
|
||||
let animationsLoading = false;
|
||||
let animationErrors: Record<string, string> = {};
|
||||
let avatarPreview: VoiceAvatar;
|
||||
let previewContainer: HTMLDivElement;
|
||||
let animationInput: HTMLInputElement;
|
||||
let animationTarget: AvatarState | number = 'idle';
|
||||
$: gestures = value?.gestures ?? [];
|
||||
$: namesValid =
|
||||
gestures.every(
|
||||
(g) => /^[a-z][a-z0-9_]{0,47}$/.test(g.name) && g.description.trim().length > 0 && g.file_id
|
||||
) && new Set(gestures.map((g) => g.name)).size === gestures.length;
|
||||
$: valid =
|
||||
avatarValid && !animationsLoading && !Object.keys(animationErrors).length && namesValid;
|
||||
const uploadAnimation = (target: AvatarState | number) => {
|
||||
animationTarget = target;
|
||||
animationInput.click();
|
||||
};
|
||||
const chooseAnimation = () => {
|
||||
const selected = animationInput.files?.[0];
|
||||
animationInput.value = '';
|
||||
if (!selected || !value) return;
|
||||
if (!/\.vrma$/i.test(selected.name) || selected.size > 10 * 1024 * 1024) {
|
||||
error = $i18n.t('Choose a VRMA file, up to 10 MiB.');
|
||||
return;
|
||||
}
|
||||
const id = crypto.randomUUID();
|
||||
animationFiles = { ...animationFiles, [id]: selected };
|
||||
if (typeof animationTarget === 'number') {
|
||||
value = {
|
||||
...value,
|
||||
gestures: gestures.map((g, i) => (i === animationTarget ? { ...g, file_id: id } : g))
|
||||
};
|
||||
} else value = { ...value, states: { ...value.states, [animationTarget]: { file_id: id } } };
|
||||
error = '';
|
||||
};
|
||||
const removeState = (mode: AvatarState) => {
|
||||
if (!value) return;
|
||||
const states = { ...value.states };
|
||||
delete states[mode];
|
||||
value = { ...value, states };
|
||||
};
|
||||
const addGesture = () => {
|
||||
if (!value) return;
|
||||
let n = 1;
|
||||
while (gestures.some((g) => g.name === `gesture_${n}`)) n++;
|
||||
value = {
|
||||
...value,
|
||||
gestures: [...gestures, { name: `gesture_${n}`, description: '', file_id: '' }]
|
||||
};
|
||||
};
|
||||
const previewGesture = (name: string) => {
|
||||
previewContainer?.scrollIntoView({ block: 'nearest', behavior: 'smooth' });
|
||||
const result = avatarPreview?.playAnimation(name);
|
||||
if (result !== 'started')
|
||||
error = $i18n.t(
|
||||
result === 'busy'
|
||||
? 'Wait for the current gesture to finish.'
|
||||
: 'Animation is unavailable or reduced motion is enabled.'
|
||||
);
|
||||
else error = '';
|
||||
};
|
||||
export let disabled = false;
|
||||
const i18n = getContext<any>('i18n');
|
||||
let input: HTMLInputElement;
|
||||
let error = '';
|
||||
let ready = false;
|
||||
let capabilities = { mouth: true, blink: true };
|
||||
let state: 'idle' | 'listening' | 'speaking' = 'idle';
|
||||
let level = 0;
|
||||
let previewTimer: ReturnType<typeof setInterval> | undefined;
|
||||
const stopPreview = () => {
|
||||
clearInterval(previewTimer);
|
||||
state = 'idle';
|
||||
level = 0;
|
||||
};
|
||||
onDestroy(stopPreview);
|
||||
const preview = (next: typeof state) => {
|
||||
previewContainer?.scrollIntoView({ block: 'nearest', behavior: 'smooth' });
|
||||
stopPreview();
|
||||
avatarPreview?.cancelAnimation();
|
||||
state = next;
|
||||
if (next === 'speaking') {
|
||||
let step = 0;
|
||||
previewTimer = setInterval(() => {
|
||||
step++;
|
||||
level = step % 17 < 3 ? 0 : 0.04 + Math.abs(Math.sin(step * 0.9)) * 0.13;
|
||||
}, 60);
|
||||
}
|
||||
};
|
||||
const choose = () => {
|
||||
const selected = input.files?.[0];
|
||||
input.value = '';
|
||||
if (!selected) return;
|
||||
if (selected.size > AVATAR_MAX_BYTES || !/\.(vrm|glb)$/i.test(selected.name)) {
|
||||
error = $i18n.t('Choose a VRM file, up to 25 MiB.');
|
||||
return;
|
||||
}
|
||||
stopPreview();
|
||||
error = '';
|
||||
ready = false;
|
||||
avatarValid = false;
|
||||
file = selected;
|
||||
value = { ...value, file_id: '' };
|
||||
};
|
||||
const remove = () => {
|
||||
stopPreview();
|
||||
file = null;
|
||||
animationFiles = {};
|
||||
animationErrors = {};
|
||||
animationsLoading = false;
|
||||
value = null;
|
||||
error = '';
|
||||
ready = false;
|
||||
avatarValid = true;
|
||||
};
|
||||
</script>
|
||||
|
||||
<fieldset {disabled} class="avatar-editor my-4 min-w-0 w-full">
|
||||
<legend class="text-xs text-gray-500">{$i18n.t('Voice avatar')}</legend>
|
||||
<p class="text-xs text-gray-500 mt-1 mb-3">
|
||||
{$i18n.t('A character for bridge voice calls. The orb is used by default.')}
|
||||
</p>
|
||||
<input
|
||||
bind:this={input}
|
||||
type="file"
|
||||
accept=".vrm,.glb"
|
||||
class="hidden"
|
||||
on:change={choose}
|
||||
aria-label={$i18n.t('Upload voice avatar')}
|
||||
/>
|
||||
{#if value}
|
||||
<div
|
||||
bind:this={previewContainer}
|
||||
class="preview rounded-xl bg-gray-50 dark:bg-gray-900"
|
||||
aria-label={$i18n.t('Avatar preview')}
|
||||
>
|
||||
{#key file ?? value.file_id}
|
||||
<VoiceAvatar
|
||||
bind:this={avatarPreview}
|
||||
{animationFiles}
|
||||
on:animations={(event) => {
|
||||
animationsLoading = event.detail.loading;
|
||||
animationErrors = event.detail.errors;
|
||||
}}
|
||||
config={value}
|
||||
{file}
|
||||
speaking={state === 'speaking'}
|
||||
listening={state === 'listening'}
|
||||
{level}
|
||||
on:ready={(event) => {
|
||||
capabilities = event.detail;
|
||||
ready = true;
|
||||
avatarValid = true;
|
||||
error = '';
|
||||
}}
|
||||
on:error={(event) => {
|
||||
error = event.detail?.message ?? $i18n.t('Could not load avatar.');
|
||||
ready = false;
|
||||
avatarValid = !file;
|
||||
stopPreview();
|
||||
}}
|
||||
/>
|
||||
{/key}
|
||||
{#if !ready && !error}<span class="preview-message text-xs text-gray-500" role="status"
|
||||
>{$i18n.t('Loading avatar…')}</span
|
||||
>{/if}
|
||||
</div>
|
||||
<div
|
||||
class="flex gap-1 justify-center my-2"
|
||||
role="group"
|
||||
aria-label={$i18n.t('Preview behavior')}
|
||||
>
|
||||
{#each ['idle', 'listening', 'speaking'] as mode}
|
||||
<button
|
||||
type="button"
|
||||
class="preview-state text-xs px-3 py-1.5 rounded-lg"
|
||||
class:selected={state === mode}
|
||||
disabled={!ready || disabled}
|
||||
aria-pressed={state === mode}
|
||||
on:click={() => preview(mode as typeof state)}
|
||||
>
|
||||
{$i18n.t(mode === 'idle' ? 'Idle' : mode === 'listening' ? 'Listening' : 'Speaking')}
|
||||
</button>
|
||||
{/each}
|
||||
</div>
|
||||
<p class="text-xs text-gray-500 text-center mb-3">
|
||||
{$i18n.t('Silent motion preview. During calls, the mouth follows assistant audio.')}
|
||||
</p>
|
||||
{#if ready && (!capabilities.mouth || !capabilities.blink)}
|
||||
<p class="text-xs text-amber-700 dark:text-amber-400 mb-3" role="status">
|
||||
{#if !capabilities.mouth}{$i18n.t(
|
||||
'This avatar has no mouth expression; speech will use body motion only.'
|
||||
)}{/if}
|
||||
{#if !capabilities.blink}
|
||||
{$i18n.t('This avatar has no blink expressions.')}{/if}
|
||||
</p>
|
||||
{/if}
|
||||
<input
|
||||
bind:this={animationInput}
|
||||
type="file"
|
||||
accept=".vrma"
|
||||
class="hidden"
|
||||
on:change={chooseAnimation}
|
||||
aria-label={$i18n.t('Upload animation')}
|
||||
/>
|
||||
<section class="my-4 text-xs" aria-label={$i18n.t('State animations')}>
|
||||
<div class="font-medium mb-1">{$i18n.t('State animations')}</div>
|
||||
<p class="text-gray-500 mb-2">
|
||||
{$i18n.t('Optional clips replace the built-in movement for each state.')}
|
||||
</p>
|
||||
{#each ['idle', 'listening', 'speaking'] as mode}
|
||||
{@const asset = value.states?.[mode as AvatarState]}
|
||||
<div class="flex items-center justify-between gap-3 py-2">
|
||||
<div class="min-w-0">
|
||||
<div>
|
||||
{$i18n.t(mode === 'idle' ? 'Idle' : mode === 'listening' ? 'Listening' : 'Speaking')}
|
||||
</div>
|
||||
<div class="text-gray-500 truncate max-w-44">
|
||||
{asset
|
||||
? (animationFiles[asset.file_id]?.name ?? $i18n.t('Custom animation'))
|
||||
: $i18n.t('Built-in')}
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex shrink-0 gap-3 text-gray-500">
|
||||
{#if asset}<button
|
||||
type="button"
|
||||
disabled={!ready || animationsLoading || !!animationErrors[asset.file_id]}
|
||||
on:click={() => preview(mode as AvatarState)}>{$i18n.t('Preview')}</button
|
||||
>{/if}
|
||||
<button
|
||||
type="button"
|
||||
aria-label={`${$i18n.t('Upload')} ${mode} VRMA`}
|
||||
on:click={() => uploadAnimation(mode as AvatarState)}
|
||||
>{$i18n.t(asset ? 'Replace' : 'Upload')}</button
|
||||
>
|
||||
{#if asset}<button
|
||||
type="button"
|
||||
aria-label={`${$i18n.t('Remove')} ${mode} VRMA`}
|
||||
on:click={() => removeState(mode as AvatarState)}>{$i18n.t('Remove')}</button
|
||||
>{/if}
|
||||
</div>
|
||||
</div>
|
||||
{#if asset && animationErrors[asset.file_id]}<p
|
||||
class="text-red-600 dark:text-red-400"
|
||||
role="alert"
|
||||
>
|
||||
{animationErrors[asset.file_id]}
|
||||
</p>{/if}
|
||||
{/each}
|
||||
</section>
|
||||
<section class="my-4 text-xs" aria-label={$i18n.t('Named gestures')}>
|
||||
<div class="flex items-center justify-between">
|
||||
<div class="font-medium">{$i18n.t('Named gestures')}</div>
|
||||
<button
|
||||
type="button"
|
||||
class="text-gray-500"
|
||||
disabled={gestures.length >= 16}
|
||||
on:click={addGesture}>{$i18n.t('Add gesture')}</button
|
||||
>
|
||||
</div>
|
||||
<p class="text-gray-500 mt-1">
|
||||
{$i18n.t('The voice model chooses when to play a gesture using its name and description.')}
|
||||
</p>
|
||||
{#each gestures as gesture, i}
|
||||
<div class="mt-3 space-y-2">
|
||||
<div class="flex items-center gap-3">
|
||||
<input
|
||||
aria-label={$i18n.t('Gesture name')}
|
||||
class="setting-input min-w-0 flex-1"
|
||||
placeholder="wave"
|
||||
maxlength="48"
|
||||
bind:value={gesture.name}
|
||||
on:input={() => {
|
||||
if (value) value = { ...value, gestures };
|
||||
}}
|
||||
/><button
|
||||
type="button"
|
||||
class="text-gray-500"
|
||||
aria-label={`${$i18n.t('Remove gesture')} ${i + 1}`}
|
||||
on:click={() => {
|
||||
if (value)
|
||||
value = { ...value, gestures: gestures.filter((_, index) => index !== i) };
|
||||
}}>{$i18n.t('Remove')}</button
|
||||
>
|
||||
</div>
|
||||
<input
|
||||
aria-label={$i18n.t('Gesture description')}
|
||||
class="setting-input"
|
||||
placeholder={$i18n.t('When should the model use this gesture?')}
|
||||
maxlength="500"
|
||||
bind:value={gesture.description}
|
||||
on:input={() => {
|
||||
if (value) value = { ...value, gestures };
|
||||
}}
|
||||
/>
|
||||
<div class="flex items-center gap-3 text-gray-500">
|
||||
<button
|
||||
type="button"
|
||||
aria-label={`${$i18n.t('Upload gesture')} ${i + 1} VRMA`}
|
||||
on:click={() => uploadAnimation(i)}
|
||||
>{$i18n.t(gesture.file_id ? 'Replace clip' : 'Upload VRMA')}</button
|
||||
>{#if gesture.file_id}<button
|
||||
type="button"
|
||||
aria-label={`${$i18n.t('Preview gesture')} ${i + 1}`}
|
||||
disabled={!ready || animationsLoading || !!animationErrors[gesture.file_id]}
|
||||
on:click={() => previewGesture(gesture.name)}>{$i18n.t('Preview')}</button
|
||||
><span class="truncate"
|
||||
>{animationFiles[gesture.file_id]?.name ?? $i18n.t('Custom animation')}</span
|
||||
>{/if}
|
||||
</div>
|
||||
{#if animationErrors[gesture.file_id]}<p
|
||||
class="text-red-600 dark:text-red-400"
|
||||
role="alert"
|
||||
>
|
||||
{animationErrors[gesture.file_id]}
|
||||
</p>{/if}
|
||||
</div>
|
||||
{/each}
|
||||
{#if !namesValid}<p class="mt-2 text-gray-500">
|
||||
{$i18n.t(
|
||||
'Each gesture needs a clip, a description, and a unique lowercase name using letters, numbers or underscores.'
|
||||
)}
|
||||
</p>{/if}
|
||||
</section>
|
||||
<p class="text-xs text-gray-500 mb-4">
|
||||
{$i18n.t(
|
||||
'VRMA 1.0 · 10 MiB max · 60 seconds max. Body motion only, played in place. Mouth, blinking and gaze remain automatic.'
|
||||
)}
|
||||
</p>
|
||||
{#if animationsLoading}<p class="text-xs text-gray-500" role="status">
|
||||
{$i18n.t('Loading animations…')}
|
||||
</p>{/if}
|
||||
{/if}
|
||||
{#if error}<p class="text-xs text-red-600 dark:text-red-400 my-2" role="alert">{error}</p>{/if}
|
||||
<div class="flex gap-3 items-center mt-3 text-xs">
|
||||
<button
|
||||
type="button"
|
||||
class="px-3 py-2 rounded-lg bg-gray-100 dark:bg-gray-850"
|
||||
on:click={() => input.click()}>{$i18n.t(value ? 'Replace avatar' : 'Upload VRM')}</button
|
||||
>
|
||||
{#if value}<button type="button" class="text-gray-500" on:click={remove}
|
||||
>{$i18n.t('Use orb')}</button
|
||||
>{/if}
|
||||
{#if file}<span class="text-gray-500 truncate">{file.name}</span>{/if}
|
||||
</div>
|
||||
<p class="text-xs text-gray-500 mt-2">
|
||||
{$i18n.t(
|
||||
'VRM 0.x or 1.0 · 25 MiB max · Embedded textures. Upload a character you have permission to use.'
|
||||
)}
|
||||
</p>
|
||||
</fieldset>
|
||||
|
||||
<style>
|
||||
.preview {
|
||||
height: 320px;
|
||||
position: relative;
|
||||
overflow: hidden;
|
||||
}
|
||||
.preview-message {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
display: grid;
|
||||
place-items: center;
|
||||
}
|
||||
.preview-state.selected {
|
||||
background: rgb(127 127 127 / 0.15);
|
||||
}
|
||||
.setting-input {
|
||||
padding: 7px 8px;
|
||||
background: rgb(127 127 127 / 0.08);
|
||||
border-radius: 7px;
|
||||
width: 100%;
|
||||
}
|
||||
</style>
|
||||
|
|
@ -0,0 +1,90 @@
|
|||
<script lang="ts">
|
||||
import { getContext } from 'svelte';
|
||||
import Modal from '$lib/components/common/Modal.svelte';
|
||||
import XMark from '$lib/components/icons/XMark.svelte';
|
||||
import type { AnimationFiles, VoiceAvatarConfig } from '$lib/utils/voice-avatar';
|
||||
import VoiceAvatarEditor from './VoiceAvatarEditor.svelte';
|
||||
|
||||
export let value: VoiceAvatarConfig | null = null;
|
||||
export let file: File | null = null;
|
||||
export let disabled = false;
|
||||
export let animationFiles: AnimationFiles = {};
|
||||
let draftAnimationFiles: AnimationFiles = {};
|
||||
const i18n = getContext<any>('i18n');
|
||||
let show = false;
|
||||
let draft: VoiceAvatarConfig | null = null;
|
||||
let draftFile: File | null = null;
|
||||
let valid = true;
|
||||
const open = () => {
|
||||
draft = value ? structuredClone(value) : null;
|
||||
draftFile = file;
|
||||
draftAnimationFiles = { ...animationFiles };
|
||||
valid = !file;
|
||||
show = true;
|
||||
};
|
||||
const apply = () => {
|
||||
if (!valid) return;
|
||||
value = draft;
|
||||
file = draftFile;
|
||||
animationFiles = draftAnimationFiles;
|
||||
show = false;
|
||||
};
|
||||
</script>
|
||||
|
||||
<div class="my-3 flex items-center justify-between gap-3">
|
||||
<div>
|
||||
<div class="text-xs text-gray-500">{$i18n.t('Voice avatar')}</div>
|
||||
<div class="text-sm mt-0.5">{$i18n.t(value ? 'Custom avatar' : 'Default orb')}</div>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
{disabled}
|
||||
class="text-xs text-gray-500 transition hover:text-gray-700 dark:hover:text-gray-300"
|
||||
on:click={open}
|
||||
>
|
||||
{$i18n.t('Configure')}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<Modal bind:show size="sm">
|
||||
<div class="p-5 text-gray-900 dark:text-gray-100">
|
||||
<div class="flex items-center justify-between gap-3">
|
||||
<h2 class="text-base font-medium">{$i18n.t('Avatar setup')}</h2>
|
||||
<button
|
||||
type="button"
|
||||
class="p-1 rounded-lg text-gray-500 hover:bg-gray-100 dark:hover:bg-gray-850"
|
||||
aria-label={$i18n.t('Close avatar setup')}
|
||||
on:click={() => {
|
||||
show = false;
|
||||
}}
|
||||
>
|
||||
<XMark className="size-4" />
|
||||
</button>
|
||||
</div>
|
||||
{#if show}
|
||||
<div class="max-h-[70dvh] overflow-y-auto scrollbar-hidden">
|
||||
<VoiceAvatarEditor
|
||||
bind:value={draft}
|
||||
bind:file={draftFile}
|
||||
bind:animationFiles={draftAnimationFiles}
|
||||
bind:valid
|
||||
/>
|
||||
</div>
|
||||
{/if}
|
||||
<div class="flex justify-end gap-2 mt-5">
|
||||
<button
|
||||
type="button"
|
||||
class="text-sm px-3 py-2 rounded-lg"
|
||||
on:click={() => {
|
||||
show = false;
|
||||
}}>{$i18n.t('Cancel')}</button
|
||||
>
|
||||
<button
|
||||
type="button"
|
||||
disabled={!valid}
|
||||
class="text-sm px-4 py-2 rounded-lg bg-black text-white dark:bg-white dark:text-black disabled:opacity-40"
|
||||
on:click={apply}>{$i18n.t('Apply')}</button
|
||||
>
|
||||
</div>
|
||||
</div>
|
||||
</Modal>
|
||||
|
|
@ -16,6 +16,8 @@ class RealtimeAudioProcessor extends AudioWorkletProcessor {
|
|||
this.inputEnergy = 0;
|
||||
this.outputEnergy = 0;
|
||||
this.levelSamples = 0;
|
||||
this.playbackSamples = 0;
|
||||
this.clearId = 0;
|
||||
this.port.onmessage = ({ data }) => {
|
||||
if (data.type === 'capture') {
|
||||
this.enabled = data.enabled;
|
||||
|
|
@ -31,6 +33,8 @@ class RealtimeAudioProcessor extends AudioWorkletProcessor {
|
|||
} else if (data.type === 'done') {
|
||||
this.ended.add(data.response_id);
|
||||
} else if (data.type === 'clear') {
|
||||
this.clearId = data.id;
|
||||
this.playbackSamples = this.outputEnergy = 0;
|
||||
this.port.postMessage({
|
||||
type: 'cleared',
|
||||
id: data.id,
|
||||
|
|
@ -76,6 +80,7 @@ class RealtimeAudioProcessor extends AudioWorkletProcessor {
|
|||
offset += count;
|
||||
chunk.offset += count;
|
||||
this.queued -= count;
|
||||
this.playbackSamples += count;
|
||||
const key = `${chunk.item_id}:${chunk.content_index}`;
|
||||
const position = this.rendered.get(key) ?? {
|
||||
response_id: chunk.response_id,
|
||||
|
|
@ -98,12 +103,14 @@ class RealtimeAudioProcessor extends AudioWorkletProcessor {
|
|||
if (++this.ticks % 8 === 0) {
|
||||
this.port.postMessage({
|
||||
type: 'playback',
|
||||
clearId: this.clearId,
|
||||
playbackActive: this.playbackSamples > 0,
|
||||
queued: this.queued,
|
||||
received: this.received,
|
||||
inputLevel: Math.sqrt(this.inputEnergy / this.levelSamples),
|
||||
outputLevel: Math.sqrt(this.outputEnergy / this.levelSamples)
|
||||
});
|
||||
this.inputEnergy = this.outputEnergy = this.levelSamples = 0;
|
||||
this.inputEnergy = this.outputEnergy = this.levelSamples = this.playbackSamples = 0;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -80,6 +80,9 @@ export class RealtimeCall {
|
|||
inputLevel = 0;
|
||||
outputLevel = 0;
|
||||
error = '';
|
||||
animationPlayer?: (name: string) => string;
|
||||
animationInterruption = 0;
|
||||
private animationCalls = new Set<string>();
|
||||
private ws?: WebSocket;
|
||||
private context?: AudioContext;
|
||||
private stream?: MediaStream;
|
||||
|
|
@ -441,6 +444,29 @@ export class RealtimeCall {
|
|||
event.item.status === 'completed'
|
||||
) {
|
||||
const response = this.responses.get(event.response_id);
|
||||
if (event.item.name === 'play_animation') {
|
||||
if (this.animationCalls.has(event.item.call_id)) return;
|
||||
this.animationCalls.add(event.item.call_id);
|
||||
let status = 'unavailable';
|
||||
if (!response || this.interrupted.has(event.response_id) || this.receivingSpeech)
|
||||
status = 'cancelled';
|
||||
else {
|
||||
try {
|
||||
const args = JSON.parse(event.item.arguments);
|
||||
if (
|
||||
event.animation_valid !== false &&
|
||||
Object.keys(args).length === 1 &&
|
||||
typeof args.name === 'string'
|
||||
)
|
||||
status = this.animationPlayer?.(args.name) ?? 'unavailable';
|
||||
} catch {
|
||||
/* Bad or unavailable gestures never interrupt a voice call. */
|
||||
}
|
||||
response.animation = true;
|
||||
}
|
||||
this.send({ type: 'bridge.animation.result', call_id: event.item.call_id, status });
|
||||
return;
|
||||
}
|
||||
if (
|
||||
!response ||
|
||||
event.item.name !== 'generate_chat_completion' ||
|
||||
|
|
@ -521,6 +547,14 @@ export class RealtimeCall {
|
|||
if (response) {
|
||||
response.done = true;
|
||||
this.saveSpeech(response);
|
||||
if (
|
||||
response.animation &&
|
||||
!response.delegated &&
|
||||
event.response.status === 'completed' &&
|
||||
!this.interrupted.has(response.id)
|
||||
) {
|
||||
this.enqueue({ type: 'bridge.animation.respond', response_id: response.id });
|
||||
}
|
||||
}
|
||||
if (['failed', 'incomplete'].includes(event.response.status))
|
||||
this.options.error('The voice response did not complete.');
|
||||
|
|
@ -619,6 +653,7 @@ export class RealtimeCall {
|
|||
}
|
||||
|
||||
stopSpeaking() {
|
||||
this.animationInterruption++;
|
||||
if (this.responseRequested) this.cancelRequested = true;
|
||||
const responses = new Set(
|
||||
[...this.speakingResponses].filter((id) => !this.interrupted.has(id))
|
||||
|
|
@ -639,7 +674,9 @@ export class RealtimeCall {
|
|||
const id = ++this.clearId;
|
||||
this.clears.set(id, responses);
|
||||
this.audio?.port.postMessage({ type: 'clear', id });
|
||||
this.commands = this.commands.filter((command) => command.type !== 'bridge.status');
|
||||
this.commands = this.commands.filter(
|
||||
(command) => command.type !== 'bridge.status' && command.type !== 'bridge.animation.respond'
|
||||
);
|
||||
this.options.change();
|
||||
}
|
||||
|
||||
|
|
@ -723,6 +760,7 @@ export class RealtimeCall {
|
|||
this.responses.clear();
|
||||
this.inputs.clear();
|
||||
this.calls.clear();
|
||||
this.animationCalls.clear();
|
||||
this.clears.clear();
|
||||
this.clearId = 0;
|
||||
this.options.change();
|
||||
|
|
|
|||
194
src/lib/utils/voice-avatar-animation.ts
Normal file
194
src/lib/utils/voice-avatar-animation.ts
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
import * as THREE from 'three';
|
||||
import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
|
||||
import {
|
||||
VRMAnimationLoaderPlugin,
|
||||
createVRMAnimationHumanoidTracks
|
||||
} from '@pixiv/three-vrm-animation';
|
||||
import type { VRM } from '@pixiv/three-vrm';
|
||||
|
||||
export const ANIMATION_MAX_BYTES = 10 * 1024 * 1024;
|
||||
|
||||
export function validateAnimation(data: ArrayBuffer) {
|
||||
if (data.byteLength > ANIMATION_MAX_BYTES) throw new Error('Animation must be at most 10 MiB.');
|
||||
if (data.byteLength < 28) throw new Error('Upload a binary VRMA file.');
|
||||
const view = new DataView(data);
|
||||
const length = view.getUint32(12, true);
|
||||
if (
|
||||
view.getUint32(0, true) !== 0x46546c67 ||
|
||||
view.getUint32(4, true) !== 2 ||
|
||||
view.getUint32(8, true) !== data.byteLength ||
|
||||
view.getUint32(16, true) !== 0x4e4f534a ||
|
||||
length > 2 * 1024 * 1024 ||
|
||||
length + 28 > data.byteLength
|
||||
)
|
||||
throw new Error('Invalid VRMA container.');
|
||||
const start = 28 + length;
|
||||
const bytes = view.getUint32(20 + length, true);
|
||||
if (view.getUint32(24 + length, true) !== 0x004e4942 || start + bytes !== data.byteLength)
|
||||
throw new Error('Invalid animation binary data.');
|
||||
const json = JSON.parse(new TextDecoder().decode(new Uint8Array(data, 20, length)));
|
||||
const ext = json.extensions?.VRMC_vrm_animation;
|
||||
const bones = Object.values(ext?.humanoid?.humanBones ?? {}) as { node: number }[];
|
||||
if (
|
||||
ext?.specVersion !== '1.0' ||
|
||||
!bones.length ||
|
||||
!json.nodes?.length ||
|
||||
json.nodes.length > 512 ||
|
||||
bones.some((b) => !Number.isInteger(b.node) || !json.nodes[b.node])
|
||||
)
|
||||
throw new Error('Use a VRMA 1.0 humanoid animation.');
|
||||
const parents = new Set<number>(),
|
||||
visiting = new Set<number>(),
|
||||
visited = new Set<number>();
|
||||
const visit = (index: number) => {
|
||||
if (!Number.isInteger(index) || !json.nodes[index] || visiting.has(index))
|
||||
throw new Error('Invalid animation node hierarchy.');
|
||||
if (visited.has(index)) return;
|
||||
visiting.add(index);
|
||||
const node = json.nodes[index];
|
||||
for (const [key, length] of Object.entries({
|
||||
translation: 3,
|
||||
rotation: 4,
|
||||
scale: 3,
|
||||
matrix: 16
|
||||
})) {
|
||||
if (
|
||||
node[key] &&
|
||||
(node[key].length !== length || node[key].some((v: number) => !Number.isFinite(v)))
|
||||
)
|
||||
throw new Error('Invalid animation transform.');
|
||||
}
|
||||
for (const child of node.children ?? []) {
|
||||
if (parents.has(child)) throw new Error('Animation nodes must have a single parent.');
|
||||
parents.add(child);
|
||||
visit(child);
|
||||
}
|
||||
visiting.delete(index);
|
||||
visited.add(index);
|
||||
};
|
||||
json.nodes.forEach((_: unknown, index: number) => visit(index));
|
||||
for (const scene of json.scenes ?? [])
|
||||
if ((scene.nodes ?? []).some((n: number) => !Number.isInteger(n) || !json.nodes[n]))
|
||||
throw new Error('Invalid animation scene.');
|
||||
if (
|
||||
json.buffers?.length !== 1 ||
|
||||
'uri' in json.buffers[0] ||
|
||||
json.buffers[0].byteLength > bytes ||
|
||||
json.buffers[0].byteLength < bytes - 3 ||
|
||||
['images', 'textures', 'meshes', 'skins'].some((k) => json[k]?.length) ||
|
||||
(json.extensionsRequired ?? []).some((e: string) => e !== 'VRMC_vrm_animation')
|
||||
)
|
||||
throw new Error('Export embedded animation only, without meshes or textures.');
|
||||
for (const buffer of json.bufferViews ?? []) {
|
||||
if (
|
||||
(buffer.buffer ?? 0) !== 0 ||
|
||||
(buffer.byteOffset ?? 0) < 0 ||
|
||||
!(buffer.byteLength > 0) ||
|
||||
(buffer.byteOffset ?? 0) + buffer.byteLength > bytes
|
||||
)
|
||||
throw new Error('Invalid animation buffer.');
|
||||
}
|
||||
let count = 0;
|
||||
const values = (json.accessors ?? []).map((a: any) => {
|
||||
count += a.count;
|
||||
const components = ({ SCALAR: 1, VEC3: 3, VEC4: 4 } as Record<string, number>)[
|
||||
a.type
|
||||
] as number;
|
||||
const buffer = json.bufferViews?.[a.bufferView];
|
||||
const stride = buffer?.byteStride ?? components * 4;
|
||||
const offset = a.byteOffset ?? 0;
|
||||
if (
|
||||
!Number.isInteger(a.count) ||
|
||||
a.count <= 0 ||
|
||||
count > 500000 ||
|
||||
a.sparse ||
|
||||
a.componentType !== 5126 ||
|
||||
!components ||
|
||||
!buffer ||
|
||||
stride < components * 4 ||
|
||||
stride % 4 ||
|
||||
offset < 0 ||
|
||||
offset + (a.count - 1) * stride + components * 4 > buffer.byteLength
|
||||
)
|
||||
throw new Error('Invalid animation keyframes.');
|
||||
return Array.from({ length: a.count }, (_, i) =>
|
||||
Array.from({ length: components }, (_, c) => {
|
||||
const value = view.getFloat32(
|
||||
start + (buffer.byteOffset ?? 0) + offset + i * stride + c * 4,
|
||||
true
|
||||
);
|
||||
if (!Number.isFinite(value)) throw new Error('Animation keyframes must be finite.');
|
||||
return value;
|
||||
})
|
||||
);
|
||||
});
|
||||
const animation = json.animations?.[0];
|
||||
if (
|
||||
json.animations?.length !== 1 ||
|
||||
!animation?.channels?.length ||
|
||||
animation.channels.length > 256
|
||||
)
|
||||
throw new Error('Export exactly one animation per VRMA file.');
|
||||
let duration = 0,
|
||||
body = false;
|
||||
for (const channel of animation.channels) {
|
||||
const target = channel.target;
|
||||
const sampler = animation.samplers?.[channel.sampler];
|
||||
const times = values[sampler?.input],
|
||||
output = values[sampler?.output];
|
||||
const interpolation = sampler?.interpolation ?? 'LINEAR';
|
||||
if (!['LINEAR', 'STEP'].includes(interpolation))
|
||||
throw new Error('Export baked animation with linear or stepped keyframes.');
|
||||
if (
|
||||
!json.nodes[target?.node] ||
|
||||
!['rotation', 'translation'].includes(target.path) ||
|
||||
!times ||
|
||||
!output ||
|
||||
json.accessors[sampler.input].type !== 'SCALAR' ||
|
||||
!['LINEAR', 'STEP'].includes(interpolation) ||
|
||||
output.length !== times.length ||
|
||||
output[0].length !== (target.path === 'rotation' ? 4 : 3) ||
|
||||
times.some((t: number[], i: number) => t[0] < 0 || (i > 0 && t[0] <= times[i - 1][0]))
|
||||
)
|
||||
throw new Error('Invalid animation channels or timing.');
|
||||
duration = Math.max(duration, times[times.length - 1][0]);
|
||||
body ||= bones.some((b) => b.node === target.node);
|
||||
}
|
||||
if (!body || duration <= 0 || duration > 60)
|
||||
throw new Error('Use a body animation between 0 and 60 seconds.');
|
||||
}
|
||||
|
||||
export async function loadBodyAnimation(data: ArrayBuffer, vrm: VRM) {
|
||||
validateAnimation(data);
|
||||
const manager = new THREE.LoadingManager();
|
||||
manager.setURLModifier(() => {
|
||||
throw new Error('External animation resources are not supported.');
|
||||
});
|
||||
const loader = new GLTFLoader(manager);
|
||||
loader.register((parser) => new VRMAnimationLoaderPlugin(parser));
|
||||
const gltf = await loader.parseAsync(data, '');
|
||||
const animation = gltf.userData.vrmAnimations?.[0];
|
||||
if (
|
||||
!animation ||
|
||||
!Number.isFinite(animation.restHipsPosition.y) ||
|
||||
animation.restHipsPosition.y <= 0
|
||||
)
|
||||
throw new Error('Animation needs a valid humanoid rest pose.');
|
||||
const humanoid = createVRMAnimationHumanoidTracks(animation, vrm.humanoid, vrm.meta.metaVersion);
|
||||
for (const [name, track] of humanoid.rotation)
|
||||
track.setInterpolation(animation.humanoidTracks.rotation.get(name).getInterpolation());
|
||||
const tracks: THREE.KeyframeTrack[] = [
|
||||
...humanoid.rotation.values(),
|
||||
...humanoid.translation.values()
|
||||
];
|
||||
if (!tracks.length) throw new Error('Animation has no compatible body tracks.');
|
||||
const hips = vrm.humanoid.getNormalizedBoneNode('hips')!;
|
||||
for (const track of tracks) {
|
||||
if (!track.name.endsWith('.position')) continue;
|
||||
for (let i = 0; i < track.values.length; i += 3) {
|
||||
track.values[i] = hips.position.x;
|
||||
track.values[i + 2] = hips.position.z;
|
||||
}
|
||||
}
|
||||
return new THREE.AnimationClip('body', animation.duration, tracks);
|
||||
}
|
||||
308
src/lib/utils/voice-avatar-renderer.ts
Normal file
308
src/lib/utils/voice-avatar-renderer.ts
Normal file
|
|
@ -0,0 +1,308 @@
|
|||
import * as THREE from 'three';
|
||||
import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
|
||||
import { VRMLoaderPlugin, VRMUtils, type VRM } from '@pixiv/three-vrm';
|
||||
import { loadBodyAnimation } from './voice-avatar-animation';
|
||||
import { validateAvatar, type VoiceAvatarConfig } from './voice-avatar';
|
||||
|
||||
export type AvatarSignals = {
|
||||
speaking: boolean;
|
||||
listening: boolean;
|
||||
level: number;
|
||||
active: boolean;
|
||||
};
|
||||
|
||||
export async function createAvatarRenderer(
|
||||
canvas: HTMLCanvasElement,
|
||||
data: ArrayBuffer,
|
||||
signal: AbortSignal
|
||||
) {
|
||||
await validateAvatar(data);
|
||||
signal.throwIfAborted();
|
||||
const manager = new THREE.LoadingManager();
|
||||
manager.setURLModifier((url) => {
|
||||
if (!url.startsWith('blob:')) throw new Error('External avatar resources are not supported.');
|
||||
return url;
|
||||
});
|
||||
const loader = new GLTFLoader(manager);
|
||||
loader.register((parser) => new VRMLoaderPlugin(parser));
|
||||
const gltf = await loader.parseAsync(data, '');
|
||||
const vrm: VRM | undefined = gltf.userData.vrm;
|
||||
if (!vrm || signal.aborted) {
|
||||
VRMUtils.deepDispose(gltf.scene);
|
||||
signal.throwIfAborted();
|
||||
throw new Error('This VRM could not be loaded.');
|
||||
}
|
||||
VRMUtils.rotateVRM0(vrm);
|
||||
const scene = new THREE.Scene();
|
||||
scene.add(vrm.scene);
|
||||
scene.add(new THREE.HemisphereLight(0xffffff, 0x8b929f, 2));
|
||||
const light = new THREE.DirectionalLight(0xffffff, 2.2);
|
||||
light.position.set(-1, 2, 3);
|
||||
scene.add(light);
|
||||
let renderer: THREE.WebGLRenderer;
|
||||
try {
|
||||
renderer = new THREE.WebGLRenderer({
|
||||
canvas,
|
||||
alpha: true,
|
||||
antialias: true,
|
||||
powerPreference: 'low-power'
|
||||
});
|
||||
} catch (error) {
|
||||
VRMUtils.deepDispose(vrm.scene);
|
||||
throw error;
|
||||
}
|
||||
renderer.setPixelRatio(Math.min(window.devicePixelRatio || 1, 2));
|
||||
renderer.setClearColor(0x000000, 0);
|
||||
renderer.outputColorSpace = THREE.SRGBColorSpace;
|
||||
const camera = new THREE.PerspectiveCamera(30, 1, 0.01, 100);
|
||||
const bone = (name: Parameters<typeof vrm.humanoid.getNormalizedBoneNode>[0]) =>
|
||||
vrm.humanoid.getNormalizedBoneNode(name);
|
||||
const head = bone('head');
|
||||
const spine = bone('spine');
|
||||
const leftArm = bone('leftUpperArm');
|
||||
const rightArm = bone('rightUpperArm');
|
||||
// VRM's normalized skeleton gives both VRM 0 and 1 a predictable T-pose.
|
||||
if (leftArm) leftArm.rotation.z = -1.22;
|
||||
if (rightArm) rightArm.rotation.z = 1.22;
|
||||
const leftElbow = bone('leftLowerArm');
|
||||
const rightElbow = bone('rightLowerArm');
|
||||
if (leftElbow) leftElbow.rotation.y = -0.12;
|
||||
if (rightElbow) rightElbow.rotation.y = 0.12;
|
||||
vrm.update(0);
|
||||
vrm.scene.updateMatrixWorld(true);
|
||||
const bounds = new THREE.Box3().setFromObject(vrm.scene);
|
||||
const size = bounds.getSize(new THREE.Vector3());
|
||||
const center = bounds.getCenter(new THREE.Vector3());
|
||||
const headPosition = head?.getWorldPosition(new THREE.Vector3()) ?? center.clone();
|
||||
const bodyHeight = Math.max(size.y, 0.5);
|
||||
const target = new THREE.Object3D();
|
||||
scene.add(target);
|
||||
if (vrm.lookAt) vrm.lookAt.target = target;
|
||||
const expressions = vrm.expressionManager;
|
||||
const mouth = !!expressions?.getExpression('aa');
|
||||
const blink = !!expressions?.getExpression('blink');
|
||||
const separateBlink =
|
||||
!!expressions?.getExpression('blinkLeft') && !!expressions?.getExpression('blinkRight');
|
||||
let speaking = 0,
|
||||
listening = 0,
|
||||
speechTime = 0,
|
||||
listenTime = 0,
|
||||
mouthWeight = 0,
|
||||
elapsed = 0,
|
||||
nextBlink = 2.8,
|
||||
blinkStart = -10;
|
||||
let width = 0,
|
||||
height = 0;
|
||||
|
||||
const mixer = new THREE.AnimationMixer(vrm.scene);
|
||||
const poses = Object.values(vrm.humanoid.normalizedHumanBones).map(({ node }) => ({
|
||||
node,
|
||||
restQ: node.quaternion.clone(),
|
||||
restP: node.position.clone(),
|
||||
q: node.quaternion.clone(),
|
||||
p: node.position.clone()
|
||||
}));
|
||||
const clips = new Map<string, THREE.AnimationClip>();
|
||||
let current: THREE.AnimationAction | null = null;
|
||||
let outgoing: THREE.AnimationAction | null = null;
|
||||
let fadeTime = 0,
|
||||
bodyWeight = 0,
|
||||
speechHold = 0;
|
||||
let gesture = false,
|
||||
gestureElapsed = 0;
|
||||
let enabled = true;
|
||||
let cameraDistance = 0;
|
||||
const switchClip = (clip: THREE.AnimationClip, once: boolean) => {
|
||||
outgoing?.stop();
|
||||
outgoing = current;
|
||||
current = mixer.clipAction(clip);
|
||||
current.reset().setLoop(once ? THREE.LoopOnce : THREE.LoopRepeat, once ? 1 : Infinity);
|
||||
current.clampWhenFinished = true;
|
||||
current.enabled = true;
|
||||
current.setEffectiveWeight(1).play();
|
||||
if (outgoing && outgoing !== current) current.crossFadeFrom(outgoing, 0.3, false);
|
||||
fadeTime = 0;
|
||||
};
|
||||
const cancelGesture = () => {
|
||||
gesture = false;
|
||||
gestureElapsed = 0;
|
||||
speechHold = 0;
|
||||
};
|
||||
|
||||
return {
|
||||
capabilities: { mouth, blink: blink || separateBlink },
|
||||
async loadAnimation(id: string, data: ArrayBuffer) {
|
||||
const clip = await loadBodyAnimation(data, vrm);
|
||||
signal.throwIfAborted();
|
||||
// Every body clip owns the full pose, including bones omitted by its author.
|
||||
const names = new Set(clip.tracks.map((track) => track.name));
|
||||
for (const { node, restQ, restP } of poses) {
|
||||
if (!names.has(`${node.name}.quaternion`))
|
||||
clip.tracks.push(
|
||||
new THREE.QuaternionKeyframeTrack(`${node.name}.quaternion`, [0], restQ.toArray())
|
||||
);
|
||||
if (!names.has(`${node.name}.position`))
|
||||
clip.tracks.push(
|
||||
new THREE.VectorKeyframeTrack(`${node.name}.position`, [0], restP.toArray())
|
||||
);
|
||||
}
|
||||
clips.set(id, clip);
|
||||
},
|
||||
playGesture(id: string) {
|
||||
if (!enabled) return 'unavailable' as const;
|
||||
if (gesture) return 'busy' as const;
|
||||
const clip = clips.get(id);
|
||||
if (!clip) return 'unavailable' as const;
|
||||
switchClip(clip, true);
|
||||
gesture = true;
|
||||
gestureElapsed = 0;
|
||||
return 'started' as const;
|
||||
},
|
||||
cancelGesture,
|
||||
render(delta: number, config: VoiceAvatarConfig, input: AvatarSignals, reducedMotion: boolean) {
|
||||
const dt = Math.min(delta, 0.05);
|
||||
elapsed += dt;
|
||||
enabled = input.active && !reducedMotion;
|
||||
if (!enabled) cancelGesture();
|
||||
if (input.speaking && input.active) speechHold = 0.2;
|
||||
else speechHold = Math.max(0, speechHold - dt);
|
||||
if (input.listening) speechHold = 0;
|
||||
const state =
|
||||
input.speaking || speechHold > 0 ? 'speaking' : input.listening ? 'listening' : 'idle';
|
||||
if (gesture) {
|
||||
gestureElapsed += dt;
|
||||
if (gestureElapsed >= current!.getClip().duration) cancelGesture();
|
||||
}
|
||||
const stateClip = enabled ? clips.get(config.states?.[state]?.file_id ?? '') : undefined;
|
||||
if (
|
||||
!gesture &&
|
||||
stateClip &&
|
||||
(current?.getClip() !== stateClip || current.loop === THREE.LoopOnce)
|
||||
)
|
||||
switchClip(stateClip, false);
|
||||
const authored = enabled && (gesture || !!stateClip);
|
||||
bodyWeight = THREE.MathUtils.clamp(bodyWeight + (authored ? dt : -dt) / 0.3, 0, 1);
|
||||
if (!enabled) bodyWeight = 0;
|
||||
// Start the procedural pose from rest, never from last frame's authored pose.
|
||||
for (const pose of poses) {
|
||||
pose.node.quaternion.copy(pose.restQ);
|
||||
pose.node.position.copy(pose.restP);
|
||||
}
|
||||
const rect = canvas.getBoundingClientRect();
|
||||
if (!rect.width || !rect.height) return;
|
||||
if (rect.width !== width || rect.height !== height) {
|
||||
width = rect.width;
|
||||
height = rect.height;
|
||||
renderer.setSize(width, height, false);
|
||||
camera.aspect = width / height;
|
||||
camera.updateProjectionMatrix();
|
||||
}
|
||||
|
||||
const smooth = 1 - Math.exp(-dt * 5);
|
||||
speaking += ((input.active && input.speaking ? 1 : 0) - speaking) * smooth;
|
||||
listening +=
|
||||
((input.active && input.listening && !input.speaking ? 1 : 0) - listening) * smooth;
|
||||
speechTime = input.active && input.speaking ? speechTime + dt : 0;
|
||||
listenTime = input.active && input.listening ? listenTime + dt : 0;
|
||||
// Full-body framing needs readable poses, not faster oscillation.
|
||||
const amount = reducedMotion ? 0 : Math.sqrt(0.35);
|
||||
const breath = Math.sin(elapsed * 1.5);
|
||||
const phrase = 0.5 + 0.5 * Math.sin(speechTime * 1.1);
|
||||
const leftGesture = speaking * (0.45 + 0.55 * phrase);
|
||||
const rightGesture = speaking * (0.15 + 0.55 * (1 - phrase));
|
||||
// One small acknowledgment every few seconds, with stillness between nods.
|
||||
const nodPhase = listenTime % 5;
|
||||
const nod = nodPhase < 1.2 ? Math.sin((nodPhase / 1.2) * Math.PI) : 0;
|
||||
if (spine) {
|
||||
spine.rotation.x = amount * (0.012 * breath + 0.065 * listening);
|
||||
spine.rotation.y = amount * speaking * 0.045 * Math.sin(speechTime * 0.8);
|
||||
}
|
||||
if (head) {
|
||||
head.rotation.x =
|
||||
amount *
|
||||
(0.018 * Math.sin(elapsed * 0.7) +
|
||||
speaking * 0.085 * Math.sin(speechTime * 2.2) +
|
||||
listening * (0.08 + 0.12 * nod));
|
||||
head.rotation.y =
|
||||
amount *
|
||||
(0.025 * Math.sin(elapsed * 0.47) + speaking * 0.09 * Math.sin(speechTime * 0.9));
|
||||
head.rotation.z = amount * (0.02 * Math.sin(elapsed * 0.63) + listening * 0.18);
|
||||
}
|
||||
if (leftArm) leftArm.rotation.z = -1.22 + amount * (0.015 * breath + 0.22 * leftGesture);
|
||||
if (rightArm) rightArm.rotation.z = 1.22 - amount * (0.015 * breath + 0.22 * rightGesture);
|
||||
if (leftElbow) leftElbow.rotation.z = amount * 1.25 * leftGesture;
|
||||
if (rightElbow) rightElbow.rotation.z = -amount * 1.25 * rightGesture;
|
||||
if (current && bodyWeight > 0) {
|
||||
for (const pose of poses) {
|
||||
pose.q.copy(pose.node.quaternion);
|
||||
pose.p.copy(pose.node.position);
|
||||
}
|
||||
mixer.update(dt);
|
||||
fadeTime += dt;
|
||||
if (outgoing && fadeTime >= 0.3) {
|
||||
if (outgoing !== current) outgoing.stop();
|
||||
outgoing = null;
|
||||
}
|
||||
for (const pose of poses) {
|
||||
pose.node.quaternion.slerpQuaternions(pose.q, pose.node.quaternion, bodyWeight);
|
||||
pose.node.position.lerpVectors(pose.p, pose.node.position, bodyWeight);
|
||||
}
|
||||
} else if (current) {
|
||||
mixer.stopAllAction();
|
||||
current = outgoing = null;
|
||||
}
|
||||
// Only assistant PCM drives the mouth. Silence and interruptions close it immediately.
|
||||
const desiredMouth =
|
||||
input.active && input.speaking ? Math.min(1, Math.max(0, input.level) * 7) * 0.7 : 0;
|
||||
mouthWeight =
|
||||
desiredMouth === 0
|
||||
? 0
|
||||
: mouthWeight + (desiredMouth - mouthWeight) * (1 - Math.exp(-dt * 24));
|
||||
if (mouth) expressions!.setValue('aa', mouthWeight);
|
||||
if (elapsed >= nextBlink) {
|
||||
blinkStart = elapsed;
|
||||
nextBlink = elapsed + 3.2 + Math.random() * 2.8;
|
||||
}
|
||||
const blinkPhase = (elapsed - blinkStart) / 0.17;
|
||||
const blinkWeight = blinkPhase >= 0 && blinkPhase <= 1 ? Math.sin(blinkPhase * Math.PI) : 0;
|
||||
if (blink) expressions!.setValue('blink', blinkWeight);
|
||||
else if (separateBlink) {
|
||||
expressions!.setValue('blinkLeft', blinkWeight);
|
||||
expressions!.setValue('blinkRight', blinkWeight);
|
||||
}
|
||||
if (vrm.lookAt) vrm.lookAt.autoUpdate = true;
|
||||
target.position.set(
|
||||
camera.position.x + amount * 0.1 * Math.sin(elapsed * 0.4),
|
||||
headPosition.y,
|
||||
camera.position.z
|
||||
);
|
||||
vrm.update(reducedMotion ? 0 : dt);
|
||||
vrm.scene.updateMatrixWorld(true);
|
||||
const animatedBounds = bounds.clone();
|
||||
const bonePosition = new THREE.Vector3();
|
||||
for (const { node } of poses)
|
||||
animatedBounds.expandByPoint(node.getWorldPosition(bonePosition));
|
||||
animatedBounds.expandByScalar(bodyHeight * 0.06);
|
||||
const extent = animatedBounds.getSize(new THREE.Vector3());
|
||||
const animatedCenter = animatedBounds.getCenter(new THREE.Vector3());
|
||||
const fitHeight = Math.max(bodyHeight, extent.y + 2 * Math.abs(animatedCenter.y - center.y));
|
||||
const fitWidth = Math.max(size.x, extent.x + 2 * Math.abs(animatedCenter.x - center.x));
|
||||
const distance =
|
||||
Math.max(size.z, extent.z) / 2 +
|
||||
(Math.max(fitHeight, fitWidth / camera.aspect) * 1.2) / (2 * Math.tan(Math.PI / 12));
|
||||
// Widen before a limb reaches the edge; never zoom in/out on each gesture.
|
||||
cameraDistance = Math.max(distance, cameraDistance);
|
||||
camera.position.set(center.x, center.y, center.z + cameraDistance);
|
||||
camera.lookAt(center);
|
||||
renderer.render(scene, camera);
|
||||
},
|
||||
dispose() {
|
||||
mixer.stopAllAction();
|
||||
mixer.uncacheRoot(vrm.scene);
|
||||
clips.clear();
|
||||
VRMUtils.deepDispose(vrm.scene);
|
||||
renderer.dispose();
|
||||
renderer.forceContextLoss();
|
||||
}
|
||||
};
|
||||
}
|
||||
112
src/lib/utils/voice-avatar.ts
Normal file
112
src/lib/utils/voice-avatar.ts
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
export const AVATAR_MAX_BYTES = 25 * 1024 * 1024;
|
||||
|
||||
export type AvatarState = 'idle' | 'listening' | 'speaking';
|
||||
export type AvatarAnimation = { file_id: string };
|
||||
export type AvatarGesture = AvatarAnimation & { name: string; description: string };
|
||||
export type AnimationFiles = Record<string, File>;
|
||||
export function avatarAssetIds(config?: VoiceAvatarConfig | null): string[] {
|
||||
return config
|
||||
? [
|
||||
config.file_id,
|
||||
...Object.values(config.states ?? {}).map((a) => a.file_id),
|
||||
...(config.gestures ?? []).map((a) => a.file_id)
|
||||
].filter(Boolean)
|
||||
: [];
|
||||
}
|
||||
|
||||
export type VoiceAvatarConfig = {
|
||||
file_id: string;
|
||||
states?: Partial<Record<AvatarState, AvatarAnimation>>;
|
||||
gestures?: AvatarGesture[];
|
||||
};
|
||||
|
||||
// Validate before GLTFLoader can fetch resources or allocate GPU buffers.
|
||||
export function readAvatar(data: ArrayBuffer) {
|
||||
if (data.byteLength > AVATAR_MAX_BYTES) throw new Error('Avatar must be at most 25 MiB.');
|
||||
if (data.byteLength < 20) throw new Error('Upload a VRM 0.x or VRM 1.0 file.');
|
||||
const view = new DataView(data);
|
||||
if (
|
||||
view.getUint32(0, true) !== 0x46546c67 ||
|
||||
view.getUint32(4, true) !== 2 ||
|
||||
view.getUint32(8, true) !== data.byteLength ||
|
||||
view.getUint32(16, true) !== 0x4e4f534a
|
||||
) {
|
||||
throw new Error('Upload a binary VRM file.');
|
||||
}
|
||||
const length = view.getUint32(12, true);
|
||||
if (length > 2 * 1024 * 1024 || 20 + length + 8 > data.byteLength)
|
||||
throw new Error('Invalid avatar container.');
|
||||
const json = JSON.parse(new TextDecoder().decode(new Uint8Array(data, 20, length)));
|
||||
const binaryStart = 28 + length;
|
||||
if (
|
||||
view.getUint32(24 + length, true) !== 0x004e4942 ||
|
||||
binaryStart + view.getUint32(20 + length, true) !== data.byteLength
|
||||
)
|
||||
throw new Error('Invalid avatar binary data.');
|
||||
const vrm = json.extensions?.VRMC_vrm ?? json.extensions?.VRM;
|
||||
if (!vrm?.humanoid?.humanBones)
|
||||
throw new Error('This model needs a VRM humanoid rig. A plain GLB is not enough.');
|
||||
const bones = Array.isArray(vrm.humanoid.humanBones)
|
||||
? Object.fromEntries(vrm.humanoid.humanBones.map((bone: any) => [bone.bone, bone]))
|
||||
: vrm.humanoid.humanBones;
|
||||
for (const name of [
|
||||
'hips',
|
||||
'spine',
|
||||
'head',
|
||||
'leftUpperArm',
|
||||
'rightUpperArm',
|
||||
'leftLowerArm',
|
||||
'rightLowerArm'
|
||||
]) {
|
||||
if (!Number.isInteger(bones[name]?.node) || !json.nodes?.[bones[name].node])
|
||||
throw new Error(`Avatar is missing its ${name} bone.`);
|
||||
}
|
||||
if (
|
||||
json.buffers?.length !== 1 ||
|
||||
json.buffers[0].uri ||
|
||||
json.buffers[0].byteLength > data.byteLength - binaryStart ||
|
||||
(json.images ?? []).some((image: any) => image.uri || !Number.isInteger(image.bufferView))
|
||||
) {
|
||||
throw new Error(
|
||||
'Embed all textures and buffers in the VRM file. External resources are not supported.'
|
||||
);
|
||||
}
|
||||
const vertexCount = (json.accessors ?? []).reduce(
|
||||
(sum: number, a: any) => sum + (a.count ?? 0),
|
||||
0
|
||||
);
|
||||
if (
|
||||
(json.nodes?.length ?? 0) > 512 ||
|
||||
(json.images?.length ?? 0) > 32 ||
|
||||
vertexCount > 5_000_000
|
||||
) {
|
||||
throw new Error('This avatar is too complex for a voice call. Use a lighter export.');
|
||||
}
|
||||
return { json, binaryStart };
|
||||
}
|
||||
|
||||
export async function validateAvatar(data: ArrayBuffer) {
|
||||
const { json, binaryStart } = readAvatar(data);
|
||||
let pixels = 0;
|
||||
for (const image of json.images ?? []) {
|
||||
const buffer = json.bufferViews?.[image.bufferView];
|
||||
const start = binaryStart + (buffer?.byteOffset ?? 0);
|
||||
if (
|
||||
!buffer ||
|
||||
(buffer.buffer ?? 0) !== 0 ||
|
||||
!Number.isInteger(buffer.byteLength) ||
|
||||
start < binaryStart ||
|
||||
start + buffer.byteLength > data.byteLength ||
|
||||
!['image/png', 'image/jpeg', 'image/webp'].includes(image.mimeType)
|
||||
)
|
||||
throw new Error('Invalid embedded avatar texture.');
|
||||
const bitmap = await createImageBitmap(
|
||||
new Blob([data.slice(start, start + buffer.byteLength)], { type: image.mimeType })
|
||||
);
|
||||
const tooLarge = bitmap.width > 4096 || bitmap.height > 4096;
|
||||
pixels += bitmap.width * bitmap.height;
|
||||
bitmap.close();
|
||||
if (tooLarge || pixels > 32 * 1024 * 1024)
|
||||
throw new Error('Avatar textures are too large. Export at 2048px or below.');
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue