This commit is contained in:
Timothy Jaeryang Baek 2026-11-02 01:47:06 +09:00
parent 72091df6d4
commit 03401a6e33
4 changed files with 208 additions and 29 deletions

View file

@ -1,5 +1,6 @@
<script lang="ts">
import { onMount } from 'svelte';
import { onMount, tick } from 'svelte';
import FrameCanvas from './FrameCanvas.svelte';
import type RFB from '@novnc/novnc';
export let streamUrl: string;
export let active = true;
@ -15,6 +16,7 @@
export let onStatus: (status: string) => void = () => {};
let surface: HTMLDivElement;
let frame = '';
let frames: FrameCanvas;
let transport = '';
let vncSurface: HTMLDivElement;
let viewer: RFB | null = null;
@ -109,7 +111,7 @@
(e.altKey ? 1 : 0) | (e.ctrlKey ? 2 : 0) | (e.metaKey ? 4 : 0) | (e.shiftKey ? 8 : 0);
const mouse = (event: MouseEvent, type: string) => {
if (transport === 'vnc') return;
const image = surface.querySelector(transport === 'vnc' ? 'canvas' : 'img');
const image = surface.querySelector(transport === 'jpeg' ? 'img' : 'canvas');
if (!image) return;
const box = image.getBoundingClientRect();
event.preventDefault();
@ -165,8 +167,11 @@
const url = new URL(streamUrl, window.location.origin);
url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
socket = new WebSocket(url);
const channel = socket;
socket.binaryType = 'arraybuffer';
socket.onopen = () => {
onError('');
command('frames');
resize();
command('quality', { quality });
command('visibility', { visible: active });
@ -175,8 +180,17 @@
pendingNavigation = null;
}
};
socket.onmessage = (event) => {
socket.onmessage = async (event) => {
try {
if (event.data instanceof ArrayBuffer) {
await tick();
if (disposed || socket !== channel) return;
await frames.draw(event.data);
if (disposed || socket !== channel) return;
command('frameAck');
onStatus('');
return;
}
const message = JSON.parse(event.data);
if (message.type === 'frame' || message.type === 'state') {
hasConnected = true;
@ -189,8 +203,8 @@
onState(message);
onError(message.error || '');
}
} catch {
/* Ignore malformed stream messages. */
} catch (error) {
onError(error instanceof Error ? error.message : String(error));
}
};
socket.onclose = (event) => {
@ -247,7 +261,7 @@
on:wheel|preventDefault={(event) => {
if (transport === 'vnc') return;
const box = surface
.querySelector(transport === 'vnc' ? 'canvas' : 'img')
.querySelector(transport === 'jpeg' ? 'img' : 'canvas')
?.getBoundingClientRect();
if (!box) return;
command('mouse', {
@ -262,6 +276,8 @@
>
{#if transport === 'vnc'}
<div bind:this={vncSurface} class="vnc h-full w-full"></div>
{:else if transport === 'frames'}
<FrameCanvas bind:this={frames} />
{:else if frame}<img
src={frame}
alt="Browser page"

View file

@ -1,5 +1,6 @@
<script lang="ts">
import { getContext, onMount } from 'svelte';
import { getContext, onMount, tick } from 'svelte';
import NativeDesktop from './NativeDesktop.svelte';
import { toast } from 'svelte-sonner';
import type RFB from '@novnc/novnc';
import Icon from '../FileNav/Icon.svelte';
@ -16,6 +17,11 @@
let fullscreen = false;
let fullscreenEnabled = false;
let surface: HTMLDivElement;
let native: NativeDesktop;
let transport = 'vnc';
const send = (message: Record<string, unknown>) => {
if (socket?.readyState === WebSocket.OPEN) socket.send(JSON.stringify(message));
};
let socket: WebSocket | null = null;
let viewer: RFB | null = null;
let error = '';
@ -58,30 +64,51 @@
error = '';
status = $i18n.t('Connecting…');
try {
const { default: RFB } = await import('@novnc/novnc');
await terminalRequest(connection, chatId, '/desktop');
const info = await terminalRequest(connection, chatId, '/desktop');
if (version !== generation) return;
socket = openTerminalSocket(connection, chatId, '/desktop/stream', (channel) => {
if (version !== generation) return;
viewer = new RFB(surface, channel);
const cursor = viewer._cursor;
const detachCursor = cursor.detach.bind(cursor);
cursor.detach = () => {
// noVNC removes this canvas from body, including on a server disconnect.
if (cursor._canvas.parentElement === container) document.body.appendChild(cursor._canvas);
detachCursor();
transport = info.transport ?? 'vnc';
await tick();
if (version !== generation) return;
if (transport === 'frames') {
socket = openTerminalSocket(connection, chatId, '/desktop/stream');
socket.onmessage = async (event) => {
if (version !== generation) return;
try {
await native.draw(event.data);
if (version !== generation) return;
send({ type: 'frameAck' });
status = '';
} catch (reason) {
if (version === generation)
error = reason instanceof Error ? reason.message : String(reason);
}
};
viewer.scaleViewport = true;
viewer.resizeSession = false;
viewer.qualityLevel = 6;
viewer.addEventListener('connect', () => {
status = '';
syncFullscreenCursor();
} else {
const { default: RFB } = await import('@novnc/novnc');
if (version !== generation) return;
socket = openTerminalSocket(connection, chatId, '/desktop/stream', (channel) => {
if (version !== generation) return;
viewer = new RFB(surface, channel);
const cursor = viewer._cursor;
const detachCursor = cursor.detach.bind(cursor);
cursor.detach = () => {
// noVNC removes this canvas from body, including on a server disconnect.
if (cursor._canvas.parentElement === container)
document.body.appendChild(cursor._canvas);
detachCursor();
};
viewer.scaleViewport = true;
viewer.resizeSession = false;
viewer.qualityLevel = 6;
viewer.addEventListener('connect', () => {
status = '';
syncFullscreenCursor();
});
viewer.addEventListener('securityfailure', () => {
error = $i18n.t('Desktop connection failed.');
});
});
viewer.addEventListener('securityfailure', () => {
error = $i18n.t('Desktop connection failed.');
});
});
}
socket.addEventListener('close', (event) => {
if (version !== generation) return;
viewer?.disconnect();
@ -105,6 +132,10 @@
}
};
const paste = (text: string, shift = false) => {
if (transport === 'frames') {
send({ type: 'paste', text });
return;
}
if (!viewer) return;
viewer.clipboardPasteFrom(text);
viewer.sendKey(0xffe3, 'ControlLeft', true);
@ -154,7 +185,9 @@
.catch(() => {});
}
}}
></div>
>
{#if transport === 'frames'}<NativeDesktop bind:this={native} {send} />{/if}
</div>
{#if status || error}
<div
class="absolute inset-0 flex flex-col items-center justify-center gap-2 bg-white/90 px-4 text-center text-sm text-gray-500 dark:bg-gray-900/90"

View file

@ -0,0 +1,26 @@
<script lang="ts">
let canvas: HTMLCanvasElement;
export const draw = async (packet: ArrayBuffer) => {
if (packet.byteLength < 13) throw new Error('Invalid screen update');
const header = new DataView(packet);
const [width, height, x, y, w, h] = Array.from({ length: 6 }, (_, i) =>
header.getUint16(i * 2)
);
if (!width || !height || width > 8192 || height > 8192 || x + w > width || y + h > height)
throw new Error('Invalid screen dimensions');
const image = await createImageBitmap(new Blob([packet.slice(12)], { type: 'image/jpeg' }));
try {
if (!canvas) return;
if (canvas.width !== width || canvas.height !== height) {
canvas.width = width;
canvas.height = height;
}
canvas.getContext('2d')?.drawImage(image, x, y, w, h);
} finally {
image.close();
}
};
</script>
<canvas bind:this={canvas} class="max-h-full max-w-full select-none" style="object-fit:contain"
></canvas>

View file

@ -0,0 +1,104 @@
<script lang="ts">
import { onDestroy } from 'svelte';
import FrameCanvas from './FrameCanvas.svelte';
export let send: (message: Record<string, unknown>) => void;
let surface: HTMLDivElement;
let frames: FrameCanvas;
let pending: Record<string, unknown> | null = null;
let frame = 0;
export const draw = (packet: ArrayBuffer) => frames.draw(packet);
const flush = () => {
cancelAnimationFrame(frame);
if (pending) send(pending);
pending = null;
frame = 0;
};
const pointer = (event: PointerEvent) => {
const canvas = surface.querySelector('canvas');
if (!canvas) return;
const box = canvas.getBoundingClientRect();
const x = Math.max(
0,
Math.min(
canvas.width - 1,
Math.floor(((event.clientX - box.left) * canvas.width) / box.width)
)
);
const y = Math.max(
0,
Math.min(
canvas.height - 1,
Math.floor(((event.clientY - box.top) * canvas.height) / box.height)
)
);
if (event.type === 'pointerdown') {
surface.focus();
surface.setPointerCapture(event.pointerId);
}
pending = { type: 'mouse', x, y, buttons: event.buttons };
if (event.type !== 'pointermove') flush();
else if (!frame) frame = requestAnimationFrame(flush);
};
const release = () => {
cancelAnimationFrame(frame);
frame = 0;
pending = null;
send({ type: 'release' });
};
const key = (event: KeyboardEvent, down: boolean) => {
if ((event.ctrlKey || event.metaKey) && event.key.toLowerCase() === 'v') return;
if (event.isComposing) return;
event.preventDefault();
flush();
send({ type: 'key', code: event.code, down });
};
onDestroy(release);
</script>
<svelte:window on:blur={release} />
<div
bind:this={surface}
class="flex h-full w-full touch-none items-start justify-center overflow-hidden outline-none"
role="application"
aria-label="Desktop screen"
tabindex="0"
on:pointerdown|preventDefault={pointer}
on:pointermove={pointer}
on:pointerup={pointer}
on:pointercancel={release}
on:blur={release}
on:keydown={(event) => key(event, true)}
on:keyup={(event) => key(event, false)}
on:contextmenu|preventDefault
on:paste|preventDefault|stopPropagation={(event) =>
send({ type: 'paste', text: event.clipboardData?.getData('text/plain') ?? '' })}
on:wheel|preventDefault={(event) => {
const canvas = surface.querySelector('canvas');
if (!canvas) return;
const box = canvas.getBoundingClientRect();
flush();
send({
type: 'mouse',
x: Math.max(
0,
Math.min(
canvas.width - 1,
Math.floor(((event.clientX - box.left) * canvas.width) / box.width)
)
),
y: Math.max(
0,
Math.min(
canvas.height - 1,
Math.floor(((event.clientY - box.top) * canvas.height) / box.height)
)
),
buttons: event.buttons,
deltaX: Math.sign(event.deltaX) * Math.ceil(Math.abs(event.deltaX) / 100),
deltaY: Math.sign(event.deltaY) * Math.ceil(Math.abs(event.deltaY) / 100)
});
}}
>
<FrameCanvas bind:this={frames} />
</div>