From 3bd0cdd0f3cc075420540462f2bf246c73236495 Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Mon, 2 Nov 2026 01:16:43 +0900 Subject: [PATCH] refac --- backend/open_webui/routers/terminals.py | 7 +- package-lock.json | 7 + package.json | 1 + src/lib/apis/terminal/index.ts | 2 + src/lib/components/chat/ChatTabs.svelte | 83 +++++++-- .../components/chat/ChatTabs/Browser.svelte | 1 + .../chat/ChatTabs/ChromeBrowser.svelte | 107 ++++++++++- .../components/chat/ChatTabs/Desktop.svelte | 166 ++++++++++++++++++ src/lib/components/chat/ChatTabs/tabs.ts | 1 + src/lib/types/novnc.d.ts | 12 ++ vite.config.ts | 8 + 11 files changed, 369 insertions(+), 26 deletions(-) create mode 100644 src/lib/components/chat/ChatTabs/Desktop.svelte create mode 100644 src/lib/types/novnc.d.ts diff --git a/backend/open_webui/routers/terminals.py b/backend/open_webui/routers/terminals.py index a8dfc2f3e7..e4af6e09a8 100644 --- a/backend/open_webui/routers/terminals.py +++ b/backend/open_webui/routers/terminals.py @@ -216,7 +216,7 @@ async def browser_ws(ws: WebSocket, server_id: str, sid: str, ticket: str, kind: connections = await Config.get('terminal_server.connections', []) or [] connection = next((c for c in connections if c.get('id') == server_id), None) if ( - kind not in ('ws', 'stream') + kind not in ('ws', 'stream', 'vnc') or not ENABLE_TOOL_SERVERS or not connection or not connection.get('enabled', True) @@ -559,6 +559,11 @@ async def ws_events(ws: WebSocket, server_id: str): await _proxy_terminal_websocket(ws, server_id) +@router.websocket('/{server_id}/desktop/stream') +async def desktop_ws(ws: WebSocket, server_id: str): + await _proxy_terminal_websocket(ws, server_id, stream_path='desktop/stream') + + @router.websocket('/{server_id}/execute') async def ws_processes(ws: WebSocket, server_id: str): await _proxy_terminal_websocket(ws, server_id, stream_path='execute') diff --git a/package-lock.json b/package-lock.json index eee8df45ea..4968150126 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "@joplin/turndown-plugin-gfm": "^1.0.62", "@mediapipe/tasks-vision": "^0.10.17", "@modelcontextprotocol/ext-apps": "2.0.4", + "@novnc/novnc": "^1.7.0", "@pixiv/three-vrm": "3.5.5", "@pixiv/three-vrm-animation": "3.5.5", "@pyscript/core": "^0.4.32", @@ -2656,6 +2657,12 @@ "node": ">= 8" } }, + "node_modules/@novnc/novnc": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@novnc/novnc/-/novnc-1.7.0.tgz", + "integrity": "sha512-ucEJOx4T2avIRCleodk7YobZj5O2Ga2AeLfQ69A/yjG9HHba2+PDgwSkN3FttrmG+70ZGx21sElNFouK13RzyA==", + "license": "MPL-2.0" + }, "node_modules/@parcel/watcher": { "version": "2.5.6", "resolved": "https://registry.npmjs.org/@parcel/watcher/-/watcher-2.5.6.tgz", diff --git a/package.json b/package.json index 73d0109fbb..5c0455c5cd 100644 --- a/package.json +++ b/package.json @@ -71,6 +71,7 @@ "@joplin/turndown-plugin-gfm": "^1.0.62", "@mediapipe/tasks-vision": "^0.10.17", "@modelcontextprotocol/ext-apps": "2.0.4", + "@novnc/novnc": "^1.7.0", "@pixiv/three-vrm": "3.5.5", "@pixiv/three-vrm-animation": "3.5.5", "@pyscript/core": "^0.4.32", diff --git a/src/lib/apis/terminal/index.ts b/src/lib/apis/terminal/index.ts index de89648054..9d9d7cbe3f 100644 --- a/src/lib/apis/terminal/index.ts +++ b/src/lib/apis/terminal/index.ts @@ -81,6 +81,8 @@ export type ListeningPort = { export type TerminalFeatures = { terminal?: boolean; + browser?: boolean; + desktop?: boolean; git?: boolean; events?: boolean; }; diff --git a/src/lib/components/chat/ChatTabs.svelte b/src/lib/components/chat/ChatTabs.svelte index f393a83169..413693349f 100644 --- a/src/lib/components/chat/ChatTabs.svelte +++ b/src/lib/components/chat/ChatTabs.svelte @@ -53,6 +53,7 @@ import { fileIconName } from './FileNav/fileIcon'; import UntitledFile from './ChatTabs/UntitledFile.svelte'; import Browser from './ChatTabs/Browser.svelte'; + import Desktop from './ChatTabs/Desktop.svelte'; import XTerminal from './XTerminal.svelte'; import CallPanel from './MessageInput/CallPanel.svelte'; import Drawer from '../common/Drawer.svelte'; @@ -109,14 +110,16 @@ let dirtyFiles: Record = {}; let fileContexts: Record = {}; let panelTarget: HTMLElement | null = null; - let initialized = false; let tabsChatId = chatId; + let hasOpenedTabs = false; let terminalSequence = 0; let commentTarget: { comment_id?: string } = {}; let closing = new Set(); let navigationApproved = false; let selectedConnection: TerminalConnection | null = null; let terminalEnabled = false; + let desktopEnabled = false; + let browserEnabled = false; let featureVersion = 0; let releasePanel: (target: HTMLElement) => void = () => {}; const preservePanel = (target: HTMLElement) => ({ destroy: () => releasePanel(target) }); @@ -155,6 +158,7 @@ file: 'page', terminal: 'terminal', browser: 'browser', + desktop: 'monitor', artifacts: 'code', preview: 'browser', voice: 'microphone', @@ -187,7 +191,7 @@ showTabs.set(true); }; const openPanel = (tab: PanelTab) => { - initialized = true; + hasOpenedTabs = true; tabs = openTab(tabs, tab); showTabs.set(true); }; @@ -250,12 +254,24 @@ connection: { ...selectedConnection } }); }; + const openDesktop = () => { + if (!desktopEnabled || !selectedConnection) return; + openPanel({ + id: JSON.stringify(['desktop', selectedConnection.selector]), + kind: 'desktop', + label: $i18n.t('Desktop'), + terminalId: selectedConnection.selector, + connection: { ...selectedConnection } + }); + }; const openBrowser = ( port?: number, selector = $selectedTerminalId, baseUrl?: string, initialUrl?: string ) => { + if (!selector || !terminalAccessible(selector)) return; + if (selector === $selectedTerminalId && !browserEnabled) return; const url = initialUrl ?? (port ? `http://localhost:${port}/` : undefined); const existing = url && @@ -274,7 +290,7 @@ localStorage.token ?? '' ) : null; - if (selector && !terminalAccessible(selector)) return; + if (!connection) return; const terminal = ($terminalServers ?? []).find((item) => item.id === selector); const proxyBase = baseUrl ?? @@ -467,8 +483,7 @@ for (const tab of tabs.tabs) { const available = tab.terminalId ? !!$selectedTerminalId && terminalAccessible(tab.terminalId) - : tab.kind === 'browser' || - availableViews.some((view) => view.kind === tab.kind && view.available); + : availableViews.some((view) => view.kind === tab.kind && view.available); if (!available) { tabs = removeTab(tabs, tab.id); delete dirtyFiles[tab.id]; @@ -489,6 +504,8 @@ const checkTerminal = async () => { const version = ++featureVersion; terminalEnabled = false; + desktopEnabled = false; + browserEnabled = false; selectedConnection = terminalFilesAvailable && mounted ? resolveTerminalConnection( @@ -500,12 +517,14 @@ : null; if (!selectedConnection) return; try { - const result = await terminalRequest<{ features?: { terminal?: boolean } }>( - selectedConnection, - chatId, - '/api/config' - ); - if (version === featureVersion) terminalEnabled = result.features?.terminal !== false; + const result = await terminalRequest<{ + features?: { terminal?: boolean; desktop?: boolean; browser?: boolean }; + }>(selectedConnection, chatId, '/api/config'); + if (version === featureVersion) { + terminalEnabled = result.features?.terminal !== false; + desktopEnabled = result.features?.desktop === true; + browserEnabled = result.features?.browser === true; + } } catch { /* Files remain available when interactive terminals are disabled. */ } @@ -521,18 +540,29 @@ } $: if (chatId !== tabsChatId) { - if (tabsChatId !== null || !chatId) { + if (tabsChatId || !chatId) { tabs = emptyTabs(); + hasOpenedTabs = false; dirtyFiles = {}; fileContexts = {}; - initialized = false; commentTarget = {}; } tabsChatId = chatId; } - $: if ($showTabs === true && !initialized && showFilesTab) { - initialized = true; - openFiles(); + $: if ($showTabs === true && !tabs.tabs.length) { + const options = [ + ...availableViews.filter((view) => view.available).map((view) => () => { + if (view.kind === 'files') openFiles(); + else openFeature(view.kind as 'artifacts' | 'preview' | 'voice' | 'comments'); + }), + ...(terminalFilesAvailable ? [newFile] : []), + ...(terminalEnabled ? [newTerminal] : []), + ...(browserEnabled ? [openBrowser] : []), + ...(desktopEnabled ? [openDesktop] : []) + ]; + if (terminalFilesAvailable || (!hasOpenedTabs && showFilesTab)) openFiles(); + else if (options.length === 1) options[0](); + hasOpenedTabs = true; } $: if ($showTabs === 'files' && showFilesTab) openFiles(); $: if ( @@ -760,15 +790,23 @@ >
+ {#if showFilesTab && !tabs.tabs.some((tab) => tab.id === ($selectedTerminalId ? `files:${$selectedTerminalId}` : 'files'))} + + {/if} {#if terminalFilesAvailable}{/if} {#if terminalEnabled}{/if} - + {#if browserEnabled}{/if} + {#if desktopEnabled}{/if} {#each availableViews.filter((view) => view.kind !== 'files' && view.kind !== 'voice' && view.available) as view}
+ + diff --git a/src/lib/components/chat/ChatTabs/Desktop.svelte b/src/lib/components/chat/ChatTabs/Desktop.svelte new file mode 100644 index 0000000000..87b3960ca2 --- /dev/null +++ b/src/lib/components/chat/ChatTabs/Desktop.svelte @@ -0,0 +1,166 @@ + + + (fullscreen = document.fullscreenElement === container)} +/> + +
+ +
{ + event.preventDefault(); + paste(event.clipboardData?.getData('text/plain') ?? ''); + }} + on:keydown|capture={(event) => { + if ( + (event.ctrlKey || event.metaKey) && + event.key.toLowerCase() === 'v' && + navigator.clipboard?.readText + ) { + event.preventDefault(); + event.stopPropagation(); + void navigator.clipboard + .readText() + .then((text) => paste(text, event.shiftKey)) + .catch(() => {}); + } + }} + >
+ {#if status || error} +
+ {error || status} + {#if error}{/if} +
+ {/if} + {#if fullscreenEnabled} + + {/if} + {#if overlay}
{/if} +
diff --git a/src/lib/components/chat/ChatTabs/tabs.ts b/src/lib/components/chat/ChatTabs/tabs.ts index 0184fdbf27..afee1dad67 100644 --- a/src/lib/components/chat/ChatTabs/tabs.ts +++ b/src/lib/components/chat/ChatTabs/tabs.ts @@ -6,6 +6,7 @@ export type PanelTabKind = | 'file' | 'terminal' | 'browser' + | 'desktop' | 'artifacts' | 'preview' | 'voice' diff --git a/src/lib/types/novnc.d.ts b/src/lib/types/novnc.d.ts new file mode 100644 index 0000000000..e51fd2c1e9 --- /dev/null +++ b/src/lib/types/novnc.d.ts @@ -0,0 +1,12 @@ +declare module '@novnc/novnc' { + export default class RFB extends EventTarget { + constructor(target: HTMLElement, channel: WebSocket); + scaleViewport: boolean; + resizeSession: boolean; + qualityLevel: number; + viewOnly: boolean; + disconnect(): void; + clipboardPasteFrom(text: string): void; + sendKey(keysym: number, code: string, down?: boolean): void; + } +} diff --git a/vite.config.ts b/vite.config.ts index 5216d38835..a2cbcd8d7b 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -19,6 +19,10 @@ export default defineConfig({ src: 'src/lib/encryption/vendor/THIRD_PARTY_LICENSES.txt', dest: 'licenses/encryption' }, + { + src: ['node_modules/@novnc/novnc/docs/LICENSE*', 'node_modules/@novnc/novnc/AUTHORS'], + dest: 'licenses/novnc' + }, { src: 'node_modules/onnxruntime-web/dist/*.jsep.*', @@ -33,8 +37,12 @@ export default defineConfig({ APP_BUILD_CHANNEL: JSON.stringify(buildInfo.channel) }, build: { + target: 'es2022', sourcemap: true }, + optimizeDeps: { + esbuildOptions: { target: 'es2022' } + }, server: { proxy: { '/api': {