From b51e68fbb3e641e7f9027f5164fcf6b0e111cf27 Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Mon, 2 Nov 2026 00:31:46 +0900 Subject: [PATCH] refac --- backend/open_webui/main.py | 5 +- backend/open_webui/routers/terminals.py | 131 +++++++- backend/open_webui/utils/asgi_middleware.py | 10 + backend/open_webui/utils/browser.py | 42 +++ src/lib/apis/terminal/browser.ts | 56 ++++ src/lib/apis/terminal/index.ts | 11 +- src/lib/components/chat/ChatTabs.svelte | 10 +- .../components/chat/ChatTabs/Browser.svelte | 317 +++++++++++++++--- .../chat/ChatTabs/ChromeBrowser.svelte | 186 ++++++++++ 9 files changed, 719 insertions(+), 49 deletions(-) create mode 100644 backend/open_webui/utils/browser.py create mode 100644 src/lib/apis/terminal/browser.ts create mode 100644 src/lib/components/chat/ChatTabs/ChromeBrowser.svelte diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index 04c5bfbeae..e2b762527e 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -22,7 +22,6 @@ from fastapi import ( applications, status, ) -from fastapi.middleware.cors import CORSMiddleware from fastapi.openapi.docs import get_swagger_ui_html from fastapi.responses import FileResponse, JSONResponse from fastapi.staticfiles import StaticFiles @@ -221,7 +220,7 @@ from open_webui.utils import logger from open_webui.utils.access_control import has_permission from open_webui.utils.access_control.folders import has_folder_write_access from open_webui.utils.actions import chat_action as chat_action_handler -from open_webui.utils.asgi_middleware import AppHTTPMiddleware +from open_webui.utils.asgi_middleware import AppHTTPMiddleware, BrowserCORSMiddleware from open_webui.utils.audit import AuditLevel, AuditLoggingMiddleware from open_webui.utils.auth import ( create_admin_user, @@ -874,7 +873,7 @@ app.add_middleware(AppHTTPMiddleware) app.add_middleware( - CORSMiddleware, + BrowserCORSMiddleware, allow_origins=CORS_ALLOW_ORIGIN, allow_credentials=True, allow_methods=['*'], diff --git a/backend/open_webui/routers/terminals.py b/backend/open_webui/routers/terminals.py index 0018fcf5ae..a8dfc2f3e7 100644 --- a/backend/open_webui/routers/terminals.py +++ b/backend/open_webui/routers/terminals.py @@ -9,12 +9,13 @@ import asyncio import json import logging import re +from contextlib import suppress from html import escape import posixpath -from urllib.parse import unquote, urljoin, urlsplit +from urllib.parse import quote, unquote, urljoin, urlsplit import aiohttp -from fastapi import APIRouter, Depends, Request, Response, WebSocket +from fastapi import APIRouter, Depends, HTTPException, Request, Response, WebSocket from fastapi.responses import JSONResponse, StreamingResponse from open_webui.config import TERMINAL_PROXY_HEADERS from open_webui.env import AIOHTTP_CLIENT_SESSION_SSL, ENABLE_TOOL_SERVERS @@ -23,6 +24,7 @@ from open_webui.models.config import Config from open_webui.models.groups import Groups from open_webui.utils.access_control import has_connection_access from open_webui.utils.auth import get_verified_user, get_verified_user_by_token +from open_webui.utils.browser import browser_access, browser_ticket from open_webui.utils.headers import bearer_auth_header, normalize_bearer_token from open_webui.utils.json_codec import JSONCodec from open_webui.utils.terminals import ( @@ -43,7 +45,7 @@ log = logging.getLogger(__name__) router = APIRouter() -STREAMING_CONTENT_TYPES = ('application/octet-stream', 'image/', 'application/pdf') +STREAMING_CONTENT_TYPES = ('application/octet-stream', 'image/', 'application/pdf', 'text/event-stream') ADMIN_API_PATHS = ('api/v1/policies', 'api/v1/status', 'api/v1/terminals') # Drop the upstream's server and date: uvicorn adds its own and forwarding both duplicates them. STRIPPED_RESPONSE_HEADERS = frozenset( @@ -159,6 +161,112 @@ def _sanitize_proxy_path(path: str) -> str | None: return cleaned +@router.api_route( + '/{server_id}/browser/view/{sid}/{ticket}/{path:path}', + methods=['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS'], +) +async def browser_http(request: Request, server_id: str, sid: str, ticket: str, path: str): + user, data = await browser_access(server_id, sid, ticket) + request.state.browser_auth = data['relay'] + request.scope['path_params']['chat_id'] = data.get('chat', '') + return await proxy_terminal(server_id, f'browser/view/{sid}/{data["capability"]}/{path}', request, user) + + +async def relay_from_browser(ws, upstream): + while True: + msg = await ws.receive() + if msg['type'] == 'websocket.disconnect': + return + if msg.get('bytes') is not None: + await upstream.send_bytes(msg['bytes']) + elif msg.get('text') is not None: + await upstream.send_str(msg['text']) + + +async def relay_to_browser(ws, upstream): + async for msg in upstream: + if msg.type == aiohttp.WSMsgType.TEXT: + await ws.send_text(msg.data) + elif msg.type == aiohttp.WSMsgType.BINARY: + await ws.send_bytes(msg.data) + else: + return + + +async def watch_access(server_id, sid, ticket): + while True: + await asyncio.sleep(10) + current_user, _ = await browser_access(server_id, sid, ticket) + latest = await Config.get('terminal_server.connections', []) or [] + current = next((c for c in latest if c.get('id') == server_id), None) + if ( + not current + or not current.get('enabled', True) + or not ENABLE_TOOL_SERVERS + or not await has_connection_access(current_user, current) + or not terminal_context_available(current, 'chat') + ): + return + + +@router.websocket('/{server_id}/browser/view/{sid}/{ticket}/{kind}') +async def browser_ws(ws: WebSocket, server_id: str, sid: str, ticket: str, kind: str): + try: + user, data = await browser_access(server_id, sid, ticket) + connections = await Config.get('terminal_server.connections', []) or [] + connection = next((c for c in connections if c.get('id') == server_id), None) + if ( + kind not in ('ws', 'stream') + or not ENABLE_TOOL_SERVERS + or not connection + or not connection.get('enabled', True) + or not await has_connection_access(user, connection) + or not terminal_context_available(connection, 'chat') + ): + raise HTTPException(403, 'Browser access denied') + chat = data.get('chat', '') + context = terminal_context_id(connection, {'chat_id': chat}, 'chat') + if terminal_context_config(connection, 'chat').get('context_id') == 'chat_id' and not context: + raise HTTPException(403, 'Saved chat required') + except HTTPException: + await ws.close(code=4001) + return + await ws.accept() + base = get_terminal_server_url(connection).replace('https://', 'wss://').replace('http://', 'ws://') + url = f'{base}/browser/view/{sid}/{quote(data["capability"], safe="")}/{kind}' + headers = {'X-User-Id': user.id, 'X-Session-Id': chat} + if context: + headers[TERMINAL_CONTEXT_HEADER] = context + if data['relay'].get('authorization'): + headers['Authorization'] = data['relay']['authorization'] + tasks = [] + try: + async with aiohttp.ClientSession(cookies=data['relay'].get('cookies', {}), trust_env=True) as client: + async with client.ws_connect( + url, + headers=headers, + ssl=AIOHTTP_CLIENT_SESSION_SSL, + max_msg_size=16 * 1024 * 1024, + ) as upstream: + + tasks = [ + asyncio.create_task(relay_from_browser(ws, upstream)), + asyncio.create_task(relay_to_browser(ws, upstream)), + asyncio.create_task(watch_access(server_id, sid, ticket)), + ] + await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) + except aiohttp.WSServerHandshakeError as exc: + await ws.close(code=4004 if exc.status in (401, 403, 404) else 1011) + except Exception as exc: + log.debug('Browser relay disconnected: %s', type(exc).__name__) + finally: + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + with suppress(Exception): + await ws.close(code=1011) + + @router.get('/') async def list_terminal_servers(request: Request, user=Depends(get_verified_user)): """Return terminal servers the authenticated user has access to.""" @@ -247,7 +355,12 @@ async def proxy_terminal( cookies = getattr(request, 'cookies', {}) if connection.get('forward_cookies', False) else {} auth_type = connection.get('auth_type', 'bearer') - if auth_type == 'bearer': + browser_auth = getattr(request.state, 'browser_auth', None) + if browser_auth is not None: + cookies = browser_auth.get('cookies', {}) + if browser_auth.get('authorization'): + headers['Authorization'] = browser_auth['authorization'] + elif auth_type == 'bearer': headers.update(bearer_auth_header(connection.get('key', ''))) elif auth_type == 'session': headers.update(bearer_auth_header(request.state.token.credentials)) @@ -266,6 +379,11 @@ async def proxy_terminal( headers.update(bearer_auth_header(oauth_token.get('access_token', ''))) # auth_type == "none": no Authorization header + if browser_auth is not None: + for name in ('accept', 'origin', 'access-control-request-method', 'access-control-request-headers', 'last-event-id', 'range'): + if value := request.headers.get(name): + headers[name] = value + content_type = request.headers.get('content-type') if content_type: headers['Content-Type'] = content_type @@ -336,6 +454,11 @@ async def proxy_terminal( # Buffer text/JSON responses response_body = await upstream_response.read() + if safe_path.startswith('browser/sessions') and upstream_response.status < 300 and request.method in ('GET', 'POST', 'PATCH'): + browser_session = json.loads(response_body) + if isinstance(browser_session, dict) and browser_session.get('id') and browser_session.get('token'): + browser_session['token'] = browser_ticket(server_id, user.id, session_id or '', browser_session, headers, cookies) + response_body = json.dumps(browser_session).encode() if port_prefix and 'text/html' in upstream_content_type: response_body = _port_browser_html(response_body, port_prefix, int(port_match[1])) status_code = upstream_response.status diff --git a/backend/open_webui/utils/asgi_middleware.py b/backend/open_webui/utils/asgi_middleware.py index 2d9f4e190e..58caa4cd60 100644 --- a/backend/open_webui/utils/asgi_middleware.py +++ b/backend/open_webui/utils/asgi_middleware.py @@ -42,12 +42,22 @@ from open_webui.internal.db import ScopedSession from open_webui.utils.auth import get_http_authorization_cred from open_webui.utils.security_headers import set_security_headers from starlette.datastructures import MutableHeaders +from starlette.middleware.cors import CORSMiddleware from starlette.requests import Request from starlette.types import ASGIApp, Message, Receive, Scope, Send log = logging.getLogger(__name__) +class BrowserCORSMiddleware(CORSMiddleware): + async def __call__(self, scope, receive, send): + # Sandboxed pages have an opaque origin; capability routes handle their own CORS. + if re.match(r'^/api/v1/terminals/[^/]+/browser/view/', scope.get('path', '')): + await self.app(scope, receive, send) + else: + await super().__call__(scope, receive, send) + + class AppHTTPMiddleware: """Open WebUI's pure-ASGI HTTP middleware. diff --git a/backend/open_webui/utils/browser.py b/backend/open_webui/utils/browser.py new file mode 100644 index 0000000000..2bc3ebf3d7 --- /dev/null +++ b/backend/open_webui/utils/browser.py @@ -0,0 +1,42 @@ +from fastapi import HTTPException +from open_webui.utils.auth import create_token, decode_token, get_verified_user_by_id + + +def browser_ticket(server_id, user_id, chat_id, session, headers=None, cookies=None): + from open_webui.utils.oauth import encrypt_data + + relay = encrypt_data( + { + 'authorization': (headers or {}).get('Authorization'), + 'cookies': cookies or {}, + } + ) + return create_token( + { + 'purpose': 'terminal-browser', + 'server': server_id, + 'user': user_id, + 'chat': chat_id, + 'session': session['id'], + 'capability': session['token'], + 'relay': relay, + }, + ) + + +async def browser_access(server_id, sid, ticket): + data = decode_token(ticket) + if ( + not data + or data.get('purpose') != 'terminal-browser' + or data.get('server') != server_id + or data.get('session') != sid + ): + raise HTTPException(401, 'Browser session expired') + user = await get_verified_user_by_id(data.get('user')) + if user is None: + raise HTTPException(401, 'Browser session expired') + from open_webui.utils.oauth import decrypt_data + + data['relay'] = decrypt_data(data['relay']) + return user, data diff --git a/src/lib/apis/terminal/browser.ts b/src/lib/apis/terminal/browser.ts new file mode 100644 index 0000000000..be831effa4 --- /dev/null +++ b/src/lib/apis/terminal/browser.ts @@ -0,0 +1,56 @@ +import { terminalRequest, type TerminalConnection } from './index'; + +export type BrowserSession = { + id: string; + token: string; + url: string; + title: string; + mode: 'proxy' | 'chrome'; + chrome_available: boolean; +}; + +export const createBrowserSession = ( + connection: TerminalConnection, + chatId: string | null, + url = '' +) => + terminalRequest(connection, chatId, '/browser/sessions', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ url }) + }); + +export const updateBrowserSession = ( + connection: TerminalConnection, + chatId: string | null, + id: string, + fields: Partial> +) => + terminalRequest( + connection, + chatId, + `/browser/sessions/${encodeURIComponent(id)}`, + { + method: 'PATCH', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(fields) + } + ); + +export const closeBrowserSession = ( + connection: TerminalConnection, + chatId: string | null, + id: string +) => + terminalRequest(connection, chatId, `/browser/sessions/${encodeURIComponent(id)}`, { + method: 'DELETE', + keepalive: true + }); + +export const browserViewUrl = (connection: TerminalConnection, session: BrowserSession) => + `${connection.baseUrl.replace(/\/$/, '')}/browser/view/${encodeURIComponent(session.id)}/${encodeURIComponent(session.token)}`; + +export function browserFrameUrl(prefix: string, value: string) { + const url = new URL(value); + return `${prefix}/frame/${url.protocol.slice(0, -1)}/${encodeURIComponent(url.host)}${url.pathname}${url.search}${url.hash}`; +} diff --git a/src/lib/apis/terminal/index.ts b/src/lib/apis/terminal/index.ts index b76cb35dfa..de89648054 100644 --- a/src/lib/apis/terminal/index.ts +++ b/src/lib/apis/terminal/index.ts @@ -183,7 +183,16 @@ export const terminalRequest = async ( ...options.headers } }); - if (!response.ok) throw new Error(`Terminal request failed (${response.status})`); + if (!response.ok) { + const body = await response.json().catch(() => null); + throw new Error( + typeof body?.detail === 'string' + ? body.detail + : typeof body?.error === 'string' + ? body.error + : `Terminal request failed (${response.status})` + ); + } return response.json(); }; diff --git a/src/lib/components/chat/ChatTabs.svelte b/src/lib/components/chat/ChatTabs.svelte index fc4bd5870e..f393a83169 100644 --- a/src/lib/components/chat/ChatTabs.svelte +++ b/src/lib/components/chat/ChatTabs.svelte @@ -250,8 +250,13 @@ connection: { ...selectedConnection } }); }; - const openBrowser = (port?: number, selector = $selectedTerminalId, baseUrl?: string) => { - const url = port ? `http://localhost:${port}/` : undefined; + const openBrowser = ( + port?: number, + selector = $selectedTerminalId, + baseUrl?: string, + initialUrl?: string + ) => { + const url = initialUrl ?? (port ? `http://localhost:${port}/` : undefined); const existing = url && tabs.tabs.find( @@ -825,6 +830,7 @@ {chatId} active={!!$showTabs && tabs.activeId === tab.id} initialUrl={tab.path ?? ''} + onOpenBrowser={(url) => openBrowser(undefined, tab.terminalId, tab.browserBaseUrl, url)} overlay={dragged} onNavigate={(path, label) => { tabs = { diff --git a/src/lib/components/chat/ChatTabs/Browser.svelte b/src/lib/components/chat/ChatTabs/Browser.svelte index 6251131ed0..e1ca453034 100644 --- a/src/lib/components/chat/ChatTabs/Browser.svelte +++ b/src/lib/components/chat/ChatTabs/Browser.svelte @@ -1,5 +1,17 @@ -
-
- {#each [{ name: 'Back', icon: 'chevron-left', disabled: !canGoBack, action: () => move(-1) }, { name: 'Forward', icon: 'chevron-right', disabled: !canGoForward, action: () => move(1) }, { name: 'Refresh', icon: 'refresh', disabled: !currentUrl, action: () => navigate(currentUrl, false) }] as button} +
+
+ {#each [{ name: 'Back', icon: 'chevron-left', disabled: !canGoBack, action: () => move(-1) }, { name: 'Forward', icon: 'chevron-right', disabled: !canGoForward, action: () => move(1) }, { name: 'Refresh', icon: 'refresh', disabled: !currentUrl, action: reload }] as button} {/each} -
navigate()}> + navigate()}> +
- + {#if session} + + +
+ + {#each session.chrome_available ? ['proxy', 'chrome'] : ['proxy'] as mode} + + {/each} + {#if session.mode === 'chrome'} +
+ {#each [{ value: 'low', label: 'Low' }, { value: 'balanced', label: 'Balanced' }, { value: 'crisp', label: 'Crisp' }] as item} + + {/each} +
+ + {/if} + +
+ + {/if} + {#if onClose}{/if}
- {#if error}{/if} + {#if error} + + {:else if connectionStatus} +

+ {$i18n.t(connectionStatus)} +

+ {/if}
{#if overlay}
{/if} - {#if frameUrl} + {#if session?.mode === 'chrome'} +
+ (error = message)} + onStatus={(message) => (connectionStatus = message)} + onState={(state) => { + if (!state.url) return; + if (currentUrl !== state.url || document.activeElement !== addressInput) + urlInput = state.url; + currentUrl = state.url; + chromeBack = state.canGoBack; + chromeForward = state.canGoForward; + loading = false; + onNavigate(state.url, state.title.trim() || new URL(state.url).host); + }} + /> +
+ {:else if frameUrl} {#key frameKey}{/key} - {:else}
+ {/if} + {#if !currentUrl && !frameUrl}
diff --git a/src/lib/components/chat/ChatTabs/ChromeBrowser.svelte b/src/lib/components/chat/ChatTabs/ChromeBrowser.svelte new file mode 100644 index 0000000000..cb7e31ed4a --- /dev/null +++ b/src/lib/components/chat/ChatTabs/ChromeBrowser.svelte @@ -0,0 +1,186 @@ + + +
key(event, 'keyDown')} + on:keyup={(event) => key(event, 'keyUp')} + on:paste={(event) => { + event.preventDefault(); + command('paste', { text: event.clipboardData?.getData('text/plain') ?? '' }); + }} + on:mousedown={(event) => mouse(event, 'mousePressed')} + on:mouseup={(event) => mouse(event, 'mouseReleased')} + on:mousemove={(event) => mouse(event, 'mouseMoved')} + on:contextmenu|preventDefault + on:wheel|preventDefault={(event) => { + const box = surface.querySelector('img')?.getBoundingClientRect(); + if (!box) return; + command('mouse', { + event: 'mouseWheel', + x: ((event.clientX - box.left) * width) / box.width, + y: ((event.clientY - box.top) * height) / box.height, + deltaX: event.deltaX, + deltaY: event.deltaY, + modifiers: modifiers(event) + }); + }} +> + {#if frame}Browser page{/if} +