diff --git a/.dockerignore b/.dockerignore index f3a80fee3e4..a57d7a1e3ff 100644 --- a/.dockerignore +++ b/.dockerignore @@ -52,6 +52,7 @@ build/ ui/litellm-dashboard/.next ui/litellm-dashboard/out litellm-rust/target/ +playground/workspace/target/ litellm/rust_bridge/_native*.so *.log .env diff --git a/playground/DEPLOY.md b/playground/DEPLOY.md new file mode 100644 index 00000000000..953b7cdcc27 --- /dev/null +++ b/playground/DEPLOY.md @@ -0,0 +1,32 @@ +# Fly deployment + +The Fly Machine runs the playground and OpenCode Web against the same workspace on one persistent volume + +From the repository root, update the `app` name in `playground/fly.toml`, then create the app and volume + +```sh +fly apps create litellm-rust-playground +fly volumes create playground_data --app litellm-rust-playground --region sjc --size 20 +``` + +Set the OpenCode password and whichever provider key OpenCode should use + +```sh +fly secrets set --app litellm-rust-playground \ + OPENCODE_SERVER_PASSWORD=replace-me \ + OPENAI_API_KEY=replace-me +``` + +Deploy with the repository root as the Docker build context + +```sh +fly deploy --config playground/fly.toml . +``` + +The playground is served at `https://litellm-rust-playground.fly.dev` with the username `opencode` + +OpenCode Web is served from the same Machine at `https://litellm-rust-playground.fly.dev:8443` with the username `opencode` + +The first boot copies the LiteLLM Rust source, examples, and guide into `/data/workspace/litellm`. Later deployments leave that workspace unchanged + +OpenCode session data is stored under `/data/opencode`. Provider keys and the OpenCode password stay in Fly secrets diff --git a/playground/Dockerfile b/playground/Dockerfile new file mode 100644 index 00000000000..35db4b6ebef --- /dev/null +++ b/playground/Dockerfile @@ -0,0 +1,46 @@ +FROM node:22-bookworm-slim AS frontend + +WORKDIR /build/playground +COPY playground/package.json playground/package-lock.json ./ +RUN npm ci +COPY playground/index.html ./index.html +COPY playground/src ./src +RUN npm run build + +FROM rust:1.98.0-bookworm AS rust-toolchain +RUN rustup component add rust-analyzer + +FROM node:22-bookworm-slim + +ARG OPENCODE_VERSION=1.18.30 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends build-essential ca-certificates clang cmake git libssl-dev pkg-config tini util-linux-extra \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd workspace \ + && useradd --create-home --gid workspace opencode \ + && useradd --create-home --gid workspace playground \ + && npm install --global "opencode-ai@${OPENCODE_VERSION}" + +COPY --from=rust-toolchain /usr/local/cargo /usr/local/cargo +COPY --from=rust-toolchain /usr/local/rustup /usr/local/rustup + +ENV PATH="/usr/local/cargo/bin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +ENV RUSTUP_HOME="/usr/local/rustup" +ENV CARGO_HOME="/usr/local/cargo" + +WORKDIR /opt/playground +COPY --from=frontend /build/playground/node_modules ./node_modules +COPY --from=frontend /build/playground/dist ./dist +COPY playground/package.json playground/package-lock.json playground/server.mjs ./ +COPY playground/deploy/opencode.json /etc/opencode/opencode.json +COPY playground/deploy/xdg-open /usr/local/bin/xdg-open +COPY playground/deploy/start.sh /usr/local/bin/start-playground +COPY litellm-rust /opt/litellm-seed/litellm-rust +COPY playground/workspace /opt/litellm-seed/playground/workspace + +RUN chmod 0755 /usr/local/bin/start-playground /usr/local/bin/xdg-open + +EXPOSE 8080 4096 + +ENTRYPOINT ["tini", "--", "/usr/local/bin/start-playground"] diff --git a/playground/deploy/opencode.json b/playground/deploy/opencode.json new file mode 100644 index 00000000000..6945e6cbecb --- /dev/null +++ b/playground/deploy/opencode.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://opencode.ai/config.json", + "autoupdate": false, + "share": "disabled", + "snapshot": true, + "permission": { + "read": "allow", + "edit": "allow", + "bash": "ask", + "external_directory": "deny" + }, + "watcher": { + "ignore": [ + ".git/**", + "node_modules/**", + "target/**" + ] + } +} diff --git a/playground/deploy/start.sh b/playground/deploy/start.sh new file mode 100644 index 00000000000..330544f21c4 --- /dev/null +++ b/playground/deploy/start.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${OPENCODE_SERVER_PASSWORD:?Set OPENCODE_SERVER_PASSWORD as a Fly secret}" +umask 0002 + +volume_root="${PLAYGROUND_VOLUME_ROOT:-/data}" +repository_directory="${volume_root}/workspace/litellm" +workspace_directory="${repository_directory}/playground/workspace" +opencode_data_directory="${volume_root}/opencode" +cargo_cache_directory="${volume_root}/cache/cargo" + +mkdir -p "${repository_directory}" "${opencode_data_directory}" "${cargo_cache_directory}" /home/playground +cp -a -n /opt/litellm-seed/. "${repository_directory}/" + +if [[ ! -d "${repository_directory}/.git" ]]; then + git -C "${repository_directory}" init --quiet --initial-branch=main + git -C "${repository_directory}" config user.name "Playground" + git -C "${repository_directory}" config user.email "playground@localhost" + git -C "${repository_directory}" add . + git -C "${repository_directory}" commit --quiet -m "Initial volume workspace" +fi + +chown -R opencode:workspace "${opencode_data_directory}" +chown -R playground:workspace "${cargo_cache_directory}" +chown -R root:workspace "${repository_directory}" +chmod -R g+rwX "${repository_directory}" +find "${repository_directory}" -type d -exec chmod g+s {} + +git config --system --get-all safe.directory | grep -Fxq "${repository_directory}" \ + || git config --system --add safe.directory "${repository_directory}" + +( + cd "${repository_directory}" + exec runuser -u opencode --preserve-environment -- env \ + HOME=/home/opencode \ + BROWSER=/bin/true \ + XDG_DATA_HOME="${opencode_data_directory}" \ + OPENCODE_CONFIG=/etc/opencode/opencode.json \ + opencode web --hostname 0.0.0.0 --port 4096 +) & +opencode_pid=$! + +runuser -u playground -- env -i \ + PATH=/usr/local/cargo/bin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + HOME=/home/playground \ + CARGO_HOME="${cargo_cache_directory}" \ + RUSTUP_HOME=/usr/local/rustup \ + NODE_ENV=production \ + HOST=0.0.0.0 \ + PORT=8080 \ + PLAYGROUND_PASSWORD="${OPENCODE_SERVER_PASSWORD}" \ + PLAYGROUND_REPOSITORY_DIR="${repository_directory}" \ + PLAYGROUND_WORKSPACE_DIR="${workspace_directory}" \ + node /opt/playground/server.mjs & +playground_pid=$! + +trap 'kill "${opencode_pid}" "${playground_pid}" 2>/dev/null || true' EXIT INT TERM +wait -n "${opencode_pid}" "${playground_pid}" diff --git a/playground/deploy/xdg-open b/playground/deploy/xdg-open new file mode 100644 index 00000000000..039e4d0069c --- /dev/null +++ b/playground/deploy/xdg-open @@ -0,0 +1,2 @@ +#!/bin/sh +exit 0 diff --git a/playground/fly.toml b/playground/fly.toml new file mode 100644 index 00000000000..d09355d3f2a --- /dev/null +++ b/playground/fly.toml @@ -0,0 +1,49 @@ +app = "litellm-rust-playground" +primary_region = "sjc" + +[build] + dockerfile = "playground/Dockerfile" + +[env] + PLAYGROUND_VOLUME_ROOT = "/data" + +[mounts] + source = "playground_data" + destination = "/data" + initial_size = "20gb" + snapshot_retention = 14 + +[http_service] + internal_port = 8080 + force_https = true + auto_stop_machines = "off" + auto_start_machines = true + min_machines_running = 1 + + [http_service.concurrency] + type = "connections" + soft_limit = 20 + hard_limit = 30 + + [http_service.http_options] + idle_timeout = 3600 + + [http_service.checks] + grace_period = "30s" + interval = "30s" + method = "GET" + timeout = "5s" + path = "/healthz" + +[[services]] + internal_port = 4096 + protocol = "tcp" + + [[services.ports]] + handlers = ["tls", "http"] + port = 8443 + +[[vm]] + size = "shared-cpu-2x" + memory = "4gb" + persist_rootfs = "never" diff --git a/playground/server.mjs b/playground/server.mjs index d26d03ba06a..ff8bde3c27a 100644 --- a/playground/server.mjs +++ b/playground/server.mjs @@ -1,4 +1,5 @@ import { spawn } from 'node:child_process'; +import { timingSafeEqual } from 'node:crypto'; import { readdir, readFile, writeFile } from 'node:fs/promises'; import http from 'node:http'; import path from 'node:path'; @@ -8,10 +9,28 @@ import { createServer as createViteServer } from 'vite'; import { WebSocketServer } from 'ws'; const playgroundDirectory = path.dirname(fileURLToPath(import.meta.url)); -const repositoryDirectory = path.resolve(playgroundDirectory, '..'); -const workspaceDirectory = path.join(playgroundDirectory, 'workspace'); +const repositoryDirectory = path.resolve(process.env.PLAYGROUND_REPOSITORY_DIR ?? path.resolve(playgroundDirectory, '..')); +const workspaceDirectory = path.resolve(process.env.PLAYGROUND_WORKSPACE_DIR ?? path.join(playgroundDirectory, 'workspace')); const examplesDirectory = path.join(workspaceDirectory, 'examples'); const port = Number(process.env.PORT ?? 5173); +const host = process.env.HOST ?? '127.0.0.1'; +const isProduction = process.env.NODE_ENV === 'production'; +const staticDirectory = path.join(playgroundDirectory, 'dist'); +const playgroundPassword = process.env.PLAYGROUND_PASSWORD; +const commandEnvironment = { ...process.env }; +delete commandEnvironment.PLAYGROUND_PASSWORD; + +const isAuthorized = request => { + if (!playgroundPassword) { + return true; + } + const expected = Buffer.from(`opencode:${playgroundPassword}`); + const authorization = request.headers.authorization ?? ''; + const actual = authorization.startsWith('Basic ') + ? Buffer.from(authorization.slice('Basic '.length), 'base64') + : Buffer.alloc(0); + return actual.length === expected.length && timingSafeEqual(actual, expected); +}; const readRequestBody = request => new Promise((resolve, reject) => { @@ -26,6 +45,35 @@ const sendJson = (response, status, value) => { response.end(JSON.stringify(value)); }; +const contentTypes = new Map([ + ['.css', 'text/css; charset=utf-8'], + ['.html', 'text/html; charset=utf-8'], + ['.js', 'text/javascript; charset=utf-8'], + ['.json', 'application/json; charset=utf-8'], + ['.map', 'application/json; charset=utf-8'], + ['.svg', 'image/svg+xml'], +]); + +const serveStaticFile = async (request, response) => { + const pathname = new URL(request.url ?? '/', 'http://localhost').pathname; + const relativePath = pathname === '/' ? 'index.html' : pathname.slice(1); + const filePath = path.resolve(staticDirectory, relativePath); + if (!filePath.startsWith(`${staticDirectory}${path.sep}`)) { + return false; + } + try { + const body = await readFile(filePath); + response.writeHead(200, { + 'cache-control': relativePath === 'index.html' ? 'no-cache' : 'public, max-age=31536000, immutable', + 'content-type': contentTypes.get(path.extname(filePath)) ?? 'application/octet-stream', + }); + response.end(body); + return true; + } catch { + return false; + } +}; + const runCommand = (command, args, options) => new Promise(resolve => { const child = spawn(command, args, options); @@ -96,13 +144,34 @@ const readExamples = async () => { })); }; -const vite = await createViteServer({ +const writableFilePath = async target => { + if (target === 'GUIDE.md') { + return path.join(workspaceDirectory, 'GUIDE.md'); + } + const examples = await readExamples(); + const match = examples.flatMap(example => example.files.map(file => ({ example, file }))) + .find(({ file }) => file.target === target); + return match ? path.join(match.example.directory, match.file.path) : null; +}; + +const vite = isProduction ? null : await createViteServer({ root: playgroundDirectory, server: { middlewareMode: true }, appType: 'spa', }); const server = http.createServer(async (request, response) => { + if (request.method === 'GET' && request.url === '/healthz') { + sendJson(response, 200, { healthy: true }); + return; + } + + if (!isAuthorized(request)) { + response.writeHead(401, { 'www-authenticate': 'Basic realm="LiteLLM Playground"' }); + response.end('Authentication required'); + return; + } + if (request.method === 'GET' && request.url === '/api/info') { const [examples, guideSource, revision] = await Promise.all([ readExamples(), @@ -118,6 +187,25 @@ const server = http.createServer(async (request, response) => { return; } + if (request.method === 'PUT' && request.url === '/api/file') { + try { + const body = JSON.parse(await readRequestBody(request)); + const filePath = typeof body.target === 'string' ? await writableFilePath(body.target) : null; + if (!filePath || typeof body.source !== 'string') { + sendJson(response, 400, { success: false, output: 'A valid playground file is required' }); + return; + } + await writeFile(filePath, body.source, 'utf8'); + sendJson(response, 200, { success: true }); + } catch (error) { + sendJson(response, 500, { + success: false, + output: error instanceof Error ? error.message : 'Unable to save file', + }); + } + return; + } + if (request.method === 'POST' && request.url === '/api/run') { try { const body = JSON.parse(await readRequestBody(request)); @@ -136,7 +224,7 @@ const server = http.createServer(async (request, response) => { const result = await runCommand( 'cargo', ['run', '--quiet', '--manifest-path', path.join(example.directory, 'Cargo.toml')], - { cwd: workspaceDirectory, stdio: ['ignore', 'pipe', 'pipe'] }, + { cwd: workspaceDirectory, env: commandEnvironment, stdio: ['ignore', 'pipe', 'pipe'] }, ); sendJson(response, 200, result); } catch (error) { @@ -148,18 +236,32 @@ const server = http.createServer(async (request, response) => { return; } - vite.middlewares(request, response, () => { - response.writeHead(404); - response.end('Not found'); - }); + if (vite) { + vite.middlewares(request, response, () => { + response.writeHead(404); + response.end('Not found'); + }); + return; + } + + if (await serveStaticFile(request, response)) { + return; + } + + response.writeHead(404); + response.end('Not found'); }); -const websocketServer = new WebSocketServer({ server, path: '/lsp' }); +const websocketServer = new WebSocketServer({ + server, + path: '/lsp', + verifyClient: ({ req }) => isAuthorized(req), +}); websocketServer.on('connection', socket => { const analyzer = spawn('rust-analyzer', [], { cwd: workspaceDirectory, - env: process.env, + env: commandEnvironment, stdio: ['pipe', 'pipe', 'pipe'], }); let pending = Buffer.alloc(0); @@ -201,6 +303,6 @@ websocketServer.on('connection', socket => { socket.on('close', () => analyzer.kill()); }); -server.listen(port, '127.0.0.1', () => { - process.stdout.write(`LiteLLM Rust Playground: http://localhost:${port}\n`); +server.listen(port, host, () => { + process.stdout.write(`LiteLLM Rust Playground: http://${host}:${port}\n`); }); diff --git a/playground/src/main.ts b/playground/src/main.ts index 6a0d04d54c3..fdcf8ea7836 100644 --- a/playground/src/main.ts +++ b/playground/src/main.ts @@ -16,8 +16,6 @@ import { EditorView, basicSetup } from 'codemirror'; import './style.css'; -const STORAGE_KEY = 'litellm-rust-playground-files'; - type PlaygroundFile = { languageId: 'rust' | 'toml'; path: string; @@ -240,17 +238,29 @@ const runCode = async (exampleId: string, files: StoredFiles): Promise; }; -const loadStoredFiles = (): StoredFiles => { - try { - const value = localStorage.getItem(STORAGE_KEY); - return value ? (JSON.parse(value) as StoredFiles) : {}; - } catch { - return {}; - } -}; +const saveTimers = new Map(); -const saveStoredFile = (path: string, source: string) => { - localStorage.setItem(STORAGE_KEY, JSON.stringify({ ...loadStoredFiles(), [path]: source })); +const saveFile = (target: string, source: string) => { + const pendingTimer = saveTimers.get(target); + if (pendingTimer !== undefined) { + window.clearTimeout(pendingTimer); + } + saveTimers.set(target, window.setTimeout(async () => { + saveTimers.delete(target); + try { + const response = await fetch('/api/file', { + method: 'PUT', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ target, source }), + }); + if (!response.ok) { + throw new Error(`Unable to save ${target}`); + } + } catch { + runStatus.textContent = `Save failed: ${target}`; + runStatus.dataset.state = 'error'; + } + }, 350)); }; const main = async () => { @@ -260,7 +270,6 @@ const main = async () => { throw new Error('No runnable examples found'); } - const storedFiles = loadStoredFiles(); const allFiles = info.examples.flatMap(example => example.files.map(file => ({ example, file }))); const fileByUri = new Map(allFiles.map(entry => [entry.file.uri, entry])); const views = new Map(); @@ -414,7 +423,7 @@ const main = async () => { ? [rust(), client.plugin(file.uri, 'rust'), commandClickDefinition] : []; const view = new EditorView({ - doc: storedFiles[file.target] ?? file.source, + doc: file.source, extensions: [ basicSetup, oneDark, @@ -422,7 +431,7 @@ const main = async () => { EditorView.lineWrapping, EditorView.updateListener.of(update => { if (update.docChanged) { - saveStoredFile(file.target, update.state.doc.toString()); + saveFile(file.target, update.state.doc.toString()); } }), ], @@ -432,7 +441,7 @@ const main = async () => { }); const markdownView = new EditorView({ - doc: storedFiles[info.guide.target] ?? info.guide.source, + doc: info.guide.source, extensions: [ basicSetup, markdown(), @@ -442,7 +451,7 @@ const main = async () => { EditorView.lineWrapping, EditorView.updateListener.of(update => { if (update.docChanged) { - saveStoredFile(info.guide.target, update.state.doc.toString()); + saveFile(info.guide.target, update.state.doc.toString()); } }), ],