mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
feat: add persistent Fly playground deployment
This commit is contained in:
parent
de0834b590
commit
fa9cb9f906
9 changed files with 347 additions and 29 deletions
|
|
@ -52,6 +52,7 @@ build/
|
|||
ui/litellm-dashboard/.next
|
||||
ui/litellm-dashboard/out
|
||||
litellm-rust/target/
|
||||
playground/workspace/target/
|
||||
litellm/rust_bridge/_native*.so
|
||||
*.log
|
||||
.env
|
||||
|
|
|
|||
32
playground/DEPLOY.md
Normal file
32
playground/DEPLOY.md
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
# Fly deployment
|
||||
|
||||
The Fly Machine runs the playground and OpenCode Web against the same workspace on one persistent volume
|
||||
|
||||
From the repository root, update the `app` name in `playground/fly.toml`, then create the app and volume
|
||||
|
||||
```sh
|
||||
fly apps create litellm-rust-playground
|
||||
fly volumes create playground_data --app litellm-rust-playground --region sjc --size 20
|
||||
```
|
||||
|
||||
Set the OpenCode password and whichever provider key OpenCode should use
|
||||
|
||||
```sh
|
||||
fly secrets set --app litellm-rust-playground \
|
||||
OPENCODE_SERVER_PASSWORD=replace-me \
|
||||
OPENAI_API_KEY=replace-me
|
||||
```
|
||||
|
||||
Deploy with the repository root as the Docker build context
|
||||
|
||||
```sh
|
||||
fly deploy --config playground/fly.toml .
|
||||
```
|
||||
|
||||
The playground is served at `https://litellm-rust-playground.fly.dev` with the username `opencode`
|
||||
|
||||
OpenCode Web is served from the same Machine at `https://litellm-rust-playground.fly.dev:8443` with the username `opencode`
|
||||
|
||||
The first boot copies the LiteLLM Rust source, examples, and guide into `/data/workspace/litellm`. Later deployments leave that workspace unchanged
|
||||
|
||||
OpenCode session data is stored under `/data/opencode`. Provider keys and the OpenCode password stay in Fly secrets
|
||||
46
playground/Dockerfile
Normal file
46
playground/Dockerfile
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
FROM node:22-bookworm-slim AS frontend
|
||||
|
||||
WORKDIR /build/playground
|
||||
COPY playground/package.json playground/package-lock.json ./
|
||||
RUN npm ci
|
||||
COPY playground/index.html ./index.html
|
||||
COPY playground/src ./src
|
||||
RUN npm run build
|
||||
|
||||
FROM rust:1.98.0-bookworm AS rust-toolchain
|
||||
RUN rustup component add rust-analyzer
|
||||
|
||||
FROM node:22-bookworm-slim
|
||||
|
||||
ARG OPENCODE_VERSION=1.18.30
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends build-essential ca-certificates clang cmake git libssl-dev pkg-config tini util-linux-extra \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& groupadd workspace \
|
||||
&& useradd --create-home --gid workspace opencode \
|
||||
&& useradd --create-home --gid workspace playground \
|
||||
&& npm install --global "opencode-ai@${OPENCODE_VERSION}"
|
||||
|
||||
COPY --from=rust-toolchain /usr/local/cargo /usr/local/cargo
|
||||
COPY --from=rust-toolchain /usr/local/rustup /usr/local/rustup
|
||||
|
||||
ENV PATH="/usr/local/cargo/bin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
ENV RUSTUP_HOME="/usr/local/rustup"
|
||||
ENV CARGO_HOME="/usr/local/cargo"
|
||||
|
||||
WORKDIR /opt/playground
|
||||
COPY --from=frontend /build/playground/node_modules ./node_modules
|
||||
COPY --from=frontend /build/playground/dist ./dist
|
||||
COPY playground/package.json playground/package-lock.json playground/server.mjs ./
|
||||
COPY playground/deploy/opencode.json /etc/opencode/opencode.json
|
||||
COPY playground/deploy/xdg-open /usr/local/bin/xdg-open
|
||||
COPY playground/deploy/start.sh /usr/local/bin/start-playground
|
||||
COPY litellm-rust /opt/litellm-seed/litellm-rust
|
||||
COPY playground/workspace /opt/litellm-seed/playground/workspace
|
||||
|
||||
RUN chmod 0755 /usr/local/bin/start-playground /usr/local/bin/xdg-open
|
||||
|
||||
EXPOSE 8080 4096
|
||||
|
||||
ENTRYPOINT ["tini", "--", "/usr/local/bin/start-playground"]
|
||||
19
playground/deploy/opencode.json
Normal file
19
playground/deploy/opencode.json
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
{
|
||||
"$schema": "https://opencode.ai/config.json",
|
||||
"autoupdate": false,
|
||||
"share": "disabled",
|
||||
"snapshot": true,
|
||||
"permission": {
|
||||
"read": "allow",
|
||||
"edit": "allow",
|
||||
"bash": "ask",
|
||||
"external_directory": "deny"
|
||||
},
|
||||
"watcher": {
|
||||
"ignore": [
|
||||
".git/**",
|
||||
"node_modules/**",
|
||||
"target/**"
|
||||
]
|
||||
}
|
||||
}
|
||||
58
playground/deploy/start.sh
Normal file
58
playground/deploy/start.sh
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
: "${OPENCODE_SERVER_PASSWORD:?Set OPENCODE_SERVER_PASSWORD as a Fly secret}"
|
||||
umask 0002
|
||||
|
||||
volume_root="${PLAYGROUND_VOLUME_ROOT:-/data}"
|
||||
repository_directory="${volume_root}/workspace/litellm"
|
||||
workspace_directory="${repository_directory}/playground/workspace"
|
||||
opencode_data_directory="${volume_root}/opencode"
|
||||
cargo_cache_directory="${volume_root}/cache/cargo"
|
||||
|
||||
mkdir -p "${repository_directory}" "${opencode_data_directory}" "${cargo_cache_directory}" /home/playground
|
||||
cp -a -n /opt/litellm-seed/. "${repository_directory}/"
|
||||
|
||||
if [[ ! -d "${repository_directory}/.git" ]]; then
|
||||
git -C "${repository_directory}" init --quiet --initial-branch=main
|
||||
git -C "${repository_directory}" config user.name "Playground"
|
||||
git -C "${repository_directory}" config user.email "playground@localhost"
|
||||
git -C "${repository_directory}" add .
|
||||
git -C "${repository_directory}" commit --quiet -m "Initial volume workspace"
|
||||
fi
|
||||
|
||||
chown -R opencode:workspace "${opencode_data_directory}"
|
||||
chown -R playground:workspace "${cargo_cache_directory}"
|
||||
chown -R root:workspace "${repository_directory}"
|
||||
chmod -R g+rwX "${repository_directory}"
|
||||
find "${repository_directory}" -type d -exec chmod g+s {} +
|
||||
git config --system --get-all safe.directory | grep -Fxq "${repository_directory}" \
|
||||
|| git config --system --add safe.directory "${repository_directory}"
|
||||
|
||||
(
|
||||
cd "${repository_directory}"
|
||||
exec runuser -u opencode --preserve-environment -- env \
|
||||
HOME=/home/opencode \
|
||||
BROWSER=/bin/true \
|
||||
XDG_DATA_HOME="${opencode_data_directory}" \
|
||||
OPENCODE_CONFIG=/etc/opencode/opencode.json \
|
||||
opencode web --hostname 0.0.0.0 --port 4096
|
||||
) &
|
||||
opencode_pid=$!
|
||||
|
||||
runuser -u playground -- env -i \
|
||||
PATH=/usr/local/cargo/bin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
|
||||
HOME=/home/playground \
|
||||
CARGO_HOME="${cargo_cache_directory}" \
|
||||
RUSTUP_HOME=/usr/local/rustup \
|
||||
NODE_ENV=production \
|
||||
HOST=0.0.0.0 \
|
||||
PORT=8080 \
|
||||
PLAYGROUND_PASSWORD="${OPENCODE_SERVER_PASSWORD}" \
|
||||
PLAYGROUND_REPOSITORY_DIR="${repository_directory}" \
|
||||
PLAYGROUND_WORKSPACE_DIR="${workspace_directory}" \
|
||||
node /opt/playground/server.mjs &
|
||||
playground_pid=$!
|
||||
|
||||
trap 'kill "${opencode_pid}" "${playground_pid}" 2>/dev/null || true' EXIT INT TERM
|
||||
wait -n "${opencode_pid}" "${playground_pid}"
|
||||
2
playground/deploy/xdg-open
Normal file
2
playground/deploy/xdg-open
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
#!/bin/sh
|
||||
exit 0
|
||||
49
playground/fly.toml
Normal file
49
playground/fly.toml
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
app = "litellm-rust-playground"
|
||||
primary_region = "sjc"
|
||||
|
||||
[build]
|
||||
dockerfile = "playground/Dockerfile"
|
||||
|
||||
[env]
|
||||
PLAYGROUND_VOLUME_ROOT = "/data"
|
||||
|
||||
[mounts]
|
||||
source = "playground_data"
|
||||
destination = "/data"
|
||||
initial_size = "20gb"
|
||||
snapshot_retention = 14
|
||||
|
||||
[http_service]
|
||||
internal_port = 8080
|
||||
force_https = true
|
||||
auto_stop_machines = "off"
|
||||
auto_start_machines = true
|
||||
min_machines_running = 1
|
||||
|
||||
[http_service.concurrency]
|
||||
type = "connections"
|
||||
soft_limit = 20
|
||||
hard_limit = 30
|
||||
|
||||
[http_service.http_options]
|
||||
idle_timeout = 3600
|
||||
|
||||
[http_service.checks]
|
||||
grace_period = "30s"
|
||||
interval = "30s"
|
||||
method = "GET"
|
||||
timeout = "5s"
|
||||
path = "/healthz"
|
||||
|
||||
[[services]]
|
||||
internal_port = 4096
|
||||
protocol = "tcp"
|
||||
|
||||
[[services.ports]]
|
||||
handlers = ["tls", "http"]
|
||||
port = 8443
|
||||
|
||||
[[vm]]
|
||||
size = "shared-cpu-2x"
|
||||
memory = "4gb"
|
||||
persist_rootfs = "never"
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
import { spawn } from 'node:child_process';
|
||||
import { timingSafeEqual } from 'node:crypto';
|
||||
import { readdir, readFile, writeFile } from 'node:fs/promises';
|
||||
import http from 'node:http';
|
||||
import path from 'node:path';
|
||||
|
|
@ -8,10 +9,28 @@ import { createServer as createViteServer } from 'vite';
|
|||
import { WebSocketServer } from 'ws';
|
||||
|
||||
const playgroundDirectory = path.dirname(fileURLToPath(import.meta.url));
|
||||
const repositoryDirectory = path.resolve(playgroundDirectory, '..');
|
||||
const workspaceDirectory = path.join(playgroundDirectory, 'workspace');
|
||||
const repositoryDirectory = path.resolve(process.env.PLAYGROUND_REPOSITORY_DIR ?? path.resolve(playgroundDirectory, '..'));
|
||||
const workspaceDirectory = path.resolve(process.env.PLAYGROUND_WORKSPACE_DIR ?? path.join(playgroundDirectory, 'workspace'));
|
||||
const examplesDirectory = path.join(workspaceDirectory, 'examples');
|
||||
const port = Number(process.env.PORT ?? 5173);
|
||||
const host = process.env.HOST ?? '127.0.0.1';
|
||||
const isProduction = process.env.NODE_ENV === 'production';
|
||||
const staticDirectory = path.join(playgroundDirectory, 'dist');
|
||||
const playgroundPassword = process.env.PLAYGROUND_PASSWORD;
|
||||
const commandEnvironment = { ...process.env };
|
||||
delete commandEnvironment.PLAYGROUND_PASSWORD;
|
||||
|
||||
const isAuthorized = request => {
|
||||
if (!playgroundPassword) {
|
||||
return true;
|
||||
}
|
||||
const expected = Buffer.from(`opencode:${playgroundPassword}`);
|
||||
const authorization = request.headers.authorization ?? '';
|
||||
const actual = authorization.startsWith('Basic ')
|
||||
? Buffer.from(authorization.slice('Basic '.length), 'base64')
|
||||
: Buffer.alloc(0);
|
||||
return actual.length === expected.length && timingSafeEqual(actual, expected);
|
||||
};
|
||||
|
||||
const readRequestBody = request =>
|
||||
new Promise((resolve, reject) => {
|
||||
|
|
@ -26,6 +45,35 @@ const sendJson = (response, status, value) => {
|
|||
response.end(JSON.stringify(value));
|
||||
};
|
||||
|
||||
const contentTypes = new Map([
|
||||
['.css', 'text/css; charset=utf-8'],
|
||||
['.html', 'text/html; charset=utf-8'],
|
||||
['.js', 'text/javascript; charset=utf-8'],
|
||||
['.json', 'application/json; charset=utf-8'],
|
||||
['.map', 'application/json; charset=utf-8'],
|
||||
['.svg', 'image/svg+xml'],
|
||||
]);
|
||||
|
||||
const serveStaticFile = async (request, response) => {
|
||||
const pathname = new URL(request.url ?? '/', 'http://localhost').pathname;
|
||||
const relativePath = pathname === '/' ? 'index.html' : pathname.slice(1);
|
||||
const filePath = path.resolve(staticDirectory, relativePath);
|
||||
if (!filePath.startsWith(`${staticDirectory}${path.sep}`)) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
const body = await readFile(filePath);
|
||||
response.writeHead(200, {
|
||||
'cache-control': relativePath === 'index.html' ? 'no-cache' : 'public, max-age=31536000, immutable',
|
||||
'content-type': contentTypes.get(path.extname(filePath)) ?? 'application/octet-stream',
|
||||
});
|
||||
response.end(body);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
const runCommand = (command, args, options) =>
|
||||
new Promise(resolve => {
|
||||
const child = spawn(command, args, options);
|
||||
|
|
@ -96,13 +144,34 @@ const readExamples = async () => {
|
|||
}));
|
||||
};
|
||||
|
||||
const vite = await createViteServer({
|
||||
const writableFilePath = async target => {
|
||||
if (target === 'GUIDE.md') {
|
||||
return path.join(workspaceDirectory, 'GUIDE.md');
|
||||
}
|
||||
const examples = await readExamples();
|
||||
const match = examples.flatMap(example => example.files.map(file => ({ example, file })))
|
||||
.find(({ file }) => file.target === target);
|
||||
return match ? path.join(match.example.directory, match.file.path) : null;
|
||||
};
|
||||
|
||||
const vite = isProduction ? null : await createViteServer({
|
||||
root: playgroundDirectory,
|
||||
server: { middlewareMode: true },
|
||||
appType: 'spa',
|
||||
});
|
||||
|
||||
const server = http.createServer(async (request, response) => {
|
||||
if (request.method === 'GET' && request.url === '/healthz') {
|
||||
sendJson(response, 200, { healthy: true });
|
||||
return;
|
||||
}
|
||||
|
||||
if (!isAuthorized(request)) {
|
||||
response.writeHead(401, { 'www-authenticate': 'Basic realm="LiteLLM Playground"' });
|
||||
response.end('Authentication required');
|
||||
return;
|
||||
}
|
||||
|
||||
if (request.method === 'GET' && request.url === '/api/info') {
|
||||
const [examples, guideSource, revision] = await Promise.all([
|
||||
readExamples(),
|
||||
|
|
@ -118,6 +187,25 @@ const server = http.createServer(async (request, response) => {
|
|||
return;
|
||||
}
|
||||
|
||||
if (request.method === 'PUT' && request.url === '/api/file') {
|
||||
try {
|
||||
const body = JSON.parse(await readRequestBody(request));
|
||||
const filePath = typeof body.target === 'string' ? await writableFilePath(body.target) : null;
|
||||
if (!filePath || typeof body.source !== 'string') {
|
||||
sendJson(response, 400, { success: false, output: 'A valid playground file is required' });
|
||||
return;
|
||||
}
|
||||
await writeFile(filePath, body.source, 'utf8');
|
||||
sendJson(response, 200, { success: true });
|
||||
} catch (error) {
|
||||
sendJson(response, 500, {
|
||||
success: false,
|
||||
output: error instanceof Error ? error.message : 'Unable to save file',
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (request.method === 'POST' && request.url === '/api/run') {
|
||||
try {
|
||||
const body = JSON.parse(await readRequestBody(request));
|
||||
|
|
@ -136,7 +224,7 @@ const server = http.createServer(async (request, response) => {
|
|||
const result = await runCommand(
|
||||
'cargo',
|
||||
['run', '--quiet', '--manifest-path', path.join(example.directory, 'Cargo.toml')],
|
||||
{ cwd: workspaceDirectory, stdio: ['ignore', 'pipe', 'pipe'] },
|
||||
{ cwd: workspaceDirectory, env: commandEnvironment, stdio: ['ignore', 'pipe', 'pipe'] },
|
||||
);
|
||||
sendJson(response, 200, result);
|
||||
} catch (error) {
|
||||
|
|
@ -148,18 +236,32 @@ const server = http.createServer(async (request, response) => {
|
|||
return;
|
||||
}
|
||||
|
||||
vite.middlewares(request, response, () => {
|
||||
response.writeHead(404);
|
||||
response.end('Not found');
|
||||
});
|
||||
if (vite) {
|
||||
vite.middlewares(request, response, () => {
|
||||
response.writeHead(404);
|
||||
response.end('Not found');
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (await serveStaticFile(request, response)) {
|
||||
return;
|
||||
}
|
||||
|
||||
response.writeHead(404);
|
||||
response.end('Not found');
|
||||
});
|
||||
|
||||
const websocketServer = new WebSocketServer({ server, path: '/lsp' });
|
||||
const websocketServer = new WebSocketServer({
|
||||
server,
|
||||
path: '/lsp',
|
||||
verifyClient: ({ req }) => isAuthorized(req),
|
||||
});
|
||||
|
||||
websocketServer.on('connection', socket => {
|
||||
const analyzer = spawn('rust-analyzer', [], {
|
||||
cwd: workspaceDirectory,
|
||||
env: process.env,
|
||||
env: commandEnvironment,
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
});
|
||||
let pending = Buffer.alloc(0);
|
||||
|
|
@ -201,6 +303,6 @@ websocketServer.on('connection', socket => {
|
|||
socket.on('close', () => analyzer.kill());
|
||||
});
|
||||
|
||||
server.listen(port, '127.0.0.1', () => {
|
||||
process.stdout.write(`LiteLLM Rust Playground: http://localhost:${port}\n`);
|
||||
server.listen(port, host, () => {
|
||||
process.stdout.write(`LiteLLM Rust Playground: http://${host}:${port}\n`);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -16,8 +16,6 @@ import { EditorView, basicSetup } from 'codemirror';
|
|||
|
||||
import './style.css';
|
||||
|
||||
const STORAGE_KEY = 'litellm-rust-playground-files';
|
||||
|
||||
type PlaygroundFile = {
|
||||
languageId: 'rust' | 'toml';
|
||||
path: string;
|
||||
|
|
@ -240,17 +238,29 @@ const runCode = async (exampleId: string, files: StoredFiles): Promise<RunResult
|
|||
return response.json() as Promise<RunResult>;
|
||||
};
|
||||
|
||||
const loadStoredFiles = (): StoredFiles => {
|
||||
try {
|
||||
const value = localStorage.getItem(STORAGE_KEY);
|
||||
return value ? (JSON.parse(value) as StoredFiles) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
};
|
||||
const saveTimers = new Map<string, number>();
|
||||
|
||||
const saveStoredFile = (path: string, source: string) => {
|
||||
localStorage.setItem(STORAGE_KEY, JSON.stringify({ ...loadStoredFiles(), [path]: source }));
|
||||
const saveFile = (target: string, source: string) => {
|
||||
const pendingTimer = saveTimers.get(target);
|
||||
if (pendingTimer !== undefined) {
|
||||
window.clearTimeout(pendingTimer);
|
||||
}
|
||||
saveTimers.set(target, window.setTimeout(async () => {
|
||||
saveTimers.delete(target);
|
||||
try {
|
||||
const response = await fetch('/api/file', {
|
||||
method: 'PUT',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ target, source }),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Unable to save ${target}`);
|
||||
}
|
||||
} catch {
|
||||
runStatus.textContent = `Save failed: ${target}`;
|
||||
runStatus.dataset.state = 'error';
|
||||
}
|
||||
}, 350));
|
||||
};
|
||||
|
||||
const main = async () => {
|
||||
|
|
@ -260,7 +270,6 @@ const main = async () => {
|
|||
throw new Error('No runnable examples found');
|
||||
}
|
||||
|
||||
const storedFiles = loadStoredFiles();
|
||||
const allFiles = info.examples.flatMap(example => example.files.map(file => ({ example, file })));
|
||||
const fileByUri = new Map(allFiles.map(entry => [entry.file.uri, entry]));
|
||||
const views = new Map<string, EditorView>();
|
||||
|
|
@ -414,7 +423,7 @@ const main = async () => {
|
|||
? [rust(), client.plugin(file.uri, 'rust'), commandClickDefinition]
|
||||
: [];
|
||||
const view = new EditorView({
|
||||
doc: storedFiles[file.target] ?? file.source,
|
||||
doc: file.source,
|
||||
extensions: [
|
||||
basicSetup,
|
||||
oneDark,
|
||||
|
|
@ -422,7 +431,7 @@ const main = async () => {
|
|||
EditorView.lineWrapping,
|
||||
EditorView.updateListener.of(update => {
|
||||
if (update.docChanged) {
|
||||
saveStoredFile(file.target, update.state.doc.toString());
|
||||
saveFile(file.target, update.state.doc.toString());
|
||||
}
|
||||
}),
|
||||
],
|
||||
|
|
@ -432,7 +441,7 @@ const main = async () => {
|
|||
});
|
||||
|
||||
const markdownView = new EditorView({
|
||||
doc: storedFiles[info.guide.target] ?? info.guide.source,
|
||||
doc: info.guide.source,
|
||||
extensions: [
|
||||
basicSetup,
|
||||
markdown(),
|
||||
|
|
@ -442,7 +451,7 @@ const main = async () => {
|
|||
EditorView.lineWrapping,
|
||||
EditorView.updateListener.of(update => {
|
||||
if (update.docChanged) {
|
||||
saveStoredFile(info.guide.target, update.state.doc.toString());
|
||||
saveFile(info.guide.target, update.state.doc.toString());
|
||||
}
|
||||
}),
|
||||
],
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue