From e82adf4145d9a7165368847ce930aca7c9c66f26 Mon Sep 17 00:00:00 2001 From: Chris Estreich Date: Sun, 29 Jun 2025 07:26:27 -0700 Subject: [PATCH] Fly workers (#153) --- .docker/Dockerfile.roomote-worker | 28 +++++++++++++++++++--- .docker/entrypoints/worker.sh | 6 +++++ .env.development | 2 +- .env.preview | 2 +- .env.production | 2 +- apps/roomote/package.json | 1 + apps/roomote/src/lib/controller.ts | 38 +++++++++++++++++++----------- apps/roomote/src/lib/runTask.ts | 23 +++++++++++++----- apps/roomote/src/lib/slack.ts | 6 +++++ apps/roomote/src/lib/utils.ts | 4 ++++ apps/roomote/src/lib/worker.ts | 10 ++++---- fly.roomote-worker.toml | 33 ++++++++++++++++++++++++++ package.json | 3 ++- packages/db/src/types.ts | 1 + 14 files changed, 127 insertions(+), 32 deletions(-) create mode 100644 .docker/entrypoints/worker.sh create mode 100644 fly.roomote-worker.toml diff --git a/.docker/Dockerfile.roomote-worker b/.docker/Dockerfile.roomote-worker index 8e53c8e215..a9f71c37c3 100644 --- a/.docker/Dockerfile.roomote-worker +++ b/.docker/Dockerfile.roomote-worker @@ -1,4 +1,4 @@ -# docker compose build roomote-worker && docker compose up roomote-worker +# docker compose build --build-arg GH_TOKEN=$(npx dotenvx get GH_TOKEN -f .env.development) roomote-worker FROM node:20-slim AS base @@ -44,6 +44,18 @@ RUN wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor && apt update && apt install -y code \ && rm -rf /var/lib/apt/lists/* +# Install ssh server +# RUN apt-get update \ +# && apt-get install -y openssh-server \ +# && cp /etc/ssh/sshd_config /etc/ssh/sshd_config-original \ +# && sed -i 's/^#\s*Port.*/Port 2222/' /etc/ssh/sshd_config \ +# && sed -i 's/^#\s*PasswordAuthentication yes/PasswordAuthentication no/' /etc/ssh/sshd_config \ +# && mkdir -p /root/.ssh \ +# && chmod 700 /root/.ssh \ +# && mkdir /var/run/sshd \ +# && chmod 755 /var/run/sshd \ +# && rm -rf /var/lib/apt/lists /var/cache/apt/archives + WORKDIR /roo # Install extensions @@ -56,16 +68,22 @@ RUN mkdir -p /roo/.vscode \ # Clone repo (requires $GH_TOKEN) ARG GH_TOKEN ENV GH_TOKEN=${GH_TOKEN} -WORKDIR /roo/repos RUN git config --global user.email "chris@roocode.com" RUN git config --global user.name "Roo Code" RUN git config --global credential.helper store RUN echo "https://oauth2:${GH_TOKEN}@github.com" > ~/.git-credentials + +WORKDIR /roo/repos RUN gh repo clone RooCodeInc/Roo-Code WORKDIR /roo/repos/Roo-Code RUN gh repo set-default RooCodeInc/Roo-Code RUN pnpm install +WORKDIR /roo/repos +RUN gh repo clone RooCodeInc/Roo-Code-Cloud +WORKDIR /roo/repos/Roo-Code-Cloud +RUN pnpm install + WORKDIR /roo COPY package.json pnpm-lock.yaml pnpm-workspace.yaml turbo.json .env.* ./ @@ -87,6 +105,10 @@ COPY apps/roomote ./apps/roomote/ WORKDIR /roo/apps/roomote +# COPY .docker/entrypoints/worker.sh /usr/local/bin/worker.sh +# RUN chmod +x /usr/local/bin/worker.sh + ENV NODE_ENV=production ENV HOST_EXECUTION_METHOD=docker -CMD ["sh", "-c", "npx dotenvx run -f ../../.env.${APP_ENV:-development} -- tsx src/lib/controller.ts"] +# ENTRYPOINT ["/usr/local/bin/worker.sh"] +CMD ["sleep", "infinity"] diff --git a/.docker/entrypoints/worker.sh b/.docker/entrypoints/worker.sh new file mode 100644 index 0000000000..5ac619f750 --- /dev/null +++ b/.docker/entrypoints/worker.sh @@ -0,0 +1,6 @@ +#!/bin/bash -e + +echo "$AUTHORIZED_KEYS" > /root/.ssh/authorized_keys +/usr/sbin/sshd + +exec "$@" diff --git a/.env.development b/.env.development index 4623173cda..8dd15bb1d9 100644 --- a/.env.development +++ b/.env.development @@ -18,7 +18,7 @@ GH_TOKEN=encrypted:BGQvKArqcFk+y2UmBmzUZVBo6cNtDH20i0kHiqoGDRqAoFUJkc2hEumU6ukK+ OPENROUTER_API_KEY=encrypted:BA3bYeSEXFA0jnw3lmzLKz53ekkGr0K6sDLN6+nFGsa0TdJrYoKI5IcRPCMWuXIoLR1dlRi61eziWMQt0e2MgLU8zcR+zKif/2twlIzFlTU7OtFSwC1DPknxAG0pf3e2LSTDiLU4hRKgYbdJn8coA3rf/YY9NRVlmJV1gDLb9KVL89IWgT+fEn3Tzh383dSolr1ec/9RyAYVGpIYTg4O57OzQ9TaZp6InWs= -SLACK_API_TOKEN=encrypted:BI5T2ZFY35FaOfSzPT8qQyUdhJs+fmL6wOFp8cVd4hM0xcSJ8va+vk1CCpltNrk25uXCd8D+fQnITb3G+L/1z1X8eqZVobqRwxO39wTcqVv1d9AQSLxAmYcEoAEWowiXQei2qS+AMcBSjZhDfURbdOfYo1+XHj0NdfspvsB1YwC+3CN3iZC/TSVBkzcQDy82Db541HnfFxhIQg== +SLACK_API_TOKEN=encrypted:BJyTFw3gugKZr6uaNZaqCH0UpW/mP4S8P1oL5NzeCvoZzSMJEcDrKsW+b+3Ojp0EuqBFI7T8SPUfRMr6ZTSI880+pKYWkmfAAl2jYYE5aw48LGkg+WJTjIW8g2jTEc3PXPYeMzszrJ2Igf8NuyDMLmhI9S9ew5OfhxD6bKwr73WdUKVDJ+MMVBQGIkbrt1PBX30kRuKZq5oqiA== diff --git a/.env.preview b/.env.preview index a8b5776404..60006006cd 100644 --- a/.env.preview +++ b/.env.preview @@ -18,5 +18,5 @@ GH_TOKEN=encrypted:BMB+HR14pdhTojK7exBUDaWpiQGDWziDR5cdUMM/I24MpkdCtZrvHsf+uTc/g OPENROUTER_API_KEY=encrypted:BDoXCMz3fV5DXa89cwd77/pZhw6tVIw96LvtyZEFwFQtQV1Qk+ZmtQoObiKZkfI4y1ZvpOSL2ivXfS7YY8QtLnl8uEG550FHq6s0FZsVdua+SlmoUhJOK0zV341GadCG3bc5aSMB5Wha6ml51CJzy/AKud4U4Zi3eXaj+nMxeXF8wtnHsw51JQ3F7N9VnYklyrb5FEin2SVovg3TPZk40saP8Q/YyDyU8e8= -SLACK_API_TOKEN=encrypted:BFmTkof1HoUg2UTgpm2wWdv5ZqVuldkgU5xLhGjgGFqRG7sOnAF5QdUVAfjV+890FbGF/BxBcb7bJsjMdf6+krq5U8p8Il2lucELdg1BoWisGcoy1+OCYoLxWm4RDrI/zovC09vaSnIdgIpz4T/4sNRbY5U8r7KaKXSCDPp2rCTUAC9Fc5cODlurbnMbfmPaFJIxKktqm5rfXg== +SLACK_API_TOKEN=encrypted:BEmAqaFftmVNlZdx19KGXBDtJ1E1SFBJrAH52pwhFR00Eq6DJAOJSBLY+9lV4wsDRnhTaR8QBQ5fx4kX6751mqJ/Ldu9n7zYV+NTV5cWowvTh4jD+w7yufbUvNgnqs1CpvGXysaJoyQCQqQlUJ+wxlf+rb3yEN5CMR9y1Es/d+cXF2d73H2VfBDtVcKf1pCPn+kVeQW9XHquiw== diff --git a/.env.production b/.env.production index 0e416615c0..07e35210c4 100644 --- a/.env.production +++ b/.env.production @@ -18,5 +18,5 @@ GH_TOKEN=encrypted:BN93qO9v+VfEZFaIdRwyhwR2CPM0rw1QUE9SSm/qjIT6W9XnL4Gg6EStB9zfl OPENROUTER_API_KEY=encrypted:BLBANC6AI/i5gHIeiBaY5lOogEEahxSLAvsxUpk6Z1FXtfrIEj/6EKA/CE1LzHOJEZNIXHTtlb41ONGQWuRFtgDcvbhSthZ5XZB4c9dcdPz0cxEU4RfWSRwjQxYZ8OMs1oLdr7NV8oR6IYwJ43lVfuePBYit7zX125Mp/25R7Kl50oSVREHnlfhtOMTOTQqLI7LutgGvQiDym1Ji1BFDaTBd4rTrkoymYxo= -SLACK_API_TOKEN=encrypted:BHrN/c76f9fFC6XlzfgRnrb2NXK1jlGrq5XbSqk0FM+t4BaLFGILW4RruQRnbBVVm4dGNVCP395H0rHXAQsFd25nPYbBUXiHY42Ap2jBfy3xVqm3dWhapffv4+MiTMdEJvpVY+yyxJwNAu2RFEEnI5I4UpogpY7yMFvHyzV2Jym3/9pJ1svQpCPqAzz79SXRLDQq8w1b7LmQhA== +SLACK_API_TOKEN=encrypted:BHQC5Lv0k3VnZ0Ez4TcVvL0meJs8R5ysPFa47U3kkjOJstrtDa26F8Fjvd8/Qr7ztiKdJzGXpr9k4/Tm8mL9jZvG7HNH9D+fVHkyGZ2Mf7B97FdXjiDxA3f+Pifj0aTfM7U58tDrKTLeQ/6UzE3Emcxyg4CxKjAjEaRFOAiQxs1lVAtqzbeF7OCV6GHScpSe4zoAaFXvtQ07FA== diff --git a/apps/roomote/package.json b/apps/roomote/package.json index 28318af923..3abf84e919 100644 --- a/apps/roomote/package.json +++ b/apps/roomote/package.json @@ -14,6 +14,7 @@ "worker": "dotenvx run -f ../../.env.development -- tsx src/lib/worker.ts", "worker:production": "dotenvx run -f ../../.env.production -- tsx src/lib/worker.ts", "cli": "dotenvx run -f ../../.env.development -- tsx src/lib/cli.ts", + "cli:production": "dotenvx run -f ../../.env.production -- tsx src/lib/cli.ts", "clean": "rimraf .next .turbo" }, "dependencies": { diff --git a/apps/roomote/src/lib/controller.ts b/apps/roomote/src/lib/controller.ts index adfcefc5d8..7de9c2a1ed 100644 --- a/apps/roomote/src/lib/controller.ts +++ b/apps/roomote/src/lib/controller.ts @@ -1,8 +1,8 @@ import { spawn } from 'child_process'; -import fs from 'fs'; import { Queue } from 'bullmq'; import { redis } from './redis'; +import { isFlyMachine, isDockerContainer } from './utils'; export class WorkerController { private readonly POLL_INTERVAL_MS = 5000; @@ -75,22 +75,32 @@ export class WorkerController { try { console.log(`Spawning worker: ${workerId}`); - const isRunningInDocker = fs.existsSync('/.dockerenv'); - const dockerArgs = [ - `--name roomote-${workerId}`, - '--rm', - '--network roo-code-cloud_default', - `-e GH_TOKEN=${process.env.GH_TOKEN}`, - '-v /var/run/docker.sock:/var/run/docker.sock', - '-v /tmp/roomote:/var/log/roomote', - ]; + const cliCommand = + process.env.APP_ENV === 'production' + ? 'pnpm worker:production' + : 'pnpm worker'; - const cliCommand = 'pnpm worker'; + let command; - const command = isRunningInDocker - ? `docker run ${dockerArgs.join(' ')} roomote-worker sh -c "${cliCommand}"` - : cliCommand; + if (isFlyMachine()) { + command = `fly machine run $(fly releases --image -a roomote-worker -j 2>/dev/null | jq -r '.[0].ImageRef') --vm-size performance-16x --rm --shell --command "pnpm worker:production" -a roomote-worker`; + } else if (isDockerContainer()) { + const dockerArgs = [ + `--name roomote-${workerId}`, + '--rm', + '--network roo-code-cloud_default', + `-e APP_ENV=${process.env.APP_ENV || 'development'}`, + `-e GH_TOKEN=${process.env.GH_TOKEN}`, + `-e DOTENV_PRIVATE_KEY_PRODUCTION=${process.env.DOTENV_PRIVATE_KEY_PRODUCTION}`, + '-v /var/run/docker.sock:/var/run/docker.sock', + '-v /tmp/roomote:/var/log/roomote', + ]; + + command = `docker run ${dockerArgs.join(' ')} roomote-worker sh -c "${cliCommand}"`; + } else { + command = cliCommand; + } console.log('Spawning worker with command:', command); diff --git a/apps/roomote/src/lib/runTask.ts b/apps/roomote/src/lib/runTask.ts index 252969c814..7762a427a8 100644 --- a/apps/roomote/src/lib/runTask.ts +++ b/apps/roomote/src/lib/runTask.ts @@ -1,6 +1,7 @@ import * as path from 'path'; import * as os from 'node:os'; import * as crypto from 'node:crypto'; +import * as fs from 'node:fs'; import pWaitFor from 'p-wait-for'; import { execa } from 'execa'; @@ -17,7 +18,7 @@ import { IpcClient } from '@roo-code-cloud/ipc'; import type { JobPayload, JobType } from '@roo-code-cloud/db'; import { Logger } from './logger'; -import { isDockerContainer } from './utils'; +import { isFlyMachine, isDockerContainer } from './utils'; import { SlackNotifier } from './slack'; const TIMEOUT = 30 * 60 * 1_000; @@ -71,19 +72,17 @@ export const runTask = async ({ `${crypto.randomUUID().slice(0, 8)}.sock`, ); - const env = { ROO_CODE_IPC_SOCKET_PATH: ipcSocketPath }; const controller = new AbortController(); const cancelSignal = controller.signal; - const containerized = isDockerContainer(); + const containerized = isFlyMachine() || isDockerContainer(); const codeCommand = containerized - ? `xvfb-run --auto-servernum --server-num=1 code --wait --log trace --disable-workspace-trust --disable-gpu --disable-lcd-text --no-sandbox --user-data-dir /roo/.vscode --password-store="basic" -n ${workspacePath}` - : `code --disable-workspace-trust -n ${workspacePath}`; + ? `ROO_CODE_IPC_SOCKET_PATH=${ipcSocketPath} xvfb-run --auto-servernum --server-num=1 code --wait --log trace --disable-workspace-trust --disable-gpu --disable-lcd-text --no-sandbox --user-data-dir /roo/.vscode --password-store="basic" -n ${workspacePath}` + : `ROO_CODE_IPC_SOCKET_PATH=${ipcSocketPath} code --disable-workspace-trust -n ${workspacePath}`; logger.info(codeCommand); const subprocess = execa({ - env, shell: '/bin/bash', cancelSignal, })`${codeCommand}`; @@ -91,6 +90,18 @@ export const runTask = async ({ // If debugging, add `--verbose` to `command` and uncomment the following line. // subprocess.stdout.pipe(process.stdout) + try { + await pWaitFor(() => fs.existsSync(ipcSocketPath), { + interval: 250, + timeout: 10_000, + }); + } catch (_error) { + logger.error(`IPC socket was not created within timeout: ${ipcSocketPath}`); + throw new Error( + `IPC socket was not created within timeout -> ${ipcSocketPath}`, + ); + } + let client: IpcClient | undefined = undefined; let attempts = 5; diff --git a/apps/roomote/src/lib/slack.ts b/apps/roomote/src/lib/slack.ts index f3f9a7ca90..8299b9c467 100644 --- a/apps/roomote/src/lib/slack.ts +++ b/apps/roomote/src/lib/slack.ts @@ -47,10 +47,12 @@ export class SlackNotifier { this.logger?.error( `Slack API failed: ${response.status} ${response.statusText}`, ); + return null; } const result: SlackResponse = await response.json(); + console.log('🔗 Slack API Response ->', result); if (!result.ok) { this.logger?.error(`Slack API error: ${result.error}`); @@ -157,4 +159,8 @@ export class SlackNotifier { thread_ts: threadTs, }); } + + public async sendMessage(message: SlackMessage): Promise { + return await this.postMessage(message); + } } diff --git a/apps/roomote/src/lib/utils.ts b/apps/roomote/src/lib/utils.ts index ab6b1a671a..815cbfcdd1 100644 --- a/apps/roomote/src/lib/utils.ts +++ b/apps/roomote/src/lib/utils.ts @@ -1,6 +1,10 @@ import * as fs from 'fs'; import * as path from 'path'; +export const isFlyMachine = () => { + return !!process.env.FLY_IMAGE_REF; +}; + export const isDockerContainer = () => { try { return fs.existsSync('/.dockerenv'); diff --git a/apps/roomote/src/lib/worker.ts b/apps/roomote/src/lib/worker.ts index c41c8355b4..2a0cfe4750 100644 --- a/apps/roomote/src/lib/worker.ts +++ b/apps/roomote/src/lib/worker.ts @@ -3,14 +3,14 @@ import { Worker } from 'bullmq'; import { redis } from './redis'; import { processJob } from './job'; -// docker compose build worker +// docker compose build --build-arg GH_TOKEN=$(npx dotenvx get GH_TOKEN -f .env.development) roomote-worker // docker run \ // --name roomote-worker \ -// --rm \ -// --interactive \ -// --tty \ +// --rm --interactive --tty \ // --network roo-code-cloud_default \ -// -e GH_TOKEN=$GH_TOKEN \ +// -e APP_ENV=production \ +// -e GH_TOKEN=$(npx dotenvx get GH_TOKEN -f .env.production) \ +// -e DOTENV_PRIVATE_KEY_PRODUCTION=$(npx dotenvx get DOTENV_PRIVATE_KEY_PRODUCTION -f .env.keys) \ // -v /var/run/docker.sock:/var/run/docker.sock \ // -v /tmp/roomote:/var/log/roomote \ // roomote-worker sh -c "bash" diff --git a/fly.roomote-worker.toml b/fly.roomote-worker.toml new file mode 100644 index 0000000000..0a75b679e4 --- /dev/null +++ b/fly.roomote-worker.toml @@ -0,0 +1,33 @@ +# See https://fly.io/docs/reference/configuration/ for information about how to use this file. +# +# Deploy with: `fly deploy --config fly.roomote-worker.toml --build-arg GH_TOKEN=$(npx dotenvx get GH_TOKEN -f .env.production)` +# Test with: `fly machine run $(fly releases --image -a roomote-worker -j 2>/dev/null | jq -r '.[0].ImageRef') --vm-size performance-16x --rm --shell --command "pnpm cli:production prompt --text 'Tell me a pirate joke.' --mode code --workspace-path /roo/repos/Roo-Code-Cloud" -a roomote-worker` + +app = "roomote-worker" +primary_region = "sjc" # See `fly platform regions` + +[env] + APP_ENV = "production" + +[build] + dockerfile = ".docker/Dockerfile.roomote-worker" + ignorefile = ".dockerignore" + +# [scaling] +# min_count = 0 +# max_count = 1 + +# [[services]] +# internal_port = 2222 +# protocol = "tcp" +# auto_stop_machines = true +# auto_start_machines = true +# [[services.ports]] +# port = 22 + +[[vm]] + size = "performance-16x" # See `fly platform vm-sizes` + +[[restart]] + policy = "on-failure" + retries = 1 diff --git a/package.json b/package.json index 220bf7da53..d00035bb80 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,8 @@ "env:decrypt:production": "dotenvx decrypt -f .env.production --stdout", "deploy:roomote-dashboard": "fly deploy --config fly.roomote-dashboard.toml", "deploy:roomote-api": "fly deploy --config fly.roomote-api.toml", - "deploy:roomote-controller": "fly deploy --config fly.roomote-controller.toml" + "deploy:roomote-controller": "fly deploy --config fly.roomote-controller.toml", + "deploy:roomote-worker": "fly deploy --config fly.roomote-worker.toml --build-arg GH_TOKEN=$(npx dotenvx get GH_TOKEN -f .env.production) && fly scale count 0 -a roomote-worker -y" }, "devDependencies": { "@dotenvx/dotenvx": "^1.44.2", diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 82206cd30a..acdc453050 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -135,6 +135,7 @@ export type CreateJob = z.infer; export type JobTypes = { [K in CreateJob['type']]: Extract['payload']; }; + /** * JobType, JobStatus, JobPayload, JobParams */