diff --git a/.docker/Dockerfile.roomote-controller b/.docker/Dockerfile.roomote-controller index 95b533c768..3deec55fcd 100644 --- a/.docker/Dockerfile.roomote-controller +++ b/.docker/Dockerfile.roomote-controller @@ -30,6 +30,11 @@ RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o / && apt update && apt install -y docker-ce-cli \ && rm -rf /var/lib/apt/lists/* +# Install flyctl +RUN curl -fsSL https://fly.io/install.sh | sh +RUN echo 'export FLYCTL_INSTALL="/root/.fly"' >> ~/.bashrc +RUN echo "export PATH=\"\$FLYCTL_INSTALL/bin:\$PATH\"" >> ~/.bashrc + WORKDIR /roo COPY package.json pnpm-lock.yaml pnpm-workspace.yaml turbo.json .env.* ./ diff --git a/.env.development b/.env.development index 8dd15bb1d9..8d45226450 100644 --- a/.env.development +++ b/.env.development @@ -22,3 +22,4 @@ SLACK_API_TOKEN=encrypted:BJyTFw3gugKZr6uaNZaqCH0UpW/mP4S8P1oL5NzeCvoZzSMJEcDrKs +FLY_ACCESS_TOKEN="encrypted:BDm2qpf85nXBtq7KENGMiwcsFg3Ln/gRcZoUm5busQ8tRgEDXhlXUB4gSUT+Z/XRvwurBcBmYZhkjkDycNQLME53htE8lMLfBqmPTIWFgHhKVZWtlk3EhC7GZcycuLYKZVFuEsvKxu4kH5JibqUtJ5fRjKEbQG1HXuTvwuwGjSahgZC3I+Q1xb/83+nFmHKAr0ibaLHHO/mbZZ4YLSS91dw/BiVPmhaY2eDpdDvrLkYZVJZ28f+JkYocYyNUdaCnUqlzfpQAa6xBTelk9OaNwm3XnCYlm+VssMQpN6z5iOyBDG4Jr2Wx8eNyg9PC5lOptweYRYh2ho3RYW0gDwYujfFfRpMKGaEnCB2l/++iDWB3LY9p3xh8ysJ59OTMb8+luOHJbZdiwbnOgY4WcaF0nJyPNX8aAq22Z/imvFqb5hFnytg5kD2mn4h6KG36NmldAw/zJeyat8j4KQEZSW9pBv/vXN2UtTWnA+vsgnua5SHaHoa2GHN/8HyODbdNPWv1tWnjOhZGbxa1+MzkX/4w6BS9jNG+6Pe4m0YVZMjX/Bv6B8qIrcKqVouBrsKIjN9BSFdQzqlDLk6GsP25Nu/eTQS8+/fuSbI+mn7eo84xHWtawlgej4ydkNDOBVmUV9xgoST8LGq6H6Gq5zvuNyY8IStglpybN1yQzEP2RT+JNM112qQK5EWuWTEGuMq4C8ZIIO8ETgDgDjVBGuEeOVFIHCpXzrwB9HTWf3W4DO7JKeNRBmbYkNBTbr3AqzqRKa/+aTw4G8YTyn0T++AaYKUv9KZAajtZ6TQKL1bkGOvsqBBzRW1/NCyuRxEt7+r/ffx0zo7qLW6ir2gTZuDoPi8kQIu8RxkGJypg8QYZHGtr3yGXxm8IEjPlCYINx2RWZjFr0RDO6FRqlexCnnnj10n+TZJgeEQb0esGXJd20klhYwngi9BnvIWoOPLPPeC+JHeILVAUgAxKjsspF5ck3I/Td2u/" diff --git a/.env.preview b/.env.preview index 60006006cd..ea1d6e4a44 100644 --- a/.env.preview +++ b/.env.preview @@ -20,3 +20,4 @@ OPENROUTER_API_KEY=encrypted:BDoXCMz3fV5DXa89cwd77/pZhw6tVIw96LvtyZEFwFQtQV1Qk+Z SLACK_API_TOKEN=encrypted:BEmAqaFftmVNlZdx19KGXBDtJ1E1SFBJrAH52pwhFR00Eq6DJAOJSBLY+9lV4wsDRnhTaR8QBQ5fx4kX6751mqJ/Ldu9n7zYV+NTV5cWowvTh4jD+w7yufbUvNgnqs1CpvGXysaJoyQCQqQlUJ+wxlf+rb3yEN5CMR9y1Es/d+cXF2d73H2VfBDtVcKf1pCPn+kVeQW9XHquiw== +FLY_ACCESS_TOKEN="encrypted:BCoSnp5DqB0Td27HYtDwIS/dZfZkyvjsgUz+0XcKdovFMr3/ZRZkjLW8GgBpNIdvxYz+/iS8wGPtnFHmD73Kf1+CU5eKNWUTwRaxoD7GyCTXvWwYnNt55unoNpr51rCWTgQ66M8aQGm85gC+948woiL64i5ozwWezkxbPtk4vOM5dbbeIObzfPoPKOqFDOIdFjfbaEFpd6Yuw2BrxCT9HExiK+IOp2xUNEvB4/w4xbzenDDkkyw+kNx0hcFvkefX2EkuSHRS3nPVKMFVH2Pcfc7HuccAbmjIvlvuKMulyiAyJi7XrkyW46ZN7BCRokYR0j7zIqAhQmNgYgs37tbiq+CvqDHbkOEqGIZZIxV+pzONbJ07moAbDBpBeMJ6WeH6gAURJ3mqa3tCyCEFk8TxHMe5lbfjJGuYe4SakrOTMPOMH451gtw5KdUN6m7XFKfL1+MbklpIy/lVgOWAKiyMlxxUmeDShn4ElMIKEGYxY8gOcI5Akyn/3B3tk1RgutTQ7p3gjya5GCf2hKNnOAAjlRzOfDbnObslQZv/ZG1QQ81VLqpvj0S2zMDITwjuGCey6MT9UXrVs1JYaLPsJHiEKlylWuXRenWBaVz2iRdFugRhx1PbisOPkgrqhU5K/B5d8vpsuKOCX65ty8CkJq3KoW6TzzkO2nAm8LJ10Ihv3y+fGRCgUUmjLmSOzg7yG6a89wklCUW3BS3bDdvgEfBMGcdPH/9BTVOQ5wpXZWB9CX4tvPLlXnLyWqH6VWShsfhRRL0MoQ9PXMD9NK61jHIw8RXB1xAASL8B+NeVthj490ymhUHuaL0nBetS38W+tD6jE5IVAyvaR4tvz/4eJz/uxbvgqCJyd4GFW8kyp6blTziksryr4TlLzS7d1F84WKIjUcNyaG5te/L4GixvnPQ5nxFo2Ib464knuljvjdDRxLiOk4H8k7S45OLx2seSsV+KTGlKeN3S013rpXFnG5APUEvz" diff --git a/.env.production b/.env.production index 07e35210c4..c854f7f02b 100644 --- a/.env.production +++ b/.env.production @@ -20,3 +20,4 @@ OPENROUTER_API_KEY=encrypted:BLBANC6AI/i5gHIeiBaY5lOogEEahxSLAvsxUpk6Z1FXtfrIEj/ SLACK_API_TOKEN=encrypted:BHQC5Lv0k3VnZ0Ez4TcVvL0meJs8R5ysPFa47U3kkjOJstrtDa26F8Fjvd8/Qr7ztiKdJzGXpr9k4/Tm8mL9jZvG7HNH9D+fVHkyGZ2Mf7B97FdXjiDxA3f+Pifj0aTfM7U58tDrKTLeQ/6UzE3Emcxyg4CxKjAjEaRFOAiQxs1lVAtqzbeF7OCV6GHScpSe4zoAaFXvtQ07FA== +FLY_ACCESS_TOKEN="encrypted:BFcS7pt6Evj8OG1khLFJkqzg2MI/pP4g4tP1l85vbsiTL1x1OvjsIqhZvGUm/PkamInYMTVOhWOWHx/s0Iv3F++FfXeWCzRTaaGvCupMgGB5BEEckODT+4GLrsc14NUCT6p+DE/iREzkJFlHjPwj8NergqnUwWmxhNyFDxFK0AZKpZAKSf69/WrAHDN5I907mOm4wUkPag03L9SuhaotCedsv/zqvIfvbr8gxyk7N1vphXaiihVtE+GIaBnHkHjz3eiS/kDtKFPPeZT1lS9yE9T00Bah2hZkSIxpIu4Cd3zAAWYIplqlhDrO32tm89gPSeQfn7ORaHPogK5I8n9ctlKJV8PFO9YoYqaq8TUnkDzvLj7BNk7GpPibodl2mSX8eVspJXCSewKhnbVFA1RNWJ6B3Bqp5n3PmmBaF4n0VCYMzxUHT4UvOfoQGH1GhzVQiQkV0+s8jfD3eRFiovpvOc89WXmuhtGGMwZ0pYrFWg1qTgmUndcfKzob7yzvrlLpJ+kqp6X0c+cO26yOfN+7LfnZt9BeuzzHSEN9+aEWfHv8IK/xqDjT/V8ZoctOGJG/nFFqF7OeYk2Er0AvD+53kwXBCKiBRjx/b2w6s+Z4f71+62alH/Pm62Q3vWGpXIuj8GF7CaHIQD9FH6cegOa7klabrdXjyIETQLPtpKniwVJbaRhEj9p+WOB0dhibZVFX/LtQ8hIt1HuV0TcoB1jLToN835gPuPV5db+Zs0qf3v0lgM4dRM2N3R7d+XdTVZVvfpdJTTNpN9P6AMrHjTHGp2sCDkInymjrhZlpwD3h3XcsnbcGkjTraJsoNsfGGoH1JUyX0AVyF4ec61eFiaFec4RGH2Oxn/rdCq/X4xbqM7ZQhxhhBhkyIMlztRONXYfz8Jqn+f9KK4lxuhg5Ao/xWWyqv5UCeGh2KB32CO1ez3ei9QqD72wyw/NJLLh8XPW1f9TPuFQsfQTG1ftljC2w/XGu" diff --git a/.env.test b/.env.test index 2ada575edb..65e97a051d 100644 --- a/.env.test +++ b/.env.test @@ -19,3 +19,4 @@ GH_TOKEN=encrypted:BGefstTA/5VjgS/e9iy7TCvM64YaKDDgp3SSr6IUGZfWrfqAhyMZe5VmxyNlN OPENROUTER_API_KEY=encrypted:BJHyYNaUsHiU6By54yUA2lG48W6NFuafGOr6Sc4XDopmTB9WulF1LbWrP0hiHY+GcWNhb3JR6V4XSOoqQDjNPza65uJCVOob1bWdC1vVoLBjxLev19OUI53jCi0kQVuNYz7P7zI5/YTWk0noSDU= SLACK_API_TOKEN=encrypted:BDkskSYEb9SYQLpS0LyyOMgS7pVOyha9agMrS15iQUBoeULTzXbfGxEc+w3oEUa7mFyuRW+gHpo1jqEYicc2Nm9xXawDFsfhoTgXkhUdDlAVsJY/jctWtw97Hb9qLvigRrJWyx7ha5wUMg== +FLY_ACCESS_TOKEN="encrypted:BIobJarcf4MpmWDkSjVGjXUH87dNj5rbMqow0/7QNpa1s3eMZeRNoi8oMGn3lBSgPv/hZjCUJJ9t8+5mZNgZk0CrrLUz9W1VxD1hIJwzYlKA6vhR8Ef+A8z6W4H+XBl3zFXPxfvfdy+v" diff --git a/ENV_VARS.md b/ENV_VARS.md new file mode 100644 index 0000000000..af2863771a --- /dev/null +++ b/ENV_VARS.md @@ -0,0 +1,19 @@ +# ENV Vars + +Add a new encrypted variable: + +```sh +npx dotenvx set FLY_ACCESS_TOKEN fm2_... -f .env.production +``` + +View an existing encrypted variable: + +```sh +npx dotenvx get FLY_ACCESS_TOKEN -f .env.production +``` + +These secrets can also be set on Fly.io containers: + +```sh +fly secrets set FLY_ACCESS_TOKEN=fm2_... -a roomote-worker +``` diff --git a/apps/roomote/src/lib/checkStalledJobs.ts b/apps/roomote/src/lib/checkStalledJobs.ts new file mode 100644 index 0000000000..d59cb4509b --- /dev/null +++ b/apps/roomote/src/lib/checkStalledJobs.ts @@ -0,0 +1,37 @@ +// npx dotenvx run -f ../../.env.production -- tsx src/lib/checkStalledJobs.ts + +import { Worker } from 'bullmq'; + +import { redis } from './redis'; + +async function checkStalledJobs() { + const worker = new Worker('roomote', undefined, { + autorun: false, + connection: redis, + }); + + while (true) { + console.log('startStalledCheckTimer()'); + await worker.startStalledCheckTimer(); + await new Promise((resolve) => setTimeout(resolve, 30_000)); + } +} + +process.on('SIGTERM', async () => { + console.log('SIGTERM'); + process.exit(0); +}); + +process.on('SIGINT', async () => { + console.log('SIGINT'); + process.exit(0); +}); + +checkStalledJobs() + .then(() => { + process.exit(0); + }) + .catch((error) => { + console.error(error); + process.exit(1); + }); diff --git a/apps/roomote/src/lib/controller.ts b/apps/roomote/src/lib/controller.ts index 7de9c2a1ed..3375583966 100644 --- a/apps/roomote/src/lib/controller.ts +++ b/apps/roomote/src/lib/controller.ts @@ -1,12 +1,12 @@ -import { spawn } from 'child_process'; import { Queue } from 'bullmq'; +import { execa } from 'execa'; import { redis } from './redis'; import { isFlyMachine, isDockerContainer } from './utils'; export class WorkerController { private readonly POLL_INTERVAL_MS = 5000; - private readonly MAX_WORKERS = 5; + private readonly MAX_WORKERS = 2; private queue: Queue; public isRunning = false; @@ -84,7 +84,7 @@ export class WorkerController { let command; if (isFlyMachine()) { - command = `fly machine run $(fly releases --image -a roomote-worker -j 2>/dev/null | jq -r '.[0].ImageRef') --vm-size performance-16x --rm --shell --command "pnpm worker:production" -a roomote-worker`; + command = `fly machine run $(fly releases --image -a roomote-worker -j 2>/dev/null | jq -r '.[0].ImageRef') --vm-size performance-16x --restart on-fail --rm --shell --command "pnpm worker:production" -a roomote-worker`; } else if (isDockerContainer()) { const dockerArgs = [ `--name roomote-${workerId}`, @@ -103,38 +103,21 @@ export class WorkerController { } console.log('Spawning worker with command:', command); - - const childProcess = spawn('sh', ['-c', command], { - detached: true, - stdio: ['ignore', 'pipe', 'pipe'], - }); - - if (childProcess.stdout) { - childProcess.stdout.on('data', (data) => { - console.log(data.toString()); - }); - } - - if (childProcess.stderr) { - childProcess.stderr.on('data', (data) => { - console.error(data.toString()); - }); - } + const subprocess = execa({ shell: '/bin/bash' })`${command}`; + subprocess.stdout?.pipe(process.stdout); + subprocess.stderr?.pipe(process.stderr); this.activeWorkers.add(workerId); - childProcess.on('exit', (code) => { - console.log(`Worker ${workerId} exited with code ${code}`); - this.activeWorkers.delete(workerId); - }); - - childProcess.on('error', (error) => { - console.error(`Worker ${workerId} error:`, error); - this.activeWorkers.delete(workerId); - }); - - // Detach the process so it can run independently. - childProcess.unref(); + subprocess + .then((result) => { + console.log(`Worker ${workerId} exited with code ${result.exitCode}`); + this.activeWorkers.delete(workerId); + }) + .catch((error) => { + console.error(`Worker ${workerId} error:`, error); + this.activeWorkers.delete(workerId); + }); } catch (error) { console.error(`Failed to spawn worker ${workerId}:`, error); this.activeWorkers.delete(workerId); @@ -142,7 +125,6 @@ export class WorkerController { } } -// Only run if this file is executed directly (not imported). if (import.meta.url === `file://${process.argv[1]}`) { const controller = new WorkerController(); diff --git a/fly.roomote-controller.toml b/fly.roomote-controller.toml index 6abc77a094..b4206d8f7d 100644 --- a/fly.roomote-controller.toml +++ b/fly.roomote-controller.toml @@ -1,6 +1,7 @@ # See https://fly.io/docs/reference/configuration/ for information about how to use this file. # # Deploy with: `fly deploy --config fly.roomote-controller.toml` +# Connect with: `fly machine run $(fly releases --image -a roomote-controller -j 2>/dev/null | jq -r '.[0].ImageRef') --rm --shell -a roomote-controller` app = "roomote-controller" primary_region = "sjc" # See `fly platform regions` diff --git a/fly.roomote-worker.toml b/fly.roomote-worker.toml index 0a75b679e4..17c3e8ed71 100644 --- a/fly.roomote-worker.toml +++ b/fly.roomote-worker.toml @@ -1,7 +1,7 @@ # See https://fly.io/docs/reference/configuration/ for information about how to use this file. # # Deploy with: `fly deploy --config fly.roomote-worker.toml --build-arg GH_TOKEN=$(npx dotenvx get GH_TOKEN -f .env.production)` -# Test with: `fly machine run $(fly releases --image -a roomote-worker -j 2>/dev/null | jq -r '.[0].ImageRef') --vm-size performance-16x --rm --shell --command "pnpm cli:production prompt --text 'Tell me a pirate joke.' --mode code --workspace-path /roo/repos/Roo-Code-Cloud" -a roomote-worker` +# Test with: `fly machine run $(fly releases --image -a roomote-worker -j 2>/dev/null | jq -r '.[0].ImageRef') --vm-size performance-16x --restart on-fail --rm --shell --command "pnpm cli:production prompt --text 'Tell me a pirate joke.' --mode code --workspace-path /roo/repos/Roo-Code-Cloud" -a roomote-worker` app = "roomote-worker" primary_region = "sjc" # See `fly platform regions`