Spawn fly machine workers on production (#154)

Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com>

.docker/Dockerfile.roomote-controller

@@ -30,6 +30,11 @@ RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /
   && apt update && apt install -y docker-ce-cli \
   && rm -rf /var/lib/apt/lists/*
 
+# Install flyctl
+RUN curl -fsSL https://fly.io/install.sh | sh
+RUN echo 'export FLYCTL_INSTALL="/root/.fly"' >> ~/.bashrc
+RUN echo "export PATH=\"\$FLYCTL_INSTALL/bin:\$PATH\"" >> ~/.bashrc
+
 WORKDIR /roo
 
 COPY package.json pnpm-lock.yaml pnpm-workspace.yaml turbo.json .env.* ./

.env.development

@@ -22,3 +22,4 @@ SLACK_API_TOKEN=encrypted:BJyTFw3gugKZr6uaNZaqCH0UpW/mP4S8P1oL5NzeCvoZzSMJEcDrKs
 
 
 
+FLY_ACCESS_TOKEN="encrypted:BDm2qpf85nXBtq7KENGMiwcsFg3Ln/gRcZoUm5busQ8tRgEDXhlXUB4gSUT+Z/XRvwurBcBmYZhkjkDycNQLME53htE8lMLfBqmPTIWFgHhKVZWtlk3EhC7GZcycuLYKZVFuEsvKxu4kH5JibqUtJ5fRjKEbQG1HXuTvwuwGjSahgZC3I+Q1xb/83+nFmHKAr0ibaLHHO/mbZZ4YLSS91dw/BiVPmhaY2eDpdDvrLkYZVJZ28f+JkYocYyNUdaCnUqlzfpQAa6xBTelk9OaNwm3XnCYlm+VssMQpN6z5iOyBDG4Jr2Wx8eNyg9PC5lOptweYRYh2ho3RYW0gDwYujfFfRpMKGaEnCB2l/++iDWB3LY9p3xh8ysJ59OTMb8+luOHJbZdiwbnOgY4WcaF0nJyPNX8aAq22Z/imvFqb5hFnytg5kD2mn4h6KG36NmldAw/zJeyat8j4KQEZSW9pBv/vXN2UtTWnA+vsgnua5SHaHoa2GHN/8HyODbdNPWv1tWnjOhZGbxa1+MzkX/4w6BS9jNG+6Pe4m0YVZMjX/Bv6B8qIrcKqVouBrsKIjN9BSFdQzqlDLk6GsP25Nu/eTQS8+/fuSbI+mn7eo84xHWtawlgej4ydkNDOBVmUV9xgoST8LGq6H6Gq5zvuNyY8IStglpybN1yQzEP2RT+JNM112qQK5EWuWTEGuMq4C8ZIIO8ETgDgDjVBGuEeOVFIHCpXzrwB9HTWf3W4DO7JKeNRBmbYkNBTbr3AqzqRKa/+aTw4G8YTyn0T++AaYKUv9KZAajtZ6TQKL1bkGOvsqBBzRW1/NCyuRxEt7+r/ffx0zo7qLW6ir2gTZuDoPi8kQIu8RxkGJypg8QYZHGtr3yGXxm8IEjPlCYINx2RWZjFr0RDO6FRqlexCnnnj10n+TZJgeEQb0esGXJd20klhYwngi9BnvIWoOPLPPeC+JHeILVAUgAxKjsspF5ck3I/Td2u/"

.env.preview

@@ -20,3 +20,4 @@ OPENROUTER_API_KEY=encrypted:BDoXCMz3fV5DXa89cwd77/pZhw6tVIw96LvtyZEFwFQtQV1Qk+Z
 
 SLACK_API_TOKEN=encrypted:BEmAqaFftmVNlZdx19KGXBDtJ1E1SFBJrAH52pwhFR00Eq6DJAOJSBLY+9lV4wsDRnhTaR8QBQ5fx4kX6751mqJ/Ldu9n7zYV+NTV5cWowvTh4jD+w7yufbUvNgnqs1CpvGXysaJoyQCQqQlUJ+wxlf+rb3yEN5CMR9y1Es/d+cXF2d73H2VfBDtVcKf1pCPn+kVeQW9XHquiw==
 
+FLY_ACCESS_TOKEN="encrypted:BCoSnp5DqB0Td27HYtDwIS/dZfZkyvjsgUz+0XcKdovFMr3/ZRZkjLW8GgBpNIdvxYz+/iS8wGPtnFHmD73Kf1+CU5eKNWUTwRaxoD7GyCTXvWwYnNt55unoNpr51rCWTgQ66M8aQGm85gC+948woiL64i5ozwWezkxbPtk4vOM5dbbeIObzfPoPKOqFDOIdFjfbaEFpd6Yuw2BrxCT9HExiK+IOp2xUNEvB4/w4xbzenDDkkyw+kNx0hcFvkefX2EkuSHRS3nPVKMFVH2Pcfc7HuccAbmjIvlvuKMulyiAyJi7XrkyW46ZN7BCRokYR0j7zIqAhQmNgYgs37tbiq+CvqDHbkOEqGIZZIxV+pzONbJ07moAbDBpBeMJ6WeH6gAURJ3mqa3tCyCEFk8TxHMe5lbfjJGuYe4SakrOTMPOMH451gtw5KdUN6m7XFKfL1+MbklpIy/lVgOWAKiyMlxxUmeDShn4ElMIKEGYxY8gOcI5Akyn/3B3tk1RgutTQ7p3gjya5GCf2hKNnOAAjlRzOfDbnObslQZv/ZG1QQ81VLqpvj0S2zMDITwjuGCey6MT9UXrVs1JYaLPsJHiEKlylWuXRenWBaVz2iRdFugRhx1PbisOPkgrqhU5K/B5d8vpsuKOCX65ty8CkJq3KoW6TzzkO2nAm8LJ10Ihv3y+fGRCgUUmjLmSOzg7yG6a89wklCUW3BS3bDdvgEfBMGcdPH/9BTVOQ5wpXZWB9CX4tvPLlXnLyWqH6VWShsfhRRL0MoQ9PXMD9NK61jHIw8RXB1xAASL8B+NeVthj490ymhUHuaL0nBetS38W+tD6jE5IVAyvaR4tvz/4eJz/uxbvgqCJyd4GFW8kyp6blTziksryr4TlLzS7d1F84WKIjUcNyaG5te/L4GixvnPQ5nxFo2Ib464knuljvjdDRxLiOk4H8k7S45OLx2seSsV+KTGlKeN3S013rpXFnG5APUEvz"

.env.production

@@ -20,3 +20,4 @@ OPENROUTER_API_KEY=encrypted:BLBANC6AI/i5gHIeiBaY5lOogEEahxSLAvsxUpk6Z1FXtfrIEj/
 
 SLACK_API_TOKEN=encrypted:BHQC5Lv0k3VnZ0Ez4TcVvL0meJs8R5ysPFa47U3kkjOJstrtDa26F8Fjvd8/Qr7ztiKdJzGXpr9k4/Tm8mL9jZvG7HNH9D+fVHkyGZ2Mf7B97FdXjiDxA3f+Pifj0aTfM7U58tDrKTLeQ/6UzE3Emcxyg4CxKjAjEaRFOAiQxs1lVAtqzbeF7OCV6GHScpSe4zoAaFXvtQ07FA==
 
+FLY_ACCESS_TOKEN="encrypted:BFcS7pt6Evj8OG1khLFJkqzg2MI/pP4g4tP1l85vbsiTL1x1OvjsIqhZvGUm/PkamInYMTVOhWOWHx/s0Iv3F++FfXeWCzRTaaGvCupMgGB5BEEckODT+4GLrsc14NUCT6p+DE/iREzkJFlHjPwj8NergqnUwWmxhNyFDxFK0AZKpZAKSf69/WrAHDN5I907mOm4wUkPag03L9SuhaotCedsv/zqvIfvbr8gxyk7N1vphXaiihVtE+GIaBnHkHjz3eiS/kDtKFPPeZT1lS9yE9T00Bah2hZkSIxpIu4Cd3zAAWYIplqlhDrO32tm89gPSeQfn7ORaHPogK5I8n9ctlKJV8PFO9YoYqaq8TUnkDzvLj7BNk7GpPibodl2mSX8eVspJXCSewKhnbVFA1RNWJ6B3Bqp5n3PmmBaF4n0VCYMzxUHT4UvOfoQGH1GhzVQiQkV0+s8jfD3eRFiovpvOc89WXmuhtGGMwZ0pYrFWg1qTgmUndcfKzob7yzvrlLpJ+kqp6X0c+cO26yOfN+7LfnZt9BeuzzHSEN9+aEWfHv8IK/xqDjT/V8ZoctOGJG/nFFqF7OeYk2Er0AvD+53kwXBCKiBRjx/b2w6s+Z4f71+62alH/Pm62Q3vWGpXIuj8GF7CaHIQD9FH6cegOa7klabrdXjyIETQLPtpKniwVJbaRhEj9p+WOB0dhibZVFX/LtQ8hIt1HuV0TcoB1jLToN835gPuPV5db+Zs0qf3v0lgM4dRM2N3R7d+XdTVZVvfpdJTTNpN9P6AMrHjTHGp2sCDkInymjrhZlpwD3h3XcsnbcGkjTraJsoNsfGGoH1JUyX0AVyF4ec61eFiaFec4RGH2Oxn/rdCq/X4xbqM7ZQhxhhBhkyIMlztRONXYfz8Jqn+f9KK4lxuhg5Ao/xWWyqv5UCeGh2KB32CO1ez3ei9QqD72wyw/NJLLh8XPW1f9TPuFQsfQTG1ftljC2w/XGu"

.env.test

@@ -19,3 +19,4 @@ GH_TOKEN=encrypted:BGefstTA/5VjgS/e9iy7TCvM64YaKDDgp3SSr6IUGZfWrfqAhyMZe5VmxyNlN
 OPENROUTER_API_KEY=encrypted:BJHyYNaUsHiU6By54yUA2lG48W6NFuafGOr6Sc4XDopmTB9WulF1LbWrP0hiHY+GcWNhb3JR6V4XSOoqQDjNPza65uJCVOob1bWdC1vVoLBjxLev19OUI53jCi0kQVuNYz7P7zI5/YTWk0noSDU=
 
 SLACK_API_TOKEN=encrypted:BDkskSYEb9SYQLpS0LyyOMgS7pVOyha9agMrS15iQUBoeULTzXbfGxEc+w3oEUa7mFyuRW+gHpo1jqEYicc2Nm9xXawDFsfhoTgXkhUdDlAVsJY/jctWtw97Hb9qLvigRrJWyx7ha5wUMg==
+FLY_ACCESS_TOKEN="encrypted:BIobJarcf4MpmWDkSjVGjXUH87dNj5rbMqow0/7QNpa1s3eMZeRNoi8oMGn3lBSgPv/hZjCUJJ9t8+5mZNgZk0CrrLUz9W1VxD1hIJwzYlKA6vhR8Ef+A8z6W4H+XBl3zFXPxfvfdy+v"

ENV_VARS.md

@@ -0,0 +1,19 @@
+# ENV Vars
+
+Add a new encrypted variable:
+
+```sh
+npx dotenvx set FLY_ACCESS_TOKEN fm2_... -f .env.production
+```
+
+View an existing encrypted variable:
+
+```sh
+npx dotenvx get FLY_ACCESS_TOKEN -f .env.production
+```
+
+These secrets can also be set on Fly.io containers:
+
+```sh
+fly secrets set FLY_ACCESS_TOKEN=fm2_... -a roomote-worker
+```

apps/roomote/src/lib/checkStalledJobs.ts

@@ -0,0 +1,37 @@
+// npx dotenvx run -f ../../.env.production -- tsx src/lib/checkStalledJobs.ts
+
+import { Worker } from 'bullmq';
+
+import { redis } from './redis';
+
+async function checkStalledJobs() {
+  const worker = new Worker('roomote', undefined, {
+    autorun: false,
+    connection: redis,
+  });
+
+  while (true) {
+    console.log('startStalledCheckTimer()');
+    await worker.startStalledCheckTimer();
+    await new Promise((resolve) => setTimeout(resolve, 30_000));
+  }
+}
+
+process.on('SIGTERM', async () => {
+  console.log('SIGTERM');
+  process.exit(0);
+});
+
+process.on('SIGINT', async () => {
+  console.log('SIGINT');
+  process.exit(0);
+});
+
+checkStalledJobs()
+  .then(() => {
+    process.exit(0);
+  })
+  .catch((error) => {
+    console.error(error);
+    process.exit(1);
+  });

apps/roomote/src/lib/controller.ts

@@ -1,12 +1,12 @@
-import { spawn } from 'child_process';
 import { Queue } from 'bullmq';
+import { execa } from 'execa';
 
 import { redis } from './redis';
 import { isFlyMachine, isDockerContainer } from './utils';
 
 export class WorkerController {
   private readonly POLL_INTERVAL_MS = 5000;
-  private readonly MAX_WORKERS = 5;
+  private readonly MAX_WORKERS = 2;
 
   private queue: Queue;
   public isRunning = false;
@@ -84,7 +84,7 @@ export class WorkerController {
       let command;
 
       if (isFlyMachine()) {
-        command = `fly machine run $(fly releases --image -a roomote-worker -j 2>/dev/null | jq -r '.[0].ImageRef') --vm-size performance-16x --rm --shell --command "pnpm worker:production" -a roomote-worker`;
+        command = `fly machine run $(fly releases --image -a roomote-worker -j 2>/dev/null | jq -r '.[0].ImageRef') --vm-size performance-16x --restart on-fail --rm --shell --command "pnpm worker:production" -a roomote-worker`;
       } else if (isDockerContainer()) {
         const dockerArgs = [
           `--name roomote-${workerId}`,
@@ -103,38 +103,21 @@ export class WorkerController {
       }
 
       console.log('Spawning worker with command:', command);
-
-      const childProcess = spawn('sh', ['-c', command], {
-        detached: true,
-        stdio: ['ignore', 'pipe', 'pipe'],
-      });
-
-      if (childProcess.stdout) {
-        childProcess.stdout.on('data', (data) => {
-          console.log(data.toString());
-        });
-      }
-
-      if (childProcess.stderr) {
-        childProcess.stderr.on('data', (data) => {
-          console.error(data.toString());
-        });
-      }
+      const subprocess = execa({ shell: '/bin/bash' })`${command}`;
+      subprocess.stdout?.pipe(process.stdout);
+      subprocess.stderr?.pipe(process.stderr);
 
       this.activeWorkers.add(workerId);
 
-      childProcess.on('exit', (code) => {
-        console.log(`Worker ${workerId} exited with code ${code}`);
-        this.activeWorkers.delete(workerId);
-      });
-
-      childProcess.on('error', (error) => {
-        console.error(`Worker ${workerId} error:`, error);
-        this.activeWorkers.delete(workerId);
-      });
-
-      // Detach the process so it can run independently.
-      childProcess.unref();
+      subprocess
+        .then((result) => {
+          console.log(`Worker ${workerId} exited with code ${result.exitCode}`);
+          this.activeWorkers.delete(workerId);
+        })
+        .catch((error) => {
+          console.error(`Worker ${workerId} error:`, error);
+          this.activeWorkers.delete(workerId);
+        });
     } catch (error) {
       console.error(`Failed to spawn worker ${workerId}:`, error);
       this.activeWorkers.delete(workerId);
@@ -142,7 +125,6 @@ export class WorkerController {
   }
 }
 
-// Only run if this file is executed directly (not imported).
 if (import.meta.url === `file://${process.argv[1]}`) {
   const controller = new WorkerController();
 

fly.roomote-controller.toml

@@ -1,6 +1,7 @@
 # See https://fly.io/docs/reference/configuration/ for information about how to use this file.
 #
 # Deploy with: `fly deploy --config fly.roomote-controller.toml`
+# Connect with: `fly machine run $(fly releases --image -a roomote-controller -j 2>/dev/null | jq -r '.[0].ImageRef') --rm --shell -a roomote-controller`
 
 app = "roomote-controller"
 primary_region = "sjc" # See `fly platform regions`

fly.roomote-worker.toml

@@ -1,7 +1,7 @@
 # See https://fly.io/docs/reference/configuration/ for information about how to use this file.
 #
 # Deploy with: `fly deploy --config fly.roomote-worker.toml --build-arg GH_TOKEN=$(npx dotenvx get GH_TOKEN -f .env.production)`
-# Test with: `fly machine run $(fly releases --image -a roomote-worker -j 2>/dev/null | jq -r '.[0].ImageRef') --vm-size performance-16x --rm --shell --command "pnpm cli:production prompt --text 'Tell me a pirate joke.' --mode code --workspace-path /roo/repos/Roo-Code-Cloud" -a roomote-worker`
+# Test with: `fly machine run $(fly releases --image -a roomote-worker -j 2>/dev/null | jq -r '.[0].ImageRef') --vm-size performance-16x --restart on-fail --rm --shell --command "pnpm cli:production prompt --text 'Tell me a pirate joke.' --mode code --workspace-path /roo/repos/Roo-Code-Cloud" -a roomote-worker`
 
 app = "roomote-worker"
 primary_region = "sjc" # See `fly platform regions`