Merge pull request #903 from fabro-sh/codex/fix-nightly-docker-timeouts

Fix Docker test timeouts in nightly and release CI

.config/nextest.toml

@@ -7,6 +7,18 @@ leak-timeout = "500ms"
     filter = "package(fabro-cli)"
     slow-timeout = { period = "6s", terminate-after = 4 }
 
+    # Docker scenarios allow 60s for a cold image pull; leave time for cleanup.
+    # Keep this before the general server override because the first match wins.
+    [[profile.default.overrides]]
+    filter = """
+        package(fabro-server) & (
+            test(=scenario::petri::a_runs_projection_carries_its_docker_sandbox_instance)
+            | test(=scenario::petri::the_server_attaches_to_the_container_petri_created)
+            | test(=scenario::petri::a_bundle_naming_a_catalog_environment_runs_on_docker_with_its_image)
+        )
+    """
+    slow-timeout = { period = "30s", terminate-after = 3 }
+
     [[profile.default.overrides]]
     filter = "package(fabro-server)"
     slow-timeout = { period = "5s", terminate-after = 4 }

.github/workflows/nightly.yml

@@ -61,6 +61,15 @@ jobs:
       - uses: taiki-e/install-action@773334c0e05d7e699e4d78234494308223f3a2cf # nextest
         if: steps.skip.outputs.skip != 'true'
 
+      - name: Pull the images the Docker tests run
+        if: steps.skip.outputs.skip != 'true'
+        run: |
+          backend="$(dirname "$(cargo metadata --locked --format-version 1 | jq -r '.packages[] | select(.name == "petri-executor-sandbox") | .manifest_path')")/src/backend.rs"
+          pin="$(sed -n 's/^const RUNNER_PIN: &str = "\([0-9a-f]*\)";$/\1/p' "$backend")"
+          test -n "$pin"
+          docker pull "ghcr.io/lithoscomputer/ubuntu-24.04:slim-$pin"
+          docker pull ghcr.io/lithoscomputer/ubuntu-22.04:slim
+
       - name: Release nightly
         if: steps.skip.outputs.skip != 'true'
         run: |

.github/workflows/release.yml

@@ -88,6 +88,15 @@ jobs:
       - name: Refresh embedded SPA
         run: cargo --locked dev spa refresh
 
+      - name: Pull the images the Docker tests run
+        if: runner.os == 'Linux' && matrix.target != 'aarch64-unknown-linux-musl'
+        run: |
+          backend="$(dirname "$(cargo metadata --locked --format-version 1 | jq -r '.packages[] | select(.name == "petri-executor-sandbox") | .manifest_path')")/src/backend.rs"
+          pin="$(sed -n 's/^const RUNNER_PIN: &str = "\([0-9a-f]*\)";$/\1/p' "$backend")"
+          test -n "$pin"
+          docker pull "ghcr.io/lithoscomputer/ubuntu-24.04:slim-$pin"
+          docker pull ghcr.io/lithoscomputer/ubuntu-22.04:slim
+
       - name: Test (x86_64-musl)
         # nextest still shells through cargo test for this target, so
         # build.rs C code needs an explicit musl compiler/linker.

lib/apps/fabro-server/tests/it/helpers.rs

@@ -16,7 +16,7 @@ use fabro_test::{
     expect_axum_status_in, expect_axum_text,
 };
 use fabro_types::ServerSettings;
-use tokio::time::sleep;
+use tokio::time::{self, sleep};
 use tower::ServiceExt;
 
 pub(crate) const MINIMAL_DOT: &str = r#"digraph Test {
@@ -307,18 +307,36 @@ pub(crate) async fn wait_for_run_status(
     run_id: &str,
     expected: &[&str],
 ) -> String {
-    for _ in 0..POLL_ATTEMPTS {
-        let body = run_json(app, run_id).await;
-        let status = body["lifecycle"]["status"]["kind"]
-            .as_str()
-            .expect("run response should include a tagged status kind")
-            .to_string();
-        if expected.iter().any(|candidate| *candidate == status) {
-            return status;
+    wait_for_run_status_with_timeout(app, run_id, expected, Duration::from_secs(5)).await
+}
+
+pub(crate) async fn wait_for_run_status_with_timeout(
+    app: &axum::Router,
+    run_id: &str,
+    expected: &[&str],
+    timeout: Duration,
+) -> String {
+    let mut last_body = serde_json::Value::Null;
+    time::timeout(timeout, async {
+        loop {
+            last_body = run_json(app, run_id).await;
+            let status = last_body["lifecycle"]["status"]["kind"]
+                .as_str()
+                .expect("run response should include a tagged status kind");
+            if expected.contains(&status) {
+                return status.to_owned();
+            }
+            sleep(POLL_INTERVAL).await;
         }
-        sleep(POLL_INTERVAL).await;
-    }
-    panic!("run {run_id} did not reach any of {expected:?}");
+    })
+    .await
+    .unwrap_or_else(|_| {
+        panic!(
+            "run {run_id} did not reach any of {expected:?} within {timeout:?}; \
+             last lifecycle: {}; last sandbox: {}",
+            last_body["lifecycle"], last_body["sandbox"]
+        )
+    })
 }
 
 pub(crate) async fn wait_for_run_status_not_in(

lib/apps/fabro-server/tests/it/scenario/petri.rs

@@ -18,6 +18,7 @@ use std::collections::BTreeMap;
 use std::path::{Path, PathBuf};
 use std::process::{Command, Stdio};
 use std::sync::Arc;
+use std::time::Duration;
 
 use axum::body::Body;
 use axum::http::{Request, StatusCode};
@@ -36,12 +37,15 @@ use fabro_types::{RunId, WorkflowPath, WorkflowVersion};
 use tower::ServiceExt;
 
 use crate::helpers::{
-    api, create_and_start_run_from_intent, minimal_manifest_json, read_repo_file, response_json,
-    run_json, settings_from_toml, test_app_state_with_options, test_app_with_scheduler,
-    test_settings, wait_for_run_status,
+    self, api, create_and_start_run_from_intent, minimal_manifest_json, read_repo_file,
+    response_json, run_json, settings_from_toml, test_app_state_with_options,
+    test_app_with_scheduler, test_settings, wait_for_run_status,
 };
 
 const OPENAI_MODEL: &str = "gpt-5.4";
+// A cold Docker daemon may need to download the runner image before the run
+// starts.
+const DOCKER_RUN_TIMEOUT: Duration = Duration::from_mins(1);
 
 /// A command-only workflow: one script stage between start and exit.
 const COMMAND_DOT: &str = r#"digraph Command {
@@ -919,7 +923,13 @@ async fn a_runs_projection_carries_its_docker_sandbox_instance() {
         "args": {},
     });
     let run_id = create_and_start_run_from_intent(&app, intent).await;
-    let status = wait_for_run_status(&app, &run_id, &["succeeded", "failed"]).await;
+    let status = helpers::wait_for_run_status_with_timeout(
+        &app,
+        &run_id,
+        &["succeeded", "failed"],
+        DOCKER_RUN_TIMEOUT,
+    )
+    .await;
     assert_eq!(
         status,
         "succeeded",
@@ -1022,7 +1032,13 @@ async fn the_server_attaches_to_the_container_petri_created() {
         "args": {},
     });
     let run_id = create_and_start_run_from_intent(&app, intent).await;
-    let status = wait_for_run_status(&app, &run_id, &["succeeded", "failed"]).await;
+    let status = helpers::wait_for_run_status_with_timeout(
+        &app,
+        &run_id,
+        &["succeeded", "failed"],
+        DOCKER_RUN_TIMEOUT,
+    )
+    .await;
     assert_eq!(
         status,
         "succeeded",
@@ -1350,7 +1366,13 @@ async fn a_bundle_naming_a_catalog_environment_runs_on_docker_with_its_image() {
         return;
     }
     start_run(&app, &run_id).await;
-    let status = wait_for_run_status(&app, &run_id, &["succeeded", "failed"]).await;
+    let status = helpers::wait_for_run_status_with_timeout(
+        &app,
+        &run_id,
+        &["succeeded", "failed"],
+        DOCKER_RUN_TIMEOUT,
+    )
+    .await;
     let projection = settled_state(&state, &app, &run_id).await;
     assert_eq!(status, "succeeded", "run: {projection}");
     let instance = &projection["sandbox"]["instance"];