From c56f1173c906a56a7a14461c5fae7391d69d0e2b Mon Sep 17 00:00:00 2001 From: Scott Werner Date: Sat, 26 Sep 2026 15:24:51 -0400 Subject: [PATCH] fix(ci): prepare Docker images for nightly release tests --- .config/nextest.toml | 12 ++++++ .github/workflows/nightly.yml | 9 ++++ .github/workflows/release.yml | 9 ++++ lib/apps/fabro-server/tests/it/helpers.rs | 42 +++++++++++++------ .../fabro-server/tests/it/scenario/petri.rs | 34 ++++++++++++--- 5 files changed, 88 insertions(+), 18 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index 43c024c88..65282c997 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -7,6 +7,18 @@ leak-timeout = "500ms" filter = "package(fabro-cli)" slow-timeout = { period = "6s", terminate-after = 4 } + # Docker scenarios allow 60s for a cold image pull; leave time for cleanup. + # Keep this before the general server override because the first match wins. + [[profile.default.overrides]] + filter = """ + package(fabro-server) & ( + test(=scenario::petri::a_runs_projection_carries_its_docker_sandbox_instance) + | test(=scenario::petri::the_server_attaches_to_the_container_petri_created) + | test(=scenario::petri::a_bundle_naming_a_catalog_environment_runs_on_docker_with_its_image) + ) + """ + slow-timeout = { period = "30s", terminate-after = 3 } + [[profile.default.overrides]] filter = "package(fabro-server)" slow-timeout = { period = "5s", terminate-after = 4 } diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index b5469bf9f..b80b429f6 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -61,6 +61,15 @@ jobs: - uses: taiki-e/install-action@773334c0e05d7e699e4d78234494308223f3a2cf # nextest if: steps.skip.outputs.skip != 'true' + - name: Pull the images the Docker tests run + if: steps.skip.outputs.skip != 'true' + run: | + backend="$(dirname "$(cargo metadata --locked --format-version 1 | jq -r '.packages[] | select(.name == "petri-executor-sandbox") | .manifest_path')")/src/backend.rs" + pin="$(sed -n 's/^const RUNNER_PIN: &str = "\([0-9a-f]*\)";$/\1/p' "$backend")" + test -n "$pin" + docker pull "ghcr.io/lithoscomputer/ubuntu-24.04:slim-$pin" + docker pull ghcr.io/lithoscomputer/ubuntu-22.04:slim + - name: Release nightly if: steps.skip.outputs.skip != 'true' run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6900c4dea..ed9f08e47 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -88,6 +88,15 @@ jobs: - name: Refresh embedded SPA run: cargo --locked dev spa refresh + - name: Pull the images the Docker tests run + if: runner.os == 'Linux' && matrix.target != 'aarch64-unknown-linux-musl' + run: | + backend="$(dirname "$(cargo metadata --locked --format-version 1 | jq -r '.packages[] | select(.name == "petri-executor-sandbox") | .manifest_path')")/src/backend.rs" + pin="$(sed -n 's/^const RUNNER_PIN: &str = "\([0-9a-f]*\)";$/\1/p' "$backend")" + test -n "$pin" + docker pull "ghcr.io/lithoscomputer/ubuntu-24.04:slim-$pin" + docker pull ghcr.io/lithoscomputer/ubuntu-22.04:slim + - name: Test (x86_64-musl) # nextest still shells through cargo test for this target, so # build.rs C code needs an explicit musl compiler/linker. diff --git a/lib/apps/fabro-server/tests/it/helpers.rs b/lib/apps/fabro-server/tests/it/helpers.rs index 0c2aa2ff7..969e02e7a 100644 --- a/lib/apps/fabro-server/tests/it/helpers.rs +++ b/lib/apps/fabro-server/tests/it/helpers.rs @@ -16,7 +16,7 @@ use fabro_test::{ expect_axum_status_in, expect_axum_text, }; use fabro_types::ServerSettings; -use tokio::time::sleep; +use tokio::time::{self, sleep}; use tower::ServiceExt; pub(crate) const MINIMAL_DOT: &str = r#"digraph Test { @@ -307,18 +307,36 @@ pub(crate) async fn wait_for_run_status( run_id: &str, expected: &[&str], ) -> String { - for _ in 0..POLL_ATTEMPTS { - let body = run_json(app, run_id).await; - let status = body["lifecycle"]["status"]["kind"] - .as_str() - .expect("run response should include a tagged status kind") - .to_string(); - if expected.iter().any(|candidate| *candidate == status) { - return status; + wait_for_run_status_with_timeout(app, run_id, expected, Duration::from_secs(5)).await +} + +pub(crate) async fn wait_for_run_status_with_timeout( + app: &axum::Router, + run_id: &str, + expected: &[&str], + timeout: Duration, +) -> String { + let mut last_body = serde_json::Value::Null; + time::timeout(timeout, async { + loop { + last_body = run_json(app, run_id).await; + let status = last_body["lifecycle"]["status"]["kind"] + .as_str() + .expect("run response should include a tagged status kind"); + if expected.contains(&status) { + return status.to_owned(); + } + sleep(POLL_INTERVAL).await; } - sleep(POLL_INTERVAL).await; - } - panic!("run {run_id} did not reach any of {expected:?}"); + }) + .await + .unwrap_or_else(|_| { + panic!( + "run {run_id} did not reach any of {expected:?} within {timeout:?}; \ + last lifecycle: {}; last sandbox: {}", + last_body["lifecycle"], last_body["sandbox"] + ) + }) } pub(crate) async fn wait_for_run_status_not_in( diff --git a/lib/apps/fabro-server/tests/it/scenario/petri.rs b/lib/apps/fabro-server/tests/it/scenario/petri.rs index dd5a68422..7541469ae 100644 --- a/lib/apps/fabro-server/tests/it/scenario/petri.rs +++ b/lib/apps/fabro-server/tests/it/scenario/petri.rs @@ -18,6 +18,7 @@ use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::sync::Arc; +use std::time::Duration; use axum::body::Body; use axum::http::{Request, StatusCode}; @@ -36,12 +37,15 @@ use fabro_types::{RunId, WorkflowPath, WorkflowVersion}; use tower::ServiceExt; use crate::helpers::{ - api, create_and_start_run_from_intent, minimal_manifest_json, read_repo_file, response_json, - run_json, settings_from_toml, test_app_state_with_options, test_app_with_scheduler, - test_settings, wait_for_run_status, + self, api, create_and_start_run_from_intent, minimal_manifest_json, read_repo_file, + response_json, run_json, settings_from_toml, test_app_state_with_options, + test_app_with_scheduler, test_settings, wait_for_run_status, }; const OPENAI_MODEL: &str = "gpt-5.4"; +// A cold Docker daemon may need to download the runner image before the run +// starts. +const DOCKER_RUN_TIMEOUT: Duration = Duration::from_mins(1); /// A command-only workflow: one script stage between start and exit. const COMMAND_DOT: &str = r#"digraph Command { @@ -919,7 +923,13 @@ async fn a_runs_projection_carries_its_docker_sandbox_instance() { "args": {}, }); let run_id = create_and_start_run_from_intent(&app, intent).await; - let status = wait_for_run_status(&app, &run_id, &["succeeded", "failed"]).await; + let status = helpers::wait_for_run_status_with_timeout( + &app, + &run_id, + &["succeeded", "failed"], + DOCKER_RUN_TIMEOUT, + ) + .await; assert_eq!( status, "succeeded", @@ -1022,7 +1032,13 @@ async fn the_server_attaches_to_the_container_petri_created() { "args": {}, }); let run_id = create_and_start_run_from_intent(&app, intent).await; - let status = wait_for_run_status(&app, &run_id, &["succeeded", "failed"]).await; + let status = helpers::wait_for_run_status_with_timeout( + &app, + &run_id, + &["succeeded", "failed"], + DOCKER_RUN_TIMEOUT, + ) + .await; assert_eq!( status, "succeeded", @@ -1350,7 +1366,13 @@ async fn a_bundle_naming_a_catalog_environment_runs_on_docker_with_its_image() { return; } start_run(&app, &run_id).await; - let status = wait_for_run_status(&app, &run_id, &["succeeded", "failed"]).await; + let status = helpers::wait_for_run_status_with_timeout( + &app, + &run_id, + &["succeeded", "failed"], + DOCKER_RUN_TIMEOUT, + ) + .await; let projection = settled_state(&state, &app, &run_id).await; assert_eq!(status, "succeeded", "run: {projection}"); let instance = &projection["sandbox"]["instance"];