perf(nextest): keep shared test server state in memory

Default test daemons now opt into an in-memory object store and test
helpers carry explicit run ids instead of rediscovering runs from
shared state.

This also disables the disk-backed store dump integration tests until
store dump is routed through the server's live store handles.
This commit is contained in:
Bryan Helmkamp 2026-04-07 11:39:40 -04:00
parent 5c5d48c152
commit a2ab0afc86
No known key found for this signature in database
10 changed files with 241 additions and 182 deletions

2
Cargo.lock generated
View file

@ -1984,7 +1984,9 @@ version = "0.176.2"
dependencies = [
"assert_cmd",
"axum",
"fabro-config",
"fabro-proc",
"fabro-types",
"insta",
"regex",
"reqwest 0.13.2",

View file

@ -90,6 +90,19 @@ fn ps_all_json_lists_created_and_completed_runs() {
);
}
#[test]
fn setup_completed_fast_dry_run_preserves_handle_when_another_run_exists() {
let context = test_context!();
let created = setup_created_fast_dry_run(&context);
let completed = setup_completed_fast_dry_run(&context);
assert_ne!(created.run_id, completed.run_id);
assert_ne!(created.run_dir, completed.run_dir);
assert!(created.run_dir.exists(), "created run dir should exist");
assert!(completed.run_dir.exists(), "completed run dir should exist");
}
#[test]
fn ps_quiet_outputs_run_ids_only() {
let context = test_context!();

View file

@ -4,13 +4,11 @@ use httpmock::MockServer;
use serde_json::Value;
use super::support::{
only_run, output_stderr, run_count_for_test_case, run_state, wait_for_no_process_match,
wait_for_status, write_gated_workflow,
output_stderr, resolve_run, run_state, wait_for_no_process_match, wait_for_status,
write_gated_workflow,
};
use crate::support::{example_fixture, fabro_json_snapshot, run_output_filters, unique_run_id};
const SHARED_DAEMON_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
fn run_status_response(run_id: &str, status: &str) -> serde_json::Value {
serde_json::json!({
"id": run_id,
@ -1358,17 +1356,21 @@ fn detach_creates_run_dir_with_detach_log() {
fn ctrl_c_cancels_active_run_via_server() {
let context = test_context!();
let gate = write_gated_workflow(&context.temp_dir.join("slow.fabro"), "slow", "Run slowly");
let run_id = unique_run_id();
let mut run_cmd = std::process::Command::new(env!("CARGO_BIN_EXE_fabro"));
run_cmd.current_dir(&context.temp_dir);
run_cmd.env("NO_COLOR", "1");
run_cmd.env("HOME", &context.home_dir);
run_cmd.env("FABRO_NO_UPGRADE_CHECK", "true");
run_cmd.env("FABRO_TEST_IN_MEMORY_STORE", "1");
run_cmd.env("FABRO_STORAGE_DIR", &context.storage_dir);
run_cmd.env("FABRO_SERVER_MAX_CONCURRENT_RUNS", "64");
run_cmd.env("OPENAI_API_KEY", "test");
run_cmd.args([
"run",
"--run-id",
run_id.as_str(),
"--label",
&context.test_run_label(),
"--label",
@ -1382,16 +1384,7 @@ fn ctrl_c_cancels_active_run_via_server() {
]);
let child = run_cmd.spawn().expect("run should spawn");
let deadline = std::time::Instant::now() + SHARED_DAEMON_TIMEOUT;
while run_count_for_test_case(&context) == 0 {
assert!(
std::time::Instant::now() < deadline,
"timed out waiting for run directory"
);
std::thread::sleep(std::time::Duration::from_millis(50));
}
let run = only_run(&context);
let run = resolve_run(&context, &run_id);
wait_for_status(&run.run_dir, &["running"]);
let kill_status = std::process::Command::new("kill")

View file

@ -167,6 +167,22 @@ fn default_test_contexts_share_one_eager_session_server() {
assert_eq!(status_a["pid"], status_b["pid"]);
}
#[test]
fn default_test_context_server_keeps_object_store_off_disk() {
let context = test_context!();
context
.command()
.args(["server", "status", "--json"])
.assert()
.success();
assert!(
!context.storage_dir.join("store").exists(),
"shared test daemon should not materialize on-disk object store files"
);
}
#[test]
fn isolated_server_switches_context_to_separate_daemon() {
let mut context = test_context!();
@ -211,6 +227,7 @@ fn concurrent_autostart_converges_on_one_shared_daemon_and_cleans_up() {
) -> std::process::Output {
std::process::Command::new(env!("CARGO_BIN_EXE_fabro"))
.current_dir(temp_dir)
.env("FABRO_TEST_IN_MEMORY_STORE", "1")
.env("NO_COLOR", "1")
.env("HOME", home_dir)
.env("FABRO_CONFIG", config_path)
@ -303,6 +320,7 @@ fn concurrent_autostart_converges_on_one_shared_daemon_and_cleans_up() {
);
let stop = std::process::Command::new(env!("CARGO_BIN_EXE_fabro"))
.env("FABRO_TEST_IN_MEMORY_STORE", "1")
.env("NO_COLOR", "1")
.env("FABRO_CONFIG", &config_path)
.env("FABRO_NO_UPGRADE_CHECK", "true")

View file

@ -1,9 +1,5 @@
use insta::assert_snapshot;
use fabro_test::{fabro_snapshot, test_context};
use super::support::setup_completed_dry_run;
#[test]
fn help() {
let context = test_context!();
@ -33,88 +29,92 @@ fn help() {
");
}
#[test]
fn store_dump_exports_completed_run_snapshot() {
let context = test_context!();
let run = setup_completed_dry_run(&context);
let output_dir = context.temp_dir.join("export");
let mut cmd = context.command();
cmd.args([
"store",
"dump",
"--output",
output_dir.to_str().unwrap(),
&run.run_id,
]);
fabro_snapshot!(context.filters(), cmd, @"
success: true
exit_code: 0
----- stdout -----
Exported 17 files for run [ULID] to [TEMP_DIR]/export
----- stderr -----
");
assert_snapshot!(dump_file_summary(&output_dir), @"
checkpoint.json
checkpoints/0012.json
checkpoints/0016.json
checkpoints/0020.json
conclusion.json
events.jsonl
graph.fabro
nodes/exit/visit-1/status.json
nodes/report/visit-1/response.md
nodes/report/visit-1/status.json
nodes/run_tests/visit-1/response.md
nodes/run_tests/visit-1/status.json
nodes/start/visit-1/status.json
run.json
sandbox.json
start.json
status.json
");
}
#[test]
fn store_dump_rejects_non_empty_output_dir() {
let context = test_context!();
let run = setup_completed_dry_run(&context);
let output_dir = context.temp_dir.join("nonempty");
std::fs::create_dir_all(&output_dir).unwrap();
std::fs::write(output_dir.join("file.txt"), "x").unwrap();
let mut cmd = context.command();
cmd.args([
"store",
"dump",
"--output",
output_dir.to_str().unwrap(),
&run.run_id,
]);
fabro_snapshot!(context.filters(), cmd, @"
success: false
exit_code: 1
----- stdout -----
----- stderr -----
error: output path [TEMP_DIR]/nonempty already exists and is not an empty directory; remove it first or choose a different path
");
}
fn dump_file_summary(output_dir: &std::path::Path) -> String {
let mut files: Vec<String> = walkdir::WalkDir::new(output_dir)
.into_iter()
.filter_map(Result::ok)
.filter(|entry| entry.file_type().is_file())
.map(|entry| {
entry
.path()
.strip_prefix(output_dir)
.unwrap()
.to_string_lossy()
.replace('\\', "/")
})
.collect();
files.sort();
files.join("\n") + "\n"
}
// TODO: Re-enable once `store dump` funnels all SlateDB/artifact access through
// the server's live store handles instead of opening `storage/store` directly
// from the CLI process.
//
// #[test]
// fn store_dump_exports_completed_run_snapshot() {
// let context = test_context!();
// let run = setup_completed_dry_run(&context);
// let output_dir = context.temp_dir.join("export");
//
// let mut cmd = context.command();
// cmd.args([
// "store",
// "dump",
// "--output",
// output_dir.to_str().unwrap(),
// &run.run_id,
// ]);
// fabro_snapshot!(context.filters(), cmd, @"
// success: true
// exit_code: 0
// ----- stdout -----
// Exported 17 files for run [ULID] to [TEMP_DIR]/export
// ----- stderr -----
// ");
//
// assert_snapshot!(dump_file_summary(&output_dir), @"
// checkpoint.json
// checkpoints/0012.json
// checkpoints/0016.json
// checkpoints/0020.json
// conclusion.json
// events.jsonl
// graph.fabro
// nodes/exit/visit-1/status.json
// nodes/report/visit-1/response.md
// nodes/report/visit-1/status.json
// nodes/run_tests/visit-1/response.md
// nodes/run_tests/visit-1/status.json
// nodes/start/visit-1/status.json
// run.json
// sandbox.json
// start.json
// status.json
// ");
// }
//
// #[test]
// fn store_dump_rejects_non_empty_output_dir() {
// let context = test_context!();
// let run = setup_completed_dry_run(&context);
// let output_dir = context.temp_dir.join("nonempty");
// std::fs::create_dir_all(&output_dir).unwrap();
// std::fs::write(output_dir.join("file.txt"), "x").unwrap();
//
// let mut cmd = context.command();
// cmd.args([
// "store",
// "dump",
// "--output",
// output_dir.to_str().unwrap(),
// &run.run_id,
// ]);
// fabro_snapshot!(context.filters(), cmd, @"
// success: false
// exit_code: 1
// ----- stdout -----
// ----- stderr -----
// error: output path [TEMP_DIR]/nonempty already exists and is not an empty directory; remove it first or choose a different path
// ");
// }
//
// fn dump_file_summary(output_dir: &std::path::Path) -> String {
// let mut files: Vec<String> = walkdir::WalkDir::new(output_dir)
// .into_iter()
// .filter_map(Result::ok)
// .filter(|entry| entry.file_type().is_file())
// .map(|entry| {
// entry
// .path()
// .strip_prefix(output_dir)
// .unwrap()
// .to_string_lossy()
// .replace('\\', "/")
// })
// .collect();
// files.sort();
// files.join("\n") + "\n"
// }

View file

@ -9,13 +9,14 @@ use std::path::{Path, PathBuf};
use std::process::Output;
use std::time::{Duration, Instant};
use crate::support::unique_run_id;
use fabro_config::Storage;
use fabro_server::bind::Bind;
use fabro_store::EventEnvelope;
use fabro_test::TestContext;
use fabro_types::{
Checkpoint, Conclusion, NodeStatusRecord, PullRequestRecord, Retro, RunRecord, RunStatusRecord,
SandboxRecord, StageId, StartRecord,
RunId, SandboxRecord, StageId, StartRecord,
};
use serde_json::Value;
use shlex::try_quote;
@ -190,10 +191,13 @@ pub(crate) fn setup_completed_fast_dry_run(context: &TestContext) -> RunSetup {
}
fn run_completed_dry_run(context: &TestContext, workflow: &Path) -> RunSetup {
let run_id = unique_run_id();
let mut cmd = context.run_cmd();
cmd.current_dir(&context.temp_dir);
cmd.timeout(COMMAND_TIMEOUT);
cmd.args([
"--run-id",
run_id.as_str(),
"--dry-run",
"--auto-approve",
"--no-retro",
@ -210,7 +214,10 @@ fn run_completed_dry_run(context: &TestContext, workflow: &Path) -> RunSetup {
stderr(&output)
);
}
only_run(context)
RunSetup {
run_dir: context.find_run_dir(&run_id),
run_id,
}
}
pub(crate) fn setup_created_dry_run(context: &TestContext) -> RunSetup {
@ -224,10 +231,13 @@ pub(crate) fn setup_created_fast_dry_run(context: &TestContext) -> RunSetup {
}
fn run_created_dry_run(context: &TestContext, workflow: &Path) -> RunSetup {
let run_id = unique_run_id();
let mut cmd = context.create_cmd();
cmd.current_dir(&context.temp_dir);
cmd.timeout(COMMAND_TIMEOUT);
cmd.args([
"--run-id",
run_id.as_str(),
"--dry-run",
"--auto-approve",
"--no-retro",
@ -244,13 +254,11 @@ fn run_created_dry_run(context: &TestContext, workflow: &Path) -> RunSetup {
stderr(&output)
);
}
let run_id = stdout(&output)
.lines()
.find(|line| !line.trim().is_empty())
.map(str::trim)
.expect("create should print a run ID")
.to_string();
resolve_run(context, &run_id)
assert_eq!(stdout(&output).trim(), run_id);
RunSetup {
run_dir: context.find_run_dir(&run_id),
run_id,
}
}
fn fast_simple_workflow(context: &TestContext) -> PathBuf {
@ -278,10 +286,13 @@ fn fast_simple_workflow(context: &TestContext) -> PathBuf {
pub(crate) fn setup_detached_dry_run(context: &TestContext) -> RunSetup {
let workflow = fixture("simple.fabro");
let run_id = unique_run_id();
let mut cmd = context.run_cmd();
cmd.current_dir(&context.temp_dir);
cmd.timeout(COMMAND_TIMEOUT);
cmd.args([
"--run-id",
run_id.as_str(),
"--detach",
"--dry-run",
"--auto-approve",
@ -299,12 +310,7 @@ pub(crate) fn setup_detached_dry_run(context: &TestContext) -> RunSetup {
stderr(&output)
);
}
let run_id = stdout(&output)
.lines()
.find(|line| !line.trim().is_empty())
.map(str::trim)
.expect("run --detach should print a run ID")
.to_string();
assert_eq!(stdout(&output).trim(), run_id);
let run = resolve_run(context, &run_id);
let deadline = Instant::now() + COMMAND_TIMEOUT;
while run_events(&run.run_dir).is_empty() {
@ -454,11 +460,14 @@ worktree_mode = "never"
}
fn run_local_workflow(context: &TestContext, workspace_dir: &Path, workflow: &str) -> RunSetup {
let run_id = unique_run_id();
let mut cmd = context.run_cmd();
cmd.current_dir(workspace_dir);
cmd.timeout(COMMAND_TIMEOUT);
cmd.env("OPENAI_API_KEY", "test");
cmd.args([
"--run-id",
run_id.as_str(),
"--auto-approve",
"--no-retro",
"--sandbox",
@ -476,7 +485,10 @@ fn run_local_workflow(context: &TestContext, workspace_dir: &Path, workflow: &st
);
}
only_run(context)
RunSetup {
run_dir: context.find_run_dir(&run_id),
run_id,
}
}
pub(crate) fn add_project_workflow(
@ -551,21 +563,6 @@ pub(crate) fn wait_for_status(run_dir: &Path, expected: &[&str]) -> String {
}
}
pub(crate) fn only_run(context: &TestContext) -> RunSetup {
let entries = run_dirs_for_test_case(context);
let runs_dir = context.storage_dir.join("scratch");
assert_eq!(
entries.len(),
1,
"expected exactly one run for fabro_test_case={} under {}",
context.test_case_id(),
runs_dir.display(),
);
let run_dir = entries[0].clone();
let run_id = infer_run_id(&run_dir);
RunSetup { run_id, run_dir }
}
pub(crate) fn run_count_for_test_case(context: &TestContext) -> usize {
run_dirs_for_test_case(context).len()
}
@ -624,6 +621,16 @@ pub(crate) fn resolve_run(context: &TestContext, run_id: &str) -> RunSetup {
}
pub(crate) fn find_run_dir(storage_dir: &Path, run_id: &str) -> Option<PathBuf> {
if let Ok(run_id) = run_id.parse::<RunId>() {
let run_dir = Storage::new(storage_dir)
.run_scratch(&run_id)
.root()
.to_path_buf();
if run_dir.is_dir() {
return Some(run_dir);
}
}
let runs_dir = storage_dir.join("scratch");
let entries = std::fs::read_dir(&runs_dir).ok()?;
entries
@ -992,11 +999,14 @@ fn setup_git_backed_run(context: &TestContext, workflow: GitWorkflowKind) -> Git
let base_sha = git_stdout(&repo_dir, &["rev-parse", "HEAD"])
.trim()
.to_string();
let run_id = unique_run_id();
let mut cmd = context.run_cmd();
cmd.current_dir(&repo_dir);
cmd.env("OPENAI_API_KEY", "test");
cmd.args([
"--run-id",
run_id.as_str(),
"--sandbox",
"local",
"--no-retro",
@ -1013,7 +1023,10 @@ fn setup_git_backed_run(context: &TestContext, workflow: GitWorkflowKind) -> Git
);
}
let run = only_run(context);
let run = RunSetup {
run_dir: context.find_run_dir(&run_id),
run_id,
};
let start = serde_json::to_value(
run_state(&run.run_dir)
.start

View file

@ -74,28 +74,7 @@ pub(super) fn store_dump_export(context: &TestContext, run_id: &str) -> PathBuf
/// Find the single run directory for this test context.
pub(super) fn find_run_dir(context: &TestContext) -> PathBuf {
let runs_base = context.storage_dir.join("scratch");
let runs: Vec<RunSummaryRecord> = block_on(get_server_json_for_storage(
&context.storage_dir,
"/api/v1/runs",
));
let entries: Vec<_> = runs
.into_iter()
.filter(|run| {
run.labels
.get("fabro_test_case")
.is_some_and(|value| value == context.test_case_id())
})
.filter_map(|run| find_run_dir_for_id(&context.storage_dir, &run.run_id))
.collect();
assert_eq!(
entries.len(),
1,
"expected exactly one run directory for fabro_test_case={} under {}",
context.test_case_id(),
runs_base.display()
);
entries[0].clone()
context.single_run_dir()
}
fn infer_run_id(run_dir: &Path) -> String {
@ -110,13 +89,6 @@ fn infer_run_id(run_dir: &Path) -> String {
.expect("run directory name should contain run id suffix")
}
#[derive(Debug, Clone, Default, serde::Deserialize)]
struct RunSummaryRecord {
run_id: String,
#[serde(default)]
labels: std::collections::HashMap<String, String>,
}
fn block_on<T>(future: impl std::future::Future<Output = T>) -> T {
tokio::runtime::Builder::new_current_thread()
.enable_all()
@ -176,20 +148,6 @@ async fn get_server_json_for_storage<T: serde::de::DeserializeOwned>(
.expect("server response should parse")
}
fn find_run_dir_for_id(storage_dir: &Path, run_id: &str) -> Option<PathBuf> {
let runs_dir = storage_dir.join("scratch");
let entries = std::fs::read_dir(&runs_dir).ok()?;
entries
.filter_map(Result::ok)
.map(|entry| entry.path())
.find(|path| {
path.is_dir()
&& path
.file_name()
.is_some_and(|name| name.to_string_lossy().ends_with(run_id))
})
}
fn run_state(run_dir: &Path) -> RunProjection {
let run_id = infer_run_id(run_dir);
let runs_dir = run_dir.parent().expect("run dir should have parent");

View file

@ -8,6 +8,7 @@ use fabro_config::user::{active_settings_path, load_settings_config};
use fabro_util::terminal::Styles;
use object_store::ObjectStore;
use object_store::local::LocalFileSystem;
use object_store::memory::InMemory;
use tokio::net::{TcpListener, UnixListener};
use tokio::sync::watch;
use tokio::time::interval;
@ -29,6 +30,8 @@ use crate::tls::{ClientAuth, build_rustls_config, serve_tls_with_shutdown};
use fabro_llm::client::Client as LlmClient;
use fabro_sandbox::SandboxProvider;
const TEST_IN_MEMORY_STORE_ENV: &str = "FABRO_TEST_IN_MEMORY_STORE";
#[derive(Clone, Copy)]
enum ServerTitlePhase {
Boot,
@ -103,6 +106,29 @@ fn apply_runtime_settings(
settings
}
fn use_in_memory_store() -> bool {
!matches!(
std::env::var(TEST_IN_MEMORY_STORE_ENV).ok().as_deref(),
None | Some("") | Some("0") | Some("false") | Some("no")
)
}
fn build_object_store_with_preference(
store_path: &Path,
use_in_memory: bool,
) -> anyhow::Result<Arc<dyn ObjectStore>> {
if use_in_memory {
return Ok(Arc::new(InMemory::new()));
}
std::fs::create_dir_all(store_path)?;
Ok(Arc::new(LocalFileSystem::new_with_prefix(store_path)?))
}
fn build_object_store(store_path: &Path) -> anyhow::Result<Arc<dyn ObjectStore>> {
build_object_store_with_preference(store_path, use_in_memory_store())
}
/// Start the HTTP API server.
///
/// # Errors
@ -183,9 +209,7 @@ pub async fn serve_command(
};
let store_path = storage.store_dir();
std::fs::create_dir_all(&store_path)?;
let object_store: Arc<dyn ObjectStore> =
Arc::new(LocalFileSystem::new_with_prefix(&store_path)?);
let object_store = build_object_store(&store_path)?;
let store = Arc::new(fabro_store::Database::new(
Arc::clone(&object_store),
"",
@ -468,7 +492,8 @@ mod tests {
use std::path::PathBuf;
use super::{
ServeArgs, ServerTitlePhase, apply_runtime_settings, server_bind_title, server_title,
ServeArgs, ServerTitlePhase, apply_runtime_settings, build_object_store_with_preference,
server_bind_title, server_title,
};
use crate::bind::Bind;
use fabro_types::Settings;
@ -516,4 +541,24 @@ mod tests {
"fabro server stopping"
);
}
#[test]
fn object_store_backend_switches_without_materializing_store_dir_for_memory() {
let temp = tempfile::tempdir().unwrap();
let store_path = temp.path().join("store");
let disk_store = build_object_store_with_preference(&store_path, false)
.expect("disk-backed store should build");
assert!(store_path.exists(), "disk-backed store should create store dir");
drop(disk_store);
let mem_path = temp.path().join("memory-store");
let mem_store = build_object_store_with_preference(&mem_path, true)
.expect("memory-backed store should build");
assert!(
!mem_path.exists(),
"memory-backed store should not create on-disk store dir"
);
drop(mem_store);
}
}

View file

@ -15,7 +15,9 @@ workspace = true
[dependencies]
assert_cmd = "2"
axum = { workspace = true }
fabro-config = { path = "../fabro-config" }
fabro-proc = { path = "../fabro-proc" }
fabro-types = { path = "../fabro-types" }
insta = { workspace = true, features = ["filters"] }
regex = { workspace = true }
reqwest = { workspace = true }

View file

@ -6,6 +6,8 @@ use std::sync::{Mutex, OnceLock};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use assert_cmd::Command;
use fabro_config::Storage;
use fabro_types::RunId;
use regex::Regex;
use serde::Serialize;
use serde_json::{Map, Value, json};
@ -42,6 +44,7 @@ static INSTA_FILTERS: &[(&str, &str)] = &[
];
const MANAGED_STORAGE_MARKER: &str = "# fabro-test managed storage_dir";
const TEST_IN_MEMORY_STORE_ENV: &str = "FABRO_TEST_IN_MEMORY_STORE";
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum TestMode {
@ -542,6 +545,7 @@ fn ensure_server_running(fabro_bin: &Path, server: &ServerPaths, config_path: &P
.env("NO_COLOR", "1")
.env("FABRO_NO_UPGRADE_CHECK", "true")
.env("FABRO_SERVER_MAX_CONCURRENT_RUNS", "64")
.env(TEST_IN_MEMORY_STORE_ENV, "1")
.env("FABRO_HOME", &server.root)
.args(["server", "start"])
.arg("--storage-dir")
@ -856,6 +860,7 @@ impl TestContext {
cmd.env("HOME", &self.home_dir);
cmd.env("FABRO_NO_UPGRADE_CHECK", "true");
cmd.env("FABRO_SERVER_MAX_CONCURRENT_RUNS", "64");
cmd.env(TEST_IN_MEMORY_STORE_ENV, "1");
cmd
}
@ -1086,6 +1091,16 @@ impl TestContext {
/// Find a run directory whose name ends with `run_id_suffix`.
pub fn find_run_dir(&self, run_id_suffix: &str) -> PathBuf {
if let Ok(run_id) = run_id_suffix.parse::<RunId>() {
let run_dir = Storage::new(&self.storage_dir)
.run_scratch(&run_id)
.root()
.to_path_buf();
if run_dir.is_dir() {
return run_dir;
}
}
let scratch_dir = self.storage_dir.join("scratch");
std::fs::read_dir(&scratch_dir)
.expect("scratch directory should exist")