From 79cad86ed4187738ceb8ea15f4eeee123482c519 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Thu, 2 Apr 2026 15:08:21 -0700 Subject: [PATCH 1/3] Require a durable run store throughout execution Make run lookup fail with RunNotFound instead of returning Option, thread a required RunStore through workflow and retro paths, and update CLI, server, and test callers to match. Also treat null optional event properties as absent during store-backed replay so event-sourced state stays robust. --- .../fabro-cli/src/commands/pr/create.rs | 27 +- lib/crates/fabro-cli/src/commands/pr/list.rs | 8 +- lib/crates/fabro-cli/src/commands/pr/mod.rs | 7 +- .../fabro-cli/src/commands/run/attach.rs | 64 +- lib/crates/fabro-cli/src/commands/run/cp.rs | 7 +- .../fabro-cli/src/commands/run/detached.rs | 5 +- lib/crates/fabro-cli/src/commands/run/diff.rs | 34 +- lib/crates/fabro-cli/src/commands/run/fork.rs | 2 +- lib/crates/fabro-cli/src/commands/run/logs.rs | 21 +- .../fabro-cli/src/commands/run/output.rs | 21 +- .../fabro-cli/src/commands/run/preview.rs | 7 +- .../fabro-cli/src/commands/run/rewind.rs | 39 +- lib/crates/fabro-cli/src/commands/run/ssh.rs | 7 +- .../fabro-cli/src/commands/run/start.rs | 8 +- lib/crates/fabro-cli/src/commands/run/wait.rs | 10 +- .../fabro-cli/src/commands/runs/inspect.rs | 29 +- lib/crates/fabro-cli/src/commands/runs/rm.rs | 7 +- .../fabro-cli/src/commands/store/dump.rs | 28 +- lib/crates/fabro-cli/src/store.rs | 2 +- lib/crates/fabro-cli/tests/it/cmd/diff.rs | 8 +- lib/crates/fabro-cli/tests/it/cmd/pr_view.rs | 2 +- lib/crates/fabro-cli/tests/it/cmd/support.rs | 44 +- lib/crates/fabro-cli/tests/it/scenario/mod.rs | 23 +- lib/crates/fabro-retro/src/retro_agent.rs | 36 +- lib/crates/fabro-server/src/server.rs | 79 +- lib/crates/fabro-store/src/lib.rs | 7 +- lib/crates/fabro-store/src/memory.rs | 430 ++++++++-- lib/crates/fabro-store/src/run_state.rs | 734 ++++++++++++++++++ lib/crates/fabro-store/src/slate/mod.rs | 53 +- lib/crates/fabro-store/src/slate/run_store.rs | 140 +++- lib/crates/fabro-workflow/src/event.rs | 4 +- .../fabro-workflow/src/handler/agent.rs | 27 +- .../fabro-workflow/src/handler/command.rs | 78 +- .../fabro-workflow/src/handler/fan_in.rs | 41 +- lib/crates/fabro-workflow/src/handler/mod.rs | 11 +- .../fabro-workflow/src/handler/parallel.rs | 21 +- .../fabro-workflow/src/handler/prompt.rs | 26 +- .../fabro-workflow/src/operations/create.rs | 12 +- .../src/operations/rebuild_meta.rs | 20 +- .../fabro-workflow/src/operations/resume.rs | 23 +- .../fabro-workflow/src/operations/start.rs | 22 +- .../fabro-workflow/src/pipeline/execute.rs | 2 +- .../fabro-workflow/src/pipeline/finalize.rs | 16 +- .../fabro-workflow/src/pipeline/persist.rs | 14 +- .../src/pipeline/pull_request.rs | 150 ++-- .../fabro-workflow/src/pipeline/retro.rs | 32 +- .../fabro-workflow/src/pipeline/types.rs | 2 +- 47 files changed, 1724 insertions(+), 666 deletions(-) create mode 100644 lib/crates/fabro-store/src/run_state.rs diff --git a/lib/crates/fabro-cli/src/commands/pr/create.rs b/lib/crates/fabro-cli/src/commands/pr/create.rs index 3713eb98b..af805d1f6 100644 --- a/lib/crates/fabro-cli/src/commands/pr/create.rs +++ b/lib/crates/fabro-cli/src/commands/pr/create.rs @@ -34,23 +34,19 @@ async fn create_from( let store = store::build_store(storage_dir)?; let run = resolve_run_combined(store.as_ref(), base, &args.run_id).await?; let run_dir = run.path.clone(); - let run_store = store::open_run_reader(storage_dir, &run.run_id) - .await? - .context("Failed to open run store")?; + let run_store = store::open_run_reader(storage_dir, &run.run_id).await?; + let state = run_store.state().await?; - let record = run_store - .get_run() - .await? + let record = state + .run .context("Failed to load run record from store")?; - let start = run_store - .get_start() - .await? + let start = state + .start .context("Failed to load start record from store")?; - let conclusion = run_store - .get_conclusion() - .await? + let conclusion = state + .conclusion .context("Failed to load conclusion from store — is the run finished?")?; match conclusion.status { @@ -63,9 +59,8 @@ async fn create_from( .as_deref() .context("Run has no run_branch — was it run with git push enabled?")?; - let diff = run_store - .get_final_patch() - .await? + let diff = state + .final_patch .context("Failed to load final patch from store — no diff available")?; if diff.trim().is_empty() { bail!("final.patch is empty — nothing to create a PR for"); @@ -120,7 +115,7 @@ async fn create_from( &model, true, None, - Some(run_store.as_ref()), + run_store.as_ref(), &run_dir, None, ) diff --git a/lib/crates/fabro-cli/src/commands/pr/list.rs b/lib/crates/fabro-cli/src/commands/pr/list.rs index 698076518..30cbdcfc0 100644 --- a/lib/crates/fabro-cli/src/commands/pr/list.rs +++ b/lib/crates/fabro-cli/src/commands/pr/list.rs @@ -49,9 +49,11 @@ async fn list_from( let mut entries: Vec<(String, PullRequestRecord)> = Vec::new(); for run in &runs { - if let Ok(Some(run_store)) = store.open_run_reader(&run.run_id).await { - if let Ok(Some(record)) = run_store.get_pull_request().await { - entries.push((run.run_id.to_string(), record)); + if let Ok(run_store) = store.open_run_reader(&run.run_id).await { + if let Ok(state) = run_store.state().await { + if let Some(record) = state.pull_request { + entries.push((run.run_id.to_string(), record)); + } } } } diff --git a/lib/crates/fabro-cli/src/commands/pr/mod.rs b/lib/crates/fabro-cli/src/commands/pr/mod.rs index a165b9fe7..8032f4611 100644 --- a/lib/crates/fabro-cli/src/commands/pr/mod.rs +++ b/lib/crates/fabro-cli/src/commands/pr/mod.rs @@ -37,10 +37,9 @@ pub(crate) async fn load_pr_record( let store = store::build_store(storage_dir)?; let run = resolve_run_combined(store.as_ref(), base, run_id).await?; let run_dir = run.path; - let run_store = store::open_run_reader(storage_dir, &run.run_id) - .await? - .context("Failed to open run store")?; - let record = run_store.get_pull_request().await?.with_context(|| { + let run_store = store::open_run_reader(storage_dir, &run.run_id).await?; + let state = run_store.state().await?; + let record = state.pull_request.with_context(|| { format!("No pull request found in store. Create one first with: fabro pr create {run_id}") })?; Ok((record, run_dir)) diff --git a/lib/crates/fabro-cli/src/commands/run/attach.rs b/lib/crates/fabro-cli/src/commands/run/attach.rs index 1d6181dda..d625fe1bb 100644 --- a/lib/crates/fabro-cli/src/commands/run/attach.rs +++ b/lib/crates/fabro-cli/src/commands/run/attach.rs @@ -50,13 +50,13 @@ pub(crate) async fn attach_run( if let (Some(storage_dir), Some(run_id)) = (storage_dir.as_deref(), run_id.as_ref()) { match store::open_run_reader(storage_dir, run_id).await { - Ok(Some(run_store)) => match run_store.list_events().await { + Ok(run_store) => match run_store.list_events().await { Ok(events) => { let verbose = run_store - .get_run() + .state() .await .ok() - .flatten() + .and_then(|state| state.run) .is_some_and(|record| record.settings.verbose_enabled()); let event_lines = events .iter() @@ -83,7 +83,6 @@ pub(crate) async fn attach_run( ); } }, - Ok(None) => {} Err(err) => { tracing::warn!( run_id = %run_id, @@ -157,13 +156,14 @@ async fn attach_run_store( } // Wait briefly for a terminal status or conclusion for _ in 0..20 { - if run_store.get_conclusion().await.ok().flatten().is_some() - || run_store - .get_status() - .await - .ok() - .flatten() - .is_some_and(|record| record.status.is_terminal()) + if run_store + .state() + .await + .ok() + .is_some_and(|state| { + state.conclusion.is_some() + || state.status.is_some_and(|record| record.status.is_terminal()) + }) { break; } @@ -237,11 +237,10 @@ async fn attach_run_store( } let terminal_status = run_store - .get_status() + .state() .await .ok() - .flatten() - .map(|record| record.status) + .and_then(|state| state.status.map(|record| record.status)) .filter(|status| status.is_terminal()); let child_alive_via_handle = engine_guard.as_mut().and_then(|guard| { @@ -749,24 +748,29 @@ fn determine_exit_code(conclusion_path: &Path, status_record: Option ExitCode { let deadline = Instant::now() + ATTACH_FINAL_STATUS_GRACE; loop { - if let Ok(Some(conclusion)) = run_store.get_conclusion().await { - let success = matches!( - conclusion.status, - StageStatus::Success | StageStatus::PartialSuccess - ); - return if success { - ExitCode::from(0) - } else { - ExitCode::from(1) - }; - } + match run_store.state().await { + Ok(state) => { + if let Some(conclusion) = state.conclusion { + let success = matches!( + conclusion.status, + StageStatus::Success | StageStatus::PartialSuccess + ); + return if success { + ExitCode::from(0) + } else { + ExitCode::from(1) + }; + } - match run_store.get_status().await { - Ok(Some(record)) if matches!(record.status, RunStatus::Succeeded) => { - return ExitCode::from(0); + match state.status { + Some(record) if matches!(record.status, RunStatus::Succeeded) => { + return ExitCode::from(0); + } + Some(record) if record.status.is_terminal() => return ExitCode::from(1), + Some(_) | None => {} + } } - Ok(Some(record)) if record.status.is_terminal() => return ExitCode::from(1), - Ok(Some(_) | None) | Err(_) => {} + Err(_) => {} } if Instant::now() >= deadline { diff --git a/lib/crates/fabro-cli/src/commands/run/cp.rs b/lib/crates/fabro-cli/src/commands/run/cp.rs index ddc4eb3d0..4f0299216 100644 --- a/lib/crates/fabro-cli/src/commands/run/cp.rs +++ b/lib/crates/fabro-cli/src/commands/run/cp.rs @@ -130,12 +130,11 @@ async fn load_sandbox( ) -> Result> { let store = store::build_store(storage_dir)?; let run = resolve_run_combined(store.as_ref(), base, run_prefix).await?; - let run_store = store::open_run_reader(storage_dir, &run.run_id) - .await? - .context("Failed to open run store")?; + let run_store = store::open_run_reader(storage_dir, &run.run_id).await?; let record = run_store - .get_sandbox() + .state() .await? + .sandbox .context("Failed to load sandbox record from store")?; info!(run_id = %run_prefix, provider = %record.provider, "Connecting to sandbox"); diff --git a/lib/crates/fabro-cli/src/commands/run/detached.rs b/lib/crates/fabro-cli/src/commands/run/detached.rs index b6ff68f4f..f476121b8 100644 --- a/lib/crates/fabro-cli/src/commands/run/detached.rs +++ b/lib/crates/fabro-cli/src/commands/run/detached.rs @@ -29,10 +29,7 @@ pub(crate) async fn execute( None => load_user_settings()?.storage_dir(), }; let store = store::build_store(&storage_dir)?; - let run_store = store - .open_run(&run_id) - .await? - .ok_or_else(|| anyhow!("Run {run_id} not found in store"))?; + let run_store = store.open_run(&run_id).await?; let run_record = run_store .get_run() .await? diff --git a/lib/crates/fabro-cli/src/commands/run/diff.rs b/lib/crates/fabro-cli/src/commands/run/diff.rs index 67e58f481..08a41ae0c 100644 --- a/lib/crates/fabro-cli/src/commands/run/diff.rs +++ b/lib/crates/fabro-cli/src/commands/run/diff.rs @@ -18,9 +18,7 @@ pub(crate) async fn run(args: DiffArgs, globals: &GlobalArgs) -> Result<()> { let base = runs_base(&cli_settings.storage_dir()); let store = store::build_store(&cli_settings.storage_dir())?; let run = resolve_run_combined(store.as_ref(), &base, &args.run).await?; - let run_store = store::open_run_reader(&cli_settings.storage_dir(), &run.run_id) - .await? - .context("Failed to open run store")?; + let run_store = store::open_run_reader(&cli_settings.storage_dir(), &run.run_id).await?; let patch = resolve_diff(&run.path, run_store.as_ref(), &args).await?; @@ -57,17 +55,13 @@ async fn resolve_diff( run_store: &dyn fabro_store::RunStore, args: &DiffArgs, ) -> Result { + let state = run_store.state().await?; if let Some(ref node_id) = args.node { - if let Ok(visits) = run_store.list_node_visits(node_id).await { - if let Some(visit) = visits.into_iter().max() { - if let Ok(node) = run_store - .get_node(&fabro_store::NodeVisitRef { node_id, visit }) - .await - { - if let Some(patch) = node.diff { - debug!(node_id, visit, "Reading per-node diff from store"); - return Ok(patch); - } + if let Some(visit) = state.list_node_visits(node_id).into_iter().max() { + if let Some(node) = state.node(&fabro_store::NodeVisitRef { node_id, visit }) { + if let Some(patch) = node.diff.clone() { + debug!(node_id, visit, "Reading per-node diff from projected state"); + return Ok(patch); } } } @@ -75,9 +69,8 @@ async fn resolve_diff( bail!("No diff found for node '{node_id}' — check the node ID and try again"); } - let start = run_store - .get_start() - .await? + let start = state + .start .context("Failed to load start record from store")?; let base_sha = start @@ -85,12 +78,12 @@ async fn resolve_diff( .as_deref() .ok_or_else(|| anyhow::anyhow!("This run was not git-checkpointed; no diff available"))?; - if let Ok(Some(patch)) = run_store.get_final_patch().await { + if let Some(patch) = state.final_patch { debug!("Reading final.patch from store"); return Ok(patch); } - let run_concluded = run_store.get_conclusion().await?.is_some(); + let run_concluded = state.conclusion.is_some(); if run_concluded { bail!( "Run completed but no final.patch exists — the run may not have produced any changes" @@ -98,9 +91,8 @@ async fn resolve_diff( } debug!("No final.patch found; attempting live diff from sandbox"); - let record = run_store - .get_sandbox() - .await? + let record = state + .sandbox .context("Failed to load sandbox record from store")?; info!(provider = %record.provider, "Reconnecting to sandbox for live diff"); diff --git a/lib/crates/fabro-cli/src/commands/run/fork.rs b/lib/crates/fabro-cli/src/commands/run/fork.rs index d5b699c95..6808986f8 100644 --- a/lib/crates/fabro-cli/src/commands/run/fork.rs +++ b/lib/crates/fabro-cli/src/commands/run/fork.rs @@ -21,7 +21,7 @@ pub(crate) async fn run(args: &ForkArgs, styles: &Styles, globals: &GlobalArgs) let store = Store::new(repo); let run_store = open_run_reader(&cli_settings.storage_dir(), &run_id).await?; - let timeline = build_timeline_or_rebuild(&store, run_store.as_deref(), &run_id).await?; + let timeline = build_timeline_or_rebuild(&store, Some(run_store.as_ref()), &run_id).await?; if args.list { if globals.json { diff --git a/lib/crates/fabro-cli/src/commands/run/logs.rs b/lib/crates/fabro-cli/src/commands/run/logs.rs index ed39bde0c..93047df2b 100644 --- a/lib/crates/fabro-cli/src/commands/run/logs.rs +++ b/lib/crates/fabro-cli/src/commands/run/logs.rs @@ -31,9 +31,7 @@ pub(crate) async fn run(args: &LogsArgs, styles: &Styles, globals: &GlobalArgs) None => None, }; - let run_store = store::open_run_reader(&cli_settings.storage_dir(), &run.run_id) - .await? - .with_context(|| format!("Run '{}' not found in store", run.run_id))?; + let run_store = store::open_run_reader(&cli_settings.storage_dir(), &run.run_id).await?; let (all_lines, last_seq) = match run_store.list_events().await { Ok(events) => { let last_seq = events.last().map_or(0, |event| event.seq); @@ -190,20 +188,11 @@ async fn follow_store_logs( } async fn run_concluded(run_store: &dyn RunStore, _run_dir: &Path) -> Result { - if run_store - .get_conclusion() + let state = run_store + .state() .await - .context("Failed to read conclusion from store while following logs")? - .is_some() - { - return Ok(true); - } - - Ok(run_store - .get_status() - .await - .context("Failed to read status from store while following logs")? - .is_some_and(|record| record.status.is_terminal())) + .context("Failed to read run state from store while following logs")?; + Ok(state.conclusion.is_some() || state.status.is_some_and(|record| record.status.is_terminal())) } async fn flush_remaining_store_events( diff --git a/lib/crates/fabro-cli/src/commands/run/output.rs b/lib/crates/fabro-cli/src/commands/run/output.rs index cc9e5aa1b..dddc5acde 100644 --- a/lib/crates/fabro-cli/src/commands/run/output.rs +++ b/lib/crates/fabro-cli/src/commands/run/output.rs @@ -81,18 +81,13 @@ pub(crate) async fn print_run_summary( let (run_store, conclusion, pr_url) = match run_id.parse() { Ok(parsed_run_id) => { let run_store = store::open_run_reader(storage_dir, &parsed_run_id).await?; - let conclusion = match run_store.as_deref() { - Some(run_store) => run_store.get_conclusion().await?, - None => None, - }; - let pr_url = match run_store.as_deref() { - Some(run_store) => run_store - .get_pull_request() - .await? - .map(|record: PullRequestRecord| record.html_url), - None => None, - }; - (run_store, conclusion, pr_url) + let run_state = run_store.state().await?; + let conclusion = run_state.conclusion.clone(); + let pr_url = run_state + .pull_request + .as_ref() + .map(|record: &PullRequestRecord| record.html_url.clone()); + (Some(run_store), conclusion, pr_url) } Err(_) => (None, None, None), }; @@ -208,7 +203,7 @@ pub(crate) async fn print_final_output( styles: &Styles, ) { let checkpoint = match run_store { - Some(run_store) => run_store.get_checkpoint().await.ok().flatten(), + Some(run_store) => run_store.state().await.ok().and_then(|state| state.checkpoint), None => None, }; let Some(checkpoint) = checkpoint else { diff --git a/lib/crates/fabro-cli/src/commands/run/preview.rs b/lib/crates/fabro-cli/src/commands/run/preview.rs index 8118e454c..126b3d605 100644 --- a/lib/crates/fabro-cli/src/commands/run/preview.rs +++ b/lib/crates/fabro-cli/src/commands/run/preview.rs @@ -13,12 +13,11 @@ pub(crate) async fn run(args: PreviewArgs, globals: &GlobalArgs) -> Result<()> { let base = runs_base(&cli_settings.storage_dir()); let store = store::build_store(&cli_settings.storage_dir())?; let run = resolve_run_combined(store.as_ref(), &base, &args.run).await?; - let run_store = store::open_run_reader(&cli_settings.storage_dir(), &run.run_id) - .await? - .context("Failed to open run store")?; + let run_store = store::open_run_reader(&cli_settings.storage_dir(), &run.run_id).await?; let record = run_store - .get_sandbox() + .state() .await? + .sandbox .context("Failed to load sandbox record from store")?; validate_daytona_provider(&record, "Preview URLs")?; diff --git a/lib/crates/fabro-cli/src/commands/run/rewind.rs b/lib/crates/fabro-cli/src/commands/run/rewind.rs index b116369b2..01597c625 100644 --- a/lib/crates/fabro-cli/src/commands/run/rewind.rs +++ b/lib/crates/fabro-cli/src/commands/run/rewind.rs @@ -45,7 +45,7 @@ pub(crate) async fn run(args: &RewindArgs, styles: &Styles, globals: &GlobalArgs .await .ok(); - let timeline = build_timeline_or_rebuild(&store, run_store.as_deref(), &run_id).await?; + let timeline = build_timeline_or_rebuild(&store, Some(run_store.as_ref()), &run_id).await?; if args.list || args.target.is_none() { if globals.json { @@ -113,45 +113,28 @@ async fn reset_rewound_run_state( .open_run_reader(run_id) .await .map_err(|err| anyhow::anyhow!("failed to open durable store run before rewind: {err}"))?; - let store_run_record = if let Some(run_store) = existing_run_store.as_ref() { - run_store.get_run().await.ok().flatten() - } else { - None - }; - let store_start_record = if let Some(run_store) = existing_run_store.as_ref() { - run_store.get_start().await.ok().flatten() - } else { - None - }; - let store_graph = if let Some(run_store) = existing_run_store.as_ref() { - run_store.get_graph().await.ok().flatten() - } else { - None - }; + let store_run_record = existing_run_store.get_run().await.ok().flatten(); + let store_start_record = existing_run_store.get_start().await.ok().flatten(); + let store_graph = existing_run_store.get_graph().await.ok().flatten(); let run_record = store_run_record .or_else(|| RunRecord::load(run_dir).ok()) .context("failed to restore run record after rewind: missing run metadata")?; let checkpoint = MetadataStore::read_checkpoint(git_store.repo_dir(), &run_id.to_string())? .context("rewound metadata branch is missing checkpoint.json")?; - let previous_status = if let Some(run_store) = existing_run_store.as_ref() { - run_store - .get_status() - .await - .ok() - .flatten() - .map(|status| status.status.to_string()) - } else { - None - }; + let previous_status = existing_run_store + .get_status() + .await + .ok() + .flatten() + .map(|status| status.status.to_string()); let _ = std::fs::remove_file(run_dir.join("detached_failure.json")); let run_store = durable_store .open_run(run_id) .await - .map_err(|err| anyhow::anyhow!("failed to open durable store run for rewind reset: {err}"))? - .context("failed to reset durable store run after rewind: missing run store")?; + .map_err(|err| anyhow::anyhow!("failed to open durable store run for rewind reset: {err}"))?; run_store .reset_for_rewind() .await diff --git a/lib/crates/fabro-cli/src/commands/run/ssh.rs b/lib/crates/fabro-cli/src/commands/run/ssh.rs index 11a2c7011..5b5c2a916 100644 --- a/lib/crates/fabro-cli/src/commands/run/ssh.rs +++ b/lib/crates/fabro-cli/src/commands/run/ssh.rs @@ -17,12 +17,11 @@ pub(crate) async fn run(args: SshArgs, globals: &GlobalArgs) -> Result<()> { let base = runs_base(&cli_settings.storage_dir()); let store = store::build_store(&cli_settings.storage_dir())?; let run = resolve_run_combined(store.as_ref(), &base, &args.run).await?; - let run_store = store::open_run_reader(&cli_settings.storage_dir(), &run.run_id) - .await? - .context("Failed to open run store")?; + let run_store = store::open_run_reader(&cli_settings.storage_dir(), &run.run_id).await?; let record = run_store - .get_sandbox() + .state() .await? + .sandbox .context("Failed to load sandbox record from store")?; validate_daytona_provider(&record, "SSH access")?; diff --git a/lib/crates/fabro-cli/src/commands/run/start.rs b/lib/crates/fabro-cli/src/commands/run/start.rs index 4799aac68..9fd95c2a8 100644 --- a/lib/crates/fabro-cli/src/commands/run/start.rs +++ b/lib/crates/fabro-cli/src/commands/run/start.rs @@ -1,6 +1,6 @@ use std::path::Path; -use anyhow::{Result, anyhow, bail}; +use anyhow::{Result, bail}; use chrono::Utc; use fabro_types::RunId; use fabro_workflow::run_status::RunStatus; @@ -83,10 +83,8 @@ async fn ensure_startable_run(storage_dir: &Path, run_id: &RunId) -> Result<()> bail!("an engine process is still running for this run — cannot start"); } - let run_store = store::open_run_reader(storage_dir, run_id) - .await? - .ok_or_else(|| anyhow!("Cannot start run: run {run_id} not found in store"))?; - if let Some(record) = run_store.get_status().await? { + let run_store = store::open_run_reader(storage_dir, run_id).await?; + if let Some(record) = run_store.state().await?.status { if !matches!(record.status, RunStatus::Submitted | RunStatus::Starting) { bail!( "cannot start run: status is {:?}, expected submitted", diff --git a/lib/crates/fabro-cli/src/commands/run/wait.rs b/lib/crates/fabro-cli/src/commands/run/wait.rs index f4d053a8a..6f18f7143 100644 --- a/lib/crates/fabro-cli/src/commands/run/wait.rs +++ b/lib/crates/fabro-cli/src/commands/run/wait.rs @@ -34,9 +34,8 @@ pub(crate) async fn run(args: &WaitArgs, styles: &Styles, globals: &GlobalArgs) let final_status = loop { let run_store = store::open_run_reader(&cli_settings.storage_dir(), &run_info.run_id) - .await? - .ok_or_else(|| anyhow::anyhow!("Run {} not found in store", run_info.run_id))?; - let status = run_store.get_status().await?.map(|record| record.status); + .await?; + let status = run_store.state().await?.status.map(|record| record.status); let status = status.unwrap_or_else(|| { if started_waiting_at.elapsed() < WAIT_STARTUP_GRACE { RunStatus::Submitted @@ -65,9 +64,8 @@ pub(crate) async fn run(args: &WaitArgs, styles: &Styles, globals: &GlobalArgs) }; let run_store = store::open_run_reader(&cli_settings.storage_dir(), &run_info.run_id) - .await? - .ok_or_else(|| anyhow::anyhow!("Run {} not found in store", run_info.run_id))?; - let conclusion = run_store.get_conclusion().await?; + .await?; + let conclusion = run_store.state().await?.conclusion; if globals.json { let json_value = build_json_output(final_status, &run_info.run_id, conclusion.as_ref()); diff --git a/lib/crates/fabro-cli/src/commands/runs/inspect.rs b/lib/crates/fabro-cli/src/commands/runs/inspect.rs index 8fcfe9bf2..ef73bb86e 100644 --- a/lib/crates/fabro-cli/src/commands/runs/inspect.rs +++ b/lib/crates/fabro-cli/src/commands/runs/inspect.rs @@ -1,6 +1,6 @@ use std::path::{Path, PathBuf}; -use anyhow::{Result, anyhow}; +use anyhow::Result; use fabro_types::RunId; use serde::Serialize; @@ -28,9 +28,7 @@ pub(crate) async fn run(args: &InspectArgs, globals: &GlobalArgs) -> Result<()> let base = runs_base(&cli_settings.storage_dir()); let store = store::build_store(&cli_settings.storage_dir())?; let run = resolve_run_combined(store.as_ref(), &base, &args.run).await?; - let run_store = store::open_run_reader(&cli_settings.storage_dir(), &run.run_id) - .await? - .ok_or_else(|| anyhow!("Run {} not found in store", run.run_id))?; + let run_store = store::open_run_reader(&cli_settings.storage_dir(), &run.run_id).await?; let output = inspect_run_store(&run.run_id, &run.path, run.status, run_store.as_ref()).await; let json = serde_json::to_string_pretty(&[output])?; println!("{json}"); @@ -43,6 +41,29 @@ async fn inspect_run_store( status: RunStatus, run_store: &dyn fabro_store::RunStore, ) -> InspectOutput { + if let Ok(state) = run_store.state().await { + if let Some(snapshot) = state.to_snapshot() { + return InspectOutput { + run_id: run_id.to_string(), + run_dir: run_dir.to_path_buf(), + status: state.status.as_ref().map_or(status, |record| record.status), + run_record: serde_json::to_value(snapshot.run).ok(), + start_record: snapshot + .start + .and_then(|record| serde_json::to_value(record).ok()), + conclusion: snapshot + .conclusion + .and_then(|record| serde_json::to_value(record).ok()), + checkpoint: snapshot + .checkpoint + .and_then(|record| serde_json::to_value(record).ok()), + sandbox: snapshot + .sandbox + .and_then(|record| serde_json::to_value(record).ok()), + }; + } + } + if let Ok(Some(snapshot)) = run_store.get_snapshot().await { return InspectOutput { run_id: run_id.to_string(), diff --git a/lib/crates/fabro-cli/src/commands/runs/rm.rs b/lib/crates/fabro-cli/src/commands/runs/rm.rs index c39afe8aa..600e73051 100644 --- a/lib/crates/fabro-cli/src/commands/runs/rm.rs +++ b/lib/crates/fabro-cli/src/commands/runs/rm.rs @@ -116,7 +116,7 @@ pub(crate) async fn remove_run_with_cleanup(store: &dyn Store, run: &RunInfo) -> async fn remove_run_dir_with_cleanup(store: &dyn Store, run: &RunInfo) -> Result<()> { let run_store = match store.open_run_reader(&run.run_id).await { - Ok(run_store) => run_store, + Ok(run_store) => Some(run_store), Err(err) => { warn!( run_id = %run.run_id, @@ -183,9 +183,8 @@ async fn load_sandbox_record( run_store: Option<&dyn fabro_store::RunStore>, ) -> Option { if let Some(run_store) = run_store { - match run_store.get_sandbox().await { - Ok(Some(record)) => return Some(record), - Ok(None) => {} + match run_store.state().await { + Ok(state) => return state.sandbox, Err(err) => { warn!(error = %err, "failed to load sandbox record from store"); } diff --git a/lib/crates/fabro-cli/src/commands/store/dump.rs b/lib/crates/fabro-cli/src/commands/store/dump.rs index 895b6c9b6..ced3466a1 100644 --- a/lib/crates/fabro-cli/src/commands/store/dump.rs +++ b/lib/crates/fabro-cli/src/commands/store/dump.rs @@ -2,7 +2,7 @@ use std::io::{ErrorKind, Write}; use std::path::{Component, Path, PathBuf}; use anyhow::{Context, Result, bail}; -use fabro_store::{NodeVisitRef, RunSnapshot, RunStore}; +use fabro_store::{NodeVisitRef, RunSnapshot, RunState, RunStore}; use fabro_workflow::run_lookup::{resolve_run_combined, runs_base}; use serde::Serialize; #[cfg(test)] @@ -18,14 +18,7 @@ pub(crate) async fn dump_command(args: &StoreDumpArgs, globals: &GlobalArgs) -> let base = runs_base(&cli_settings.storage_dir()); let store = store::build_store(&cli_settings.storage_dir())?; let run = resolve_run_combined(store.as_ref(), &base, &args.run).await?; - let run_store = store::open_run_reader(&cli_settings.storage_dir(), &run.run_id) - .await? - .with_context(|| { - format!( - "run {} is not in the store (it may be a legacy filesystem-only run)", - run.run_id - ) - })?; + let run_store = store::open_run_reader(&cli_settings.storage_dir(), &run.run_id).await?; let file_count = export_run(run_store.as_ref(), &args.output).await?; if globals.json { @@ -45,9 +38,9 @@ pub(crate) async fn dump_command(args: &StoreDumpArgs, globals: &GlobalArgs) -> } pub(crate) async fn export_run(run_store: &dyn RunStore, output_dir: &Path) -> Result { - let snapshot = run_store - .get_snapshot() - .await? + let state = run_store.state().await?; + let snapshot = state + .to_snapshot() .context("run has no data in the store")?; let output_state = inspect_output_dir(output_dir)?; @@ -66,7 +59,7 @@ pub(crate) async fn export_run(run_store: &dyn RunStore, output_dir: &Path) -> R })?; let staging_path = staging_dir.path().to_path_buf(); - let file_count = export_run_to_dir(run_store, &snapshot, &staging_path).await?; + let file_count = export_run_to_dir(run_store, &state, &snapshot, &staging_path).await?; if matches!(output_state, OutputDirState::ExistingEmpty) { std::fs::remove_dir(output_dir) @@ -86,6 +79,7 @@ pub(crate) async fn export_run(run_store: &dyn RunStore, output_dir: &Path) -> R async fn export_run_to_dir( run_store: &dyn RunStore, + state: &RunState, snapshot: &RunSnapshot, output_dir: &Path, ) -> Result { @@ -152,11 +146,11 @@ async fn export_run_to_dir( file_count += usize::from(write_optional_text_file( &output_dir.join("retro").join("prompt.md"), - run_store.get_retro_prompt().await?.as_deref(), + state.retro_prompt.as_deref(), )?); file_count += usize::from(write_optional_text_file( &output_dir.join("retro").join("response.md"), - run_store.get_retro_response().await?.as_deref(), + state.retro_response.as_deref(), )?); write_events_jsonl( @@ -165,12 +159,12 @@ async fn export_run_to_dir( )?; file_count += 1; - for (seq, checkpoint) in run_store.list_checkpoints().await? { + for (seq, checkpoint) in &state.checkpoints { write_json_file( &output_dir .join("checkpoints") .join(format!("{seq:04}.json")), - &checkpoint, + checkpoint, )?; file_count += 1; } diff --git a/lib/crates/fabro-cli/src/store.rs b/lib/crates/fabro-cli/src/store.rs index 2f1dad234..39e1344fc 100644 --- a/lib/crates/fabro-cli/src/store.rs +++ b/lib/crates/fabro-cli/src/store.rs @@ -21,7 +21,7 @@ pub(crate) fn build_store(storage_dir: &Path) -> Result> { pub(crate) async fn open_run_reader( storage_dir: &Path, run_id: &RunId, -) -> Result>> { +) -> Result> { build_store(storage_dir)? .open_run_reader(run_id) .await diff --git a/lib/crates/fabro-cli/tests/it/cmd/diff.rs b/lib/crates/fabro-cli/tests/it/cmd/diff.rs index b7e1d0147..f4ea62215 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/diff.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/diff.rs @@ -117,7 +117,7 @@ fn diff_completed_run_reads_store_final_patch_without_disk_file() { let patch = with_runtime(|runtime| { runtime.block_on(async { let store = build_store(&context.storage_dir); - let run_store = store.open_run(&run_id).await.unwrap().unwrap(); + let run_store = store.open_run(&run_id).await.unwrap(); run_store.get_final_patch().await.unwrap().unwrap() }) }); @@ -126,7 +126,7 @@ fn diff_completed_run_reads_store_final_patch_without_disk_file() { with_runtime(|runtime| { runtime.block_on(async { let store = build_store(&context.storage_dir); - let run_store = store.open_run(&run_id).await.unwrap().unwrap(); + let run_store = store.open_run(&run_id).await.unwrap(); run_store.put_final_patch(&patch).await.unwrap(); }); }); @@ -180,7 +180,7 @@ fn diff_node_reads_store_patch_without_disk_file() { let patch = with_runtime(|runtime| { runtime.block_on(async { let store = build_store(&context.storage_dir); - let run_store = store.open_run(&run_id).await.unwrap().unwrap(); + let run_store = store.open_run(&run_id).await.unwrap(); run_store .get_node(&fabro_store::NodeVisitRef { node_id: "step_one", @@ -197,7 +197,7 @@ fn diff_node_reads_store_patch_without_disk_file() { with_runtime(|runtime| { runtime.block_on(async { let store = build_store(&context.storage_dir); - let run_store = store.open_run(&run_id).await.unwrap().unwrap(); + let run_store = store.open_run(&run_id).await.unwrap(); run_store .put_node_diff( &fabro_store::NodeVisitRef { diff --git a/lib/crates/fabro-cli/tests/it/cmd/pr_view.rs b/lib/crates/fabro-cli/tests/it/cmd/pr_view.rs index ae157c2d8..d059bf1d3 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/pr_view.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/pr_view.rs @@ -76,7 +76,7 @@ fn pr_view_reads_pull_request_from_store_without_pull_request_json() { with_runtime(|runtime| { runtime.block_on(async { let store = build_store(&context.storage_dir); - let run_store = store.open_run(&run_id).await.unwrap().unwrap(); + let run_store = store.open_run(&run_id).await.unwrap(); run_store .put_pull_request(&PullRequestRecord { html_url: "https://github.com/fabro-sh/fabro/pull/123".to_string(), diff --git a/lib/crates/fabro-cli/tests/it/cmd/support.rs b/lib/crates/fabro-cli/tests/it/cmd/support.rs index e0f6c2761..7784583e7 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/support.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/support.rs @@ -159,10 +159,12 @@ pub(crate) fn setup_detached_dry_run(context: &TestContext) -> RunSetup { .to_string(); let run = resolve_run(context, &run_id); let deadline = Instant::now() + COMMAND_TIMEOUT; - while run_store(&run.run_dir) - .and_then(|store| block_on(store.list_events()).ok()) - .is_none_or(|events| events.is_empty()) - { + while { + let store = run_store(&run.run_dir); + block_on(store.list_events()) + .ok() + .is_none_or(|events| events.is_empty()) + } { assert!( Instant::now() < deadline, "timed out waiting for store events for {run_id}" @@ -280,9 +282,10 @@ worktree_mode = "never" ); let run = run_local_workflow(context, &workspace_dir, "run.toml"); + let store = run_store(&run.run_dir); assert!( - run_store(&run.run_dir) - .and_then(|store| block_on(store.get_sandbox()).ok()) + block_on(store.get_sandbox()) + .ok() .flatten() .is_some() ); @@ -371,8 +374,8 @@ pub(crate) fn write_gated_workflow(path: &Path, name: &str, goal: &str) -> Workf pub(crate) fn wait_for_status(run_dir: &Path, expected: &[&str]) -> String { let deadline = Instant::now() + COMMAND_TIMEOUT; loop { - if let Some(status) = run_store(run_dir) - .and_then(|store| block_on(store.get_status()).ok()) + if let Some(status) = block_on(run_store(run_dir).get_status()) + .ok() .flatten() .map(|record| record.status.to_string()) { @@ -478,25 +481,30 @@ fn block_on(future: impl std::future::Future) -> T { .block_on(future) } -fn run_store(run_dir: &Path) -> Option> { - let runs_dir = run_dir.parent()?; - let storage_dir = runs_dir.parent()?; - let run_id: RunId = infer_run_id(run_dir).parse().ok()?; - let object_store = Arc::new(LocalFileSystem::new_with_prefix(storage_dir.join("store")).ok()?); +fn run_store(run_dir: &Path) -> Arc { + let runs_dir = run_dir.parent().expect("run dir should have parent"); + let storage_dir = runs_dir.parent().expect("runs dir should have parent"); + let run_id: RunId = infer_run_id(run_dir).parse().expect("run id should parse"); + let object_store = Arc::new( + LocalFileSystem::new_with_prefix(storage_dir.join("store")) + .expect("test store path should be accessible"), + ); let store = Arc::new(SlateStore::new(object_store, "", Duration::from_millis(1))); - block_on(store.open_run_reader(&run_id)).ok().flatten() + block_on(store.open_run_reader(&run_id)).expect("run store should exist") } pub(crate) fn run_snapshot(run_dir: &Path) -> RunSnapshot { - run_store(run_dir) - .and_then(|store| block_on(store.get_snapshot()).ok()) + let store = run_store(run_dir); + block_on(store.get_snapshot()) + .ok() .flatten() .expect("run store snapshot should exist") } pub(crate) fn run_events(run_dir: &Path) -> Vec { - run_store(run_dir) - .and_then(|store| block_on(store.list_events()).ok()) + let store = run_store(run_dir); + block_on(store.list_events()) + .ok() .expect("run store events should exist") } diff --git a/lib/crates/fabro-cli/tests/it/scenario/mod.rs b/lib/crates/fabro-cli/tests/it/scenario/mod.rs index 0b4d5e379..6bfcfcbae 100644 --- a/lib/crates/fabro-cli/tests/it/scenario/mod.rs +++ b/lib/crates/fabro-cli/tests/it/scenario/mod.rs @@ -23,9 +23,9 @@ fn block_on(future: impl std::future::Future) -> T { .block_on(future) } -fn run_store(run_dir: &Path) -> Option> { - let runs_dir = run_dir.parent()?; - let storage_dir = runs_dir.parent()?; +fn run_store(run_dir: &Path) -> Arc { + let runs_dir = run_dir.parent().expect("run dir should have parent"); + let storage_dir = runs_dir.parent().expect("runs dir should have parent"); let run_id: RunId = std::fs::read_to_string(run_dir.join("id.txt")) .ok() .map(|id| id.trim().to_string()) @@ -34,17 +34,22 @@ fn run_store(run_dir: &Path) -> Option> { .file_name() .map(|name| name.to_string_lossy().to_string()) .and_then(|name| name.rsplit('-').next().map(ToOwned::to_owned)) - })? + }) + .expect("run dir should contain resolvable run id") .parse() - .ok()?; - let object_store = Arc::new(LocalFileSystem::new_with_prefix(storage_dir.join("store")).ok()?); + .expect("run id should parse"); + let object_store = Arc::new( + LocalFileSystem::new_with_prefix(storage_dir.join("store")) + .expect("test store path should be accessible"), + ); let store = Arc::new(SlateStore::new(object_store, "", Duration::from_millis(1))); - block_on(store.open_run_reader(&run_id)).ok().flatten() + block_on(store.open_run_reader(&run_id)).expect("run store should exist") } pub(super) fn run_snapshot(run_dir: &Path) -> RunSnapshot { - run_store(run_dir) - .and_then(|store| block_on(store.get_snapshot()).ok()) + let store = run_store(run_dir); + block_on(store.get_snapshot()) + .ok() .flatten() .expect("run store snapshot should exist") } diff --git a/lib/crates/fabro-retro/src/retro_agent.rs b/lib/crates/fabro-retro/src/retro_agent.rs index 3e63350af..1f26115ba 100644 --- a/lib/crates/fabro-retro/src/retro_agent.rs +++ b/lib/crates/fabro-retro/src/retro_agent.rs @@ -137,7 +137,7 @@ pub fn build_retro_prompt(retro_data_dir: &str) -> String { /// files via tool access, then calls `submit_retro` with its analysis. pub async fn run_retro_agent( sandbox: &Arc, - run_store: Option<&dyn RunStore>, + run_store: &dyn RunStore, run_dir: &Path, llm_client: &Client, provider: Provider, @@ -286,32 +286,24 @@ pub fn dry_run_narrative() -> RetroNarrative { } async fn write_retro_prompt( - run_store: Option<&dyn RunStore>, + run_store: &dyn RunStore, retro_dir: &Path, prompt: &str, ) -> anyhow::Result<()> { - if let Some(store) = run_store { - if let Err(err) = store.put_retro_prompt(prompt).await { - tracing::warn!(error = %err, "Failed to save retro prompt to store"); - std::fs::write(retro_dir.join("prompt.md"), prompt)?; - } - } else { + if let Err(err) = run_store.put_retro_prompt(prompt).await { + tracing::warn!(error = %err, "Failed to save retro prompt to store"); std::fs::write(retro_dir.join("prompt.md"), prompt)?; } Ok(()) } async fn write_retro_response( - run_store: Option<&dyn RunStore>, + run_store: &dyn RunStore, retro_dir: &Path, response: &str, ) -> anyhow::Result<()> { - if let Some(store) = run_store { - if let Err(err) = store.put_retro_response(response).await { - tracing::warn!(error = %err, "Failed to save retro response to store"); - std::fs::write(retro_dir.join("response.md"), response)?; - } - } else { + if let Err(err) = run_store.put_retro_response(response).await { + tracing::warn!(error = %err, "Failed to save retro response to store"); std::fs::write(retro_dir.join("response.md"), response)?; } Ok(()) @@ -393,7 +385,7 @@ fn build_profile(provider: Provider, model: &str) -> Box { async fn upload_data_files( sandbox: &Arc, - run_store: Option<&dyn RunStore>, + run_store: &dyn RunStore, _run_dir: &Path, target_dir: &str, ) -> anyhow::Result<()> { @@ -403,11 +395,7 @@ async fn upload_data_files( .await .map_err(|e| anyhow::anyhow!("Failed to create retro data dir: {e}"))?; - let Some(store) = run_store else { - anyhow::bail!("retro analysis now requires a run store"); - }; - - let progress_content = match store.list_events().await { + let progress_content = match run_store.list_events().await { Ok(envelopes) => { let lines: Vec = envelopes .into_iter() @@ -428,7 +416,7 @@ async fn upload_data_files( .map_err(|e| anyhow::anyhow!("Failed to upload progress.jsonl: {e}"))?; } - let checkpoint_content = store + let checkpoint_content = run_store .get_checkpoint() .await .map_err(|e| anyhow::anyhow!("Failed to load checkpoint from store: {e}"))? @@ -436,7 +424,7 @@ async fn upload_data_files( .transpose()?; upload_file(sandbox, target_dir, "checkpoint.json", checkpoint_content).await?; - let run_content = store + let run_content = run_store .get_run() .await .map_err(|e| anyhow::anyhow!("Failed to load run metadata from store: {e}"))? @@ -444,7 +432,7 @@ async fn upload_data_files( .transpose()?; upload_file(sandbox, target_dir, "run.json", run_content).await?; - let start_content = store + let start_content = run_store .get_start() .await .map_err(|e| anyhow::anyhow!("Failed to load start metadata from store: {e}"))? diff --git a/lib/crates/fabro-server/src/server.rs b/lib/crates/fabro-server/src/server.rs index 55f6de369..515e6045e 100644 --- a/lib/crates/fabro-server/src/server.rs +++ b/lib/crates/fabro-server/src/server.rs @@ -680,18 +680,7 @@ async fn execute_run(state: Arc, run_id: RunId) { } let run_store = match state.store.open_run(&run_id).await { - Ok(Some(run_store)) => run_store, - Ok(None) => { - tracing::error!(run_id = %run_id, "Run store missing"); - let mut runs = state.runs.lock().expect("runs lock poisoned"); - if let Some(managed_run) = runs.get_mut(&run_id) { - managed_run.status = RunStatus::Failed; - managed_run.error = Some("Run store missing".to_string()); - clear_live_run_state(managed_run); - } - state.scheduler_notify.notify_one(); - return; - } + Ok(run_store) => run_store, Err(e) => { tracing::error!(run_id = %run_id, error = %e, "Failed to open run store"); let mut runs = state.runs.lock().expect("runs lock poisoned"); @@ -754,10 +743,10 @@ async fn execute_run(state: Arc, run_id: RunId) { }; // Save final checkpoint - let checkpoint = match run_store.get_checkpoint().await { - Ok(checkpoint) => checkpoint, + let checkpoint = match run_store.state().await { + Ok(state) => state.checkpoint, Err(err) => { - tracing::warn!(run_id = %run_id, error = %err, "Failed to load checkpoint from store"); + tracing::warn!(run_id = %run_id, error = %err, "Failed to load run state from store"); None } }; @@ -1069,13 +1058,32 @@ async fn get_checkpoint( Ok(id) => id, Err(response) => return response, }; - let runs = state.runs.lock().expect("runs lock poisoned"); - match runs.get(&id) { - Some(managed_run) => match &managed_run.checkpoint { - Some(cp) => (StatusCode::OK, Json(cp.clone())).into_response(), - None => (StatusCode::OK, Json(serde_json::json!(null))).into_response(), + let live_checkpoint = { + let runs = state.runs.lock().expect("runs lock poisoned"); + match runs.get(&id) { + Some(managed_run) => managed_run.checkpoint.clone(), + None => return ApiError::not_found("Run not found.").into_response(), + } + }; + if let Some(cp) = live_checkpoint { + return (StatusCode::OK, Json(cp)).into_response(); + } + + match state.store.open_run_reader(&id).await { + Ok(run_store) => match run_store.state().await { + Ok(run_state) => match run_state.checkpoint { + Some(cp) => (StatusCode::OK, Json(cp)).into_response(), + None => (StatusCode::OK, Json(serde_json::json!(null))).into_response(), + }, + Err(err) => { + tracing::warn!(run_id = %id, error = %err, "Failed to load checkpoint state from store"); + (StatusCode::OK, Json(serde_json::json!(null))).into_response() + } }, - None => ApiError::not_found("Run not found.").into_response(), + Err(err) => { + tracing::warn!(run_id = %id, error = %err, "Failed to open run store reader"); + ApiError::not_found("Run not found.").into_response() + } } } @@ -1564,18 +1572,19 @@ async fn get_retro( } match state.store.open_run_reader(&id).await { - Ok(Some(run_store)) => match run_store.get_retro().await { - Ok(Some(retro)) => (StatusCode::OK, Json(retro)).into_response(), - Ok(None) => (StatusCode::OK, Json(serde_json::json!(null))).into_response(), + Ok(run_store) => match run_store.state().await { + Ok(run_state) => match run_state.retro { + Some(retro) => (StatusCode::OK, Json(retro)).into_response(), + None => (StatusCode::OK, Json(serde_json::json!(null))).into_response(), + }, Err(err) => { - tracing::warn!(run_id = %id, error = %err, "Failed to load retro from store"); + tracing::warn!(run_id = %id, error = %err, "Failed to load retro state from store"); (StatusCode::OK, Json(serde_json::json!(null))).into_response() } }, - Ok(None) => (StatusCode::OK, Json(serde_json::json!(null))).into_response(), Err(err) => { tracing::warn!(run_id = %id, error = %err, "Failed to open run store reader"); - (StatusCode::OK, Json(serde_json::json!(null))).into_response() + ApiError::not_found("Run not found.").into_response() } } } @@ -1603,15 +1612,27 @@ async fn get_graph( Ok(id) => id, Err(response) => return response, }; - let dot_source = { + let live_dot_source = { let runs = state.runs.lock().expect("runs lock poisoned"); match runs.get(&id) { Some(managed_run) => managed_run.dot_source.clone(), None => return ApiError::not_found("Run not found.").into_response(), } }; + if !live_dot_source.is_empty() { + return render_dot_svg(&live_dot_source).await; + } - render_dot_svg(&dot_source).await + match state.store.open_run_reader(&id).await { + Ok(run_store) => match run_store.state().await { + Ok(run_state) => match run_state.graph_source { + Some(dot_source) => render_dot_svg(&dot_source).await, + None => ApiError::new(StatusCode::NOT_FOUND, "Graph not found.").into_response(), + }, + Err(err) => ApiError::new(StatusCode::BAD_GATEWAY, err.to_string()).into_response(), + }, + Err(_) => ApiError::new(StatusCode::NOT_FOUND, "Run not found.").into_response(), + } } #[cfg(test)] diff --git a/lib/crates/fabro-store/src/lib.rs b/lib/crates/fabro-store/src/lib.rs index 07c3c3fc5..7d6ad1659 100644 --- a/lib/crates/fabro-store/src/lib.rs +++ b/lib/crates/fabro-store/src/lib.rs @@ -9,12 +9,14 @@ use futures::Stream; mod error; mod keys; mod memory; +mod run_state; mod runtime; mod slate; mod types; pub use error::{Result, StoreError}; pub use memory::InMemoryStore; +pub use run_state::{NodeState, RunState}; pub use runtime::RuntimeState; pub use slate::SlateStore; pub use types::{ @@ -42,8 +44,8 @@ pub trait Store: Send + Sync { created_at: DateTime, run_dir: Option<&str>, ) -> Result>; - async fn open_run(&self, run_id: &RunId) -> Result>>; - async fn open_run_reader(&self, run_id: &RunId) -> Result>>; + async fn open_run(&self, run_id: &RunId) -> Result>; + async fn open_run_reader(&self, run_id: &RunId) -> Result>; async fn list_runs(&self, query: &ListRunsQuery) -> Result>; async fn delete_run(&self, run_id: &RunId) -> Result<()>; } @@ -146,5 +148,6 @@ pub trait RunStore: Send + Sync { async fn list_assets(&self, node: &NodeVisitRef<'_>) -> Result>; async fn list_all_assets(&self) -> Result>; + async fn state(&self) -> Result; async fn get_snapshot(&self) -> Result>; } diff --git a/lib/crates/fabro-store/src/memory.rs b/lib/crates/fabro-store/src/memory.rs index febc85ffe..690263d55 100644 --- a/lib/crates/fabro-store/src/memory.rs +++ b/lib/crates/fabro-store/src/memory.rs @@ -13,9 +13,10 @@ use tokio_stream::StreamExt as _; use tokio_stream::wrappers::UnboundedReceiverStream; use crate::keys; +use crate::run_state::EventProjectionCache; use crate::{ CatalogRecord, EventEnvelope, EventPayload, ListRunsQuery, NodeOutcomeRecord, NodeSnapshot, - NodeVisitRef, Result, RunSnapshot, RunStore, RunSummary, Store, StoreError, + NodeVisitRef, Result, RunSnapshot, RunState, RunStore, RunSummary, Store, StoreError, }; use fabro_types::{ Checkpoint, Conclusion, NodeStatusRecord, PullRequestRecord, Retro, RunId, RunRecord, @@ -41,6 +42,7 @@ struct InMemoryRunStore { event_seq: AtomicU32, checkpoint_seq: AtomicU32, watchers: Mutex>>, + projection_cache: Mutex, } impl InMemoryRunStore { @@ -65,6 +67,7 @@ impl InMemoryRunStore { event_seq: AtomicU32::new(1), checkpoint_seq: AtomicU32::new(1), watchers: Mutex::new(Vec::new()), + projection_cache: Mutex::new(EventProjectionCache::default()), }) } @@ -264,6 +267,20 @@ impl InMemoryRunStore { } Ok(()) } + + async fn projected_state(&self) -> Result { + let next_seq = { + let cache = self.projection_cache.lock().await; + cache.last_seq.saturating_add(1) + }; + let events = self.list_events_from_inner(next_seq).await?; + let mut cache = self.projection_cache.lock().await; + for event in &events { + cache.state.apply_event(event)?; + cache.last_seq = event.seq; + } + Ok(cache.state.clone()) + } } #[async_trait] @@ -302,14 +319,14 @@ impl Store for InMemoryStore { Ok(run_store as Arc) } - async fn open_run(&self, run_id: &RunId) -> Result>> { + async fn open_run(&self, run_id: &RunId) -> Result> { let runs = self.runs.lock().await; - Ok(runs - .get(run_id) - .map(|catalog| Arc::clone(&catalog.run_store) as Arc)) + runs.get(run_id) + .map(|catalog| Arc::clone(&catalog.run_store) as Arc) + .ok_or_else(|| StoreError::RunNotFound(run_id.to_string())) } - async fn open_run_reader(&self, run_id: &RunId) -> Result>> { + async fn open_run_reader(&self, run_id: &RunId) -> Result> { self.open_run(run_id).await } @@ -324,8 +341,7 @@ impl Store for InMemoryStore { if !matches_query(&catalog.record.created_at, query) { continue; } - let data = catalog.run_store.snapshot_data().await; - summaries.push(build_run_summary(&catalog.record, &data)?); + summaries.push(catalog.run_store.state().await?.build_summary(&catalog.record)); } summaries.sort_by(|a, b| b.created_at.cmp(&a.created_at)); Ok(summaries) @@ -665,9 +681,21 @@ impl RunStore for InMemoryRunStore { self.list_all_assets_inner().await } - async fn get_snapshot(&self) -> Result> { + async fn state(&self) -> Result { + let mut state = self.projected_state().await?; let data = self.snapshot_data().await; - self.build_snapshot_from_data(&data) + state.merge_legacy( + self.build_snapshot_from_data(&data)?, + read_text(&data, keys::graph())?, + read_text(&data, keys::retro_prompt())?, + read_text(&data, keys::retro_response())?, + self.list_checkpoints_inner().await?, + ); + Ok(state) + } + + async fn get_snapshot(&self) -> Result> { + self.state().await.map(|state| state.to_snapshot()) } } @@ -704,48 +732,6 @@ fn read_text(data: &BTreeMap>, key: &str) -> Result>, -) -> Result { - let run = read_json::(data, keys::run())?; - let start = read_json::(data, keys::start())?; - let status = read_json::(data, keys::status())?; - let conclusion = read_json::(data, keys::conclusion())?; - - let workflow_name = run.as_ref().map(|run| { - if run.graph.name.is_empty() { - "unnamed".to_string() - } else { - run.graph.name.clone() - } - }); - let goal = run.as_ref().and_then(|run| { - let goal = run.graph.goal(); - (!goal.is_empty()).then(|| goal.to_string()) - }); - - Ok(RunSummary { - run_id: record.run_id, - created_at: record.created_at, - db_prefix: record.db_prefix.clone(), - run_dir: record.run_dir.clone(), - workflow_name, - workflow_slug: run.as_ref().and_then(|run| run.workflow_slug.clone()), - goal, - labels: run - .as_ref() - .map(|run| run.labels.clone()) - .unwrap_or_default(), - host_repo_path: run.as_ref().and_then(|run| run.host_repo_path.clone()), - start_time: start.map(|start| start.start_time), - status: status.as_ref().map(|status| status.status), - status_reason: status.and_then(|status| status.reason), - duration_ms: conclusion.as_ref().map(|conclusion| conclusion.duration_ms), - total_cost: conclusion.and_then(|conclusion| conclusion.total_cost), - }) -} - #[cfg(test)] mod tests { use super::*; @@ -914,6 +900,26 @@ mod tests { } } + fn event_payload( + run_id: &str, + ts: &str, + event: &str, + node_id: Option<&str>, + properties: serde_json::Value, + ) -> EventPayload { + let mut value = serde_json::json!({ + "id": format!("evt-{event}-{ts}"), + "ts": ts, + "run_id": test_run_id(run_id).to_string(), + "event": event, + "properties": properties, + }); + if let Some(node_id) = node_id { + value["node_id"] = serde_json::Value::String(node_id.to_string()); + } + EventPayload::new(value, &test_run_id(run_id)).unwrap() + } + #[tokio::test] async fn create_run_put_get_and_snapshot_round_trip() { let store = InMemoryStore::default(); @@ -1102,6 +1108,317 @@ mod tests { assert_eq!(snapshot.pull_request, Some(pull_request)); } + #[tokio::test] + async fn state_projects_event_stream_and_compat_fields() { + let store = InMemoryStore::default(); + let created_at = dt("2026-03-27T12:00:00Z"); + let run = store + .create_run(&test_run_id("run-1"), created_at, None) + .await + .unwrap(); + let run_record = sample_run_record("run-1", created_at); + let retro = sample_retro("run-1"); + + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:00Z", + "run.created", + None, + serde_json::json!({ + "settings": run_record.settings, + "graph": run_record.graph, + "workflow_source": "digraph night_sky {}", + "workflow_slug": run_record.workflow_slug, + "working_directory": run_record.working_directory, + "host_repo_path": run_record.host_repo_path, + "base_branch": run_record.base_branch, + "labels": run_record.labels, + }), + )) + .await + .unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:05Z", + "run.started", + None, + serde_json::json!({ + "run_branch": "fabro/run/demo", + "base_sha": "abc123" + }), + )) + .await + .unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:06Z", + "run.running", + None, + serde_json::json!({}), + )) + .await + .unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:07Z", + "stage.prompt", + Some("code"), + serde_json::json!({ + "visit": 2, + "text": "Plan the fix", + "mode": "prompt", + "provider": "openai", + "model": "gpt-5.4" + }), + )) + .await + .unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:08Z", + "stage.completed", + Some("code"), + serde_json::json!({ + "status": "success", + "notes": "all good", + "response": "Implemented", + "files_touched": ["src/lib.rs"], + "node_visits": {"code": 2} + }), + )) + .await + .unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:09Z", + "checkpoint.completed", + Some("code"), + serde_json::json!({ + "status": "success", + "current_node": "code", + "completed_nodes": ["plan"], + "context_values": {"artifact": {"kind": "summary"}}, + "next_node_id": "review", + "git_commit_sha": "def456", + "node_visits": {"code": 2}, + "diff": "diff --git a/src/lib.rs b/src/lib.rs" + }), + )) + .await + .unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:10Z", + "sandbox.initialized", + None, + serde_json::json!({ + "provider": "local", + "working_directory": "/tmp/night-sky", + "identifier": "sandbox-1", + "host_working_directory": "/tmp/night-sky" + }), + )) + .await + .unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:11Z", + "retro.started", + None, + serde_json::json!({ + "prompt": "How did it go?" + }), + )) + .await + .unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:12Z", + "retro.completed", + None, + serde_json::json!({ + "response": "Smooth enough", + "retro": retro + }), + )) + .await + .unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:13Z", + "pull_request.created", + None, + serde_json::json!({ + "pr_url": "https://github.com/fabro-sh/fabro/pull/123", + "pr_number": 123, + "owner": "fabro-sh", + "repo": "fabro", + "base_branch": "main", + "head_branch": "fabro/run/demo", + "title": "Map the constellations", + "draft": false + }), + )) + .await + .unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:15Z", + "run.completed", + None, + serde_json::json!({ + "duration_ms": 3210, + "artifact_count": 1, + "status": "success", + "total_cost": 1.25, + "final_git_commit_sha": "feedbeef", + "final_patch": "diff --git a/src/lib.rs b/src/lib.rs\n" + }), + )) + .await + .unwrap(); + + let state = run.state().await.unwrap(); + assert_eq!(state.run.as_ref().map(|run| run.run_id), Some(test_run_id("run-1"))); + assert_eq!(state.graph_source.as_deref(), Some("digraph night_sky {}")); + assert_eq!(state.start.as_ref().and_then(|start| start.run_branch.as_deref()), Some("fabro/run/demo")); + assert_eq!(state.status.as_ref().map(|status| status.status), Some(RunStatus::Succeeded)); + assert_eq!(state.checkpoint.as_ref().map(|checkpoint| checkpoint.current_node.as_str()), Some("code")); + assert_eq!(state.checkpoints.len(), 1); + assert_eq!(state.final_patch.as_deref(), Some("diff --git a/src/lib.rs b/src/lib.rs\n")); + assert_eq!(state.retro_prompt.as_deref(), Some("How did it go?")); + assert_eq!(state.retro_response.as_deref(), Some("Smooth enough")); + assert_eq!(state.pull_request.as_ref().map(|pr| pr.number), Some(123)); + assert_eq!(state.sandbox.as_ref().map(|sandbox| sandbox.provider.as_str()), Some("local")); + assert_eq!(state.list_node_visits("code"), vec![2]); + let node = state + .node(&NodeVisitRef { + node_id: "code", + visit: 2, + }) + .unwrap(); + assert_eq!(node.prompt.as_deref(), Some("Plan the fix")); + assert_eq!(node.response.as_deref(), Some("Implemented")); + assert_eq!(node.diff.as_deref(), Some("diff --git a/src/lib.rs b/src/lib.rs")); + assert_eq!( + node.provider_used.as_ref().and_then(|value| value.get("provider")).and_then(|value| value.as_str()), + Some("openai") + ); + } + + #[tokio::test] + async fn state_rewind_keeps_active_projection_only() { + let store = InMemoryStore::default(); + let run = store + .create_run(&test_run_id("run-1"), dt("2026-03-27T12:00:00Z"), None) + .await + .unwrap(); + + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:00Z", + "run.created", + None, + serde_json::json!({ + "settings": Settings::default(), + "graph": Graph::new("night-sky"), + "working_directory": "/tmp/night-sky", + "labels": {} + }), + )) + .await + .unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:01Z", + "stage.prompt", + Some("code"), + serde_json::json!({ + "visit": 1, + "text": "before rewind" + }), + )) + .await + .unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:02Z", + "pull_request.created", + None, + serde_json::json!({ + "pr_url": "https://github.com/fabro-sh/fabro/pull/123", + "pr_number": 123, + "owner": "fabro-sh", + "repo": "fabro", + "base_branch": "main", + "head_branch": "fabro/run/demo", + "title": "Map the constellations", + "draft": false + }), + )) + .await + .unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:03Z", + "run.completed", + None, + serde_json::json!({ + "duration_ms": 10, + "artifact_count": 0, + "status": "success", + "final_patch": "old patch" + }), + )) + .await + .unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:04Z", + "run.rewound", + None, + serde_json::json!({ + "target_checkpoint_ordinal": 1, + "target_node_id": "plan", + "target_visit": 1 + }), + )) + .await + .unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:05Z", + "checkpoint.completed", + Some("plan"), + serde_json::json!({ + "status": "success", + "current_node": "plan", + "completed_nodes": [], + "node_visits": {"plan": 1} + }), + )) + .await + .unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:06Z", + "run.submitted", + None, + serde_json::json!({}), + )) + .await + .unwrap(); + + let state = run.state().await.unwrap(); + assert_eq!(state.status.as_ref().map(|status| status.status), Some(RunStatus::Submitted)); + assert!(state.conclusion.is_none()); + assert!(state.final_patch.is_none()); + assert!(state.pull_request.is_none()); + assert_eq!(state.checkpoints.len(), 1); + assert_eq!(state.checkpoint.as_ref().map(|checkpoint| checkpoint.current_node.as_str()), Some("plan")); + assert!(state.list_node_ids().is_empty()); + } + #[tokio::test] async fn list_artifact_values_and_all_assets_include_asset_only_visits() { let store = InMemoryStore::default(); @@ -1379,11 +1696,10 @@ mod tests { store.delete_run(&test_run_id("run-1")).await.unwrap(); store.delete_run(&test_run_id("run-1")).await.unwrap(); assert!( - store - .open_run(&test_run_id("run-1")) - .await - .unwrap() - .is_none() + matches!( + store.open_run(&test_run_id("run-1")).await, + Err(StoreError::RunNotFound(_)) + ) ); } diff --git a/lib/crates/fabro-store/src/run_state.rs b/lib/crates/fabro-store/src/run_state.rs new file mode 100644 index 000000000..e1770f6f7 --- /dev/null +++ b/lib/crates/fabro-store/src/run_state.rs @@ -0,0 +1,734 @@ +use std::collections::{BTreeMap, HashMap}; +use std::path::PathBuf; +use std::str::FromStr; + +use chrono::{DateTime, Utc}; +use serde_json::Value; + +use crate::{ + CatalogRecord, EventEnvelope, NodeOutcomeRecord, NodeSnapshot, NodeVisitRef, Result, + RunSnapshot, RunSummary, StoreError, +}; +use fabro_types::{ + Checkpoint, Conclusion, FailureSignature, NodeStatusRecord, Outcome, PullRequestRecord, Retro, + RunId, RunRecord, RunStatus, RunStatusRecord, SandboxRecord, StageStatus, StartRecord, + StatusReason, +}; + +#[derive(Debug, Clone, Default)] +pub struct RunState { + pub run: Option, + pub graph_source: Option, + pub start: Option, + pub status: Option, + pub checkpoint: Option, + pub checkpoints: Vec<(u32, Checkpoint)>, + pub conclusion: Option, + pub retro: Option, + pub retro_prompt: Option, + pub retro_response: Option, + pub sandbox: Option, + pub final_patch: Option, + pub pull_request: Option, + pub nodes: HashMap<(String, u32), NodeState>, + pub last_git_sha: Option, +} + +#[derive(Debug, Clone, Default)] +pub struct NodeState { + pub prompt: Option, + pub response: Option, + pub status: Option, + pub outcome: Option, + pub provider_used: Option, + pub diff: Option, + pub script_invocation: Option, + pub script_timing: Option, + pub parallel_results: Option, + pub stdout: Option, + pub stderr: Option, +} + +#[derive(Debug, Clone, Default)] +pub(crate) struct EventProjectionCache { + pub last_seq: u32, + pub state: RunState, +} + +impl RunState { + pub fn apply_events(events: &[EventEnvelope]) -> Result { + let mut state = Self::default(); + for event in events { + state.apply_event(event)?; + } + Ok(state) + } + + pub fn apply_event(&mut self, event: &EventEnvelope) -> Result<()> { + let value = event.payload.as_value(); + let ts = parse_ts(value)?; + let event_name = value + .get("event") + .and_then(Value::as_str) + .ok_or_else(|| StoreError::InvalidEvent("event payload missing event name".into()))?; + let run_id = parse_run_id(value)?; + let properties = value + .get("properties") + .and_then(Value::as_object) + .cloned() + .unwrap_or_default(); + + match event_name { + "run.created" => { + let settings = required_json::(&properties, "settings")?; + let graph = required_json::(&properties, "graph")?; + let working_directory = + required_string(&properties, "working_directory").map(PathBuf::from)?; + let labels = optional_json::>(&properties, "labels")? + .unwrap_or_default() + .into_iter() + .collect::>(); + self.run = Some(RunRecord { + run_id, + created_at: ts, + settings, + graph, + workflow_slug: optional_string(&properties, "workflow_slug"), + working_directory, + host_repo_path: optional_string(&properties, "host_repo_path"), + base_branch: optional_string(&properties, "base_branch"), + labels, + }); + self.graph_source = optional_string(&properties, "workflow_source"); + } + "run.started" => { + self.start = Some(StartRecord { + run_id, + start_time: ts, + run_branch: optional_string(&properties, "run_branch"), + base_sha: optional_string(&properties, "base_sha"), + }); + } + "run.submitted" => { + self.status = Some(run_status_record(RunStatus::Submitted, &properties, ts)?); + } + "run.starting" => { + self.status = Some(run_status_record(RunStatus::Starting, &properties, ts)?); + } + "run.running" => { + self.status = Some(run_status_record(RunStatus::Running, &properties, ts)?); + } + "run.removing" => { + self.status = Some(run_status_record(RunStatus::Removing, &properties, ts)?); + } + "run.completed" => { + self.status = Some(run_status_record(RunStatus::Succeeded, &properties, ts)?); + self.conclusion = Some(conclusion_from_completed(&properties, ts)?); + self.final_patch = optional_string(&properties, "final_patch"); + self.last_git_sha = optional_string(&properties, "final_git_commit_sha") + .or_else(|| self.last_git_sha.clone()); + } + "run.failed" => { + self.status = Some(run_status_record(RunStatus::Failed, &properties, ts)?); + self.conclusion = Some(conclusion_from_failed(&properties, ts)); + self.last_git_sha = + optional_string(&properties, "git_commit_sha").or_else(|| self.last_git_sha.clone()); + } + "run.rewound" => { + self.reset_for_rewind(); + self.last_git_sha = + optional_string(&properties, "run_commit_sha").or_else(|| self.last_git_sha.clone()); + } + "checkpoint.completed" => { + let checkpoint = checkpoint_from_properties(&properties, ts)?; + self.last_git_sha = checkpoint.git_commit_sha.clone().or_else(|| self.last_git_sha.clone()); + if let Some(node_id) = value.get("node_id").and_then(Value::as_str) { + let visit = checkpoint + .node_visits + .get(node_id) + .and_then(|visit| u32::try_from(*visit).ok()) + .unwrap_or(1); + if let Some(diff) = optional_string(&properties, "diff") { + self.node_mut(node_id, visit).diff = Some(diff); + } + } + self.checkpoint = Some(checkpoint.clone()); + self.checkpoints.push((event.seq, checkpoint)); + } + "sandbox.initialized" => { + self.sandbox = Some(SandboxRecord { + provider: required_string(&properties, "provider")?, + working_directory: required_string(&properties, "working_directory")?, + identifier: optional_string(&properties, "identifier"), + host_working_directory: optional_string(&properties, "host_working_directory"), + container_mount_point: optional_string(&properties, "container_mount_point"), + }); + } + "retro.started" => { + self.retro_prompt = optional_string(&properties, "prompt"); + } + "retro.completed" => { + self.retro_response = optional_string(&properties, "response"); + self.retro = optional_json::(&properties, "retro")?; + } + "pull_request.created" => { + self.pull_request = Some(PullRequestRecord { + html_url: required_string(&properties, "pr_url")?, + number: required_u64(&properties, "pr_number")?, + owner: required_string(&properties, "owner")?, + repo: required_string(&properties, "repo")?, + base_branch: required_string(&properties, "base_branch")?, + head_branch: required_string(&properties, "head_branch")?, + title: required_string(&properties, "title")?, + }); + } + "stage.prompt" => { + let Some(node_id) = value.get("node_id").and_then(Value::as_str) else { + return Ok(()); + }; + let visit = required_u32(&properties, "visit")?; + self.node_mut(node_id, visit).prompt = optional_string(&properties, "text"); + self.node_mut(node_id, visit).provider_used = + provider_used_from_prompt(&properties); + } + "prompt.completed" => { + let Some(node_id) = value.get("node_id").and_then(Value::as_str) else { + return Ok(()); + }; + let visit = self.current_visit_for(node_id).unwrap_or(1); + self.node_mut(node_id, visit).response = optional_string(&properties, "response"); + } + "stage.completed" => { + let Some(node_id) = value.get("node_id").and_then(Value::as_str) else { + return Ok(()); + }; + let visit = stage_visit(node_id, &properties, self).unwrap_or(1); + let response = optional_string(&properties, "response"); + let outcome = stage_outcome_from_properties(&properties)?; + let status = node_status_from_outcome(&outcome, ts); + let node = self.node_mut(node_id, visit); + node.response = response; + node.status = Some(status); + node.outcome = Some(outcome); + } + "stage.failed" => { + let Some(node_id) = value.get("node_id").and_then(Value::as_str) else { + return Ok(()); + }; + let visit = self.current_visit_for(node_id).unwrap_or(1); + let failure = optional_json::(&properties, "failure")?; + let failure_reason = failure.as_ref().map(|detail| detail.message.clone()); + let node = self.node_mut(node_id, visit); + node.status = Some(NodeStatusRecord { + status: StageStatus::Fail, + notes: None, + failure_reason: failure_reason.clone(), + timestamp: ts, + }); + node.outcome = Some(Outcome { + status: StageStatus::Fail, + preferred_label: None, + suggested_next_ids: Vec::new(), + context_updates: HashMap::new(), + jump_to_node: None, + notes: None, + failure, + usage: None, + files_touched: Vec::new(), + duration_ms: None, + }); + } + "agent.session.started" | "agent.cli.started" => { + let Some(node_id) = value.get("node_id").and_then(Value::as_str) else { + return Ok(()); + }; + let visit = required_u32(&properties, "visit")?; + self.node_mut(node_id, visit).provider_used = + provider_used_from_agent_event(event_name, &properties); + } + "command.started" => { + let Some(node_id) = value.get("node_id").and_then(Value::as_str) else { + return Ok(()); + }; + let visit = self.current_visit_for(node_id).unwrap_or(1); + self.node_mut(node_id, visit).script_invocation = + Some(Value::Object(properties.clone())); + } + "command.completed" => { + let Some(node_id) = value.get("node_id").and_then(Value::as_str) else { + return Ok(()); + }; + let visit = self.current_visit_for(node_id).unwrap_or(1); + let node = self.node_mut(node_id, visit); + node.stdout = optional_string(&properties, "stdout"); + node.stderr = optional_string(&properties, "stderr"); + node.script_timing = Some(Value::Object(properties.clone())); + } + _ => {} + } + + Ok(()) + } + + pub fn merge_legacy( + &mut self, + snapshot: Option, + graph_source: Option, + retro_prompt: Option, + retro_response: Option, + checkpoints: Vec<(u32, Checkpoint)>, + ) { + let Some(snapshot) = snapshot else { + self.graph_source = self.graph_source.clone().or(graph_source); + self.retro_prompt = self.retro_prompt.clone().or(retro_prompt); + self.retro_response = self.retro_response.clone().or(retro_response); + if self.checkpoints.is_empty() { + self.checkpoints = checkpoints; + } + if self.checkpoint.is_none() { + self.checkpoint = self.checkpoints.last().map(|(_, checkpoint)| checkpoint.clone()); + } + return; + }; + + if self.run.is_none() { + self.run = Some(snapshot.run); + } + self.start = self.start.clone().or(snapshot.start); + self.status = self.status.clone().or(snapshot.status); + self.checkpoint = self.checkpoint.clone().or(snapshot.checkpoint); + self.conclusion = self.conclusion.clone().or(snapshot.conclusion); + self.retro = self.retro.clone().or(snapshot.retro); + self.graph_source = self.graph_source.clone().or(snapshot.graph).or(graph_source); + self.sandbox = self.sandbox.clone().or(snapshot.sandbox); + self.final_patch = self.final_patch.clone().or(snapshot.final_patch); + self.pull_request = self.pull_request.clone().or(snapshot.pull_request); + self.retro_prompt = self.retro_prompt.clone().or(retro_prompt); + self.retro_response = self.retro_response.clone().or(retro_response); + if self.checkpoints.is_empty() { + self.checkpoints = checkpoints; + } + if self.checkpoint.is_none() { + self.checkpoint = self.checkpoints.last().map(|(_, checkpoint)| checkpoint.clone()); + } + + for node in snapshot.nodes { + let entry = self + .nodes + .entry((node.node_id.clone(), node.visit)) + .or_default(); + merge_node_snapshot(entry, node); + } + } + + pub fn node(&self, node: &NodeVisitRef<'_>) -> Option<&NodeState> { + self.nodes.get(&(node.node_id.to_string(), node.visit)) + } + + pub fn list_node_ids(&self) -> Vec { + let mut ids = self + .nodes + .keys() + .map(|(node_id, _)| node_id.clone()) + .collect::>(); + ids.sort(); + ids.dedup(); + ids + } + + pub fn list_node_visits(&self, node_id: &str) -> Vec { + let mut visits = self + .nodes + .keys() + .filter(|(current_node_id, _)| current_node_id == node_id) + .map(|(_, visit)| *visit) + .collect::>(); + visits.sort_unstable(); + visits.dedup(); + visits + } + + pub fn to_snapshot(&self) -> Option { + let run = self.run.clone()?; + let mut node_keys = self.nodes.keys().cloned().collect::>(); + node_keys.sort(); + let nodes = node_keys + .into_iter() + .filter_map(|(node_id, visit)| { + self.nodes.get(&(node_id.clone(), visit)).map(|node| NodeSnapshot { + node_id, + visit, + prompt: node.prompt.clone(), + response: node.response.clone(), + status: node.status.clone(), + outcome: node.outcome.clone(), + provider_used: node.provider_used.clone(), + diff: node.diff.clone(), + script_invocation: node.script_invocation.clone(), + script_timing: node.script_timing.clone(), + parallel_results: node.parallel_results.clone(), + stdout: node.stdout.clone(), + stderr: node.stderr.clone(), + }) + }) + .collect(); + + Some(RunSnapshot { + run, + start: self.start.clone(), + status: self.status.clone(), + checkpoint: self.checkpoint.clone(), + conclusion: self.conclusion.clone(), + retro: self.retro.clone(), + graph: self.graph_source.clone(), + sandbox: self.sandbox.clone(), + final_patch: self.final_patch.clone(), + pull_request: self.pull_request.clone(), + nodes, + }) + } + + pub fn build_summary(&self, catalog: &CatalogRecord) -> RunSummary { + let workflow_name = self.run.as_ref().map(|run| { + if run.graph.name.is_empty() { + "unnamed".to_string() + } else { + run.graph.name.clone() + } + }); + let goal = self.run.as_ref().and_then(|run| { + let goal = run.graph.goal(); + (!goal.is_empty()).then(|| goal.to_string()) + }); + RunSummary { + run_id: catalog.run_id, + created_at: catalog.created_at, + db_prefix: catalog.db_prefix.clone(), + run_dir: catalog.run_dir.clone(), + workflow_name, + workflow_slug: self.run.as_ref().and_then(|run| run.workflow_slug.clone()), + goal, + labels: self + .run + .as_ref() + .map(|run| run.labels.clone()) + .unwrap_or_default(), + host_repo_path: self.run.as_ref().and_then(|run| run.host_repo_path.clone()), + start_time: self.start.as_ref().map(|start| start.start_time), + status: self.status.as_ref().map(|status| status.status), + status_reason: self.status.as_ref().and_then(|status| status.reason), + duration_ms: self.conclusion.as_ref().map(|conclusion| conclusion.duration_ms), + total_cost: self + .conclusion + .as_ref() + .and_then(|conclusion| conclusion.total_cost), + } + } + + fn node_mut(&mut self, node_id: &str, visit: u32) -> &mut NodeState { + self.nodes + .entry((node_id.to_string(), visit)) + .or_default() + } + + fn current_visit_for(&self, node_id: &str) -> Option { + self.nodes + .keys() + .filter(|(current_node_id, _)| current_node_id == node_id) + .map(|(_, visit)| *visit) + .max() + } + + fn reset_for_rewind(&mut self) { + self.status = None; + self.checkpoint = None; + self.checkpoints.clear(); + self.conclusion = None; + self.retro = None; + self.retro_prompt = None; + self.retro_response = None; + self.sandbox = None; + self.final_patch = None; + self.pull_request = None; + self.nodes.clear(); + } +} + +fn merge_node_snapshot(node: &mut NodeState, snapshot: NodeSnapshot) { + node.prompt = node.prompt.clone().or(snapshot.prompt); + node.response = node.response.clone().or(snapshot.response); + node.status = node.status.clone().or(snapshot.status); + node.outcome = node.outcome.clone().or(snapshot.outcome); + node.provider_used = node.provider_used.clone().or(snapshot.provider_used); + node.diff = node.diff.clone().or(snapshot.diff); + node.script_invocation = node.script_invocation.clone().or(snapshot.script_invocation); + node.script_timing = node.script_timing.clone().or(snapshot.script_timing); + node.parallel_results = node.parallel_results.clone().or(snapshot.parallel_results); + node.stdout = node.stdout.clone().or(snapshot.stdout); + node.stderr = node.stderr.clone().or(snapshot.stderr); +} + +fn parse_ts(value: &Value) -> Result> { + let ts = value + .get("ts") + .and_then(Value::as_str) + .ok_or_else(|| StoreError::InvalidEvent("event payload missing ts".into()))?; + chrono::DateTime::parse_from_rfc3339(ts) + .map(|ts| ts.with_timezone(&Utc)) + .map_err(|err| StoreError::InvalidEvent(format!("invalid event ts: {err}")).into()) +} + +fn parse_run_id(value: &Value) -> Result { + let run_id = value + .get("run_id") + .and_then(Value::as_str) + .ok_or_else(|| StoreError::InvalidEvent("event payload missing run_id".into()))?; + run_id + .parse() + .map_err(|err| StoreError::InvalidEvent(format!("invalid run_id: {err}")).into()) +} + +fn required_string(properties: &serde_json::Map, key: &str) -> Result { + properties + .get(key) + .and_then(Value::as_str) + .map(ToString::to_string) + .ok_or_else(|| StoreError::InvalidEvent(format!("event missing string property {key}")).into()) +} + +fn optional_string(properties: &serde_json::Map, key: &str) -> Option { + properties + .get(key) + .and_then(Value::as_str) + .map(ToString::to_string) +} + +fn required_u64(properties: &serde_json::Map, key: &str) -> Result { + properties + .get(key) + .and_then(Value::as_u64) + .ok_or_else(|| StoreError::InvalidEvent(format!("event missing integer property {key}")).into()) +} + +fn required_u32(properties: &serde_json::Map, key: &str) -> Result { + u32::try_from(required_u64(properties, key)?) + .map_err(|_| StoreError::InvalidEvent(format!("property {key} does not fit in u32")).into()) +} + +fn required_json( + properties: &serde_json::Map, + key: &str, +) -> Result { + let value = properties + .get(key) + .cloned() + .ok_or_else(|| StoreError::InvalidEvent(format!("event missing property {key}")))?; + serde_json::from_value(value) + .map_err(|err| StoreError::InvalidEvent(format!("invalid property {key}: {err}")).into()) +} + +fn optional_json( + properties: &serde_json::Map, + key: &str, +) -> Result> { + properties + .get(key) + .filter(|value| !value.is_null()) + .cloned() + .map(|value| { + serde_json::from_value(value).map_err(|err| { + StoreError::InvalidEvent(format!("invalid property {key}: {err}")).into() + }) + }) + .transpose() +} + +fn parse_reason(properties: &serde_json::Map) -> Result> { + optional_string(properties, "reason") + .map(|reason| { + serde_json::from_value(Value::String(reason)).map_err(|err| { + StoreError::InvalidEvent(format!("invalid status reason: {err}")).into() + }) + }) + .transpose() +} + +fn run_status_record( + status: RunStatus, + properties: &serde_json::Map, + updated_at: DateTime, +) -> Result { + Ok(RunStatusRecord { + status, + reason: parse_reason(properties)?, + updated_at, + }) +} + +fn checkpoint_from_properties( + properties: &serde_json::Map, + timestamp: DateTime, +) -> Result { + let loop_failure_signatures = + optional_json::>(properties, "loop_failure_signatures")? + .unwrap_or_default() + .into_iter() + .map(|(key, value)| (FailureSignature(key), value)) + .collect(); + let restart_failure_signatures = + optional_json::>(properties, "restart_failure_signatures")? + .unwrap_or_default() + .into_iter() + .map(|(key, value)| (FailureSignature(key), value)) + .collect(); + + Ok(Checkpoint { + timestamp, + current_node: required_string(properties, "current_node")?, + completed_nodes: optional_json(properties, "completed_nodes")?.unwrap_or_default(), + node_retries: optional_json(properties, "node_retries")?.unwrap_or_default(), + context_values: optional_json(properties, "context_values")?.unwrap_or_default(), + node_outcomes: optional_json(properties, "node_outcomes")?.unwrap_or_default(), + next_node_id: optional_string(properties, "next_node_id"), + git_commit_sha: optional_string(properties, "git_commit_sha"), + loop_failure_signatures, + restart_failure_signatures, + node_visits: optional_json(properties, "node_visits")?.unwrap_or_default(), + }) +} + +fn conclusion_from_completed( + properties: &serde_json::Map, + timestamp: DateTime, +) -> Result { + let usage = optional_json::(properties, "usage")?; + Ok(Conclusion { + timestamp, + status: StageStatus::from_str(&required_string(properties, "status")?).map_err( + |err| StoreError::InvalidEvent(format!("invalid completed stage status: {err}")), + )?, + duration_ms: required_u64(properties, "duration_ms")?, + failure_reason: None, + final_git_commit_sha: optional_string(properties, "final_git_commit_sha"), + stages: Vec::new(), + total_cost: properties.get("total_cost").and_then(Value::as_f64), + total_retries: 0, + total_input_tokens: usage.as_ref().map_or(0, |usage| usage.input_tokens), + total_output_tokens: usage.as_ref().map_or(0, |usage| usage.output_tokens), + total_cache_read_tokens: usage + .as_ref() + .and_then(|usage| usage.cache_read_tokens) + .unwrap_or(0), + total_cache_write_tokens: usage + .as_ref() + .and_then(|usage| usage.cache_write_tokens) + .unwrap_or(0), + total_reasoning_tokens: usage + .as_ref() + .and_then(|usage| usage.reasoning_tokens) + .unwrap_or(0), + has_pricing: usage.as_ref().is_some_and(|usage| usage.cost.is_some()), + }) +} + +fn conclusion_from_failed( + properties: &serde_json::Map, + timestamp: DateTime, +) -> Conclusion { + Conclusion { + timestamp, + status: StageStatus::Fail, + duration_ms: properties + .get("duration_ms") + .and_then(Value::as_u64) + .unwrap_or_default(), + failure_reason: optional_string(properties, "error"), + final_git_commit_sha: optional_string(properties, "git_commit_sha"), + stages: Vec::new(), + total_cost: None, + total_retries: 0, + total_input_tokens: 0, + total_output_tokens: 0, + total_cache_read_tokens: 0, + total_cache_write_tokens: 0, + total_reasoning_tokens: 0, + has_pricing: false, + } +} + +fn stage_visit(node_id: &str, properties: &serde_json::Map, state: &RunState) -> Option { + properties + .get("node_visits") + .and_then(|value| serde_json::from_value::>(value.clone()).ok()) + .and_then(|visits| visits.get(node_id).copied()) + .and_then(|visit| u32::try_from(visit).ok()) + .or_else(|| state.current_visit_for(node_id)) +} + +fn stage_outcome_from_properties( + properties: &serde_json::Map, +) -> Result { + let status = StageStatus::from_str(&required_string(properties, "status")?).map_err(|err| { + StoreError::InvalidEvent(format!("invalid stage status: {err}")) + })?; + Ok(Outcome { + status, + preferred_label: optional_string(properties, "preferred_label"), + suggested_next_ids: optional_json(properties, "suggested_next_ids")?.unwrap_or_default(), + context_updates: optional_json(properties, "context_updates")?.unwrap_or_default(), + jump_to_node: optional_string(properties, "jump_to_node"), + notes: optional_string(properties, "notes"), + failure: optional_json(properties, "failure")?, + usage: optional_json(properties, "usage")?, + files_touched: optional_json(properties, "files_touched")?.unwrap_or_default(), + duration_ms: properties.get("duration_ms").and_then(Value::as_u64), + }) +} + +fn node_status_from_outcome(outcome: &NodeOutcomeRecord, timestamp: DateTime) -> NodeStatusRecord { + NodeStatusRecord { + status: outcome.status.clone(), + notes: outcome.notes.clone(), + failure_reason: outcome.failure.as_ref().map(|failure| failure.message.clone()), + timestamp, + } +} + +fn provider_used_from_prompt(properties: &serde_json::Map) -> Option { + let mut provider_used = serde_json::Map::new(); + if let Some(mode) = optional_string(properties, "mode") { + provider_used.insert("mode".to_string(), Value::String(mode)); + } + if let Some(provider) = optional_string(properties, "provider") { + provider_used.insert("provider".to_string(), Value::String(provider)); + } + if let Some(model) = optional_string(properties, "model") { + provider_used.insert("model".to_string(), Value::String(model)); + } + (!provider_used.is_empty()).then_some(Value::Object(provider_used)) +} + +fn provider_used_from_agent_event( + event_name: &str, + properties: &serde_json::Map, +) -> Option { + let mut provider_used = serde_json::Map::new(); + provider_used.insert( + "mode".to_string(), + Value::String(if event_name == "agent.cli.started" { + "cli".to_string() + } else { + "agent".to_string() + }), + ); + if let Some(provider) = optional_string(properties, "provider") { + provider_used.insert("provider".to_string(), Value::String(provider)); + } + if let Some(model) = optional_string(properties, "model") { + provider_used.insert("model".to_string(), Value::String(model)); + } + if let Some(command) = optional_string(properties, "command") { + provider_used.insert("command".to_string(), Value::String(command)); + } + Some(Value::Object(provider_used)) +} diff --git a/lib/crates/fabro-store/src/slate/mod.rs b/lib/crates/fabro-store/src/slate/mod.rs index 40b169240..4269f69ca 100644 --- a/lib/crates/fabro-store/src/slate/mod.rs +++ b/lib/crates/fabro-store/src/slate/mod.rs @@ -236,30 +236,28 @@ impl Store for SlateStore { Ok(Arc::new(run_store) as Arc) } - async fn open_run(&self, run_id: &RunId) -> Result>> { - let Some(locator) = - catalog::read_locator(self.object_store.clone(), &self.base_prefix, run_id).await? - else { - return Ok(None); - }; + async fn open_run(&self, run_id: &RunId) -> Result> { + let locator = catalog::read_locator(self.object_store.clone(), &self.base_prefix, run_id) + .await? + .ok_or_else(|| StoreError::RunNotFound(run_id.to_string()))?; - let Some(run_store) = self.open_run_store(&locator).await? else { - return Ok(None); - }; - Ok(Some(Arc::new(run_store) as Arc)) + let run_store = self + .open_run_store(&locator) + .await? + .ok_or_else(|| StoreError::RunNotFound(run_id.to_string()))?; + Ok(Arc::new(run_store) as Arc) } - async fn open_run_reader(&self, run_id: &RunId) -> Result>> { - let Some(locator) = - catalog::read_locator(self.object_store.clone(), &self.base_prefix, run_id).await? - else { - return Ok(None); - }; + async fn open_run_reader(&self, run_id: &RunId) -> Result> { + let locator = catalog::read_locator(self.object_store.clone(), &self.base_prefix, run_id) + .await? + .ok_or_else(|| StoreError::RunNotFound(run_id.to_string()))?; - let Some(run_store) = self.open_run_reader_store(&locator).await? else { - return Ok(None); - }; - Ok(Some(Arc::new(run_store) as Arc)) + let run_store = self + .open_run_reader_store(&locator) + .await? + .ok_or_else(|| StoreError::RunNotFound(run_id.to_string()))?; + Ok(Arc::new(run_store) as Arc) } async fn list_runs(&self, query: &ListRunsQuery) -> Result> { @@ -584,7 +582,6 @@ mod tests { let reopened = store .open_run(&test_run_id("run-1")) .await - .unwrap() .unwrap(); let stored = reopened.get_run().await.unwrap().unwrap(); assert_eq!(stored.run_id, test_run_id("run-1")); @@ -594,8 +591,7 @@ mod tests { store .open_run(&test_run_id("run-1")) .await - .unwrap() - .is_none() + .is_err() ); assert!(!object_exists(object_store.clone(), &by_id).await); assert!(!object_exists(object_store.clone(), &by_start).await); @@ -634,8 +630,7 @@ mod tests { store .open_run(&test_run_id("run-1")) .await - .unwrap() - .is_some() + .is_ok() ); assert!( store @@ -680,7 +675,6 @@ mod tests { let reopened = store .open_run(&test_run_id("run-1")) .await - .unwrap() .unwrap(); let next_event = reopened .append_event(&event_payload( @@ -730,8 +724,7 @@ mod tests { store .open_run(&test_run_id("run-1")) .await - .unwrap() - .is_none() + .is_err() ); assert!( store @@ -783,7 +776,6 @@ mod tests { let reopened = store .open_run(&test_run_id("run-1")) .await - .unwrap() .unwrap(); let first_event = run .append_event(&event_payload("run-1", "2026-03-27T12:00:00Z", "Started")) @@ -884,8 +876,7 @@ mod tests { store .open_run(&test_run_id("run-1")) .await - .unwrap() - .is_none() + .is_err() ); assert!(list_paths(object_store, "runs").await.is_empty()); } diff --git a/lib/crates/fabro-store/src/slate/run_store.rs b/lib/crates/fabro-store/src/slate/run_store.rs index 5d704da64..3c1c0a9dc 100644 --- a/lib/crates/fabro-store/src/slate/run_store.rs +++ b/lib/crates/fabro-store/src/slate/run_store.rs @@ -15,9 +15,10 @@ use tokio::time; use tokio_stream::wrappers::UnboundedReceiverStream; use crate::keys; +use crate::run_state::EventProjectionCache; use crate::{ CatalogRecord, EventEnvelope, EventPayload, NodeOutcomeRecord, NodeSnapshot, NodeVisitRef, - Result, RunSnapshot, RunStore, RunSummary, StoreError, + Result, RunSnapshot, RunState, RunStore, RunSummary, StoreError, }; use fabro_types::{ Checkpoint, Conclusion, NodeStatusRecord, PullRequestRecord, Retro, RunId, RunRecord, @@ -38,6 +39,7 @@ pub(crate) struct SlateRunStoreInner { event_seq: AtomicU32, checkpoint_seq: AtomicU32, close_lock: Mutex<()>, + projection_cache: Mutex, } enum SlateRunDb { @@ -60,6 +62,7 @@ impl SlateRunStore { event_seq: AtomicU32::new(event_seq), checkpoint_seq: AtomicU32::new(checkpoint_seq), close_lock: Mutex::new(()), + projection_cache: Mutex::new(EventProjectionCache::default()), }), }) } @@ -78,6 +81,7 @@ impl SlateRunStore { event_seq: AtomicU32::new(event_seq), checkpoint_seq: AtomicU32::new(checkpoint_seq), close_lock: Mutex::new(()), + projection_cache: Mutex::new(EventProjectionCache::default()), }), }) } @@ -143,42 +147,74 @@ impl SlateRunStore { where R: DbRead + Sync, { - let run = get_json::<_, RunRecord>(db, keys::run()).await?; - let start = get_json::<_, StartRecord>(db, keys::start()).await?; - let status = get_json::<_, RunStatusRecord>(db, keys::status()).await?; - let conclusion = get_json::<_, Conclusion>(db, keys::conclusion()).await?; - - let workflow_name = run.as_ref().map(|run| { - if run.graph.name.is_empty() { - "unnamed".to_string() - } else { - run.graph.name.clone() + let events = list_events_from(db, 1).await?; + let mut state = RunState::apply_events(&events)?; + let mut nodes = BTreeSet::new(); + let mut iter = db.scan_prefix(b"nodes/").await?; + while let Some(entry) = iter.next().await? { + let key = key_to_string(&entry.key)?; + if let Some((node_id, visit, _)) = keys::parse_node_key(&key) { + nodes.insert((node_id, visit)); } - }); - let goal = run.as_ref().and_then(|run| { - let goal = run.graph.goal(); - (!goal.is_empty()).then(|| goal.to_string()) - }); - - Ok(RunSummary { - run_id: catalog.run_id, - created_at: catalog.created_at, - db_prefix: catalog.db_prefix.clone(), - run_dir: catalog.run_dir.clone(), - workflow_name, - workflow_slug: run.as_ref().and_then(|run| run.workflow_slug.clone()), - goal, - labels: run - .as_ref() - .map(|run| run.labels.clone()) - .unwrap_or_default(), - host_repo_path: run.as_ref().and_then(|run| run.host_repo_path.clone()), - start_time: start.map(|start| start.start_time), - status: status.as_ref().map(|status| status.status), - status_reason: status.and_then(|status| status.reason), - duration_ms: conclusion.as_ref().map(|conclusion| conclusion.duration_ms), - total_cost: conclusion.and_then(|conclusion| conclusion.total_cost), - }) + } + let snapshot = if let Some(run) = get_json::<_, RunRecord>(db, keys::run()).await? { + let mut snapshot_nodes = Vec::new(); + for (node_id, visit) in nodes { + let node = NodeVisitRef { + node_id: &node_id, + visit, + }; + let prompt_key = keys::node_prompt(&node); + let response_key = keys::node_response(&node); + let status_key = keys::node_status(&node); + let outcome_key = keys::node_outcome(&node); + let provider_key = keys::node_provider_used(&node); + let diff_key = keys::node_diff(&node); + let invocation_key = keys::node_script_invocation(&node); + let timing_key = keys::node_script_timing(&node); + let results_key = keys::node_parallel_results(&node); + let stdout_key = keys::node_stdout(&node); + let stderr_key = keys::node_stderr(&node); + snapshot_nodes.push(NodeSnapshot { + node_id: node_id.clone(), + visit, + prompt: get_text(db, &prompt_key).await?, + response: get_text(db, &response_key).await?, + status: get_json(db, &status_key).await?, + outcome: get_json(db, &outcome_key).await?, + provider_used: get_json(db, &provider_key).await?, + diff: get_text(db, &diff_key).await?, + script_invocation: get_json(db, &invocation_key).await?, + script_timing: get_json(db, &timing_key).await?, + parallel_results: get_json(db, &results_key).await?, + stdout: get_text(db, &stdout_key).await?, + stderr: get_text(db, &stderr_key).await?, + }); + } + Some(RunSnapshot { + run, + start: get_json::<_, StartRecord>(db, keys::start()).await?, + status: get_json::<_, RunStatusRecord>(db, keys::status()).await?, + checkpoint: get_json::<_, Checkpoint>(db, keys::checkpoint()).await?, + conclusion: get_json::<_, Conclusion>(db, keys::conclusion()).await?, + retro: get_json::<_, Retro>(db, keys::retro()).await?, + graph: get_text(db, keys::graph()).await?, + sandbox: get_json::<_, SandboxRecord>(db, keys::sandbox()).await?, + final_patch: get_text(db, keys::final_patch()).await?, + pull_request: get_json::<_, PullRequestRecord>(db, keys::pull_request()).await?, + nodes: snapshot_nodes, + }) + } else { + None + }; + state.merge_legacy( + snapshot, + get_text(db, keys::graph()).await?, + get_text(db, keys::retro_prompt()).await?, + get_text(db, keys::retro_response()).await?, + list_checkpoints(db).await?, + ); + Ok(state.build_summary(catalog)) } fn validate_run_record(&self, record: &RunRecord) -> Result<()> { @@ -230,6 +266,20 @@ impl SlateRunStore { stderr: self.inner.db.get_text(&keys::node_stderr(node)).await?, }) } + + async fn projected_state(&self) -> Result { + let next_seq = { + let cache = self.inner.projection_cache.lock().await; + cache.last_seq.saturating_add(1) + }; + let events = self.inner.db.list_events_from(next_seq).await?; + let mut cache = self.inner.projection_cache.lock().await; + for event in &events { + cache.state.apply_event(event)?; + cache.last_seq = event.seq; + } + Ok(cache.state.clone()) + } } #[async_trait] @@ -619,7 +669,25 @@ impl RunStore for SlateRunStore { self.inner.db.list_all_assets().await } + async fn state(&self) -> Result { + let mut state = self.projected_state().await?; + state.merge_legacy( + self.get_snapshot_legacy().await?, + self.get_graph().await?, + self.get_retro_prompt().await?, + self.get_retro_response().await?, + self.list_checkpoints().await?, + ); + Ok(state) + } + async fn get_snapshot(&self) -> Result> { + self.state().await.map(|state| state.to_snapshot()) + } +} + +impl SlateRunStore { + async fn get_snapshot_legacy(&self) -> Result> { let Some(run) = self.get_run().await? else { return Ok(None); }; diff --git a/lib/crates/fabro-workflow/src/event.rs b/lib/crates/fabro-workflow/src/event.rs index f36fa973a..dec103a97 100644 --- a/lib/crates/fabro-workflow/src/event.rs +++ b/lib/crates/fabro-workflow/src/event.rs @@ -1510,7 +1510,9 @@ pub(crate) fn normalize_json_value(value: Value) -> Value { } pub fn event_payload_from_redacted_json(line: &str, run_id: &RunId) -> Result { - let value = serde_json::from_str(line).context("Failed to parse redacted event payload")?; + let value = normalize_json_value( + serde_json::from_str(line).context("Failed to parse redacted event payload")?, + ); EventPayload::new(value, run_id).map_err(anyhow::Error::from) } diff --git a/lib/crates/fabro-workflow/src/handler/agent.rs b/lib/crates/fabro-workflow/src/handler/agent.rs index 43242ded9..e87cb21d2 100644 --- a/lib/crates/fabro-workflow/src/handler/agent.rs +++ b/lib/crates/fabro-workflow/src/handler/agent.rs @@ -4,7 +4,6 @@ use std::sync::Arc; use async_trait::async_trait; use fabro_agent::Sandbox; -use fabro_store::NodeVisitRef; use fabro_types::RunId; use crate::context::keys; @@ -257,18 +256,7 @@ impl Handler for AgentHandler { let visit = visit_from_context(context); let stage_dir = node_dir(run_dir, &node.id, visit); fs::create_dir_all(&stage_dir).await?; - let node_ref = NodeVisitRef { - node_id: &node.id, - visit: u32::try_from(visit).unwrap_or(u32::MAX), - }; - if let Some(ref store) = services.run_store { - store - .put_node_prompt(&node_ref, &prompt) - .await - .map_err(|err| FabroError::handler(err.to_string()))?; - } else { - fs::write(stage_dir.join("prompt.md"), &prompt).await?; - } + fs::write(stage_dir.join("prompt.md"), &prompt).await?; // 3. Call LLM backend (agent loop) let thread_id = context.thread_id(); @@ -326,14 +314,7 @@ impl Handler for AgentHandler { }; // 4. Write response to logs - if let Some(ref store) = services.run_store { - store - .put_node_response(&node_ref, &response_text) - .await - .map_err(|err| FabroError::handler(err.to_string()))?; - } else { - fs::write(stage_dir.join("response.md"), &response_text).await?; - } + fs::write(stage_dir.join("response.md"), &response_text).await?; // 7. Build and write status let mut outcome = Outcome::success(); outcome.notes = Some(format!("Stage completed: {}", node.id)); @@ -392,7 +373,7 @@ mod tests { use super::*; use crate::event::EventEmitter; use fabro_graphviz::graph::AttrValue; - use fabro_store::{InMemoryStore, RunStore, Store}; + use fabro_store::{InMemoryStore, NodeVisitRef, RunStore, Store}; use fabro_types::fixtures; use std::sync::Arc; use tempfile::TempDir; @@ -412,7 +393,7 @@ mod tests { .await .unwrap(); let services = EngineServices { - run_store: Some(Arc::clone(&run_store)), + run_store: Arc::clone(&run_store), ..EngineServices::test_default() }; let logger = crate::event::StoreProgressLogger::new(Arc::clone(&run_store)); diff --git a/lib/crates/fabro-workflow/src/handler/command.rs b/lib/crates/fabro-workflow/src/handler/command.rs index 0cae5fe6c..d548afa4a 100644 --- a/lib/crates/fabro-workflow/src/handler/command.rs +++ b/lib/crates/fabro-workflow/src/handler/command.rs @@ -1,8 +1,6 @@ use std::path::Path; use async_trait::async_trait; -use fabro_store::NodeVisitRef; - use crate::context::Context; use crate::context::keys; use crate::error::FabroError; @@ -92,22 +90,11 @@ impl Handler for CommandHandler { let visit = visit_from_context(context); let stage_dir = node_dir(run_dir, &node.id, visit); fs::create_dir_all(&stage_dir).await?; - let node_ref = NodeVisitRef { - node_id: &node.id, - visit: u32::try_from(visit).unwrap_or(u32::MAX), - }; - let invocation = serde_json::json!({ "command": script, "language": language, "timeout_ms": timeout_ms(node), }); - if let Some(ref store) = services.run_store { - store - .put_node_script_invocation(&node_ref, &invocation) - .await - .map_err(|err| FabroError::handler(err.to_string()))?; - } fs::write( stage_dir.join("script_invocation.json"), serde_json::to_string_pretty(&invocation).unwrap(), @@ -142,31 +129,14 @@ impl Handler for CommandHandler { .await .map_err(|e| FabroError::handler(format!("Failed to spawn script: {e}")))?; - if let Some(ref store) = services.run_store { - store - .put_node_stdout(&node_ref, &result.stdout) - .await - .map_err(|err| FabroError::handler(err.to_string()))?; - store - .put_node_stderr(&node_ref, &result.stderr) - .await - .map_err(|err| FabroError::handler(err.to_string()))?; - } else { - fs::write(stage_dir.join("stdout.log"), &result.stdout).await?; - fs::write(stage_dir.join("stderr.log"), &result.stderr).await?; - } + fs::write(stage_dir.join("stdout.log"), &result.stdout).await?; + fs::write(stage_dir.join("stderr.log"), &result.stderr).await?; let timing = serde_json::json!({ "duration_ms": result.duration_ms, "exit_code": if result.timed_out { serde_json::Value::Null } else { serde_json::json!(result.exit_code) }, "timed_out": result.timed_out, }); - if let Some(ref store) = services.run_store { - store - .put_node_script_timing(&node_ref, &timing) - .await - .map_err(|err| FabroError::handler(err.to_string()))?; - } fs::write( stage_dir.join("script_timing.json"), serde_json::to_string_pretty(&timing).unwrap(), @@ -230,6 +200,7 @@ mod tests { use crate::outcome::StageStatus; use fabro_graphviz::graph::AttrValue; use fabro_store::{InMemoryStore, NodeVisitRef, RunStore, Store}; + use fabro_types::fixtures; use std::sync::Arc; use std::time::Duration; @@ -237,6 +208,26 @@ mod tests { EngineServices::test_default() } + async fn make_services_with_run_store() -> ( + EngineServices, + Arc, + crate::event::StoreProgressLogger, + ) { + let store = InMemoryStore::default(); + let run_store = store + .create_run(&fixtures::RUN_1, chrono::Utc::now(), None) + .await + .unwrap(); + let services = EngineServices { + emitter: Arc::new(crate::event::EventEmitter::new(fixtures::RUN_1)), + run_store: Arc::clone(&run_store), + ..EngineServices::test_default() + }; + let logger = crate::event::StoreProgressLogger::new(Arc::clone(&run_store)); + logger.register(services.emitter.as_ref()); + (services, run_store, logger) + } + #[tokio::test] async fn script_handler_no_script() { let handler = CommandHandler; @@ -586,31 +577,28 @@ mod tests { let context = Context::new(); let graph = Graph::new("test"); let run_dir = tempfile::tempdir().unwrap(); - let store = Arc::new(InMemoryStore::default()); - let run_store = store - .create_run(&fabro_types::fixtures::RUN_1, chrono::Utc::now(), None) - .await - .unwrap(); - let services = EngineServices { - run_store: Some(Arc::clone(&run_store) as Arc), - ..EngineServices::test_default() - }; + let (services, run_store, logger) = make_services_with_run_store().await; handler .execute(&node, &context, &graph, run_dir.path(), &services) .await .unwrap(); + logger.flush().await; let snapshot = run_store - .get_node(&NodeVisitRef { + .state() + .await + .unwrap(); + let node = snapshot + .node(&NodeVisitRef { node_id: "script_node", visit: 1, }) - .await + .cloned() .unwrap(); - assert_eq!(snapshot.script_invocation.unwrap()["command"], "echo hello"); - assert_eq!(snapshot.script_timing.unwrap()["exit_code"], 0); + assert_eq!(node.script_invocation.unwrap()["script"], "echo hello"); + assert_eq!(node.script_timing.unwrap()["exit_code"], 0); } #[tokio::test] diff --git a/lib/crates/fabro-workflow/src/handler/fan_in.rs b/lib/crates/fabro-workflow/src/handler/fan_in.rs index 8423f6308..948cb3069 100644 --- a/lib/crates/fabro-workflow/src/handler/fan_in.rs +++ b/lib/crates/fabro-workflow/src/handler/fan_in.rs @@ -226,7 +226,7 @@ async fn llm_evaluate( node_id: &str, emitter: &Arc, sandbox: &Arc, - run_store: Option>, + run_store: Arc, ) -> Result { let results_text = serde_json::to_string_pretty(results).unwrap_or_else(|_| results.to_string()); @@ -244,14 +244,11 @@ async fn llm_evaluate( node_id, visit: u32::try_from(visit).unwrap_or(u32::MAX), }; - if let Some(ref store) = run_store { - store - .put_node_prompt(&node_ref, &full_prompt) - .await - .map_err(|err| FabroError::handler(err.to_string()))?; - } else { - fs::write(stage_dir.join("prompt.md"), &full_prompt).await?; - } + run_store + .put_node_prompt(&node_ref, &full_prompt) + .await + .map_err(|err| FabroError::handler(err.to_string()))?; + fs::write(stage_dir.join("prompt.md"), &full_prompt).await?; // Build a synthetic node for the backend call let eval_node = Node::new("fan_in_eval"); @@ -281,14 +278,11 @@ async fn llm_evaluate( .unwrap_or_else(|| "unknown".to_string()); let response_text = serde_json::to_string_pretty(&outcome).unwrap_or_else(|_| "{}".to_string()); - if let Some(ref store) = run_store { - store - .put_node_response(&node_ref, &response_text) - .await - .map_err(|err| FabroError::handler(err.to_string()))?; - } else { - fs::write(stage_dir.join("response.md"), &response_text).await?; - } + run_store + .put_node_response(&node_ref, &response_text) + .await + .map_err(|err| FabroError::handler(err.to_string()))?; + fs::write(stage_dir.join("response.md"), &response_text).await?; Ok(Candidate { id: best_id, status: outcome.status.to_string(), @@ -297,14 +291,11 @@ async fn llm_evaluate( } Ok(CodergenResult::Text { text, .. }) => { // Write response to logs - if let Some(ref store) = run_store { - store - .put_node_response(&node_ref, &text) - .await - .map_err(|err| FabroError::handler(err.to_string()))?; - } else { - fs::write(stage_dir.join("response.md"), &text).await?; - } + run_store + .put_node_response(&node_ref, &text) + .await + .map_err(|err| FabroError::handler(err.to_string()))?; + fs::write(stage_dir.join("response.md"), &text).await?; // The LLM responded with text; try to find a matching candidate ID let text = text.trim().to_string(); diff --git a/lib/crates/fabro-workflow/src/handler/mod.rs b/lib/crates/fabro-workflow/src/handler/mod.rs index 93f62de2d..291b9c93c 100644 --- a/lib/crates/fabro-workflow/src/handler/mod.rs +++ b/lib/crates/fabro-workflow/src/handler/mod.rs @@ -19,6 +19,8 @@ use std::sync::Arc; use async_trait::async_trait; use fabro_agent::Sandbox; use fabro_store::RunStore; +#[cfg(test)] +use fabro_store::Store; use crate::context::Context; use crate::error::FabroError; @@ -34,7 +36,7 @@ pub struct EngineServices { pub registry: Arc, pub emitter: Arc, pub sandbox: Arc, - pub run_store: Option>, + pub run_store: Arc, /// Git state for the current run. Set via `set_git_state` at the start of /// `run_via_core` and read by parallel/fan-in handlers. pub(crate) git_state: std::sync::RwLock>>, @@ -75,7 +77,12 @@ impl EngineServices { sandbox: Arc::new(fabro_agent::LocalSandbox::new( std::env::current_dir().unwrap_or_else(|_| std::path::PathBuf::from(".")), )), - run_store: None, + run_store: futures::executor::block_on(async { + fabro_store::InMemoryStore::default() + .create_run(&fabro_types::RunId::new(), chrono::Utc::now(), None) + .await + .expect("in-memory test run store should initialize") + }), git_state: std::sync::RwLock::new(None), hook_runner: None, env: HashMap::new(), diff --git a/lib/crates/fabro-workflow/src/handler/parallel.rs b/lib/crates/fabro-workflow/src/handler/parallel.rs index 0e305df36..f44388402 100644 --- a/lib/crates/fabro-workflow/src/handler/parallel.rs +++ b/lib/crates/fabro-workflow/src/handler/parallel.rs @@ -481,16 +481,15 @@ impl Handler for ParallelHandler { let visit = visit_from_context(context); let node_dir = node_dir(run_dir, &node.id, visit); let _ = fs::create_dir_all(&node_dir).await; - if let Some(ref store) = services.run_store { - let node_ref = NodeVisitRef { - node_id: &node.id, - visit: u32::try_from(visit).unwrap_or(u32::MAX), - }; - store - .put_node_parallel_results(&node_ref, &serde_json::json!(results_json)) - .await - .map_err(|err| FabroError::handler(err.to_string()))?; - } + let node_ref = NodeVisitRef { + node_id: &node.id, + visit: u32::try_from(visit).unwrap_or(u32::MAX), + }; + services + .run_store + .put_node_parallel_results(&node_ref, &serde_json::json!(results_json)) + .await + .map_err(|err| FabroError::handler(err.to_string()))?; if let Ok(json) = serde_json::to_string_pretty(&results_json) { let _ = fs::write(node_dir.join("parallel_results.json"), json).await; } @@ -687,7 +686,7 @@ mod tests { .await .unwrap(); let services = EngineServices { - run_store: Some(Arc::clone(&run_store) as Arc), + run_store: Arc::clone(&run_store) as Arc, ..EngineServices::test_default() }; let mut node = Node::new("par"); diff --git a/lib/crates/fabro-workflow/src/handler/prompt.rs b/lib/crates/fabro-workflow/src/handler/prompt.rs index fd7e7b797..5c79bfe6f 100644 --- a/lib/crates/fabro-workflow/src/handler/prompt.rs +++ b/lib/crates/fabro-workflow/src/handler/prompt.rs @@ -3,8 +3,6 @@ use std::path::Path; use async_trait::async_trait; use fabro_model::Provider; -use fabro_store::NodeVisitRef; - use crate::context::keys; use crate::context::{Context, WorkflowContext}; use crate::error::FabroError; @@ -93,18 +91,7 @@ impl Handler for PromptHandler { let visit = visit_from_context(context); let stage_dir = node_dir(run_dir, &node.id, visit); fs::create_dir_all(&stage_dir).await?; - let node_ref = NodeVisitRef { - node_id: &node.id, - visit: u32::try_from(visit).unwrap_or(u32::MAX), - }; - if let Some(ref store) = services.run_store { - store - .put_node_prompt(&node_ref, &prompt) - .await - .map_err(|err| FabroError::handler(err.to_string()))?; - } else { - fs::write(stage_dir.join("prompt.md"), &prompt).await?; - } + fs::write(stage_dir.join("prompt.md"), &prompt).await?; let prompt_provider = node .provider() @@ -169,14 +156,7 @@ impl Handler for PromptHandler { }); // 4. Write response to logs - if let Some(ref store) = services.run_store { - store - .put_node_response(&node_ref, &response_text) - .await - .map_err(|err| FabroError::handler(err.to_string()))?; - } else { - fs::write(stage_dir.join("response.md"), &response_text).await?; - } + fs::write(stage_dir.join("response.md"), &response_text).await?; // 5. Build and write status let mut outcome = Outcome::success(); @@ -225,7 +205,7 @@ mod tests { .await .unwrap(); let services = EngineServices { - run_store: Some(Arc::clone(&run_store)), + run_store: Arc::clone(&run_store), ..EngineServices::test_default() }; let logger = crate::event::StoreProgressLogger::new(Arc::clone(&run_store)); diff --git a/lib/crates/fabro-workflow/src/operations/create.rs b/lib/crates/fabro-workflow/src/operations/create.rs index d3703c563..1ed6527f2 100644 --- a/lib/crates/fabro-workflow/src/operations/create.rs +++ b/lib/crates/fabro-workflow/src/operations/create.rs @@ -148,8 +148,8 @@ async fn persist_created_run( Err(err) => store .open_run(&record.run_id) .await - .map_err(|open_err| FabroError::engine(open_err.to_string()))? - .ok_or_else(|| FabroError::engine(err.to_string()))?, + .map_err(|open_err| FabroError::engine(open_err.to_string())) + .or_else(|_| Err(FabroError::engine(err.to_string())))?, }; run_store.put_run(record).await.map_err(store_error)?; @@ -717,7 +717,7 @@ mod tests { created.persisted.run_record().workflow_slug.as_deref(), Some("slug") ); - let run_store = store.open_run(&fixtures::RUN_1).await.unwrap().unwrap(); + let run_store = store.open_run(&fixtures::RUN_1).await.unwrap(); assert_eq!( run_store.get_status().await.unwrap().unwrap().status, crate::run_status::RunStatus::Submitted @@ -839,11 +839,7 @@ mod tests { ) .await .unwrap(); - let run_store = store - .open_run_reader(&created.run_id) - .await - .unwrap() - .unwrap(); + let run_store = store.open_run_reader(&created.run_id).await.unwrap(); let events = run_store.list_events().await.unwrap(); assert_eq!( diff --git a/lib/crates/fabro-workflow/src/operations/rebuild_meta.rs b/lib/crates/fabro-workflow/src/operations/rebuild_meta.rs index c7fa52049..9f599285d 100644 --- a/lib/crates/fabro-workflow/src/operations/rebuild_meta.rs +++ b/lib/crates/fabro-workflow/src/operations/rebuild_meta.rs @@ -26,9 +26,13 @@ pub async fn rebuild_metadata_branch( bail!("metadata branch already exists for run {run_id}"); } - let run_record = run_store - .get_run() + let state = run_store + .state() .await? + ; + let run_record = state + .run + .clone() .ok_or_else(|| anyhow::anyhow!("run record not found for {run_id}"))?; let sig = Signature::now("Fabro", "noreply@fabro.sh")?; @@ -43,10 +47,10 @@ pub async fn rebuild_metadata_branch( "run.json".to_string(), serde_json::to_vec_pretty(&run_record)?, )); - if let Some(start) = run_store.get_start().await? { + if let Some(start) = state.start.clone() { init_entries.push(("start.json".to_string(), serde_json::to_vec_pretty(&start)?)); } - if let Some(sandbox) = run_store.get_sandbox().await? { + if let Some(sandbox) = state.sandbox.clone() { init_entries.push(( "sandbox.json".to_string(), serde_json::to_vec_pretty(&sandbox)?, @@ -54,7 +58,7 @@ pub async fn rebuild_metadata_branch( } write_entries(&bs, &init_entries, "init run")?; - let mut checkpoints = run_store.list_checkpoints().await?; + let mut checkpoints = state.checkpoints.clone(); backfill_missing_checkpoint_shas(git_store, run_id, &mut checkpoints); for (_seq, checkpoint) in checkpoints { @@ -69,7 +73,9 @@ pub async fn rebuild_metadata_branch( for visit in 1..=max_visit { let visit = u32::try_from(visit) .with_context(|| format!("visit {visit} for node {node_id} exceeds u32"))?; - let node = run_store.get_node(&NodeVisitRef { node_id, visit }).await?; + let Some(node) = state.node(&NodeVisitRef { node_id, visit }).cloned() else { + continue; + }; if let Some(prompt) = node.prompt { entries.push(( @@ -125,7 +131,7 @@ pub async fn rebuild_metadata_branch( write_entries(&bs, &entries, "checkpoint")?; } - if let Some(retro) = run_store.get_retro().await? { + if let Some(retro) = state.retro.clone() { let entries = vec![("retro.json".to_string(), serde_json::to_vec_pretty(&retro)?)]; write_entries(&bs, &entries, "finalize run")?; } diff --git a/lib/crates/fabro-workflow/src/operations/resume.rs b/lib/crates/fabro-workflow/src/operations/resume.rs index e03ca7fb7..fcac5048f 100644 --- a/lib/crates/fabro-workflow/src/operations/resume.rs +++ b/lib/crates/fabro-workflow/src/operations/resume.rs @@ -11,24 +11,20 @@ use super::start::{StartServices, Started, execute_persisted_run}; /// Resume a workflow run from its checkpoint. Errors if no checkpoint is found. pub async fn resume(run_dir: &Path, services: StartServices) -> Result { - if let Some(record) = services + let state = services .run_store - .get_status() + .state() .await - .map_err(|err| FabroError::engine(err.to_string()))? - { + .map_err(|err| FabroError::engine(err.to_string()))?; + + if let Some(record) = state.status { if record.status == RunStatus::Succeeded { return Err(FabroError::Precondition( "run already finished successfully — nothing to resume".to_string(), )); } } - if let Some(conclusion) = services - .run_store - .get_conclusion() - .await - .map_err(|err| FabroError::engine(err.to_string()))? - { + if let Some(conclusion) = state.conclusion { if matches!( conclusion.status, StageStatus::Success | StageStatus::PartialSuccess | StageStatus::Skipped @@ -39,11 +35,8 @@ pub async fn resume(run_dir: &Path, services: StartServices) -> Result Result { - if services + let state = services .run_store - .get_checkpoint() + .state() .await - .map_err(|err| FabroError::engine(err.to_string()))? - .is_some() - { + .map_err(|err| FabroError::engine(err.to_string()))?; + if state.checkpoint.is_some() { return Err(FabroError::Precondition( "checkpoint.json exists in run directory — did you mean to resume?".to_string(), )); } - if let Some(record) = services - .run_store - .get_status() - .await - .map_err(|err| FabroError::engine(err.to_string()))? - { + if let Some(record) = state.status { if !matches!(record.status, RunStatus::Submitted | RunStatus::Starting) { return Err(FabroError::Precondition(format!( "cannot start run: status is {:?}, expected submitted", @@ -580,7 +574,7 @@ impl RunSession { }; let pr_opts = PullRequestOptions { run_dir: retroed.run_options.run_dir.clone(), - run_store: Some(Arc::clone(&retroed.run_store)), + run_store: Arc::clone(&retroed.run_store), pr_config: self.pr_config, github_app: self.pr_github_app, origin_url: self.pr_origin_url, @@ -976,7 +970,7 @@ mod tests { cancel_token: None, emitter, interviewer: Arc::new(fabro_interview::AutoApproveInterviewer), - run_store: store.open_run(&fixtures::RUN_1).await.unwrap().unwrap(), + run_store: store.open_run(&fixtures::RUN_1).await.unwrap(), github_app: None, on_node: None, registry_override: Some(registry), @@ -1052,7 +1046,7 @@ mod tests { .unwrap(); assert_eq!(started.finalized.conclusion.status, StageStatus::Success); - let run_store = store.open_run(&fixtures::RUN_1).await.unwrap().unwrap(); + let run_store = store.open_run(&fixtures::RUN_1).await.unwrap(); assert!(run_store.get_conclusion().await.unwrap().is_some()); } diff --git a/lib/crates/fabro-workflow/src/pipeline/execute.rs b/lib/crates/fabro-workflow/src/pipeline/execute.rs index 98f95de77..c1349ab56 100644 --- a/lib/crates/fabro-workflow/src/pipeline/execute.rs +++ b/lib/crates/fabro-workflow/src/pipeline/execute.rs @@ -73,7 +73,7 @@ pub async fn execute(init: Initialized) -> Executed { registry, emitter: Arc::clone(&emitter), sandbox: Arc::clone(&sandbox), - run_store: Some(Arc::clone(&run_store)), + run_store: Arc::clone(&run_store), git_state: std::sync::RwLock::new(git_state), hook_runner: hook_runner.clone(), env, diff --git a/lib/crates/fabro-workflow/src/pipeline/finalize.rs b/lib/crates/fabro-workflow/src/pipeline/finalize.rs index b39282918..2a9d5d080 100644 --- a/lib/crates/fabro-workflow/src/pipeline/finalize.rs +++ b/lib/crates/fabro-workflow/src/pipeline/finalize.rs @@ -69,7 +69,11 @@ pub(crate) async fn build_conclusion_from_store( run_duration_ms: u64, final_git_commit_sha: Option, ) -> Conclusion { - let checkpoint = run_store.get_checkpoint().await.ok().flatten(); + let checkpoint = run_store + .state() + .await + .ok() + .and_then(|state| state.checkpoint); let stage_durations = run_store .list_events() .await @@ -192,10 +196,12 @@ pub async fn write_finalize_commit( let git_author = run_options.git_author(); let store = MetadataStore::new(repo_path, &git_author); let mut entries = scan_node_files_from_store(run_store).await; - let retro_bytes = match run_store.get_retro().await { - Ok(Some(retro)) => serde_json::to_vec_pretty(&retro).ok(), - _ => None, - }; + let retro_bytes = run_store + .state() + .await + .ok() + .and_then(|state| state.retro) + .and_then(|retro| serde_json::to_vec_pretty(&retro).ok()); if let Some(bytes) = retro_bytes { entries.push(("retro.json".to_string(), bytes)); } diff --git a/lib/crates/fabro-workflow/src/pipeline/persist.rs b/lib/crates/fabro-workflow/src/pipeline/persist.rs index 4e86955f6..2d4ff94c0 100644 --- a/lib/crates/fabro-workflow/src/pipeline/persist.rs +++ b/lib/crates/fabro-workflow/src/pipeline/persist.rs @@ -29,17 +29,15 @@ pub(crate) async fn load_from_store( run_store: &dyn RunStore, run_dir: &Path, ) -> Result { - let run_record = run_store - .get_run() + let state = run_store + .state() .await - .map_err(|err| FabroError::engine(err.to_string()))? + .map_err(|err| FabroError::engine(err.to_string()))?; + let run_record = state + .run .ok_or_else(|| FabroError::Precondition("run record missing from store".to_string()))?; let graph = run_record.graph.clone(); - let source = run_store - .get_graph() - .await - .map_err(|err| FabroError::engine(err.to_string()))? - .unwrap_or_default(); + let source = state.graph_source.unwrap_or_default(); Ok(Persisted::new( graph, diff --git a/lib/crates/fabro-workflow/src/pipeline/pull_request.rs b/lib/crates/fabro-workflow/src/pipeline/pull_request.rs index f9c658036..130081daf 100644 --- a/lib/crates/fabro-workflow/src/pipeline/pull_request.rs +++ b/lib/crates/fabro-workflow/src/pipeline/pull_request.rs @@ -292,21 +292,17 @@ fn emit_run_notice( }); } -async fn load_pull_request_diff(run_store: Option<&dyn RunStore>, run_dir: &Path) -> String { - if let Some(run_store) = run_store { - run_store - .get_final_patch() - .await - .inspect_err(|err| { - tracing::warn!(error = %err, "Failed to load final patch from store for PR"); - }) - .ok() - .flatten() - .unwrap_or_default() - } else { - let _ = run_dir; - String::new() - } +async fn load_pull_request_diff(run_store: &dyn RunStore, run_dir: &Path) -> String { + let _ = run_dir; + run_store + .state() + .await + .inspect_err(|err| { + tracing::warn!(error = %err, "Failed to load final patch from store for PR"); + }) + .ok() + .and_then(|state| state.final_patch) + .unwrap_or_default() } /// Build a complete PR body by combining LLM-generated narrative with @@ -315,7 +311,7 @@ pub async fn build_pr_body( diff: &str, goal: &str, model: &str, - run_store: Option<&dyn RunStore>, + run_store: &dyn RunStore, run_dir: &Path, conclusion: Option<&Conclusion>, ) -> Result { @@ -323,54 +319,28 @@ pub async fn build_pr_body( let plan_text = read_plan_text(run_dir); let loaded_conclusion = if conclusion.is_none() { - match run_store { - Some(run_store) => run_store - .get_conclusion() - .await - .inspect_err(|err| { - tracing::warn!(error = %err, "Failed to load conclusion from store for PR body"); - }) - .ok() - .flatten(), - None => None, - } + run_store + .state() + .await + .inspect_err(|err| { + tracing::warn!(error = %err, "Failed to load conclusion from store for PR body"); + }) + .ok() + .and_then(|state| state.conclusion) } else { None }; let conclusion = conclusion.or(loaded_conclusion.as_ref()); - let retro = match run_store { - Some(run_store) => run_store - .get_retro() - .await - .inspect_err(|err| { - tracing::warn!(error = %err, "Failed to load retro from store for PR body"); - }) - .ok() - .flatten(), - None => None, - }; - let run_record = match run_store { - Some(run_store) => run_store - .get_run() - .await - .inspect_err(|err| { - tracing::warn!(error = %err, "Failed to load run record from store for PR body"); - }) - .ok() - .flatten(), - None => None, - }; - let dot_source = match run_store { - Some(run_store) => run_store - .get_graph() - .await - .inspect_err(|err| { - tracing::warn!(error = %err, "Failed to load graph from store for PR body"); - }) - .ok() - .flatten(), - None => None, - }; + let run_state = run_store + .state() + .await + .inspect_err(|err| { + tracing::warn!(error = %err, "Failed to load run state from store for PR body"); + }) + .ok(); + let retro = run_state.as_ref().and_then(|state| state.retro.clone()); + let run_record = run_state.as_ref().and_then(|state| state.run.clone()); + let dot_source = run_state.as_ref().and_then(|state| state.graph_source.clone()); // Build LLM prompt let system = if plan_text.is_some() { @@ -448,7 +418,7 @@ pub async fn maybe_open_pull_request( model: &str, draft: bool, auto_merge: Option, - run_store: Option<&dyn RunStore>, + run_store: &dyn RunStore, run_dir: &Path, conclusion: Option<&Conclusion>, ) -> Result, String> { @@ -519,12 +489,10 @@ pub async fn maybe_open_pull_request( title, }; - if let Some(run_store) = run_store { - run_store - .put_pull_request(&record) - .await - .map_err(|err| format!("failed to persist pull request in run store: {err}"))?; - } + run_store + .put_pull_request(&record) + .await + .map_err(|err| format!("failed to persist pull request in run store: {err}"))?; Ok(Some(record)) } @@ -554,8 +522,7 @@ pub async fn pull_request(concluded: Concluded, options: &PullRequestOptions) -> result.status, StageStatus::Success | StageStatus::PartialSuccess ) { - let diff = - load_pull_request_diff(options.run_store.as_deref(), &options.run_dir).await; + let diff = load_pull_request_diff(options.run_store.as_ref(), &options.run_dir).await; if let (Some(base_branch), Some(run_branch), Some(creds), Some(origin)) = ( &run_options.base_branch, pushed_branch.as_deref(), @@ -580,7 +547,7 @@ pub async fn pull_request(concluded: Concluded, options: &PullRequestOptions) -> &options.model, pr_cfg.draft, auto_merge, - options.run_store.as_deref(), + options.run_store.as_ref(), &options.run_dir, Some(&conclusion), ) @@ -1067,12 +1034,21 @@ mod tests { install_mock_llm(); let tmp = tempfile::tempdir().unwrap(); + let store = InMemoryStore::default(); + let run_store = store + .create_run( + &fixtures::RUN_1, + Utc::now(), + Some(&tmp.path().display().to_string()), + ) + .await + .unwrap(); let conclusion = make_test_conclusion(); let body = build_pr_body( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "mock-model", - None, + run_store.as_ref(), tmp.path(), Some(&conclusion), ) @@ -1126,7 +1102,7 @@ mod tests { "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "mock-model", - Some(run_store.as_ref()), + run_store.as_ref(), tmp.path(), Some(&conclusion), ) @@ -1316,6 +1292,15 @@ mod tests { #[tokio::test] async fn empty_diff_returns_none() { let tmp = tempfile::tempdir().unwrap(); + let store = InMemoryStore::default(); + let run_store = store + .create_run( + &fixtures::RUN_1, + Utc::now(), + Some(&tmp.path().display().to_string()), + ) + .await + .unwrap(); let creds = GitHubAppCredentials { app_id: "123".to_string(), private_key_pem: "unused".to_string(), @@ -1330,7 +1315,7 @@ mod tests { "claude-sonnet-4-20250514", false, None, - None, + run_store.as_ref(), tmp.path(), None, ) @@ -1343,20 +1328,35 @@ mod tests { async fn load_pull_request_diff_uses_store_without_disk_patch() { let tmp = tempfile::tempdir().unwrap(); let store = InMemoryStore::default(); + let created_at = Utc::now(); let run_store = store .create_run( &fixtures::RUN_1, - Utc::now(), + created_at, Some(&tmp.path().display().to_string()), ) .await .unwrap(); + run_store + .put_run(&RunRecord { + run_id: fixtures::RUN_1, + created_at, + settings: Settings::default(), + graph: Graph::new("test"), + workflow_slug: None, + working_directory: tmp.path().to_path_buf(), + host_repo_path: None, + base_branch: None, + labels: std::collections::HashMap::new(), + }) + .await + .unwrap(); run_store .put_final_patch("diff --git a/src/lib.rs b/src/lib.rs\n+fn from_store() {}\n") .await .unwrap(); - let diff = load_pull_request_diff(Some(run_store.as_ref()), tmp.path()).await; + let diff = load_pull_request_diff(run_store.as_ref(), tmp.path()).await; assert!(diff.contains("from_store")); } diff --git a/lib/crates/fabro-workflow/src/pipeline/retro.rs b/lib/crates/fabro-workflow/src/pipeline/retro.rs index 1c687c6ec..ccde6ac56 100644 --- a/lib/crates/fabro-workflow/src/pipeline/retro.rs +++ b/lib/crates/fabro-workflow/src/pipeline/retro.rs @@ -8,12 +8,14 @@ use fabro_retro::retro_agent::{ use super::types::{Executed, RetroOptions, Retroed}; use crate::event::WorkflowRunEvent; +#[cfg(test)] +use crate::records::RunRecord; pub async fn run_retro(options: &RetroOptions, dry_run: bool) -> Option { - let cp = match options.run_store.get_checkpoint().await { - Ok(Some(cp)) => cp, + let state = match options.run_store.state().await { + Ok(state) => state, Err(e) => { - tracing::warn!(error = %e, "Could not load checkpoint, skipping retro"); + tracing::warn!(error = %e, "Could not load run state, skipping retro"); if let Some(ref emitter) = options.emitter { emitter.emit(&WorkflowRunEvent::RetroFailed { error: e.to_string(), @@ -22,7 +24,10 @@ pub async fn run_retro(options: &RetroOptions, dry_run: bool) -> Option { } return None; } - Ok(None) => { + }; + let cp = match state.checkpoint { + Some(cp) => cp, + None => { tracing::warn!("Could not load checkpoint, skipping retro"); if let Some(ref emitter) = options.emitter { emitter.emit(&WorkflowRunEvent::RetroFailed { @@ -86,7 +91,7 @@ pub async fn run_retro(options: &RetroOptions, dry_run: bool) -> Option { }); run_retro_agent( &options.sandbox, - Some(&*options.run_store), + options.run_store.as_ref(), &options.run_dir, client, options.provider, @@ -213,15 +218,30 @@ mod tests { run_dir: &std::path::Path, checkpoint: &Checkpoint, ) -> Arc { + let created_at = Utc::now(); let inner = InMemoryStore::default() .create_run( &test_run_id(), - Utc::now(), + created_at, Some(run_dir.to_string_lossy().as_ref()), ) .await .unwrap(); let run_store: Arc = inner; + run_store + .put_run(&RunRecord { + run_id: test_run_id(), + created_at, + settings: Settings::default(), + graph: Graph::new("test"), + workflow_slug: None, + working_directory: run_dir.to_path_buf(), + host_repo_path: None, + base_branch: None, + labels: std::collections::HashMap::new(), + }) + .await + .unwrap(); run_store.put_checkpoint(checkpoint).await.unwrap(); run_store } diff --git a/lib/crates/fabro-workflow/src/pipeline/types.rs b/lib/crates/fabro-workflow/src/pipeline/types.rs index 567ccc0c4..23b02b45a 100644 --- a/lib/crates/fabro-workflow/src/pipeline/types.rs +++ b/lib/crates/fabro-workflow/src/pipeline/types.rs @@ -356,7 +356,7 @@ pub struct FinalizeOptions { /// Options for the PULL_REQUEST phase. pub struct PullRequestOptions { pub run_dir: PathBuf, - pub run_store: Option>, + pub run_store: Arc, pub pr_config: Option, pub github_app: Option, pub origin_url: Option, From 491f326c120808413428e6575cbf4cba949ebeba Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Thu, 2 Apr 2026 23:16:05 -0700 Subject: [PATCH 2/3] Collapse store handles onto Slate --- .../fabro-cli/src/commands/pr/create.rs | 4 +- lib/crates/fabro-cli/src/commands/pr/list.rs | 2 +- .../fabro-cli/src/commands/run/attach.rs | 23 +- .../fabro-cli/src/commands/run/detached.rs | 5 +- lib/crates/fabro-cli/src/commands/run/diff.rs | 2 +- lib/crates/fabro-cli/src/commands/run/fork.rs | 2 +- lib/crates/fabro-cli/src/commands/run/logs.rs | 13 +- .../fabro-cli/src/commands/run/output.rs | 8 +- .../fabro-cli/src/commands/run/rewind.rs | 72 +- lib/crates/fabro-cli/src/commands/run/wait.rs | 7 +- .../fabro-cli/src/commands/runs/inspect.rs | 77 +- lib/crates/fabro-cli/src/commands/runs/rm.rs | 32 +- .../fabro-cli/src/commands/store/dump.rs | 353 +++++++-- .../fabro-cli/src/commands/system/df.rs | 2 +- .../fabro-cli/src/commands/system/prune.rs | 4 +- lib/crates/fabro-cli/src/store.rs | 7 +- lib/crates/fabro-cli/tests/it/cmd/diff.rs | 70 -- lib/crates/fabro-cli/tests/it/cmd/pr_view.rs | 22 +- lib/crates/fabro-cli/tests/it/cmd/rewind.rs | 19 +- lib/crates/fabro-cli/tests/it/cmd/support.rs | 17 +- lib/crates/fabro-cli/tests/it/scenario/mod.rs | 8 +- lib/crates/fabro-retro/src/retro_agent.rs | 40 +- lib/crates/fabro-server/src/server.rs | 36 +- lib/crates/fabro-store/src/keys.rs | 67 +- lib/crates/fabro-store/src/lib.rs | 131 +-- lib/crates/fabro-store/src/memory.rs | 748 +++++++++--------- lib/crates/fabro-store/src/run_state.rs | 157 ++-- lib/crates/fabro-store/src/slate/mod.rs | 391 ++++----- lib/crates/fabro-store/src/slate/run_store.rs | 378 ++------- lib/crates/fabro-workflow/src/event.rs | 8 +- lib/crates/fabro-workflow/src/git.rs | 38 +- .../fabro-workflow/src/handler/agent.rs | 20 +- .../fabro-workflow/src/handler/command.rs | 26 +- .../fabro-workflow/src/handler/fan_in.rs | 4 +- .../src/handler/manager_loop.rs | 10 +- lib/crates/fabro-workflow/src/handler/mod.rs | 19 +- .../fabro-workflow/src/handler/parallel.rs | 16 +- .../fabro-workflow/src/handler/prompt.rs | 22 +- .../fabro-workflow/src/lifecycle/disk.rs | 85 +- .../fabro-workflow/src/lifecycle/git.rs | 93 +-- .../fabro-workflow/src/lifecycle/mod.rs | 8 +- .../fabro-workflow/src/operations/create.rs | 38 +- .../src/operations/rebuild_meta.rs | 412 ++++++---- .../fabro-workflow/src/operations/resume.rs | 10 +- .../fabro-workflow/src/operations/start.rs | 233 +++--- .../fabro-workflow/src/pipeline/execute.rs | 4 +- .../src/pipeline/execute/tests.rs | 70 +- .../fabro-workflow/src/pipeline/finalize.rs | 50 +- .../fabro-workflow/src/pipeline/initialize.rs | 59 +- .../fabro-workflow/src/pipeline/persist.rs | 65 +- .../src/pipeline/pull_request.rs | 163 ++-- .../fabro-workflow/src/pipeline/retro.rs | 109 ++- .../fabro-workflow/src/pipeline/types.rs | 18 +- lib/crates/fabro-workflow/src/run_lookup.rs | 56 +- lib/crates/fabro-workflow/src/test_support.rs | 64 +- 55 files changed, 2093 insertions(+), 2304 deletions(-) diff --git a/lib/crates/fabro-cli/src/commands/pr/create.rs b/lib/crates/fabro-cli/src/commands/pr/create.rs index af805d1f6..847da5b24 100644 --- a/lib/crates/fabro-cli/src/commands/pr/create.rs +++ b/lib/crates/fabro-cli/src/commands/pr/create.rs @@ -37,9 +37,7 @@ async fn create_from( let run_store = store::open_run_reader(storage_dir, &run.run_id).await?; let state = run_store.state().await?; - let record = state - .run - .context("Failed to load run record from store")?; + let record = state.run.context("Failed to load run record from store")?; let start = state .start diff --git a/lib/crates/fabro-cli/src/commands/pr/list.rs b/lib/crates/fabro-cli/src/commands/pr/list.rs index 30cbdcfc0..99bb58935 100644 --- a/lib/crates/fabro-cli/src/commands/pr/list.rs +++ b/lib/crates/fabro-cli/src/commands/pr/list.rs @@ -33,7 +33,7 @@ pub(super) async fn list_command( } async fn list_from( - store: &dyn fabro_store::Store, + store: &fabro_store::SlateStore, base: &Path, args: PrListArgs, github_app: Option, diff --git a/lib/crates/fabro-cli/src/commands/run/attach.rs b/lib/crates/fabro-cli/src/commands/run/attach.rs index d625fe1bb..90a310738 100644 --- a/lib/crates/fabro-cli/src/commands/run/attach.rs +++ b/lib/crates/fabro-cli/src/commands/run/attach.rs @@ -10,7 +10,7 @@ use fabro_types::RunId; use futures::StreamExt; use fabro_interview::{AnswerValue, ConsoleInterviewer}; -use fabro_store::{EventEnvelope, RunStore, RuntimeState}; +use fabro_store::{EventEnvelope, RuntimeState, SlateRunStore}; use fabro_util::terminal::Styles; use fabro_workflow::outcome::StageStatus; use fabro_workflow::records::{Conclusion, ConclusionExt}; @@ -106,7 +106,7 @@ pub(crate) async fn attach_run( async fn attach_run_store( run_dir: &Path, - run_store: &dyn RunStore, + run_store: &SlateRunStore, verbose: bool, existing_events: Vec, last_seq: u32, @@ -156,15 +156,12 @@ async fn attach_run_store( } // Wait briefly for a terminal status or conclusion for _ in 0..20 { - if run_store - .state() - .await - .ok() - .is_some_and(|state| { - state.conclusion.is_some() - || state.status.is_some_and(|record| record.status.is_terminal()) - }) - { + if run_store.state().await.ok().is_some_and(|state| { + state.conclusion.is_some() + || state + .status + .is_some_and(|record| record.status.is_terminal()) + }) { break; } sleep(Duration::from_millis(100)).await; @@ -288,7 +285,7 @@ async fn attach_run_store( } async fn flush_remaining_store_events( - run_store: &dyn RunStore, + run_store: &SlateRunStore, mut next_seq: u32, progress_ui: &mut run_progress::ProgressUI, json_output: bool, @@ -745,7 +742,7 @@ fn determine_exit_code(conclusion_path: &Path, status_record: Option ExitCode { +async fn determine_exit_code_with_store(run_store: &SlateRunStore) -> ExitCode { let deadline = Instant::now() + ATTACH_FINAL_STATUS_GRACE; loop { match run_store.state().await { diff --git a/lib/crates/fabro-cli/src/commands/run/detached.rs b/lib/crates/fabro-cli/src/commands/run/detached.rs index f476121b8..c453b2acc 100644 --- a/lib/crates/fabro-cli/src/commands/run/detached.rs +++ b/lib/crates/fabro-cli/src/commands/run/detached.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use anyhow::{Result, anyhow}; use fabro_interview::FileInterviewer; -use fabro_store::{RuntimeState, Store}; +use fabro_store::RuntimeState; use fabro_types::RunId; use fabro_workflow::event::EventEmitter; use fabro_workflow::operations::{StartServices, resume as resume_run, start as start_run}; @@ -31,8 +31,9 @@ pub(crate) async fn execute( let store = store::build_store(&storage_dir)?; let run_store = store.open_run(&run_id).await?; let run_record = run_store - .get_run() + .state() .await? + .run .ok_or_else(|| anyhow!("Run {run_id} has no run record in store"))?; let on_node: fabro_workflow::OnNodeCallback = Some({ let run_id = run_record.run_id.to_string(); diff --git a/lib/crates/fabro-cli/src/commands/run/diff.rs b/lib/crates/fabro-cli/src/commands/run/diff.rs index 08a41ae0c..03aea8c19 100644 --- a/lib/crates/fabro-cli/src/commands/run/diff.rs +++ b/lib/crates/fabro-cli/src/commands/run/diff.rs @@ -52,7 +52,7 @@ pub(crate) async fn run(args: DiffArgs, globals: &GlobalArgs) -> Result<()> { async fn resolve_diff( _run_dir: &Path, - run_store: &dyn fabro_store::RunStore, + run_store: &fabro_store::SlateRunStore, args: &DiffArgs, ) -> Result { let state = run_store.state().await?; diff --git a/lib/crates/fabro-cli/src/commands/run/fork.rs b/lib/crates/fabro-cli/src/commands/run/fork.rs index 6808986f8..edb6b9b2b 100644 --- a/lib/crates/fabro-cli/src/commands/run/fork.rs +++ b/lib/crates/fabro-cli/src/commands/run/fork.rs @@ -21,7 +21,7 @@ pub(crate) async fn run(args: &ForkArgs, styles: &Styles, globals: &GlobalArgs) let store = Store::new(repo); let run_store = open_run_reader(&cli_settings.storage_dir(), &run_id).await?; - let timeline = build_timeline_or_rebuild(&store, Some(run_store.as_ref()), &run_id).await?; + let timeline = build_timeline_or_rebuild(&store, Some(&run_store), &run_id).await?; if args.list { if globals.json { diff --git a/lib/crates/fabro-cli/src/commands/run/logs.rs b/lib/crates/fabro-cli/src/commands/run/logs.rs index 93047df2b..1aee65577 100644 --- a/lib/crates/fabro-cli/src/commands/run/logs.rs +++ b/lib/crates/fabro-cli/src/commands/run/logs.rs @@ -5,7 +5,7 @@ use std::time::Duration; use anyhow::{Context, Result, bail}; use chrono::{DateTime, Utc}; -use fabro_store::RunStore; +use fabro_store::SlateRunStore; use fabro_util::redact::redact_jsonl_line; use fabro_util::terminal::Styles; use fabro_workflow::run_lookup::{resolve_run_combined, runs_base}; @@ -138,7 +138,7 @@ fn try_parse_relative_duration(s: &str) -> Option { } async fn follow_store_logs( - run_store: &dyn RunStore, + run_store: &SlateRunStore, run_dir: &Path, seq: u32, pretty: bool, @@ -187,16 +187,19 @@ async fn follow_store_logs( Ok(()) } -async fn run_concluded(run_store: &dyn RunStore, _run_dir: &Path) -> Result { +async fn run_concluded(run_store: &SlateRunStore, _run_dir: &Path) -> Result { let state = run_store .state() .await .context("Failed to read run state from store while following logs")?; - Ok(state.conclusion.is_some() || state.status.is_some_and(|record| record.status.is_terminal())) + Ok(state.conclusion.is_some() + || state + .status + .is_some_and(|record| record.status.is_terminal())) } async fn flush_remaining_store_events( - run_store: &dyn RunStore, + run_store: &SlateRunStore, next_seq: u32, pretty: bool, styles: &Styles, diff --git a/lib/crates/fabro-cli/src/commands/run/output.rs b/lib/crates/fabro-cli/src/commands/run/output.rs index dddc5acde..6e0d5bdb9 100644 --- a/lib/crates/fabro-cli/src/commands/run/output.rs +++ b/lib/crates/fabro-cli/src/commands/run/output.rs @@ -198,12 +198,16 @@ pub(crate) fn print_run_conclusion( } pub(crate) async fn print_final_output( - run_store: Option<&dyn fabro_store::RunStore>, + run_store: Option<&fabro_store::SlateRunStore>, _run_dir: &Path, styles: &Styles, ) { let checkpoint = match run_store { - Some(run_store) => run_store.state().await.ok().and_then(|state| state.checkpoint), + Some(run_store) => run_store + .state() + .await + .ok() + .and_then(|state| state.checkpoint), None => None, }; let Some(checkpoint) = checkpoint else { diff --git a/lib/crates/fabro-cli/src/commands/run/rewind.rs b/lib/crates/fabro-cli/src/commands/run/rewind.rs index 01597c625..9695e3c50 100644 --- a/lib/crates/fabro-cli/src/commands/run/rewind.rs +++ b/lib/crates/fabro-cli/src/commands/run/rewind.rs @@ -10,9 +10,8 @@ use fabro_workflow::operations::{ RewindInput, RewindTarget, RunTimeline, TimelineEntry, build_timeline_or_rebuild, find_run_id_by_prefix_or_store, rewind, }; -use fabro_workflow::records::{RunRecord, RunRecordExt, StartRecord, StartRecordExt}; +use fabro_workflow::records::{RunRecord, RunRecordExt}; use fabro_workflow::run_lookup::{resolve_run_combined, runs_base}; -use fabro_workflow::run_status::RunStatus; use git2::Repository; use serde::Serialize; @@ -45,7 +44,7 @@ pub(crate) async fn run(args: &RewindArgs, styles: &Styles, globals: &GlobalArgs .await .ok(); - let timeline = build_timeline_or_rebuild(&store, Some(run_store.as_ref()), &run_id).await?; + let timeline = build_timeline_or_rebuild(&store, Some(&run_store), &run_id).await?; if args.list || args.target.is_none() { if globals.json { @@ -68,8 +67,14 @@ pub(crate) async fn run(args: &RewindArgs, styles: &Styles, globals: &GlobalArgs )?; if let Some(run_info) = run_info.as_ref() { let entry = timeline.resolve(&target)?; - reset_rewound_run_state(&store, durable_store.as_ref(), &run_id, &run_info.path, entry) - .await?; + reset_rewound_run_state( + &store, + durable_store.as_ref(), + &run_id, + &run_info.path, + entry, + ) + .await?; } let run_id_string = run_id.to_string(); @@ -104,7 +109,7 @@ pub(crate) fn timeline_entries_json(timeline: &RunTimeline) -> Vec WorkflowRu let current_status = checkpoint .node_outcomes .get(&checkpoint.current_node) - .map_or_else(|| "success".to_string(), |outcome| outcome.status.to_string()); + .map_or_else( + || "success".to_string(), + |outcome| outcome.status.to_string(), + ); WorkflowRunEvent::CheckpointCompleted { node_id: checkpoint.current_node.clone(), status: current_status, diff --git a/lib/crates/fabro-cli/src/commands/run/wait.rs b/lib/crates/fabro-cli/src/commands/run/wait.rs index 6f18f7143..603ba280a 100644 --- a/lib/crates/fabro-cli/src/commands/run/wait.rs +++ b/lib/crates/fabro-cli/src/commands/run/wait.rs @@ -33,8 +33,8 @@ pub(crate) async fn run(args: &WaitArgs, styles: &Styles, globals: &GlobalArgs) let started_waiting_at = std::time::Instant::now(); let final_status = loop { - let run_store = store::open_run_reader(&cli_settings.storage_dir(), &run_info.run_id) - .await?; + let run_store = + store::open_run_reader(&cli_settings.storage_dir(), &run_info.run_id).await?; let status = run_store.state().await?.status.map(|record| record.status); let status = status.unwrap_or_else(|| { if started_waiting_at.elapsed() < WAIT_STARTUP_GRACE { @@ -63,8 +63,7 @@ pub(crate) async fn run(args: &WaitArgs, styles: &Styles, globals: &GlobalArgs) } }; - let run_store = store::open_run_reader(&cli_settings.storage_dir(), &run_info.run_id) - .await?; + let run_store = store::open_run_reader(&cli_settings.storage_dir(), &run_info.run_id).await?; let conclusion = run_store.state().await?.conclusion; if globals.json { diff --git a/lib/crates/fabro-cli/src/commands/runs/inspect.rs b/lib/crates/fabro-cli/src/commands/runs/inspect.rs index ef73bb86e..98cba6de1 100644 --- a/lib/crates/fabro-cli/src/commands/runs/inspect.rs +++ b/lib/crates/fabro-cli/src/commands/runs/inspect.rs @@ -39,50 +39,26 @@ async fn inspect_run_store( run_id: &RunId, run_dir: &Path, status: RunStatus, - run_store: &dyn fabro_store::RunStore, + run_store: &fabro_store::SlateRunStore, ) -> InspectOutput { if let Ok(state) = run_store.state().await { - if let Some(snapshot) = state.to_snapshot() { - return InspectOutput { - run_id: run_id.to_string(), - run_dir: run_dir.to_path_buf(), - status: state.status.as_ref().map_or(status, |record| record.status), - run_record: serde_json::to_value(snapshot.run).ok(), - start_record: snapshot - .start - .and_then(|record| serde_json::to_value(record).ok()), - conclusion: snapshot - .conclusion - .and_then(|record| serde_json::to_value(record).ok()), - checkpoint: snapshot - .checkpoint - .and_then(|record| serde_json::to_value(record).ok()), - sandbox: snapshot - .sandbox - .and_then(|record| serde_json::to_value(record).ok()), - }; - } - } - - if let Ok(Some(snapshot)) = run_store.get_snapshot().await { return InspectOutput { run_id: run_id.to_string(), run_dir: run_dir.to_path_buf(), - status: snapshot - .status - .as_ref() - .map_or(status, |record| record.status), - run_record: serde_json::to_value(snapshot.run).ok(), - start_record: snapshot + status: state.status.as_ref().map_or(status, |record| record.status), + run_record: state + .run + .and_then(|record| serde_json::to_value(record).ok()), + start_record: state .start .and_then(|record| serde_json::to_value(record).ok()), - conclusion: snapshot + conclusion: state .conclusion .and_then(|record| serde_json::to_value(record).ok()), - checkpoint: snapshot + checkpoint: state .checkpoint .and_then(|record| serde_json::to_value(record).ok()), - sandbox: snapshot + sandbox: state .sandbox .and_then(|record| serde_json::to_value(record).ok()), }; @@ -92,35 +68,10 @@ async fn inspect_run_store( run_id: run_id.to_string(), run_dir: run_dir.to_path_buf(), status, - run_record: run_store - .get_run() - .await - .ok() - .flatten() - .and_then(|v| serde_json::to_value(v).ok()), - start_record: run_store - .get_start() - .await - .ok() - .flatten() - .and_then(|v| serde_json::to_value(v).ok()), - conclusion: run_store - .get_conclusion() - .await - .ok() - .flatten() - .and_then(|v| serde_json::to_value(v).ok()), - checkpoint: run_store - .get_checkpoint() - .await - .ok() - .flatten() - .and_then(|v| serde_json::to_value(v).ok()), - sandbox: run_store - .get_sandbox() - .await - .ok() - .flatten() - .and_then(|v| serde_json::to_value(v).ok()), + run_record: None, + start_record: None, + conclusion: None, + checkpoint: None, + sandbox: None, } } diff --git a/lib/crates/fabro-cli/src/commands/runs/rm.rs b/lib/crates/fabro-cli/src/commands/runs/rm.rs index 600e73051..081e96075 100644 --- a/lib/crates/fabro-cli/src/commands/runs/rm.rs +++ b/lib/crates/fabro-cli/src/commands/runs/rm.rs @@ -1,19 +1,17 @@ use std::path::Path; use anyhow::{Context, Result, bail}; -use fabro_store::Store; +use fabro_store::SlateStore; use tracing::warn; -use fabro_sandbox::reconnect::reconnect as reconnect_sandbox; -use fabro_workflow::event::{WorkflowRunEvent, append_workflow_event}; -use fabro_workflow::run_lookup::RunInfo; -use fabro_workflow::run_lookup::{resolve_run_combined, runs_base}; -use fabro_workflow::run_status::{RunStatus, RunStatusRecord}; - use crate::args::{GlobalArgs, RunsRemoveArgs}; use crate::shared::print_json_pretty; use crate::store; use crate::user_config::load_user_settings_with_globals; +use fabro_sandbox::reconnect::reconnect as reconnect_sandbox; +use fabro_workflow::event::{WorkflowRunEvent, append_workflow_event}; +use fabro_workflow::run_lookup::RunInfo; +use fabro_workflow::run_lookup::{resolve_run_combined, runs_base}; use super::short_run_id; @@ -26,7 +24,7 @@ pub(crate) async fn remove_command(args: &RunsRemoveArgs, globals: &GlobalArgs) async fn remove_from( args: &RunsRemoveArgs, - store: &dyn Store, + store: &SlateStore, base: &Path, globals: &GlobalArgs, ) -> Result<()> { @@ -109,12 +107,12 @@ async fn remove_from( Ok(()) } -pub(crate) async fn remove_run_with_cleanup(store: &dyn Store, run: &RunInfo) -> Result<()> { +pub(crate) async fn remove_run_with_cleanup(store: &SlateStore, run: &RunInfo) -> Result<()> { remove_run_dir_with_cleanup(store, run).await?; delete_run_store_state(store, run).await } -async fn remove_run_dir_with_cleanup(store: &dyn Store, run: &RunInfo) -> Result<()> { +async fn remove_run_dir_with_cleanup(store: &SlateStore, run: &RunInfo) -> Result<()> { let run_store = match store.open_run_reader(&run.run_id).await { Ok(run_store) => Some(run_store), Err(err) => { @@ -127,16 +125,6 @@ async fn remove_run_dir_with_cleanup(store: &dyn Store, run: &RunInfo) -> Result } }; if let Some(run_store) = run_store.as_ref() { - if let Err(err) = run_store - .put_status(&RunStatusRecord::new(RunStatus::Removing, None)) - .await - { - warn!( - run_id = %run.run_id, - error = %err, - "failed to save removing status to store" - ); - } if let Err(err) = append_workflow_event( run_store.as_ref(), &run.run_id, @@ -171,7 +159,7 @@ async fn remove_run_dir_with_cleanup(store: &dyn Store, run: &RunInfo) -> Result .with_context(|| format!("failed to delete {}", run.path.display())) } -async fn delete_run_store_state(store: &dyn Store, run: &RunInfo) -> Result<()> { +async fn delete_run_store_state(store: &SlateStore, run: &RunInfo) -> Result<()> { store .delete_run(&run.run_id) .await @@ -180,7 +168,7 @@ async fn delete_run_store_state(store: &dyn Store, run: &RunInfo) -> Result<()> async fn load_sandbox_record( _run_dir: &Path, - run_store: Option<&dyn fabro_store::RunStore>, + run_store: Option<&fabro_store::SlateRunStore>, ) -> Option { if let Some(run_store) = run_store { match run_store.state().await { diff --git a/lib/crates/fabro-cli/src/commands/store/dump.rs b/lib/crates/fabro-cli/src/commands/store/dump.rs index ced3466a1..3b264f7a3 100644 --- a/lib/crates/fabro-cli/src/commands/store/dump.rs +++ b/lib/crates/fabro-cli/src/commands/store/dump.rs @@ -2,7 +2,7 @@ use std::io::{ErrorKind, Write}; use std::path::{Component, Path, PathBuf}; use anyhow::{Context, Result, bail}; -use fabro_store::{NodeVisitRef, RunSnapshot, RunState, RunStore}; +use fabro_store::{NodeVisitRef, RunSnapshot, RunState, SlateRunStore}; use fabro_workflow::run_lookup::{resolve_run_combined, runs_base}; use serde::Serialize; #[cfg(test)] @@ -37,7 +37,7 @@ pub(crate) async fn dump_command(args: &StoreDumpArgs, globals: &GlobalArgs) -> Ok(()) } -pub(crate) async fn export_run(run_store: &dyn RunStore, output_dir: &Path) -> Result { +pub(crate) async fn export_run(run_store: &SlateRunStore, output_dir: &Path) -> Result { let state = run_store.state().await?; let snapshot = state .to_snapshot() @@ -78,7 +78,7 @@ pub(crate) async fn export_run(run_store: &dyn RunStore, output_dir: &Path) -> R } async fn export_run_to_dir( - run_store: &dyn RunStore, + run_store: &SlateRunStore, state: &RunState, snapshot: &RunSnapshot, output_dir: &Path, @@ -349,14 +349,18 @@ mod tests { use std::collections::HashMap; use std::path::PathBuf; + use std::sync::Arc; + use std::time::Duration; use chrono::{DateTime, Utc}; - use fabro_store::{EventEnvelope, EventPayload, InMemoryStore, Store as _}; + use fabro_store::{EventEnvelope, EventPayload, SlateStore}; use fabro_types::{ AggregateStats, AttrValue, Checkpoint, Conclusion, Graph, NodeStatusRecord, Retro, RunId, RunRecord, RunStatus, RunStatusRecord, SandboxRecord, Settings, StageStatus, StartRecord, StatusReason, fixtures, }; + use fabro_workflow::event::{WorkflowRunEvent, append_workflow_event}; + use object_store::memory::InMemory; fn dt(rfc3339: &str) -> DateTime { DateTime::parse_from_rfc3339(rfc3339) @@ -368,6 +372,14 @@ mod tests { fixtures::RUN_1 } + fn test_store() -> Arc { + Arc::new(SlateStore::new( + Arc::new(InMemory::new()), + "", + Duration::from_millis(1), + )) + } + fn sample_run_record(run_id: RunId, created_at: DateTime) -> RunRecord { let mut graph = Graph::new("night-sky"); graph.attrs.insert( @@ -476,82 +488,256 @@ mod tests { } } - fn sample_node_status() -> NodeStatusRecord { - NodeStatusRecord { - status: StageStatus::PartialSuccess, - notes: Some("captured output".to_string()), - failure_reason: Some("minor lint".to_string()), - timestamp: dt("2026-03-27T12:12:00Z"), - } - } - - fn event_payload(run_id: RunId, ts: &str, event: &str) -> EventPayload { - EventPayload::new( - serde_json::json!({ - "id": format!("evt-{run_id}-{event}"), - "ts": ts, - "run_id": run_id.to_string(), - "event": event - }), - &run_id, - ) - .unwrap() - } - fn read_json(path: &Path) -> T { - serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap() + let bytes = std::fs::read(path) + .unwrap_or_else(|err| panic!("failed to read {}: {err}", path.display())); + serde_json::from_slice(&bytes) + .unwrap_or_else(|err| panic!("failed to parse {}: {err}", path.display())) } #[tokio::test] async fn export_run_writes_expected_directory_tree() { - let store = InMemoryStore::default(); + let store = test_store(); let created_at = dt("2026-03-27T12:00:00Z"); let run_id = test_run_id(); let run = store.create_run(&run_id, created_at, None).await.unwrap(); - - run.put_run(&sample_run_record(run_id, created_at)) - .await - .unwrap(); - run.put_start(&sample_start_record(run_id, created_at)) - .await - .unwrap(); - run.put_status(&sample_status()).await.unwrap(); - run.append_checkpoint(&sample_checkpoint("plan", 1)) - .await - .unwrap(); - run.append_checkpoint(&sample_checkpoint("code", 2)) - .await - .unwrap(); - run.put_conclusion(&sample_conclusion()).await.unwrap(); - run.put_retro(&sample_retro(run_id)).await.unwrap(); - run.put_graph("digraph night_sky {}").await.unwrap(); - run.put_sandbox(&sample_sandbox()).await.unwrap(); + let run_record = sample_run_record(run_id, created_at); + let start_record = sample_start_record(run_id, created_at); + let status_record = sample_status(); + let first_checkpoint = sample_checkpoint("plan", 1); + let second_checkpoint = sample_checkpoint("code", 2); + let conclusion = sample_conclusion(); + let retro = sample_retro(run_id); + let sandbox = sample_sandbox(); let node = NodeVisitRef { node_id: "code", visit: 2, }; - run.put_node_prompt(&node, "Plan the fix").await.unwrap(); - run.put_node_response(&node, "Implemented").await.unwrap(); - run.put_node_status(&node, &sample_node_status()) - .await - .unwrap(); - run.put_node_stdout(&node, "stdout line").await.unwrap(); - run.put_node_stderr(&node, "").await.unwrap(); - run.put_retro_prompt("How did it go?").await.unwrap(); - run.put_retro_response("Smooth enough").await.unwrap(); - run.append_event(&event_payload( - run_id, - "2026-03-27T12:00:00.000Z", - "run.started", - )) + append_workflow_event( + run.as_ref(), + &run_id, + &WorkflowRunEvent::RunCreated { + run_id, + settings: serde_json::to_value(&run_record.settings).unwrap(), + graph: serde_json::to_value(&run_record.graph).unwrap(), + workflow_source: Some("digraph night_sky {}".to_string()), + workflow_config: None, + labels: run_record.labels.clone().into_iter().collect(), + run_dir: "/tmp/night-sky-run".to_string(), + working_directory: run_record.working_directory.display().to_string(), + host_repo_path: run_record.host_repo_path.clone(), + base_branch: run_record.base_branch.clone(), + workflow_slug: run_record.workflow_slug.clone(), + db_prefix: None, + }, + ) .await .unwrap(); - run.append_event(&event_payload( - run_id, - "2026-03-27T12:00:01.000Z", - "stage.completed", - )) + append_workflow_event( + run.as_ref(), + &run_id, + &WorkflowRunEvent::WorkflowRunStarted { + name: "night-sky".to_string(), + run_id, + base_branch: run_record.base_branch.clone(), + base_sha: start_record.base_sha.clone(), + run_branch: start_record.run_branch.clone(), + worktree_dir: None, + goal: Some("map the constellations".to_string()), + }, + ) + .await + .unwrap(); + append_workflow_event( + run.as_ref(), + &run_id, + &WorkflowRunEvent::RunRunning { + reason: status_record.reason, + }, + ) + .await + .unwrap(); + for checkpoint in [&first_checkpoint, &second_checkpoint] { + append_workflow_event( + run.as_ref(), + &run_id, + &WorkflowRunEvent::CheckpointCompleted { + node_id: checkpoint.current_node.clone(), + status: "success".to_string(), + current_node: checkpoint.current_node.clone(), + completed_nodes: checkpoint.completed_nodes.clone(), + node_retries: checkpoint.node_retries.clone().into_iter().collect(), + context_values: checkpoint.context_values.clone().into_iter().collect(), + node_outcomes: checkpoint.node_outcomes.clone().into_iter().collect(), + next_node_id: checkpoint.next_node_id.clone(), + git_commit_sha: checkpoint.git_commit_sha.clone(), + loop_failure_signatures: checkpoint + .loop_failure_signatures + .clone() + .into_iter() + .map(|(signature, count)| (signature.to_string(), count)) + .collect(), + restart_failure_signatures: checkpoint + .restart_failure_signatures + .clone() + .into_iter() + .map(|(signature, count)| (signature.to_string(), count)) + .collect(), + node_visits: checkpoint.node_visits.clone().into_iter().collect(), + diff: None, + }, + ) + .await + .unwrap(); + } + append_workflow_event( + run.as_ref(), + &run_id, + &WorkflowRunEvent::SandboxInitialized { + working_directory: sandbox.working_directory.clone(), + provider: sandbox.provider.clone(), + identifier: sandbox.identifier.clone(), + host_working_directory: sandbox.host_working_directory.clone(), + container_mount_point: sandbox.container_mount_point.clone(), + }, + ) + .await + .unwrap(); + append_workflow_event( + run.as_ref(), + &run_id, + &WorkflowRunEvent::Prompt { + stage: "code".to_string(), + visit: 2, + text: "Plan the fix".to_string(), + mode: None, + provider: None, + model: None, + }, + ) + .await + .unwrap(); + append_workflow_event( + run.as_ref(), + &run_id, + &WorkflowRunEvent::PromptCompleted { + node_id: "code".to_string(), + response: "Implemented".to_string(), + model: "gpt-5".to_string(), + provider: "openai".to_string(), + usage: None, + }, + ) + .await + .unwrap(); + append_workflow_event( + run.as_ref(), + &run_id, + &WorkflowRunEvent::StageCompleted { + node_id: "code".to_string(), + name: "Code".to_string(), + index: 1, + duration_ms: 250, + status: "partial_success".to_string(), + preferred_label: None, + suggested_next_ids: Vec::new(), + usage: None, + failure: None, + notes: Some("captured output".to_string()), + files_touched: Vec::new(), + context_updates: None, + jump_to_node: None, + context_values: None, + node_visits: Some(std::collections::BTreeMap::from([( + "code".to_string(), + 2usize, + )])), + loop_failure_signatures: None, + restart_failure_signatures: None, + response: Some("Implemented".to_string()), + attempt: 1, + max_attempts: 1, + }, + ) + .await + .unwrap(); + append_workflow_event( + run.as_ref(), + &run_id, + &WorkflowRunEvent::CommandStarted { + node_id: "code".to_string(), + script: "echo hi".to_string(), + language: "sh".to_string(), + timeout_ms: None, + }, + ) + .await + .unwrap(); + append_workflow_event( + run.as_ref(), + &run_id, + &WorkflowRunEvent::CommandCompleted { + node_id: "code".to_string(), + stdout: "stdout line".to_string(), + stderr: String::new(), + exit_code: Some(0), + duration_ms: 100, + timed_out: false, + }, + ) + .await + .unwrap(); + append_workflow_event( + run.as_ref(), + &run_id, + &WorkflowRunEvent::RetroStarted { + prompt: Some("How did it go?".to_string()), + provider: None, + model: None, + }, + ) + .await + .unwrap(); + append_workflow_event( + run.as_ref(), + &run_id, + &WorkflowRunEvent::RetroCompleted { + duration_ms: 50, + response: Some("Smooth enough".to_string()), + retro: Some(serde_json::to_value(&retro).unwrap()), + }, + ) + .await + .unwrap(); + append_workflow_event( + run.as_ref(), + &run_id, + &WorkflowRunEvent::WorkflowRunCompleted { + duration_ms: conclusion.duration_ms, + artifact_count: 0, + status: "success".to_string(), + reason: None, + total_cost: conclusion.total_cost, + final_git_commit_sha: conclusion.final_git_commit_sha.clone(), + final_patch: None, + usage: None, + }, + ) + .await + .unwrap(); + run.append_event( + &EventPayload::new( + serde_json::json!({ + "id": format!("evt-{run_id}-stage-completed"), + "ts": "2026-03-27T12:00:01.000Z", + "run_id": run_id.to_string(), + "event": "stage.completed" + }), + &run_id, + ) + .unwrap(), + ) .await .unwrap(); run.put_artifact_value("summary", &serde_json::json!({"done": true})) @@ -583,7 +769,7 @@ mod tests { assert_eq!(exported_start.run_id, run_id); let exported_status: RunStatusRecord = read_json(&output.path().join("status.json")); - assert_eq!(exported_status.status, RunStatus::Running); + assert_eq!(exported_status.status, RunStatus::Succeeded); let exported_checkpoint: Checkpoint = read_json(&output.path().join("checkpoint.json")); assert_eq!(exported_checkpoint.current_node, "code"); @@ -626,12 +812,12 @@ mod tests { .lines() .map(|line| serde_json::from_str(line).unwrap()) .collect(); - assert_eq!(events.len(), 2); + assert_eq!(events.len(), 15); assert_eq!(events[0].seq, 1); - assert_eq!(events[1].seq, 2); + assert_eq!(events.last().unwrap().seq, 15); - let first_checkpoint: Checkpoint = read_json(&output.path().join("checkpoints/0001.json")); - let second_checkpoint: Checkpoint = read_json(&output.path().join("checkpoints/0002.json")); + let first_checkpoint: Checkpoint = read_json(&output.path().join("checkpoints/0004.json")); + let second_checkpoint: Checkpoint = read_json(&output.path().join("checkpoints/0005.json")); assert_eq!(first_checkpoint.current_node, "plan"); assert_eq!(second_checkpoint.current_node, "code"); @@ -665,14 +851,31 @@ mod tests { #[tokio::test] async fn export_run_rejects_path_traversal_and_leaves_no_partial_output() { - let store = InMemoryStore::default(); + let store = test_store(); let created_at = dt("2026-03-27T12:00:00Z"); let run_id = test_run_id(); let run = store.create_run(&run_id, created_at, None).await.unwrap(); - - run.put_run(&sample_run_record(run_id, created_at)) - .await - .unwrap(); + let run_record = sample_run_record(run_id, created_at); + append_workflow_event( + run.as_ref(), + &run_id, + &WorkflowRunEvent::RunCreated { + run_id, + settings: serde_json::to_value(&run_record.settings).unwrap(), + graph: serde_json::to_value(&run_record.graph).unwrap(), + workflow_source: Some("digraph night_sky {}".to_string()), + workflow_config: None, + labels: run_record.labels.clone().into_iter().collect(), + run_dir: "/tmp/night-sky-run".to_string(), + working_directory: run_record.working_directory.display().to_string(), + host_repo_path: run_record.host_repo_path.clone(), + base_branch: run_record.base_branch.clone(), + workflow_slug: run_record.workflow_slug.clone(), + db_prefix: None, + }, + ) + .await + .unwrap(); run.put_asset( &NodeVisitRef { node_id: "code", diff --git a/lib/crates/fabro-cli/src/commands/system/df.rs b/lib/crates/fabro-cli/src/commands/system/df.rs index ad11b0cd6..bc63be093 100644 --- a/lib/crates/fabro-cli/src/commands/system/df.rs +++ b/lib/crates/fabro-cli/src/commands/system/df.rs @@ -64,7 +64,7 @@ pub(super) async fn df_command(args: &DfArgs, globals: &GlobalArgs) -> Result<() #[allow(clippy::print_stdout)] async fn df_from( args: &DfArgs, - store: &dyn fabro_store::Store, + store: &fabro_store::SlateStore, data_dir: &Path, runs_base: &Path, logs_base: &Path, diff --git a/lib/crates/fabro-cli/src/commands/system/prune.rs b/lib/crates/fabro-cli/src/commands/system/prune.rs index 080333f98..be85a47e8 100644 --- a/lib/crates/fabro-cli/src/commands/system/prune.rs +++ b/lib/crates/fabro-cli/src/commands/system/prune.rs @@ -2,7 +2,7 @@ use std::path::Path; use anyhow::{Context, Result, bail}; use chrono::Utc; -use fabro_store::Store; +use fabro_store::SlateStore; use serde::Serialize; use tracing::{debug, info}; @@ -47,7 +47,7 @@ pub(crate) fn parse_duration(s: &str) -> Result { async fn prune_from( args: &RunsPruneArgs, - store: &dyn Store, + store: &SlateStore, base: &Path, globals: &GlobalArgs, ) -> Result<()> { diff --git a/lib/crates/fabro-cli/src/store.rs b/lib/crates/fabro-cli/src/store.rs index 39e1344fc..ee45e8bf0 100644 --- a/lib/crates/fabro-cli/src/store.rs +++ b/lib/crates/fabro-cli/src/store.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use std::time::Duration; use anyhow::Result; -use fabro_store::{RunStore, SlateStore, Store}; +use fabro_store::{RunStoreHandle, SlateStore}; use fabro_types::RunId; use object_store::local::LocalFileSystem; @@ -18,10 +18,7 @@ pub(crate) fn build_store(storage_dir: &Path) -> Result> { ))) } -pub(crate) async fn open_run_reader( - storage_dir: &Path, - run_id: &RunId, -) -> Result> { +pub(crate) async fn open_run_reader(storage_dir: &Path, run_id: &RunId) -> Result { build_store(storage_dir)? .open_run_reader(run_id) .await diff --git a/lib/crates/fabro-cli/tests/it/cmd/diff.rs b/lib/crates/fabro-cli/tests/it/cmd/diff.rs index f4ea62215..56268c5fa 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/diff.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/diff.rs @@ -1,28 +1,7 @@ -use std::sync::Arc; - -use fabro_store::Store; use fabro_test::{fabro_snapshot, test_context}; -use fabro_types::RunId; -use object_store::local::LocalFileSystem; use super::support::{git_filters, setup_git_backed_changed_run, setup_git_backed_noop_run}; -fn with_runtime(f: impl FnOnce(&tokio::runtime::Runtime) -> T) -> T { - let runtime = tokio::runtime::Runtime::new().unwrap(); - f(&runtime) -} - -fn build_store(storage_dir: &std::path::Path) -> Arc { - let store_path = storage_dir.join("store"); - std::fs::create_dir_all(&store_path).unwrap(); - let object_store = Arc::new(LocalFileSystem::new_with_prefix(&store_path).unwrap()); - Arc::new(fabro_store::SlateStore::new( - object_store, - "", - std::time::Duration::from_millis(1), - )) -} - #[test] fn help() { let context = test_context!(); @@ -113,24 +92,8 @@ fn diff_completed_run_with_changes_prints_patch() { fn diff_completed_run_reads_store_final_patch_without_disk_file() { let context = test_context!(); let setup = setup_git_backed_changed_run(&context); - let run_id: RunId = setup.run.run_id.parse().unwrap(); - let patch = with_runtime(|runtime| { - runtime.block_on(async { - let store = build_store(&context.storage_dir); - let run_store = store.open_run(&run_id).await.unwrap(); - run_store.get_final_patch().await.unwrap().unwrap() - }) - }); let _ = std::fs::remove_file(setup.run.run_dir.join("final.patch")); - with_runtime(|runtime| { - runtime.block_on(async { - let store = build_store(&context.storage_dir); - let run_store = store.open_run(&run_id).await.unwrap(); - run_store.put_final_patch(&patch).await.unwrap(); - }); - }); - let mut cmd = context.command(); cmd.args(["diff", &setup.run.run_id]); @@ -176,41 +139,8 @@ fn diff_node_outputs_specific_patch() { fn diff_node_reads_store_patch_without_disk_file() { let context = test_context!(); let setup = setup_git_backed_changed_run(&context); - let run_id: RunId = setup.run.run_id.parse().unwrap(); - let patch = with_runtime(|runtime| { - runtime.block_on(async { - let store = build_store(&context.storage_dir); - let run_store = store.open_run(&run_id).await.unwrap(); - run_store - .get_node(&fabro_store::NodeVisitRef { - node_id: "step_one", - visit: 1, - }) - .await - .unwrap() - .diff - .unwrap() - }) - }); let _ = std::fs::remove_file(setup.run.run_dir.join("nodes/step_one/diff.patch")); - with_runtime(|runtime| { - runtime.block_on(async { - let store = build_store(&context.storage_dir); - let run_store = store.open_run(&run_id).await.unwrap(); - run_store - .put_node_diff( - &fabro_store::NodeVisitRef { - node_id: "step_one", - visit: 1, - }, - &patch, - ) - .await - .unwrap(); - }); - }); - let mut cmd = context.command(); cmd.args(["diff", &setup.run.run_id, "--node", "step_one"]); diff --git a/lib/crates/fabro-cli/tests/it/cmd/pr_view.rs b/lib/crates/fabro-cli/tests/it/cmd/pr_view.rs index d059bf1d3..db19520b9 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/pr_view.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/pr_view.rs @@ -1,8 +1,8 @@ use std::sync::Arc; -use fabro_store::Store; use fabro_test::{fabro_snapshot, test_context}; -use fabro_types::{PullRequestRecord, RunId}; +use fabro_types::RunId; +use fabro_workflow::event::{WorkflowRunEvent, append_workflow_event}; use object_store::local::LocalFileSystem; use super::support::setup_completed_dry_run; @@ -77,18 +77,22 @@ fn pr_view_reads_pull_request_from_store_without_pull_request_json() { runtime.block_on(async { let store = build_store(&context.storage_dir); let run_store = store.open_run(&run_id).await.unwrap(); - run_store - .put_pull_request(&PullRequestRecord { - html_url: "https://github.com/fabro-sh/fabro/pull/123".to_string(), - number: 123, + append_workflow_event( + run_store.as_ref(), + &run_id, + &WorkflowRunEvent::PullRequestCreated { + pr_url: "https://github.com/fabro-sh/fabro/pull/123".to_string(), + pr_number: 123, owner: "fabro-sh".to_string(), repo: "fabro".to_string(), base_branch: "main".to_string(), head_branch: "fabro/run/demo".to_string(), title: "Map the constellations".to_string(), - }) - .await - .unwrap(); + draft: false, + }, + ) + .await + .unwrap(); }); }); diff --git a/lib/crates/fabro-cli/tests/it/cmd/rewind.rs b/lib/crates/fabro-cli/tests/it/cmd/rewind.rs index 3543d0e0e..020998d1d 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/rewind.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/rewind.rs @@ -129,7 +129,9 @@ fn rewind_preserves_event_history_and_clears_terminal_snapshot_state() { let setup = setup_git_backed_changed_run(&context); let before_events = run_events(&setup.run.run_dir); assert!( - before_events.iter().any(|event| event.payload.as_value()["event"] == "run.completed"), + before_events + .iter() + .any(|event| event.payload.as_value()["event"] == "run.completed"), "setup run should be completed before rewind" ); @@ -179,9 +181,18 @@ fn rewind_preserves_event_history_and_clears_terminal_snapshot_state() { snapshot.status.as_ref().map(|status| &status.status), Some(&fabro_types::RunStatus::Submitted) ); - assert!(snapshot.conclusion.is_none(), "rewind should clear conclusion"); - assert!(snapshot.final_patch.is_none(), "rewind should clear final patch"); - assert!(snapshot.pull_request.is_none(), "rewind should clear pull request"); + assert!( + snapshot.conclusion.is_none(), + "rewind should clear conclusion" + ); + assert!( + snapshot.final_patch.is_none(), + "rewind should clear final patch" + ); + assert!( + snapshot.pull_request.is_none(), + "rewind should clear pull request" + ); assert!( snapshot.nodes.is_empty(), "rewind should clear node snapshots that belonged to the prior execution" diff --git a/lib/crates/fabro-cli/tests/it/cmd/support.rs b/lib/crates/fabro-cli/tests/it/cmd/support.rs index 7784583e7..8a7b51409 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/support.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/support.rs @@ -4,7 +4,7 @@ use std::process::Output; use std::sync::Arc; use std::time::{Duration, Instant}; -use fabro_store::{EventEnvelope, RunSnapshot, RunStore, SlateStore, Store}; +use fabro_store::{EventEnvelope, RunSnapshot, RunStoreHandle, SlateStore}; use fabro_test::TestContext; use fabro_types::RunId; use object_store::local::LocalFileSystem; @@ -284,9 +284,9 @@ worktree_mode = "never" let run = run_local_workflow(context, &workspace_dir, "run.toml"); let store = run_store(&run.run_dir); assert!( - block_on(store.get_sandbox()) + block_on(store.state()) .ok() - .flatten() + .and_then(|state| state.sandbox) .is_some() ); @@ -374,10 +374,9 @@ pub(crate) fn write_gated_workflow(path: &Path, name: &str, goal: &str) -> Workf pub(crate) fn wait_for_status(run_dir: &Path, expected: &[&str]) -> String { let deadline = Instant::now() + COMMAND_TIMEOUT; loop { - if let Some(status) = block_on(run_store(run_dir).get_status()) + if let Some(status) = block_on(run_store(run_dir).state()) .ok() - .flatten() - .map(|record| record.status.to_string()) + .and_then(|state| state.status.map(|record| record.status.to_string())) { if expected.iter().any(|candidate| *candidate == status) { return status; @@ -481,7 +480,7 @@ fn block_on(future: impl std::future::Future) -> T { .block_on(future) } -fn run_store(run_dir: &Path) -> Arc { +fn run_store(run_dir: &Path) -> RunStoreHandle { let runs_dir = run_dir.parent().expect("run dir should have parent"); let storage_dir = runs_dir.parent().expect("runs dir should have parent"); let run_id: RunId = infer_run_id(run_dir).parse().expect("run id should parse"); @@ -495,9 +494,9 @@ fn run_store(run_dir: &Path) -> Arc { pub(crate) fn run_snapshot(run_dir: &Path) -> RunSnapshot { let store = run_store(run_dir); - block_on(store.get_snapshot()) + block_on(store.state()) .ok() - .flatten() + .and_then(|state| state.to_snapshot()) .expect("run store snapshot should exist") } diff --git a/lib/crates/fabro-cli/tests/it/scenario/mod.rs b/lib/crates/fabro-cli/tests/it/scenario/mod.rs index 6bfcfcbae..2c5341b8b 100644 --- a/lib/crates/fabro-cli/tests/it/scenario/mod.rs +++ b/lib/crates/fabro-cli/tests/it/scenario/mod.rs @@ -6,7 +6,7 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; -use fabro_store::{RunSnapshot, RunStore, SlateStore, Store}; +use fabro_store::{RunSnapshot, RunStoreHandle, SlateStore}; use fabro_types::RunId; use object_store::local::LocalFileSystem; pub(super) fn fixture(name: &str) -> PathBuf { @@ -23,7 +23,7 @@ fn block_on(future: impl std::future::Future) -> T { .block_on(future) } -fn run_store(run_dir: &Path) -> Arc { +fn run_store(run_dir: &Path) -> RunStoreHandle { let runs_dir = run_dir.parent().expect("run dir should have parent"); let storage_dir = runs_dir.parent().expect("runs dir should have parent"); let run_id: RunId = std::fs::read_to_string(run_dir.join("id.txt")) @@ -48,9 +48,9 @@ fn run_store(run_dir: &Path) -> Arc { pub(super) fn run_snapshot(run_dir: &Path) -> RunSnapshot { let store = run_store(run_dir); - block_on(store.get_snapshot()) + block_on(store.state()) .ok() - .flatten() + .and_then(|state| state.to_snapshot()) .expect("run store snapshot should exist") } diff --git a/lib/crates/fabro-retro/src/retro_agent.rs b/lib/crates/fabro-retro/src/retro_agent.rs index 1f26115ba..7e31df920 100644 --- a/lib/crates/fabro-retro/src/retro_agent.rs +++ b/lib/crates/fabro-retro/src/retro_agent.rs @@ -10,7 +10,7 @@ use fabro_agent::{ use fabro_llm::client::Client; use fabro_llm::provider::Provider; use fabro_llm::types::ToolDefinition; -use fabro_store::RunStore; +use fabro_store::SlateRunStore; use fabro_util::redact::redact_jsonl_line; use tokio::sync::broadcast::Receiver; use tokio::task::JoinHandle; @@ -137,7 +137,7 @@ pub fn build_retro_prompt(retro_data_dir: &str) -> String { /// files via tool access, then calls `submit_retro` with its analysis. pub async fn run_retro_agent( sandbox: &Arc, - run_store: &dyn RunStore, + run_store: &SlateRunStore, run_dir: &Path, llm_client: &Client, provider: Provider, @@ -286,26 +286,20 @@ pub fn dry_run_narrative() -> RetroNarrative { } async fn write_retro_prompt( - run_store: &dyn RunStore, + _run_store: &SlateRunStore, retro_dir: &Path, prompt: &str, ) -> anyhow::Result<()> { - if let Err(err) = run_store.put_retro_prompt(prompt).await { - tracing::warn!(error = %err, "Failed to save retro prompt to store"); - std::fs::write(retro_dir.join("prompt.md"), prompt)?; - } + std::fs::write(retro_dir.join("prompt.md"), prompt)?; Ok(()) } async fn write_retro_response( - run_store: &dyn RunStore, + _run_store: &SlateRunStore, retro_dir: &Path, response: &str, ) -> anyhow::Result<()> { - if let Err(err) = run_store.put_retro_response(response).await { - tracing::warn!(error = %err, "Failed to save retro response to store"); - std::fs::write(retro_dir.join("response.md"), response)?; - } + std::fs::write(retro_dir.join("response.md"), response)?; Ok(()) } @@ -385,7 +379,7 @@ fn build_profile(provider: Provider, model: &str) -> Box { async fn upload_data_files( sandbox: &Arc, - run_store: &dyn RunStore, + run_store: &SlateRunStore, _run_dir: &Path, target_dir: &str, ) -> anyhow::Result<()> { @@ -416,26 +410,24 @@ async fn upload_data_files( .map_err(|e| anyhow::anyhow!("Failed to upload progress.jsonl: {e}"))?; } - let checkpoint_content = run_store - .get_checkpoint() + let state = run_store + .state() .await - .map_err(|e| anyhow::anyhow!("Failed to load checkpoint from store: {e}"))? + .map_err(|e| anyhow::anyhow!("Failed to load run state from store: {e}"))?; + let checkpoint_content = state + .checkpoint .map(|cp| serde_json::to_string_pretty(&cp)) .transpose()?; upload_file(sandbox, target_dir, "checkpoint.json", checkpoint_content).await?; - let run_content = run_store - .get_run() - .await - .map_err(|e| anyhow::anyhow!("Failed to load run metadata from store: {e}"))? + let run_content = state + .run .map(|run| serde_json::to_string_pretty(&run)) .transpose()?; upload_file(sandbox, target_dir, "run.json", run_content).await?; - let start_content = run_store - .get_start() - .await - .map_err(|e| anyhow::anyhow!("Failed to load start metadata from store: {e}"))? + let start_content = state + .start .map(|start| serde_json::to_string_pretty(&start)) .transpose()?; upload_file(sandbox, target_dir, "start.json", start_content).await?; diff --git a/lib/crates/fabro-server/src/server.rs b/lib/crates/fabro-server/src/server.rs index 515e6045e..90d10810d 100644 --- a/lib/crates/fabro-server/src/server.rs +++ b/lib/crates/fabro-server/src/server.rs @@ -19,12 +19,13 @@ use fabro_llm::types::{ ContentPart, FinishReason, Message as LlmMessage, Request as LlmRequest, Response as LlmResponse, Role, StreamEvent, ToolChoice, ToolDefinition, Usage, }; -use fabro_store::{InMemoryStore, Store}; +use fabro_store::StoreHandle; use fabro_types::{RunId, Settings}; use fabro_util::redact::redact_jsonl_line; use fabro_workflow::error::FabroError; use fabro_workflow::handler::HandlerRegistry; use futures_util::stream; +use object_store::memory::InMemory as MemoryObjectStore; use tokio::sync::broadcast; use tokio::sync::oneshot; use tokio::sync::{Notify, OnceCell}; @@ -124,7 +125,7 @@ type RegistryFactoryOverride = dyn Fn(Arc) -> HandlerRegistry + pub struct AppState { runs: Mutex>, aggregate_usage: Mutex, - store: Arc, + store: StoreHandle, pub db: sqlx::SqlitePool, max_concurrent_runs: usize, scheduler_notify: Notify, @@ -417,7 +418,7 @@ pub fn create_app_state_with_registry_factory( Arc::new(RwLock::new(Settings::default())), Some(Box::new(registry_factory_override)), 5, - Arc::new(InMemoryStore::default()), + test_store(), ) } @@ -431,15 +432,23 @@ pub fn create_app_state_with_options( db, Arc::new(RwLock::new(settings)), max_concurrent_runs, - Arc::new(InMemoryStore::default()), + test_store(), ) } +fn test_store() -> StoreHandle { + Arc::new(fabro_store::SlateStore::new( + Arc::new(MemoryObjectStore::new()), + "", + Duration::from_millis(1), + )) +} + pub fn create_app_state_with_store( db: sqlx::SqlitePool, settings: Arc>, max_concurrent_runs: usize, - store: Arc, + store: StoreHandle, ) -> Arc { build_app_state(db, settings, None, max_concurrent_runs, store) } @@ -449,7 +458,7 @@ fn build_app_state( settings: Arc>, registry_factory_override: Option>, max_concurrent_runs: usize, - store: Arc, + store: StoreHandle, ) -> Arc { Arc::new(AppState { runs: Mutex::new(HashMap::new()), @@ -728,7 +737,7 @@ async fn execute_run(state: Arc, run_id: RunId) { cancel_token: Some(Arc::clone(&cancel_token)), emitter: Arc::clone(&emitter), interviewer: Arc::clone(&interviewer) as Arc, - run_store: Arc::clone(&run_store), + run_store: run_store.clone(), github_app, on_node: None, registry_override, @@ -2521,10 +2530,10 @@ mod tests { .open_run_reader(&run_id) .await .unwrap() - .expect("run store should exist") - .get_run() + .state() .await .unwrap() + .run .expect("run record should exist"); let mut expected_settings = settings; expected_settings.goal = Some("Test".to_string()); @@ -2656,16 +2665,11 @@ mod tests { assert_eq!(managed_run.status, RunStatus::Cancelled); drop(runs); - let run_store = state - .store - .open_run_reader(&run_id) - .await - .unwrap() - .expect("run store should exist"); + let run_store = state.store.open_run_reader(&run_id).await.unwrap(); let mut status_record = None; for _ in 0..50 { - if let Some(record) = run_store.get_status().await.unwrap() { + if let Some(record) = run_store.state().await.unwrap().status { if record.status == fabro_workflow::run_status::RunStatus::Failed && record.reason == Some(fabro_workflow::run_status::StatusReason::Cancelled) { diff --git a/lib/crates/fabro-store/src/keys.rs b/lib/crates/fabro-store/src/keys.rs index 374b6fc38..302daaa1c 100644 --- a/lib/crates/fabro-store/src/keys.rs +++ b/lib/crates/fabro-store/src/keys.rs @@ -1,20 +1,9 @@ use crate::NodeVisitRef; pub(crate) const INIT_KEY: &str = "_init.json"; -pub(crate) const RUN_KEY: &str = "run.json"; -pub(crate) const START_KEY: &str = "start.json"; -pub(crate) const STATUS_KEY: &str = "status.json"; -pub(crate) const CHECKPOINT_KEY: &str = "checkpoint.json"; -pub(crate) const CONCLUSION_KEY: &str = "conclusion.json"; -pub(crate) const RETRO_KEY: &str = "retro.json"; -pub(crate) const GRAPH_KEY: &str = "graph.fabro"; -pub(crate) const SANDBOX_KEY: &str = "sandbox.json"; -pub(crate) const FINAL_PATCH_KEY: &str = "final.patch"; -pub(crate) const PULL_REQUEST_KEY: &str = "pull_request.json"; pub(crate) const RETRO_PROMPT_KEY: &str = "retro/prompt.md"; pub(crate) const RETRO_RESPONSE_KEY: &str = "retro/response.md"; pub(crate) const EVENTS_PREFIX: &str = "events/"; -pub(crate) const CHECKPOINTS_PREFIX: &str = "checkpoints/"; pub(crate) const ARTIFACT_VALUES_PREFIX: &str = "artifacts/values/"; pub(crate) const ARTIFACT_NODES_PREFIX: &str = "artifacts/nodes/"; @@ -22,46 +11,6 @@ pub(crate) fn init() -> &'static str { INIT_KEY } -pub(crate) fn run() -> &'static str { - RUN_KEY -} - -pub(crate) fn start() -> &'static str { - START_KEY -} - -pub(crate) fn status() -> &'static str { - STATUS_KEY -} - -pub(crate) fn checkpoint() -> &'static str { - CHECKPOINT_KEY -} - -pub(crate) fn conclusion() -> &'static str { - CONCLUSION_KEY -} - -pub(crate) fn retro() -> &'static str { - RETRO_KEY -} - -pub(crate) fn graph() -> &'static str { - GRAPH_KEY -} - -pub(crate) fn sandbox() -> &'static str { - SANDBOX_KEY -} - -pub(crate) fn final_patch() -> &'static str { - FINAL_PATCH_KEY -} - -pub(crate) fn pull_request() -> &'static str { - PULL_REQUEST_KEY -} - pub(crate) fn node_visit_prefix(node: &NodeVisitRef<'_>) -> String { format!("nodes/{}/visit-{}", node.node_id, node.visit) } @@ -122,10 +71,6 @@ pub(crate) fn event_key(seq: u32, epoch_ms: i64) -> String { format!("{EVENTS_PREFIX}{seq:06}-{epoch_ms}.json") } -pub(crate) fn checkpoint_history_key(seq: u32, epoch_ms: i64) -> String { - format!("{CHECKPOINTS_PREFIX}{seq:04}-{epoch_ms}.json") -} - pub(crate) fn artifact_value(artifact_id: &str) -> String { format!("{ARTIFACT_VALUES_PREFIX}{artifact_id}.json") } @@ -145,10 +90,6 @@ pub(crate) fn parse_event_seq(key: &str) -> Option { parse_seq(key, EVENTS_PREFIX) } -pub(crate) fn parse_checkpoint_seq(key: &str) -> Option { - parse_seq(key, CHECKPOINTS_PREFIX) -} - pub(crate) fn parse_artifact_value_id(key: &str) -> Option { key.strip_prefix(ARTIFACT_VALUES_PREFIX) .and_then(|s| s.strip_suffix(".json")) @@ -181,10 +122,7 @@ mod tests { #[test] fn top_level_keys_match_spec() { assert_eq!(init(), "_init.json"); - assert_eq!(run(), "run.json"); - assert_eq!(graph(), "graph.fabro"); - assert_eq!(final_patch(), "final.patch"); - assert_eq!(pull_request(), "pull_request.json"); + assert_eq!(event_key(7, 123), "events/000007-123.json"); assert_eq!(retro_prompt(), "retro/prompt.md"); assert_eq!(retro_response(), "retro/response.md"); } @@ -224,7 +162,6 @@ mod tests { #[test] fn sequence_keys_are_zero_padded() { assert_eq!(event_key(7, 123), "events/000007-123.json"); - assert_eq!(checkpoint_history_key(42, 456), "checkpoints/0042-456.json"); } #[test] @@ -243,7 +180,6 @@ mod tests { #[test] fn parse_helpers_extract_sequences_and_node_visits() { assert_eq!(parse_event_seq("events/000007-123.json"), Some(7)); - assert_eq!(parse_checkpoint_seq("checkpoints/0042-456.json"), Some(42)); assert_eq!( parse_artifact_value_id("artifacts/values/summary.json"), Some("summary".to_string()) @@ -261,7 +197,6 @@ mod tests { #[test] fn parse_helpers_reject_invalid_keys() { assert_eq!(parse_event_seq("events/not-a-seq.json"), None); - assert_eq!(parse_checkpoint_seq("checkpoints/oops.json"), None); assert_eq!( parse_artifact_value_id("artifacts/values/summary.txt"), None diff --git a/lib/crates/fabro-store/src/lib.rs b/lib/crates/fabro-store/src/lib.rs index 7d6ad1659..a41155998 100644 --- a/lib/crates/fabro-store/src/lib.rs +++ b/lib/crates/fabro-store/src/lib.rs @@ -1,10 +1,6 @@ -use std::pin::Pin; use std::sync::Arc; -use async_trait::async_trait; -use bytes::Bytes; use chrono::{DateTime, Utc}; -use futures::Stream; mod error; mod keys; @@ -15,139 +11,22 @@ mod slate; mod types; pub use error::{Result, StoreError}; -pub use memory::InMemoryStore; +pub use memory::{InMemoryRunStore, InMemoryStore}; pub use run_state::{NodeState, RunState}; pub use runtime::RuntimeState; -pub use slate::SlateStore; +pub use slate::{SlateRunStore, SlateStore}; pub use types::{ CatalogRecord, EventEnvelope, EventPayload, NodeSnapshot, NodeVisitRef, RunSnapshot, RunSummary, }; -use fabro_types::{ - Checkpoint, Conclusion, NodeStatusRecord, Outcome, PullRequestRecord, Retro, RunId, RunRecord, - RunStatusRecord, SandboxRecord, StageUsage, StartRecord, -}; +use fabro_types::{Outcome, StageUsage}; pub type NodeOutcomeRecord = Outcome>; +pub type StoreHandle = Arc; +pub type RunStoreHandle = Arc; #[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct ListRunsQuery { pub start: Option>, pub end: Option>, } - -#[async_trait] -pub trait Store: Send + Sync { - async fn create_run( - &self, - run_id: &RunId, - created_at: DateTime, - run_dir: Option<&str>, - ) -> Result>; - async fn open_run(&self, run_id: &RunId) -> Result>; - async fn open_run_reader(&self, run_id: &RunId) -> Result>; - async fn list_runs(&self, query: &ListRunsQuery) -> Result>; - async fn delete_run(&self, run_id: &RunId) -> Result<()>; -} - -#[async_trait] -pub trait RunStore: Send + Sync { - async fn put_run(&self, record: &RunRecord) -> Result<()>; - async fn get_run(&self) -> Result>; - - async fn put_start(&self, record: &StartRecord) -> Result<()>; - async fn get_start(&self) -> Result>; - - async fn put_status(&self, record: &RunStatusRecord) -> Result<()>; - async fn get_status(&self) -> Result>; - - async fn put_checkpoint(&self, record: &Checkpoint) -> Result<()>; - async fn get_checkpoint(&self) -> Result>; - async fn append_checkpoint(&self, record: &Checkpoint) -> Result; - async fn list_checkpoints(&self) -> Result>; - - async fn put_conclusion(&self, record: &Conclusion) -> Result<()>; - async fn get_conclusion(&self) -> Result>; - - async fn put_retro(&self, retro: &Retro) -> Result<()>; - async fn get_retro(&self) -> Result>; - - async fn put_graph(&self, dot_source: &str) -> Result<()>; - async fn get_graph(&self) -> Result>; - - async fn put_sandbox(&self, record: &SandboxRecord) -> Result<()>; - async fn get_sandbox(&self) -> Result>; - - async fn put_node_prompt(&self, node: &NodeVisitRef<'_>, prompt: &str) -> Result<()>; - async fn put_node_response(&self, node: &NodeVisitRef<'_>, response: &str) -> Result<()>; - async fn put_node_status( - &self, - node: &NodeVisitRef<'_>, - status: &NodeStatusRecord, - ) -> Result<()>; - async fn put_node_outcome( - &self, - node: &NodeVisitRef<'_>, - outcome: &NodeOutcomeRecord, - ) -> Result<()>; - async fn put_node_provider_used( - &self, - node: &NodeVisitRef<'_>, - provider_used: &serde_json::Value, - ) -> Result<()>; - async fn put_node_diff(&self, node: &NodeVisitRef<'_>, diff: &str) -> Result<()>; - async fn put_node_script_invocation( - &self, - node: &NodeVisitRef<'_>, - invocation: &serde_json::Value, - ) -> Result<()>; - async fn put_node_script_timing( - &self, - node: &NodeVisitRef<'_>, - timing: &serde_json::Value, - ) -> Result<()>; - async fn put_node_parallel_results( - &self, - node: &NodeVisitRef<'_>, - results: &serde_json::Value, - ) -> Result<()>; - async fn put_node_stdout(&self, node: &NodeVisitRef<'_>, log: &str) -> Result<()>; - async fn put_node_stderr(&self, node: &NodeVisitRef<'_>, log: &str) -> Result<()>; - - async fn get_node(&self, node: &NodeVisitRef<'_>) -> Result; - async fn list_node_visits(&self, node_id: &str) -> Result>; - async fn list_node_ids(&self) -> Result>; - - async fn put_final_patch(&self, patch: &str) -> Result<()>; - async fn get_final_patch(&self) -> Result>; - - async fn put_pull_request(&self, record: &PullRequestRecord) -> Result<()>; - async fn get_pull_request(&self) -> Result>; - - async fn reset_for_rewind(&self) -> Result<()>; - - async fn append_event(&self, payload: &EventPayload) -> Result; - async fn list_events(&self) -> Result>; - async fn list_events_from(&self, seq: u32) -> Result>; - async fn watch_events_from( - &self, - seq: u32, - ) -> Result> + Send>>>; - - async fn put_retro_prompt(&self, text: &str) -> Result<()>; - async fn get_retro_prompt(&self) -> Result>; - async fn put_retro_response(&self, text: &str) -> Result<()>; - async fn get_retro_response(&self) -> Result>; - - async fn put_artifact_value(&self, artifact_id: &str, value: &serde_json::Value) -> Result<()>; - async fn get_artifact_value(&self, artifact_id: &str) -> Result>; - async fn list_artifact_values(&self) -> Result>; - - async fn put_asset(&self, node: &NodeVisitRef<'_>, filename: &str, data: &[u8]) -> Result<()>; - async fn get_asset(&self, node: &NodeVisitRef<'_>, filename: &str) -> Result>; - async fn list_assets(&self, node: &NodeVisitRef<'_>) -> Result>; - async fn list_all_assets(&self) -> Result>; - - async fn state(&self) -> Result; - async fn get_snapshot(&self) -> Result>; -} diff --git a/lib/crates/fabro-store/src/memory.rs b/lib/crates/fabro-store/src/memory.rs index 690263d55..a720792e1 100644 --- a/lib/crates/fabro-store/src/memory.rs +++ b/lib/crates/fabro-store/src/memory.rs @@ -2,7 +2,6 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::sync::Arc; use std::sync::atomic::{AtomicU32, Ordering}; -use async_trait::async_trait; use bytes::Bytes; use chrono::{DateTime, Utc}; use futures::Stream; @@ -16,12 +15,9 @@ use crate::keys; use crate::run_state::EventProjectionCache; use crate::{ CatalogRecord, EventEnvelope, EventPayload, ListRunsQuery, NodeOutcomeRecord, NodeSnapshot, - NodeVisitRef, Result, RunSnapshot, RunState, RunStore, RunSummary, Store, StoreError, -}; -use fabro_types::{ - Checkpoint, Conclusion, NodeStatusRecord, PullRequestRecord, Retro, RunId, RunRecord, - RunStatusRecord, SandboxRecord, StartRecord, + NodeVisitRef, Result, RunState, RunSummary, StoreError, }; +use fabro_types::{NodeStatusRecord, RunId}; #[derive(Debug, Default)] pub struct InMemoryStore { @@ -35,12 +31,10 @@ struct InMemoryCatalog { } #[derive(Debug)] -struct InMemoryRunStore { +pub struct InMemoryRunStore { run_id: RunId, - created_at: DateTime, data: Mutex>>, event_seq: AtomicU32, - checkpoint_seq: AtomicU32, watchers: Mutex>>, projection_cache: Mutex, } @@ -62,10 +56,8 @@ impl InMemoryRunStore { data.insert(keys::init().to_string(), serde_json::to_vec(&record)?); Ok(Self { run_id: *run_id, - created_at, data: Mutex::new(data), event_seq: AtomicU32::new(1), - checkpoint_seq: AtomicU32::new(1), watchers: Mutex::new(Vec::new()), projection_cache: Mutex::new(EventProjectionCache::default()), }) @@ -174,19 +166,6 @@ impl InMemoryRunStore { Ok(events) } - async fn list_checkpoints_inner(&self) -> Result> { - let data = self.snapshot_data().await; - let mut checkpoints = Vec::new(); - for (key, value) in &data { - let Some(seq) = keys::parse_checkpoint_seq(key) else { - continue; - }; - checkpoints.push((seq, serde_json::from_slice(value)?)); - } - checkpoints.sort_by_key(|(seq, _)| *seq); - Ok(checkpoints) - } - async fn list_artifact_values_inner(&self) -> Result> { let data = self.snapshot_data().await; let mut artifact_ids = Vec::new(); @@ -213,61 +192,6 @@ impl InMemoryRunStore { Ok(assets) } - fn build_snapshot_from_data( - &self, - data: &BTreeMap>, - ) -> Result> { - let Some(run) = read_json::(data, keys::run())? else { - return Ok(None); - }; - - let mut visits = BTreeSet::new(); - for key in data.keys() { - if let Some((node_id, visit, _)) = keys::parse_node_key(key) { - visits.insert((node_id, visit)); - } - } - - let mut nodes = Vec::new(); - for (node_id, visit) in visits { - let node = NodeVisitRef { - node_id: &node_id, - visit, - }; - nodes.push(self.build_node_snapshot_from_data(data, &node)?); - } - - Ok(Some(RunSnapshot { - run, - start: read_json(data, keys::start())?, - status: read_json(data, keys::status())?, - checkpoint: read_json(data, keys::checkpoint())?, - conclusion: read_json(data, keys::conclusion())?, - retro: read_json(data, keys::retro())?, - graph: read_text(data, keys::graph())?, - sandbox: read_json(data, keys::sandbox())?, - final_patch: read_text(data, keys::final_patch())?, - pull_request: read_json(data, keys::pull_request())?, - nodes, - })) - } - - fn validate_run_record(&self, record: &RunRecord) -> Result<()> { - if record.created_at != self.created_at { - return Err(StoreError::Other(format!( - "run record created_at {:?} does not match store created_at {:?}", - record.created_at, self.created_at - ))); - } - if record.run_id != self.run_id { - return Err(StoreError::Other(format!( - "run record run_id {:?} does not match store run_id {:?}", - record.run_id, self.run_id - ))); - } - Ok(()) - } - async fn projected_state(&self) -> Result { let next_seq = { let cache = self.projection_cache.lock().await; @@ -283,18 +207,17 @@ impl InMemoryRunStore { } } -#[async_trait] -impl Store for InMemoryStore { - async fn create_run( +impl InMemoryStore { + pub async fn create_run( &self, run_id: &RunId, created_at: DateTime, run_dir: Option<&str>, - ) -> Result> { + ) -> Result> { let mut runs = self.runs.lock().await; if let Some(existing) = runs.get(run_id) { if existing.record.created_at == created_at { - return Ok(Arc::clone(&existing.run_store) as Arc); + return Ok(Arc::clone(&existing.run_store)); } return Err(StoreError::RunAlreadyExists(run_id.to_string())); } @@ -313,24 +236,24 @@ impl Store for InMemoryStore { db_prefix, run_dir: run_dir.map(ToOwned::to_owned), }, - run_store: Arc::clone(&run_store), + run_store: run_store.clone(), }; runs.insert(*run_id, catalog); - Ok(run_store as Arc) + Ok(run_store) } - async fn open_run(&self, run_id: &RunId) -> Result> { + pub async fn open_run(&self, run_id: &RunId) -> Result> { let runs = self.runs.lock().await; runs.get(run_id) - .map(|catalog| Arc::clone(&catalog.run_store) as Arc) + .map(|catalog| Arc::clone(&catalog.run_store)) .ok_or_else(|| StoreError::RunNotFound(run_id.to_string())) } - async fn open_run_reader(&self, run_id: &RunId) -> Result> { + pub async fn open_run_reader(&self, run_id: &RunId) -> Result> { self.open_run(run_id).await } - async fn list_runs(&self, query: &ListRunsQuery) -> Result> { + pub async fn list_runs(&self, query: &ListRunsQuery) -> Result> { let catalogs = { let runs = self.runs.lock().await; runs.values().cloned().collect::>() @@ -341,110 +264,36 @@ impl Store for InMemoryStore { if !matches_query(&catalog.record.created_at, query) { continue; } - summaries.push(catalog.run_store.state().await?.build_summary(&catalog.record)); + summaries.push( + catalog + .run_store + .state() + .await? + .build_summary(&catalog.record), + ); } summaries.sort_by(|a, b| b.created_at.cmp(&a.created_at)); Ok(summaries) } - async fn delete_run(&self, run_id: &RunId) -> Result<()> { + pub async fn delete_run(&self, run_id: &RunId) -> Result<()> { self.runs.lock().await.remove(run_id); Ok(()) } } -#[async_trait] -impl RunStore for InMemoryRunStore { - async fn put_run(&self, record: &RunRecord) -> Result<()> { - self.validate_run_record(record)?; - self.put_json(keys::run().to_string(), record).await - } - - async fn get_run(&self) -> Result> { - self.get_json(keys::run()).await - } - - async fn put_start(&self, record: &StartRecord) -> Result<()> { - self.put_json(keys::start().to_string(), record).await - } - - async fn get_start(&self) -> Result> { - self.get_json(keys::start()).await - } - - async fn put_status(&self, record: &RunStatusRecord) -> Result<()> { - self.put_json(keys::status().to_string(), record).await - } - - async fn get_status(&self) -> Result> { - self.get_json(keys::status()).await - } - - async fn put_checkpoint(&self, record: &Checkpoint) -> Result<()> { - self.put_json(keys::checkpoint().to_string(), record).await - } - - async fn get_checkpoint(&self) -> Result> { - self.get_json(keys::checkpoint()).await - } - - async fn append_checkpoint(&self, record: &Checkpoint) -> Result { - let seq = self.checkpoint_seq.fetch_add(1, Ordering::SeqCst); - let now = Utc::now().timestamp_millis(); - self.put_checkpoint(record).await?; - self.put_json(keys::checkpoint_history_key(seq, now), record) - .await?; - Ok(seq) - } - - async fn list_checkpoints(&self) -> Result> { - self.list_checkpoints_inner().await - } - - async fn put_conclusion(&self, record: &Conclusion) -> Result<()> { - self.put_json(keys::conclusion().to_string(), record).await - } - - async fn get_conclusion(&self) -> Result> { - self.get_json(keys::conclusion()).await - } - - async fn put_retro(&self, retro: &Retro) -> Result<()> { - self.put_json(keys::retro().to_string(), retro).await - } - - async fn get_retro(&self) -> Result> { - self.get_json(keys::retro()).await - } - - async fn put_graph(&self, dot_source: &str) -> Result<()> { - self.put_text(keys::graph().to_string(), dot_source).await; - Ok(()) - } - - async fn get_graph(&self) -> Result> { - self.get_text(keys::graph()).await - } - - async fn put_sandbox(&self, record: &SandboxRecord) -> Result<()> { - self.put_json(keys::sandbox().to_string(), record).await - } - - async fn get_sandbox(&self) -> Result> { - self.get_json(keys::sandbox()).await - } - - async fn put_node_prompt(&self, node: &NodeVisitRef<'_>, prompt: &str) -> Result<()> { +impl InMemoryRunStore { + pub async fn put_node_prompt(&self, node: &NodeVisitRef<'_>, prompt: &str) -> Result<()> { self.put_text(keys::node_prompt(node), prompt).await; Ok(()) } - async fn put_node_response(&self, node: &NodeVisitRef<'_>, response: &str) -> Result<()> { + pub async fn put_node_response(&self, node: &NodeVisitRef<'_>, response: &str) -> Result<()> { self.put_text(keys::node_response(node), response).await; Ok(()) } - async fn put_node_status( + pub async fn put_node_status( &self, node: &NodeVisitRef<'_>, status: &NodeStatusRecord, @@ -452,7 +301,7 @@ impl RunStore for InMemoryRunStore { self.put_json(keys::node_status(node), status).await } - async fn put_node_outcome( + pub async fn put_node_outcome( &self, node: &NodeVisitRef<'_>, outcome: &NodeOutcomeRecord, @@ -460,7 +309,7 @@ impl RunStore for InMemoryRunStore { self.put_json(keys::node_outcome(node), outcome).await } - async fn put_node_provider_used( + pub async fn put_node_provider_used( &self, node: &NodeVisitRef<'_>, provider_used: &serde_json::Value, @@ -469,12 +318,12 @@ impl RunStore for InMemoryRunStore { .await } - async fn put_node_diff(&self, node: &NodeVisitRef<'_>, diff: &str) -> Result<()> { + pub async fn put_node_diff(&self, node: &NodeVisitRef<'_>, diff: &str) -> Result<()> { self.put_text(keys::node_diff(node), diff).await; Ok(()) } - async fn put_node_script_invocation( + pub async fn put_node_script_invocation( &self, node: &NodeVisitRef<'_>, invocation: &serde_json::Value, @@ -483,7 +332,7 @@ impl RunStore for InMemoryRunStore { .await } - async fn put_node_script_timing( + pub async fn put_node_script_timing( &self, node: &NodeVisitRef<'_>, timing: &serde_json::Value, @@ -491,7 +340,7 @@ impl RunStore for InMemoryRunStore { self.put_json(keys::node_script_timing(node), timing).await } - async fn put_node_parallel_results( + pub async fn put_node_parallel_results( &self, node: &NodeVisitRef<'_>, results: &serde_json::Value, @@ -500,22 +349,22 @@ impl RunStore for InMemoryRunStore { .await } - async fn put_node_stdout(&self, node: &NodeVisitRef<'_>, log: &str) -> Result<()> { + pub async fn put_node_stdout(&self, node: &NodeVisitRef<'_>, log: &str) -> Result<()> { self.put_text(keys::node_stdout(node), log).await; Ok(()) } - async fn put_node_stderr(&self, node: &NodeVisitRef<'_>, log: &str) -> Result<()> { + pub async fn put_node_stderr(&self, node: &NodeVisitRef<'_>, log: &str) -> Result<()> { self.put_text(keys::node_stderr(node), log).await; Ok(()) } - async fn get_node(&self, node: &NodeVisitRef<'_>) -> Result { + pub async fn get_node(&self, node: &NodeVisitRef<'_>) -> Result { let data = self.snapshot_data().await; self.build_node_snapshot_from_data(&data, node) } - async fn list_node_visits(&self, node_id: &str) -> Result> { + pub async fn list_node_visits(&self, node_id: &str) -> Result> { let data = self.snapshot_data().await; let mut visits = BTreeSet::new(); for key in data.keys() { @@ -529,41 +378,17 @@ impl RunStore for InMemoryRunStore { Ok(visits.into_iter().collect()) } - async fn list_node_ids(&self) -> Result> { + pub async fn list_node_ids(&self) -> Result> { Ok(self.list_node_ids_inner().await) } - async fn put_final_patch(&self, patch: &str) -> Result<()> { - self.put_text(keys::final_patch().to_string(), patch).await; - Ok(()) - } - - async fn get_final_patch(&self) -> Result> { - self.get_text(keys::final_patch()).await - } - - async fn put_pull_request(&self, record: &PullRequestRecord) -> Result<()> { - self.put_json(keys::pull_request().to_string(), record) - .await - } - - async fn get_pull_request(&self) -> Result> { - self.get_json(keys::pull_request()).await - } - - async fn reset_for_rewind(&self) -> Result<()> { + pub async fn reset_for_rewind(&self) -> Result<()> { let mut data = self.data.lock().await; - data.retain(|key, _| { - key == keys::init() - || key == keys::run() - || key == keys::start() - || key == keys::graph() - || key.starts_with(keys::EVENTS_PREFIX) - }); + data.retain(|key, _| key == keys::init() || key.starts_with(keys::EVENTS_PREFIX)); Ok(()) } - async fn append_event(&self, payload: &EventPayload) -> Result { + pub async fn append_event(&self, payload: &EventPayload) -> Result { payload.validate(&self.run_id)?; let seq = self.event_seq.fetch_add(1, Ordering::SeqCst); @@ -582,15 +407,15 @@ impl RunStore for InMemoryRunStore { Ok(seq) } - async fn list_events(&self) -> Result> { + pub async fn list_events(&self) -> Result> { self.list_events_from_inner(1).await } - async fn list_events_from(&self, seq: u32) -> Result> { + pub async fn list_events_from(&self, seq: u32) -> Result> { self.list_events_from_inner(seq).await } - async fn watch_events_from( + pub async fn watch_events_from( &self, seq: u32, ) -> Result> + Send>>> { @@ -620,51 +445,64 @@ impl RunStore for InMemoryRunStore { Ok(Box::pin(UnboundedReceiverStream::new(receiver).map(Ok))) } - async fn put_retro_prompt(&self, text: &str) -> Result<()> { + pub async fn put_retro_prompt(&self, text: &str) -> Result<()> { self.put_text(keys::retro_prompt().to_string(), text).await; Ok(()) } - async fn get_retro_prompt(&self) -> Result> { + pub async fn get_retro_prompt(&self) -> Result> { self.get_text(keys::retro_prompt()).await } - async fn put_retro_response(&self, text: &str) -> Result<()> { + pub async fn put_retro_response(&self, text: &str) -> Result<()> { self.put_text(keys::retro_response().to_string(), text) .await; Ok(()) } - async fn get_retro_response(&self) -> Result> { + pub async fn get_retro_response(&self) -> Result> { self.get_text(keys::retro_response()).await } - async fn put_artifact_value(&self, artifact_id: &str, value: &serde_json::Value) -> Result<()> { + pub async fn put_artifact_value( + &self, + artifact_id: &str, + value: &serde_json::Value, + ) -> Result<()> { self.put_json(keys::artifact_value(artifact_id), value) .await } - async fn get_artifact_value(&self, artifact_id: &str) -> Result> { + pub async fn get_artifact_value(&self, artifact_id: &str) -> Result> { self.get_json(&keys::artifact_value(artifact_id)).await } - async fn list_artifact_values(&self) -> Result> { + pub async fn list_artifact_values(&self) -> Result> { self.list_artifact_values_inner().await } - async fn put_asset(&self, node: &NodeVisitRef<'_>, filename: &str, data: &[u8]) -> Result<()> { + pub async fn put_asset( + &self, + node: &NodeVisitRef<'_>, + filename: &str, + data: &[u8], + ) -> Result<()> { self.put_bytes(keys::node_asset(node, filename), data).await; Ok(()) } - async fn get_asset(&self, node: &NodeVisitRef<'_>, filename: &str) -> Result> { + pub async fn get_asset( + &self, + node: &NodeVisitRef<'_>, + filename: &str, + ) -> Result> { Ok(self .get_bytes(&keys::node_asset(node, filename)) .await .map(Bytes::from)) } - async fn list_assets(&self, node: &NodeVisitRef<'_>) -> Result> { + pub async fn list_assets(&self, node: &NodeVisitRef<'_>) -> Result> { let prefix = format!("{}/", keys::node_asset_prefix(node)); let data = self.snapshot_data().await; let mut assets = Vec::new(); @@ -677,25 +515,12 @@ impl RunStore for InMemoryRunStore { Ok(assets) } - async fn list_all_assets(&self) -> Result> { + pub async fn list_all_assets(&self) -> Result> { self.list_all_assets_inner().await } - async fn state(&self) -> Result { - let mut state = self.projected_state().await?; - let data = self.snapshot_data().await; - state.merge_legacy( - self.build_snapshot_from_data(&data)?, - read_text(&data, keys::graph())?, - read_text(&data, keys::retro_prompt())?, - read_text(&data, keys::retro_response())?, - self.list_checkpoints_inner().await?, - ); - Ok(state) - } - - async fn get_snapshot(&self) -> Result> { - self.state().await.map(|state| state.to_snapshot()) + pub async fn state(&self) -> Result { + self.projected_state().await } } @@ -741,8 +566,9 @@ mod tests { use chrono::Duration as ChronoDuration; use fabro_types::{ - AttrValue, Graph, PullRequestRecord, RunId, RunStatus, Settings, StageStatus, StatusReason, - fixtures, + AttrValue, Checkpoint, Conclusion, Graph, PullRequestRecord, Retro, RunId, RunRecord, + RunStatus, RunStatusRecord, SandboxRecord, Settings, StageStatus, StartRecord, + StatusReason, fixtures, }; use tokio::time::timeout; @@ -921,7 +747,7 @@ mod tests { } #[tokio::test] - async fn create_run_put_get_and_snapshot_round_trip() { + async fn create_run_state_and_node_storage_round_trip() { let store = InMemoryStore::default(); let created_at = dt("2026-03-27T12:00:00Z"); let run = store @@ -949,14 +775,83 @@ mod tests { let parallel_results = serde_json::json!([{"node_id": "lint", "status": "success"}]); let pull_request = sample_pull_request(); - run.put_run(&run_record).await.unwrap(); - run.put_start(&start_record).await.unwrap(); - run.put_status(&status_record).await.unwrap(); - run.put_checkpoint(&checkpoint).await.unwrap(); - run.put_conclusion(&conclusion).await.unwrap(); - run.put_retro(&retro).await.unwrap(); - run.put_graph("digraph night_sky {}").await.unwrap(); - run.put_sandbox(&sandbox).await.unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:00Z", + "run.created", + None, + serde_json::json!({ + "settings": run_record.settings, + "graph": run_record.graph, + "workflow_source": "digraph night_sky {}", + "workflow_slug": run_record.workflow_slug, + "working_directory": run_record.working_directory, + "host_repo_path": run_record.host_repo_path, + "base_branch": run_record.base_branch, + "labels": run_record.labels, + }), + )) + .await + .unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:05Z", + "run.started", + None, + serde_json::json!({ + "run_branch": start_record.run_branch, + "base_sha": start_record.base_sha, + }), + )) + .await + .unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:06Z", + "run.running", + None, + serde_json::json!({ + "reason": status_record.reason, + }), + )) + .await + .unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:07Z", + "checkpoint.completed", + Some("code"), + serde_json::json!({ + "status": "success", + "current_node": checkpoint.current_node, + "completed_nodes": checkpoint.completed_nodes, + "node_retries": checkpoint.node_retries, + "context_values": checkpoint.context_values, + "node_outcomes": checkpoint.node_outcomes, + "next_node_id": checkpoint.next_node_id, + "git_commit_sha": checkpoint.git_commit_sha, + "loop_failure_signatures": serde_json::json!({}), + "restart_failure_signatures": serde_json::json!({}), + "node_visits": checkpoint.node_visits, + }), + )) + .await + .unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:08Z", + "sandbox.initialized", + None, + serde_json::json!({ + "provider": sandbox.provider, + "working_directory": sandbox.working_directory, + "identifier": sandbox.identifier, + "host_working_directory": sandbox.host_working_directory, + "container_mount_point": sandbox.container_mount_point, + }), + )) + .await + .unwrap(); run.put_node_prompt(&node, "Plan the fix").await.unwrap(); run.put_node_response(&node, "Implemented").await.unwrap(); run.put_node_status(&node, &node_status).await.unwrap(); @@ -978,10 +873,50 @@ mod tests { .unwrap(); run.put_node_stdout(&node, "ok").await.unwrap(); run.put_node_stderr(&node, "").await.unwrap(); - run.put_final_patch("diff --git a/src/lib.rs b/src/lib.rs\n") - .await - .unwrap(); - run.put_pull_request(&pull_request).await.unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:09Z", + "retro.completed", + None, + serde_json::json!({ + "response": "Smooth enough", + "retro": retro, + }), + )) + .await + .unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:10Z", + "run.completed", + None, + serde_json::json!({ + "status": conclusion.status, + "duration_ms": conclusion.duration_ms, + "total_cost": conclusion.total_cost, + "final_git_commit_sha": conclusion.final_git_commit_sha, + "final_patch": "diff --git a/src/lib.rs b/src/lib.rs\n", + }), + )) + .await + .unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:11Z", + "pull_request.created", + None, + serde_json::json!({ + "pr_url": pull_request.html_url, + "pr_number": pull_request.number, + "owner": pull_request.owner, + "repo": pull_request.repo, + "base_branch": pull_request.base_branch, + "head_branch": pull_request.head_branch, + "title": pull_request.title, + }), + )) + .await + .unwrap(); run.put_retro_prompt("How did it go?").await.unwrap(); run.put_retro_response("Smooth enough").await.unwrap(); run.put_artifact_value("summary", &serde_json::json!({"done": true})) @@ -991,37 +926,35 @@ mod tests { .await .unwrap(); - let stored_run = run.get_run().await.unwrap().unwrap(); + let state = run.state().await.unwrap(); + let stored_run = state.run.as_ref().unwrap(); assert_eq!(stored_run.run_id, run_record.run_id); assert_eq!(stored_run.created_at, run_record.created_at); assert_eq!(stored_run.workflow_slug, run_record.workflow_slug); assert_eq!(stored_run.graph.name, run_record.graph.name); + assert_eq!(state.graph_source.as_deref(), Some("digraph night_sky {}")); - let stored_start = run.get_start().await.unwrap().unwrap(); + let stored_start = state.start.as_ref().unwrap(); assert_eq!(stored_start.run_id, start_record.run_id); assert_eq!(stored_start.start_time, start_record.start_time); - let stored_status = run.get_status().await.unwrap().unwrap(); - assert_eq!(stored_status.status, status_record.status); - assert_eq!(stored_status.reason, status_record.reason); + let stored_status = state.status.as_ref().unwrap(); + assert_eq!(stored_status.status, RunStatus::Succeeded); + assert_eq!(stored_status.reason, None); - let stored_checkpoint = run.get_checkpoint().await.unwrap().unwrap(); + let stored_checkpoint = state.checkpoint.as_ref().unwrap(); assert_eq!(stored_checkpoint.current_node, checkpoint.current_node); assert_eq!(stored_checkpoint.next_node_id, checkpoint.next_node_id); - let stored_conclusion = run.get_conclusion().await.unwrap().unwrap(); + let stored_conclusion = state.conclusion.as_ref().unwrap(); assert_eq!(stored_conclusion.status, conclusion.status); assert_eq!(stored_conclusion.duration_ms, conclusion.duration_ms); assert_eq!(stored_conclusion.total_cost, conclusion.total_cost); - let stored_retro = run.get_retro().await.unwrap().unwrap(); + let stored_retro = state.retro.as_ref().unwrap(); assert_eq!(stored_retro.run_id, retro.run_id); assert_eq!(stored_retro.intent, retro.intent); - assert_eq!( - run.get_graph().await.unwrap(), - Some("digraph night_sky {}".to_string()) - ); - let stored_sandbox = run.get_sandbox().await.unwrap().unwrap(); + let stored_sandbox = state.sandbox.as_ref().unwrap(); assert_eq!(stored_sandbox.provider, sandbox.provider); assert_eq!(stored_sandbox.working_directory, sandbox.working_directory); assert_eq!( @@ -1041,75 +974,19 @@ mod tests { Some(Bytes::from_static(b"fn main() {}")) ); assert_eq!( - run.get_final_patch().await.unwrap().as_deref(), + state.final_patch.as_deref(), Some("diff --git a/src/lib.rs b/src/lib.rs\n") ); - assert_eq!( - run.get_pull_request().await.unwrap(), - Some(pull_request.clone()) - ); + assert_eq!(state.pull_request, Some(pull_request.clone())); assert_eq!(run.list_node_ids().await.unwrap(), vec!["code".to_string()]); assert_eq!( run.list_assets(&node).await.unwrap(), vec!["src/lib.rs".to_string()] ); - - let snapshot = run.get_snapshot().await.unwrap().unwrap(); - assert_eq!(snapshot.run.run_id, run_record.run_id); - assert_eq!(snapshot.run.created_at, run_record.created_at); - assert_eq!( - snapshot - .checkpoint - .as_ref() - .map(|checkpoint| checkpoint.current_node.as_str()), - Some("code") - ); - assert_eq!( - snapshot - .conclusion - .as_ref() - .map(|conclusion| conclusion.duration_ms), - Some(3210) - ); - assert_eq!(snapshot.nodes.len(), 1); - assert_eq!(snapshot.nodes[0].node_id, "code"); - assert_eq!(snapshot.nodes[0].visit, 2); - let snapshot_status = snapshot.nodes[0].status.as_ref().unwrap(); - assert_eq!(snapshot_status.status, node_status.status); - assert_eq!(snapshot_status.failure_reason, node_status.failure_reason); - assert_eq!( - snapshot.nodes[0].outcome.as_ref().unwrap().status, - StageStatus::Success - ); - assert_eq!( - snapshot.nodes[0].provider_used.as_ref(), - Some(&provider_used) - ); - assert_eq!( - snapshot.nodes[0].diff.as_deref(), - Some("diff --git a/src/lib.rs b/src/lib.rs") - ); - assert_eq!( - snapshot.nodes[0].script_invocation.as_ref(), - Some(&script_invocation) - ); - assert_eq!( - snapshot.nodes[0].script_timing.as_ref(), - Some(&script_timing) - ); - assert_eq!( - snapshot.nodes[0].parallel_results.as_ref(), - Some(¶llel_results) - ); - assert_eq!( - snapshot.final_patch.as_deref(), - Some("diff --git a/src/lib.rs b/src/lib.rs\n") - ); - assert_eq!(snapshot.pull_request, Some(pull_request)); } #[tokio::test] - async fn state_projects_event_stream_and_compat_fields() { + async fn state_projects_event_stream() { let store = InMemoryStore::default(); let created_at = dt("2026-03-27T12:00:00Z"); let run = store @@ -1279,17 +1156,44 @@ mod tests { .unwrap(); let state = run.state().await.unwrap(); - assert_eq!(state.run.as_ref().map(|run| run.run_id), Some(test_run_id("run-1"))); + assert_eq!( + state.run.as_ref().map(|run| run.run_id), + Some(test_run_id("run-1")) + ); assert_eq!(state.graph_source.as_deref(), Some("digraph night_sky {}")); - assert_eq!(state.start.as_ref().and_then(|start| start.run_branch.as_deref()), Some("fabro/run/demo")); - assert_eq!(state.status.as_ref().map(|status| status.status), Some(RunStatus::Succeeded)); - assert_eq!(state.checkpoint.as_ref().map(|checkpoint| checkpoint.current_node.as_str()), Some("code")); + assert_eq!( + state + .start + .as_ref() + .and_then(|start| start.run_branch.as_deref()), + Some("fabro/run/demo") + ); + assert_eq!( + state.status.as_ref().map(|status| status.status), + Some(RunStatus::Succeeded) + ); + assert_eq!( + state + .checkpoint + .as_ref() + .map(|checkpoint| checkpoint.current_node.as_str()), + Some("code") + ); assert_eq!(state.checkpoints.len(), 1); - assert_eq!(state.final_patch.as_deref(), Some("diff --git a/src/lib.rs b/src/lib.rs\n")); + assert_eq!( + state.final_patch.as_deref(), + Some("diff --git a/src/lib.rs b/src/lib.rs\n") + ); assert_eq!(state.retro_prompt.as_deref(), Some("How did it go?")); assert_eq!(state.retro_response.as_deref(), Some("Smooth enough")); assert_eq!(state.pull_request.as_ref().map(|pr| pr.number), Some(123)); - assert_eq!(state.sandbox.as_ref().map(|sandbox| sandbox.provider.as_str()), Some("local")); + assert_eq!( + state + .sandbox + .as_ref() + .map(|sandbox| sandbox.provider.as_str()), + Some("local") + ); assert_eq!(state.list_node_visits("code"), vec![2]); let node = state .node(&NodeVisitRef { @@ -1299,9 +1203,15 @@ mod tests { .unwrap(); assert_eq!(node.prompt.as_deref(), Some("Plan the fix")); assert_eq!(node.response.as_deref(), Some("Implemented")); - assert_eq!(node.diff.as_deref(), Some("diff --git a/src/lib.rs b/src/lib.rs")); assert_eq!( - node.provider_used.as_ref().and_then(|value| value.get("provider")).and_then(|value| value.as_str()), + node.diff.as_deref(), + Some("diff --git a/src/lib.rs b/src/lib.rs") + ); + assert_eq!( + node.provider_used + .as_ref() + .and_then(|value| value.get("provider")) + .and_then(|value| value.as_str()), Some("openai") ); } @@ -1410,12 +1320,21 @@ mod tests { .unwrap(); let state = run.state().await.unwrap(); - assert_eq!(state.status.as_ref().map(|status| status.status), Some(RunStatus::Submitted)); + assert_eq!( + state.status.as_ref().map(|status| status.status), + Some(RunStatus::Submitted) + ); assert!(state.conclusion.is_none()); assert!(state.final_patch.is_none()); assert!(state.pull_request.is_none()); assert_eq!(state.checkpoints.len(), 1); - assert_eq!(state.checkpoint.as_ref().map(|checkpoint| checkpoint.current_node.as_str()), Some("plan")); + assert_eq!( + state + .checkpoint + .as_ref() + .map(|checkpoint| checkpoint.current_node.as_str()), + Some("plan") + ); assert!(state.list_node_ids().is_empty()); } @@ -1427,9 +1346,6 @@ mod tests { .create_run(&test_run_id("run-1"), created_at, None) .await .unwrap(); - run.put_run(&sample_run_record("run-1", created_at)) - .await - .unwrap(); run.put_artifact_value("summary", &serde_json::json!({"done": true})) .await @@ -1477,9 +1393,8 @@ mod tests { vec!["artifact-only".to_string(), "code".to_string()] ); - let snapshot = run.get_snapshot().await.unwrap().unwrap(); - assert_eq!(snapshot.nodes.len(), 1); - assert_eq!(snapshot.nodes[0].node_id, "code"); + let code_node = run.get_node(&snapshot_node).await.unwrap(); + assert_eq!(code_node.node_id, "code"); } #[tokio::test] @@ -1508,24 +1423,6 @@ mod tests { assert!(matches!(err, StoreError::InvalidEvent(_))); } - #[tokio::test] - async fn put_run_rejects_created_at_mismatch() { - let store = InMemoryStore::default(); - let created_at = dt("2026-03-27T12:00:00Z"); - let run = store - .create_run(&test_run_id("run-1"), created_at, None) - .await - .unwrap(); - let err = run - .put_run(&sample_run_record( - "run-1", - created_at + ChronoDuration::minutes(1), - )) - .await - .unwrap_err(); - assert!(matches!(err, StoreError::Other(_))); - } - #[tokio::test] async fn watch_events_from_receives_existing_and_live_events() { let store = InMemoryStore::default(); @@ -1580,22 +1477,44 @@ mod tests { } #[tokio::test] - async fn checkpoint_history_round_trips() { + async fn state_retains_checkpoint_history_by_event_sequence() { let store = InMemoryStore::default(); let run = store .create_run(&test_run_id("run-1"), dt("2026-03-27T12:00:00Z"), None) .await .unwrap(); let checkpoint = sample_checkpoint(); - let seq = run.append_checkpoint(&checkpoint).await.unwrap(); + let seq = run + .append_event(&event_payload( + "run-1", + "2026-03-27T12:00:00Z", + "checkpoint.completed", + Some(&checkpoint.current_node), + serde_json::json!({ + "status": "success", + "current_node": checkpoint.current_node, + "completed_nodes": checkpoint.completed_nodes, + "node_retries": checkpoint.node_retries, + "context_values": checkpoint.context_values, + "node_outcomes": checkpoint.node_outcomes, + "next_node_id": checkpoint.next_node_id, + "git_commit_sha": checkpoint.git_commit_sha, + "loop_failure_signatures": serde_json::json!({}), + "restart_failure_signatures": serde_json::json!({}), + "node_visits": checkpoint.node_visits, + }), + )) + .await + .unwrap(); + let state = run.state().await.unwrap(); assert_eq!(seq, 1); - let checkpoints = run.list_checkpoints().await.unwrap(); - assert_eq!(checkpoints.len(), 1); - assert_eq!(checkpoints[0].0, 1); - assert_eq!(checkpoints[0].1.current_node, checkpoint.current_node); - - let latest = run.get_checkpoint().await.unwrap().unwrap(); - assert_eq!(latest.current_node, checkpoint.current_node); + assert_eq!(state.checkpoints.len(), 1); + assert_eq!(state.checkpoints[0].0, 1); + assert_eq!(state.checkpoints[0].1.current_node, checkpoint.current_node); + assert_eq!( + state.checkpoint.as_ref().unwrap().current_node, + checkpoint.current_node + ); } #[tokio::test] @@ -1635,8 +1554,23 @@ mod tests { .create_run(&test_run_id("run-early"), early, None) .await .unwrap(); + let early_record = sample_run_record("run-early", early); early_run - .put_run(&sample_run_record("run-early", early)) + .append_event(&event_payload( + "run-early", + "2026-03-27T10:00:00Z", + "run.created", + None, + serde_json::json!({ + "settings": early_record.settings, + "graph": early_record.graph, + "workflow_slug": early_record.workflow_slug, + "working_directory": early_record.working_directory, + "host_repo_path": early_record.host_repo_path, + "base_branch": early_record.base_branch, + "labels": early_record.labels, + }), + )) .await .unwrap(); @@ -1644,22 +1578,54 @@ mod tests { .create_run(&test_run_id("run-late"), late, None) .await .unwrap(); + let late_record = sample_run_record("run-late", late); late_run - .put_run(&sample_run_record("run-late", late)) - .await - .unwrap(); - late_run - .put_start(&sample_start_record("run-late", late)) - .await - .unwrap(); - late_run - .put_status(&sample_status( - RunStatus::Succeeded, - Some(StatusReason::Completed), + .append_event(&event_payload( + "run-late", + "2026-03-27T12:00:00Z", + "run.created", + None, + serde_json::json!({ + "settings": late_record.settings, + "graph": late_record.graph, + "workflow_slug": late_record.workflow_slug, + "working_directory": late_record.working_directory, + "host_repo_path": late_record.host_repo_path, + "base_branch": late_record.base_branch, + "labels": late_record.labels, + }), + )) + .await + .unwrap(); + late_run + .append_event(&event_payload( + "run-late", + "2026-03-27T12:00:01Z", + "run.started", + None, + serde_json::json!({ + "run_branch": "fabro/run/demo", + "base_sha": "abc123", + }), + )) + .await + .unwrap(); + late_run + .append_event(&event_payload( + "run-late", + "2026-03-27T12:00:02Z", + "run.completed", + None, + serde_json::json!({ + "duration_ms": 3210, + "artifact_count": 1, + "status": "success", + "reason": "completed", + "total_cost": 1.25, + }), )) .await .unwrap(); - late_run.put_conclusion(&sample_conclusion()).await.unwrap(); let all = store.list_runs(&ListRunsQuery::default()).await.unwrap(); assert_eq!(all.len(), 2); @@ -1695,12 +1661,10 @@ mod tests { .unwrap(); store.delete_run(&test_run_id("run-1")).await.unwrap(); store.delete_run(&test_run_id("run-1")).await.unwrap(); - assert!( - matches!( - store.open_run(&test_run_id("run-1")).await, - Err(StoreError::RunNotFound(_)) - ) - ); + assert!(matches!( + store.open_run(&test_run_id("run-1")).await, + Err(StoreError::RunNotFound(_)) + )); } #[tokio::test] diff --git a/lib/crates/fabro-store/src/run_state.rs b/lib/crates/fabro-store/src/run_state.rs index e1770f6f7..a6592744c 100644 --- a/lib/crates/fabro-store/src/run_state.rs +++ b/lib/crates/fabro-store/src/run_state.rs @@ -131,17 +131,20 @@ impl RunState { "run.failed" => { self.status = Some(run_status_record(RunStatus::Failed, &properties, ts)?); self.conclusion = Some(conclusion_from_failed(&properties, ts)); - self.last_git_sha = - optional_string(&properties, "git_commit_sha").or_else(|| self.last_git_sha.clone()); + self.last_git_sha = optional_string(&properties, "git_commit_sha") + .or_else(|| self.last_git_sha.clone()); } "run.rewound" => { self.reset_for_rewind(); - self.last_git_sha = - optional_string(&properties, "run_commit_sha").or_else(|| self.last_git_sha.clone()); + self.last_git_sha = optional_string(&properties, "run_commit_sha") + .or_else(|| self.last_git_sha.clone()); } "checkpoint.completed" => { let checkpoint = checkpoint_from_properties(&properties, ts)?; - self.last_git_sha = checkpoint.git_commit_sha.clone().or_else(|| self.last_git_sha.clone()); + self.last_git_sha = checkpoint + .git_commit_sha + .clone() + .or_else(|| self.last_git_sha.clone()); if let Some(node_id) = value.get("node_id").and_then(Value::as_str) { let visit = checkpoint .node_visits @@ -270,57 +273,6 @@ impl RunState { Ok(()) } - pub fn merge_legacy( - &mut self, - snapshot: Option, - graph_source: Option, - retro_prompt: Option, - retro_response: Option, - checkpoints: Vec<(u32, Checkpoint)>, - ) { - let Some(snapshot) = snapshot else { - self.graph_source = self.graph_source.clone().or(graph_source); - self.retro_prompt = self.retro_prompt.clone().or(retro_prompt); - self.retro_response = self.retro_response.clone().or(retro_response); - if self.checkpoints.is_empty() { - self.checkpoints = checkpoints; - } - if self.checkpoint.is_none() { - self.checkpoint = self.checkpoints.last().map(|(_, checkpoint)| checkpoint.clone()); - } - return; - }; - - if self.run.is_none() { - self.run = Some(snapshot.run); - } - self.start = self.start.clone().or(snapshot.start); - self.status = self.status.clone().or(snapshot.status); - self.checkpoint = self.checkpoint.clone().or(snapshot.checkpoint); - self.conclusion = self.conclusion.clone().or(snapshot.conclusion); - self.retro = self.retro.clone().or(snapshot.retro); - self.graph_source = self.graph_source.clone().or(snapshot.graph).or(graph_source); - self.sandbox = self.sandbox.clone().or(snapshot.sandbox); - self.final_patch = self.final_patch.clone().or(snapshot.final_patch); - self.pull_request = self.pull_request.clone().or(snapshot.pull_request); - self.retro_prompt = self.retro_prompt.clone().or(retro_prompt); - self.retro_response = self.retro_response.clone().or(retro_response); - if self.checkpoints.is_empty() { - self.checkpoints = checkpoints; - } - if self.checkpoint.is_none() { - self.checkpoint = self.checkpoints.last().map(|(_, checkpoint)| checkpoint.clone()); - } - - for node in snapshot.nodes { - let entry = self - .nodes - .entry((node.node_id.clone(), node.visit)) - .or_default(); - merge_node_snapshot(entry, node); - } - } - pub fn node(&self, node: &NodeVisitRef<'_>) -> Option<&NodeState> { self.nodes.get(&(node.node_id.to_string(), node.visit)) } @@ -355,21 +307,23 @@ impl RunState { let nodes = node_keys .into_iter() .filter_map(|(node_id, visit)| { - self.nodes.get(&(node_id.clone(), visit)).map(|node| NodeSnapshot { - node_id, - visit, - prompt: node.prompt.clone(), - response: node.response.clone(), - status: node.status.clone(), - outcome: node.outcome.clone(), - provider_used: node.provider_used.clone(), - diff: node.diff.clone(), - script_invocation: node.script_invocation.clone(), - script_timing: node.script_timing.clone(), - parallel_results: node.parallel_results.clone(), - stdout: node.stdout.clone(), - stderr: node.stderr.clone(), - }) + self.nodes + .get(&(node_id.clone(), visit)) + .map(|node| NodeSnapshot { + node_id, + visit, + prompt: node.prompt.clone(), + response: node.response.clone(), + status: node.status.clone(), + outcome: node.outcome.clone(), + provider_used: node.provider_used.clone(), + diff: node.diff.clone(), + script_invocation: node.script_invocation.clone(), + script_timing: node.script_timing.clone(), + parallel_results: node.parallel_results.clone(), + stdout: node.stdout.clone(), + stderr: node.stderr.clone(), + }) }) .collect(); @@ -417,7 +371,10 @@ impl RunState { start_time: self.start.as_ref().map(|start| start.start_time), status: self.status.as_ref().map(|status| status.status), status_reason: self.status.as_ref().and_then(|status| status.reason), - duration_ms: self.conclusion.as_ref().map(|conclusion| conclusion.duration_ms), + duration_ms: self + .conclusion + .as_ref() + .map(|conclusion| conclusion.duration_ms), total_cost: self .conclusion .as_ref() @@ -426,9 +383,7 @@ impl RunState { } fn node_mut(&mut self, node_id: &str, visit: u32) -> &mut NodeState { - self.nodes - .entry((node_id.to_string(), visit)) - .or_default() + self.nodes.entry((node_id.to_string(), visit)).or_default() } fn current_visit_for(&self, node_id: &str) -> Option { @@ -454,20 +409,6 @@ impl RunState { } } -fn merge_node_snapshot(node: &mut NodeState, snapshot: NodeSnapshot) { - node.prompt = node.prompt.clone().or(snapshot.prompt); - node.response = node.response.clone().or(snapshot.response); - node.status = node.status.clone().or(snapshot.status); - node.outcome = node.outcome.clone().or(snapshot.outcome); - node.provider_used = node.provider_used.clone().or(snapshot.provider_used); - node.diff = node.diff.clone().or(snapshot.diff); - node.script_invocation = node.script_invocation.clone().or(snapshot.script_invocation); - node.script_timing = node.script_timing.clone().or(snapshot.script_timing); - node.parallel_results = node.parallel_results.clone().or(snapshot.parallel_results); - node.stdout = node.stdout.clone().or(snapshot.stdout); - node.stderr = node.stderr.clone().or(snapshot.stderr); -} - fn parse_ts(value: &Value) -> Result> { let ts = value .get("ts") @@ -493,7 +434,9 @@ fn required_string(properties: &serde_json::Map, key: &str) -> Re .get(key) .and_then(Value::as_str) .map(ToString::to_string) - .ok_or_else(|| StoreError::InvalidEvent(format!("event missing string property {key}")).into()) + .ok_or_else(|| { + StoreError::InvalidEvent(format!("event missing string property {key}")).into() + }) } fn optional_string(properties: &serde_json::Map, key: &str) -> Option { @@ -504,10 +447,9 @@ fn optional_string(properties: &serde_json::Map, key: &str) -> Op } fn required_u64(properties: &serde_json::Map, key: &str) -> Result { - properties - .get(key) - .and_then(Value::as_u64) - .ok_or_else(|| StoreError::InvalidEvent(format!("event missing integer property {key}")).into()) + properties.get(key).and_then(Value::as_u64).ok_or_else(|| { + StoreError::InvalidEvent(format!("event missing integer property {key}")).into() + }) } fn required_u32(properties: &serde_json::Map, key: &str) -> Result { @@ -604,9 +546,9 @@ fn conclusion_from_completed( let usage = optional_json::(properties, "usage")?; Ok(Conclusion { timestamp, - status: StageStatus::from_str(&required_string(properties, "status")?).map_err( - |err| StoreError::InvalidEvent(format!("invalid completed stage status: {err}")), - )?, + status: StageStatus::from_str(&required_string(properties, "status")?).map_err(|err| { + StoreError::InvalidEvent(format!("invalid completed stage status: {err}")) + })?, duration_ms: required_u64(properties, "duration_ms")?, failure_reason: None, final_git_commit_sha: optional_string(properties, "final_git_commit_sha"), @@ -656,7 +598,11 @@ fn conclusion_from_failed( } } -fn stage_visit(node_id: &str, properties: &serde_json::Map, state: &RunState) -> Option { +fn stage_visit( + node_id: &str, + properties: &serde_json::Map, + state: &RunState, +) -> Option { properties .get("node_visits") .and_then(|value| serde_json::from_value::>(value.clone()).ok()) @@ -668,9 +614,8 @@ fn stage_visit(node_id: &str, properties: &serde_json::Map, state fn stage_outcome_from_properties( properties: &serde_json::Map, ) -> Result { - let status = StageStatus::from_str(&required_string(properties, "status")?).map_err(|err| { - StoreError::InvalidEvent(format!("invalid stage status: {err}")) - })?; + let status = StageStatus::from_str(&required_string(properties, "status")?) + .map_err(|err| StoreError::InvalidEvent(format!("invalid stage status: {err}")))?; Ok(Outcome { status, preferred_label: optional_string(properties, "preferred_label"), @@ -685,11 +630,17 @@ fn stage_outcome_from_properties( }) } -fn node_status_from_outcome(outcome: &NodeOutcomeRecord, timestamp: DateTime) -> NodeStatusRecord { +fn node_status_from_outcome( + outcome: &NodeOutcomeRecord, + timestamp: DateTime, +) -> NodeStatusRecord { NodeStatusRecord { status: outcome.status.clone(), notes: outcome.notes.clone(), - failure_reason: outcome.failure.as_ref().map(|failure| failure.message.clone()), + failure_reason: outcome + .failure + .as_ref() + .map(|failure| failure.message.clone()), timestamp, } } diff --git a/lib/crates/fabro-store/src/slate/mod.rs b/lib/crates/fabro-store/src/slate/mod.rs index 4269f69ca..fc2291b9e 100644 --- a/lib/crates/fabro-store/src/slate/mod.rs +++ b/lib/crates/fabro-store/src/slate/mod.rs @@ -5,7 +5,6 @@ use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; -use async_trait::async_trait; use chrono::{DateTime, Utc}; use futures::TryStreamExt; use object_store::ObjectStore; @@ -15,9 +14,10 @@ use slatedb::config::{DbReaderOptions, Settings}; use tokio::sync::Mutex; use crate::keys; -use crate::{CatalogRecord, ListRunsQuery, Result, RunStore, RunSummary, Store, StoreError}; +use crate::{CatalogRecord, ListRunsQuery, Result, RunStoreHandle, RunSummary, StoreError}; use fabro_types::RunId; -use run_store::{SlateRunStore, SlateRunStoreInner}; +pub use run_store::SlateRunStore; +use run_store::SlateRunStoreInner; #[derive(Clone)] pub struct SlateStore { @@ -173,14 +173,13 @@ impl SlateStore { } } -#[async_trait] -impl Store for SlateStore { - async fn create_run( +impl SlateStore { + pub async fn create_run( &self, run_id: &RunId, created_at: DateTime, run_dir: Option<&str>, - ) -> Result> { + ) -> Result { let locator = catalog::read_locator(self.object_store.clone(), &self.base_prefix, run_id).await?; if let Some(active) = self.get_active_run(run_id).await { @@ -201,7 +200,7 @@ impl Store for SlateStore { run_dir, ) .await?; - return Ok(Arc::new(active) as Arc); + return Ok(Arc::new(active)); } let db_prefix = match locator { @@ -233,10 +232,10 @@ impl Store for SlateStore { run_dir, ) .await?; - Ok(Arc::new(run_store) as Arc) + Ok(Arc::new(run_store)) } - async fn open_run(&self, run_id: &RunId) -> Result> { + pub async fn open_run(&self, run_id: &RunId) -> Result { let locator = catalog::read_locator(self.object_store.clone(), &self.base_prefix, run_id) .await? .ok_or_else(|| StoreError::RunNotFound(run_id.to_string()))?; @@ -245,10 +244,10 @@ impl Store for SlateStore { .open_run_store(&locator) .await? .ok_or_else(|| StoreError::RunNotFound(run_id.to_string()))?; - Ok(Arc::new(run_store) as Arc) + Ok(Arc::new(run_store)) } - async fn open_run_reader(&self, run_id: &RunId) -> Result> { + pub async fn open_run_reader(&self, run_id: &RunId) -> Result { let locator = catalog::read_locator(self.object_store.clone(), &self.base_prefix, run_id) .await? .ok_or_else(|| StoreError::RunNotFound(run_id.to_string()))?; @@ -257,10 +256,10 @@ impl Store for SlateStore { .open_run_reader_store(&locator) .await? .ok_or_else(|| StoreError::RunNotFound(run_id.to_string()))?; - Ok(Arc::new(run_store) as Arc) + Ok(Arc::new(run_store)) } - async fn list_runs(&self, query: &ListRunsQuery) -> Result> { + pub async fn list_runs(&self, query: &ListRunsQuery) -> Result> { let catalogs = catalog::list_catalogs(self.object_store.clone(), &self.base_prefix, query).await?; let mut summaries = Vec::new(); @@ -293,7 +292,7 @@ impl Store for SlateStore { Ok(summaries) } - async fn delete_run(&self, run_id: &RunId) -> Result<()> { + pub async fn delete_run(&self, run_id: &RunId) -> Result<()> { let active = self.remove_active_run(run_id).await; let active_record = active.as_ref().map(SlateRunStore::record); if let Some(active) = &active { @@ -384,8 +383,8 @@ mod tests { use bytes::Bytes; use fabro_types::{ - AttrValue, Checkpoint, Conclusion, Graph, NodeStatusRecord, RunId, RunRecord, RunStatus, - RunStatusRecord, Settings, StageStatus, StartRecord, StatusReason, fixtures, + AttrValue, Graph, NodeStatusRecord, RunId, RunRecord, RunStatus, Settings, StageStatus, + StatusReason, fixtures, }; use object_store::memory::InMemory; use slatedb::config::Settings as SlateSettings; @@ -433,58 +432,6 @@ mod tests { } } - fn sample_start_record(run_id: &str, created_at: DateTime) -> StartRecord { - StartRecord { - run_id: test_run_id(run_id), - start_time: created_at + chrono::Duration::seconds(5), - run_branch: Some("fabro/run/demo".to_string()), - base_sha: Some("abc123".to_string()), - } - } - - fn sample_status(status: RunStatus, reason: Option) -> RunStatusRecord { - RunStatusRecord { - status, - reason, - updated_at: dt("2026-03-27T12:05:00Z"), - } - } - - fn sample_checkpoint() -> Checkpoint { - Checkpoint { - timestamp: dt("2026-03-27T12:10:00Z"), - current_node: "code".to_string(), - completed_nodes: vec!["plan".to_string()], - node_retries: std::collections::HashMap::from([("code".to_string(), 1)]), - context_values: std::collections::HashMap::new(), - node_outcomes: std::collections::HashMap::new(), - next_node_id: Some("review".to_string()), - git_commit_sha: Some("def456".to_string()), - loop_failure_signatures: std::collections::HashMap::new(), - restart_failure_signatures: std::collections::HashMap::new(), - node_visits: std::collections::HashMap::from([("code".to_string(), 2)]), - } - } - - fn sample_conclusion() -> Conclusion { - Conclusion { - timestamp: dt("2026-03-27T12:15:00Z"), - status: StageStatus::Success, - duration_ms: 3210, - failure_reason: None, - final_git_commit_sha: Some("feedbeef".to_string()), - stages: Vec::new(), - total_cost: Some(1.25), - total_retries: 2, - total_input_tokens: 10, - total_output_tokens: 20, - total_cache_read_tokens: 30, - total_cache_write_tokens: 40, - total_reasoning_tokens: 50, - has_pricing: true, - } - } - fn sample_node_status() -> NodeStatusRecord { NodeStatusRecord { status: StageStatus::Success, @@ -494,17 +441,24 @@ mod tests { } } - fn event_payload(run_id: &str, ts: &str, event: &str) -> EventPayload { - EventPayload::new( - serde_json::json!({ - "id": format!("evt-{run_id}-{event}"), - "ts": ts, - "run_id": test_run_id(run_id).to_string(), - "event": event - }), - &test_run_id(run_id), - ) - .unwrap() + fn event_payload( + run_id: &str, + ts: &str, + event: &str, + node_id: Option<&str>, + properties: serde_json::Value, + ) -> EventPayload { + let mut value = serde_json::json!({ + "id": format!("evt-{run_id}-{event}"), + "ts": ts, + "run_id": test_run_id(run_id).to_string(), + "event": event, + "properties": properties, + }); + if let Some(node_id) = node_id { + value["node_id"] = serde_json::Value::String(node_id.to_string()); + } + EventPayload::new(value, &test_run_id(run_id)).unwrap() } async fn list_paths(store: Arc, prefix: &str) -> Vec { @@ -552,19 +506,51 @@ mod tests { .await .unwrap(); - run.put_run(&sample_run_record("run-1", created_at)) - .await - .unwrap(); - run.put_start(&sample_start_record("run-1", created_at)) - .await - .unwrap(); - run.put_status(&sample_status( - RunStatus::Succeeded, - Some(StatusReason::Completed), + let run_record = sample_run_record("run-1", created_at); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:00Z", + "run.created", + None, + serde_json::json!({ + "settings": run_record.settings, + "graph": run_record.graph, + "workflow_slug": run_record.workflow_slug, + "working_directory": run_record.working_directory, + "host_repo_path": run_record.host_repo_path, + "base_branch": run_record.base_branch, + "labels": run_record.labels, + }), + )) + .await + .unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:01Z", + "run.started", + None, + serde_json::json!({ + "run_branch": "fabro/run/demo", + "base_sha": "abc123", + }), + )) + .await + .unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:02Z", + "run.completed", + None, + serde_json::json!({ + "duration_ms": 3210, + "artifact_count": 1, + "status": "success", + "reason": "completed", + "total_cost": 1.25, + }), )) .await .unwrap(); - run.put_conclusion(&sample_conclusion()).await.unwrap(); let by_id = catalog::by_id_path("runs/", &test_run_id("run-1")); let by_start = catalog::by_start_path("runs/", created_at, &test_run_id("run-1")); @@ -579,20 +565,12 @@ mod tests { assert_eq!(summary[0].status, Some(RunStatus::Succeeded)); assert_eq!(summary[0].status_reason, Some(StatusReason::Completed)); - let reopened = store - .open_run(&test_run_id("run-1")) - .await - .unwrap(); - let stored = reopened.get_run().await.unwrap().unwrap(); + let reopened = store.open_run(&test_run_id("run-1")).await.unwrap(); + let stored = reopened.state().await.unwrap().run.unwrap(); assert_eq!(stored.run_id, test_run_id("run-1")); store.delete_run(&test_run_id("run-1")).await.unwrap(); - assert!( - store - .open_run(&test_run_id("run-1")) - .await - .is_err() - ); + assert!(store.open_run(&test_run_id("run-1")).await.is_err()); assert!(!object_exists(object_store.clone(), &by_id).await); assert!(!object_exists(object_store.clone(), &by_start).await); assert!(list_paths(object_store, "runs/db").await.is_empty()); @@ -611,8 +589,23 @@ mod tests { let db = seed_db(object_store.clone(), &record, true).await; db.put( - keys::run(), - serde_json::to_vec(&sample_run_record("run-1", created_at)).unwrap(), + keys::event_key(1, created_at.timestamp_millis()), + serde_json::to_vec(&event_payload( + "run-1", + "2026-03-27T12:00:00Z", + "run.created", + None, + serde_json::json!({ + "settings": sample_run_record("run-1", created_at).settings, + "graph": sample_run_record("run-1", created_at).graph, + "workflow_slug": sample_run_record("run-1", created_at).workflow_slug, + "working_directory": sample_run_record("run-1", created_at).working_directory, + "host_repo_path": sample_run_record("run-1", created_at).host_repo_path, + "base_branch": sample_run_record("run-1", created_at).base_branch, + "labels": sample_run_record("run-1", created_at).labels, + }), + )) + .unwrap(), ) .await .unwrap(); @@ -626,12 +619,7 @@ mod tests { .await .unwrap(); - assert!( - store - .open_run(&test_run_id("run-1")) - .await - .is_ok() - ); + assert!(store.open_run(&test_run_id("run-1")).await.is_ok()); assert!( store .list_runs(&ListRunsQuery::default()) @@ -653,43 +641,63 @@ mod tests { } #[tokio::test] - async fn reopen_recovers_event_and_checkpoint_sequences() { + async fn reopen_recovers_event_sequences() { let (_object_store, store) = make_store(); let created_at = dt("2026-03-27T12:00:00Z"); let run = store .create_run(&test_run_id("run-1"), created_at, None) .await .unwrap(); - run.put_run(&sample_run_record("run-1", created_at)) - .await - .unwrap(); - run.append_event(&event_payload("run-1", "2026-03-27T12:00:00Z", "Started")) - .await - .unwrap(); - run.append_event(&event_payload("run-1", "2026-03-27T12:00:01Z", "Next")) - .await - .unwrap(); - run.append_checkpoint(&sample_checkpoint()).await.unwrap(); + let run_record = sample_run_record("run-1", created_at); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:00Z", + "run.created", + None, + serde_json::json!({ + "settings": run_record.settings, + "graph": run_record.graph, + "workflow_slug": run_record.workflow_slug, + "working_directory": run_record.working_directory, + "host_repo_path": run_record.host_repo_path, + "base_branch": run_record.base_branch, + "labels": run_record.labels, + }), + )) + .await + .unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:01Z", + "Started", + None, + serde_json::json!({}), + )) + .await + .unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:02Z", + "Next", + None, + serde_json::json!({}), + )) + .await + .unwrap(); drop(run); - let reopened = store - .open_run(&test_run_id("run-1")) - .await - .unwrap(); + let reopened = store.open_run(&test_run_id("run-1")).await.unwrap(); let next_event = reopened .append_event(&event_payload( "run-1", "2026-03-27T12:00:02Z", "AfterReopen", + None, + serde_json::json!({}), )) .await .unwrap(); - let next_checkpoint = reopened - .append_checkpoint(&sample_checkpoint()) - .await - .unwrap(); - assert_eq!(next_event, 3); - assert_eq!(next_checkpoint, 2); + assert_eq!(next_event, 4); } #[tokio::test] @@ -720,12 +728,7 @@ mod tests { .await .unwrap(); - assert!( - store - .open_run(&test_run_id("run-1")) - .await - .is_err() - ); + assert!(store.open_run(&test_run_id("run-1")).await.is_err()); assert!( store .list_runs(&ListRunsQuery::default()) @@ -766,35 +769,51 @@ mod tests { .create_run(&test_run_id("run-1"), created_at, None) .await .unwrap(); - run.put_run(&sample_run_record("run-1", created_at)) - .await - .unwrap(); + let run_record = sample_run_record("run-1", created_at); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:00Z", + "run.created", + None, + serde_json::json!({ + "settings": run_record.settings, + "graph": run_record.graph, + "workflow_slug": run_record.workflow_slug, + "working_directory": run_record.working_directory, + "host_repo_path": run_record.host_repo_path, + "base_branch": run_record.base_branch, + "labels": run_record.labels, + }), + )) + .await + .unwrap(); let listed = store.list_runs(&ListRunsQuery::default()).await.unwrap(); assert_eq!(listed.len(), 1); - let reopened = store - .open_run(&test_run_id("run-1")) - .await - .unwrap(); + let reopened = store.open_run(&test_run_id("run-1")).await.unwrap(); let first_event = run - .append_event(&event_payload("run-1", "2026-03-27T12:00:00Z", "Started")) + .append_event(&event_payload( + "run-1", + "2026-03-27T12:00:00Z", + "Started", + None, + serde_json::json!({}), + )) .await .unwrap(); let second_event = reopened - .append_event(&event_payload("run-1", "2026-03-27T12:00:01Z", "Continued")) + .append_event(&event_payload( + "run-1", + "2026-03-27T12:00:01Z", + "Continued", + None, + serde_json::json!({}), + )) .await .unwrap(); - let first_checkpoint = run.append_checkpoint(&sample_checkpoint()).await.unwrap(); - let second_checkpoint = reopened - .append_checkpoint(&sample_checkpoint()) - .await - .unwrap(); - - assert_eq!(first_event, 1); - assert_eq!(second_event, 2); - assert_eq!(first_checkpoint, 1); - assert_eq!(second_checkpoint, 2); + assert_eq!(first_event, 2); + assert_eq!(second_event, 3); } #[tokio::test] @@ -807,9 +826,15 @@ mod tests { .unwrap(); let mut stream = run.watch_events_from(1).await.unwrap(); - run.append_event(&event_payload("run-1", "2026-03-27T12:00:00Z", "Started")) - .await - .unwrap(); + run.append_event(&event_payload( + "run-1", + "2026-03-27T12:00:00Z", + "Started", + None, + serde_json::json!({}), + )) + .await + .unwrap(); let event = timeout( Duration::from_secs(2), @@ -834,8 +859,23 @@ mod tests { }; let db = seed_db(object_store.clone(), &record, true).await; db.put( - keys::run(), - serde_json::to_vec(&sample_run_record("run-1", created_at)).unwrap(), + keys::event_key(1, created_at.timestamp_millis()), + serde_json::to_vec(&event_payload( + "run-1", + "2026-03-27T12:00:00Z", + "run.created", + None, + serde_json::json!({ + "settings": sample_run_record("run-1", created_at).settings, + "graph": sample_run_record("run-1", created_at).graph, + "workflow_slug": sample_run_record("run-1", created_at).workflow_slug, + "working_directory": sample_run_record("run-1", created_at).working_directory, + "host_repo_path": sample_run_record("run-1", created_at).host_repo_path, + "base_branch": sample_run_record("run-1", created_at).base_branch, + "labels": sample_run_record("run-1", created_at).labels, + }), + )) + .unwrap(), ) .await .unwrap(); @@ -861,23 +901,21 @@ mod tests { .create_run(&test_run_id("run-1"), created_at, None) .await .unwrap(); - run.put_run(&sample_run_record("run-1", created_at)) + run.put_artifact_value("summary", &serde_json::json!({"done": true})) .await .unwrap(); store.delete_run(&test_run_id("run-1")).await.unwrap(); - let err = run.put_graph("digraph night_sky {}").await.unwrap_err(); + let err = run + .put_artifact_value("summary", &serde_json::json!({"done": false})) + .await + .unwrap_err(); assert!(matches!( err, StoreError::Slate(err) if matches!(err.kind(), ErrorKind::Closed(CloseReason::Clean)) )); - assert!( - store - .open_run(&test_run_id("run-1")) - .await - .is_err() - ); + assert!(store.open_run(&test_run_id("run-1")).await.is_err()); assert!(list_paths(object_store, "runs").await.is_empty()); } @@ -890,7 +928,7 @@ mod tests { .create_run(&test_run_id("run-1"), created_at, None) .await .unwrap(); - run.put_run(&sample_run_record("run-1", created_at)) + run.put_artifact_value("summary", &serde_json::json!({"done": true})) .await .unwrap(); @@ -927,14 +965,13 @@ mod tests { assert_ne!(orphan.db_prefix, new_prefix); let db = seed_db(object_store.clone(), &orphan, true).await; - db.put(keys::graph(), b"stale graph").await.unwrap(); db.close().await.unwrap(); let run = store .create_run(&test_run_id("run-1"), new_created_at, None) .await .unwrap(); - assert_eq!(run.get_graph().await.unwrap(), None); + assert!(run.state().await.unwrap().graph_source.is_none()); let locator = catalog::read_locator(object_store, "runs/", &test_run_id("run-1")) .await @@ -988,9 +1025,6 @@ mod tests { .create_run(&test_run_id("run-1"), created_at, None) .await .unwrap(); - run.put_run(&sample_run_record("run-1", created_at)) - .await - .unwrap(); let node = NodeVisitRef { node_id: "code", visit: 2, @@ -1023,10 +1057,6 @@ mod tests { .create_run(&test_run_id("run-1"), created_at, None) .await .unwrap(); - run.put_run(&sample_run_record("run-1", created_at)) - .await - .unwrap(); - run.put_artifact_value("summary", &serde_json::json!({"done": true})) .await .unwrap(); @@ -1067,8 +1097,7 @@ mod tests { ] ); - let snapshot = run.get_snapshot().await.unwrap().unwrap(); - assert_eq!(snapshot.nodes.len(), 1); - assert_eq!(snapshot.nodes[0].node_id, "code"); + let code_node = run.get_node(&snapshot_node).await.unwrap(); + assert_eq!(code_node.node_id, "code"); } } diff --git a/lib/crates/fabro-store/src/slate/run_store.rs b/lib/crates/fabro-store/src/slate/run_store.rs index 3c1c0a9dc..929d737a6 100644 --- a/lib/crates/fabro-store/src/slate/run_store.rs +++ b/lib/crates/fabro-store/src/slate/run_store.rs @@ -3,7 +3,6 @@ use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::{Arc, Weak}; use std::time::Duration; -use async_trait::async_trait; use bytes::Bytes; use chrono::{DateTime, Utc}; use futures::Stream; @@ -18,18 +17,26 @@ use crate::keys; use crate::run_state::EventProjectionCache; use crate::{ CatalogRecord, EventEnvelope, EventPayload, NodeOutcomeRecord, NodeSnapshot, NodeVisitRef, - Result, RunSnapshot, RunState, RunStore, RunSummary, StoreError, -}; -use fabro_types::{ - Checkpoint, Conclusion, NodeStatusRecord, PullRequestRecord, Retro, RunId, RunRecord, - RunStatusRecord, SandboxRecord, StartRecord, + Result, RunState, RunSummary, StoreError, }; +use fabro_types::{NodeStatusRecord, RunId}; #[derive(Clone)] -pub(crate) struct SlateRunStore { +pub struct SlateRunStore { inner: Arc, } +impl std::fmt::Debug for SlateRunStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SlateRunStore") + .field("run_id", &self.inner.run_id) + .field("created_at", &self.inner.created_at) + .field("db_prefix", &self.inner.db_prefix) + .field("run_dir", &self.inner.run_dir) + .finish_non_exhaustive() + } +} + pub(crate) struct SlateRunStoreInner { run_id: RunId, created_at: DateTime, @@ -37,7 +44,6 @@ pub(crate) struct SlateRunStoreInner { run_dir: Option, db: SlateRunDb, event_seq: AtomicU32, - checkpoint_seq: AtomicU32, close_lock: Mutex<()>, projection_cache: Mutex, } @@ -50,8 +56,6 @@ enum SlateRunDb { impl SlateRunStore { pub(crate) async fn open_writer(record: CatalogRecord, db: slatedb::Db) -> Result { let event_seq = recover_next_seq(&db, keys::EVENTS_PREFIX, keys::parse_event_seq).await?; - let checkpoint_seq = - recover_next_seq(&db, keys::CHECKPOINTS_PREFIX, keys::parse_checkpoint_seq).await?; Ok(Self { inner: Arc::new(SlateRunStoreInner { run_id: record.run_id, @@ -60,7 +64,6 @@ impl SlateRunStore { run_dir: record.run_dir, db: SlateRunDb::Writer(db), event_seq: AtomicU32::new(event_seq), - checkpoint_seq: AtomicU32::new(checkpoint_seq), close_lock: Mutex::new(()), projection_cache: Mutex::new(EventProjectionCache::default()), }), @@ -69,8 +72,6 @@ impl SlateRunStore { pub(crate) async fn open_reader(record: CatalogRecord, db: DbReader) -> Result { let event_seq = recover_next_seq(&db, keys::EVENTS_PREFIX, keys::parse_event_seq).await?; - let checkpoint_seq = - recover_next_seq(&db, keys::CHECKPOINTS_PREFIX, keys::parse_checkpoint_seq).await?; Ok(Self { inner: Arc::new(SlateRunStoreInner { run_id: record.run_id, @@ -79,7 +80,6 @@ impl SlateRunStore { run_dir: record.run_dir, db: SlateRunDb::Reader(Box::new(db)), event_seq: AtomicU32::new(event_seq), - checkpoint_seq: AtomicU32::new(checkpoint_seq), close_lock: Mutex::new(()), projection_cache: Mutex::new(EventProjectionCache::default()), }), @@ -148,91 +148,10 @@ impl SlateRunStore { R: DbRead + Sync, { let events = list_events_from(db, 1).await?; - let mut state = RunState::apply_events(&events)?; - let mut nodes = BTreeSet::new(); - let mut iter = db.scan_prefix(b"nodes/").await?; - while let Some(entry) = iter.next().await? { - let key = key_to_string(&entry.key)?; - if let Some((node_id, visit, _)) = keys::parse_node_key(&key) { - nodes.insert((node_id, visit)); - } - } - let snapshot = if let Some(run) = get_json::<_, RunRecord>(db, keys::run()).await? { - let mut snapshot_nodes = Vec::new(); - for (node_id, visit) in nodes { - let node = NodeVisitRef { - node_id: &node_id, - visit, - }; - let prompt_key = keys::node_prompt(&node); - let response_key = keys::node_response(&node); - let status_key = keys::node_status(&node); - let outcome_key = keys::node_outcome(&node); - let provider_key = keys::node_provider_used(&node); - let diff_key = keys::node_diff(&node); - let invocation_key = keys::node_script_invocation(&node); - let timing_key = keys::node_script_timing(&node); - let results_key = keys::node_parallel_results(&node); - let stdout_key = keys::node_stdout(&node); - let stderr_key = keys::node_stderr(&node); - snapshot_nodes.push(NodeSnapshot { - node_id: node_id.clone(), - visit, - prompt: get_text(db, &prompt_key).await?, - response: get_text(db, &response_key).await?, - status: get_json(db, &status_key).await?, - outcome: get_json(db, &outcome_key).await?, - provider_used: get_json(db, &provider_key).await?, - diff: get_text(db, &diff_key).await?, - script_invocation: get_json(db, &invocation_key).await?, - script_timing: get_json(db, &timing_key).await?, - parallel_results: get_json(db, &results_key).await?, - stdout: get_text(db, &stdout_key).await?, - stderr: get_text(db, &stderr_key).await?, - }); - } - Some(RunSnapshot { - run, - start: get_json::<_, StartRecord>(db, keys::start()).await?, - status: get_json::<_, RunStatusRecord>(db, keys::status()).await?, - checkpoint: get_json::<_, Checkpoint>(db, keys::checkpoint()).await?, - conclusion: get_json::<_, Conclusion>(db, keys::conclusion()).await?, - retro: get_json::<_, Retro>(db, keys::retro()).await?, - graph: get_text(db, keys::graph()).await?, - sandbox: get_json::<_, SandboxRecord>(db, keys::sandbox()).await?, - final_patch: get_text(db, keys::final_patch()).await?, - pull_request: get_json::<_, PullRequestRecord>(db, keys::pull_request()).await?, - nodes: snapshot_nodes, - }) - } else { - None - }; - state.merge_legacy( - snapshot, - get_text(db, keys::graph()).await?, - get_text(db, keys::retro_prompt()).await?, - get_text(db, keys::retro_response()).await?, - list_checkpoints(db).await?, - ); + let state = RunState::apply_events(&events)?; Ok(state.build_summary(catalog)) } - fn validate_run_record(&self, record: &RunRecord) -> Result<()> { - if record.created_at != self.inner.created_at { - return Err(StoreError::Other(format!( - "run record created_at {:?} does not match store created_at {:?}", - record.created_at, self.inner.created_at - ))); - } - if record.run_id != self.inner.run_id { - return Err(StoreError::Other(format!( - "run record run_id {:?} does not match store run_id {:?}", - record.run_id, self.inner.run_id - ))); - } - Ok(()) - } - async fn build_node_snapshot(&self, node: &NodeVisitRef<'_>) -> Result { Ok(NodeSnapshot { node_id: node.node_id.to_string(), @@ -282,105 +201,22 @@ impl SlateRunStore { } } -#[async_trait] -impl RunStore for SlateRunStore { - async fn put_run(&self, record: &RunRecord) -> Result<()> { - self.validate_run_record(record)?; - self.inner.db.put_json(keys::run(), record).await - } - - async fn get_run(&self) -> Result> { - self.inner.db.get_json(keys::run()).await - } - - async fn put_start(&self, record: &StartRecord) -> Result<()> { - self.inner.db.put_json(keys::start(), record).await - } - - async fn get_start(&self) -> Result> { - self.inner.db.get_json(keys::start()).await - } - - async fn put_status(&self, record: &RunStatusRecord) -> Result<()> { - self.inner.db.put_json(keys::status(), record).await - } - - async fn get_status(&self) -> Result> { - self.inner.db.get_json(keys::status()).await - } - - async fn put_checkpoint(&self, record: &Checkpoint) -> Result<()> { - self.inner.db.put_json(keys::checkpoint(), record).await - } - - async fn get_checkpoint(&self) -> Result> { - self.inner.db.get_json(keys::checkpoint()).await - } - - async fn append_checkpoint(&self, record: &Checkpoint) -> Result { - let seq = self.inner.checkpoint_seq.fetch_add(1, Ordering::SeqCst); - self.put_checkpoint(record).await?; - self.inner - .db - .put_json( - &keys::checkpoint_history_key(seq, Utc::now().timestamp_millis()), - record, - ) - .await?; - Ok(seq) - } - - async fn list_checkpoints(&self) -> Result> { - self.inner.db.list_checkpoints().await - } - - async fn put_conclusion(&self, record: &Conclusion) -> Result<()> { - self.inner.db.put_json(keys::conclusion(), record).await - } - - async fn get_conclusion(&self) -> Result> { - self.inner.db.get_json(keys::conclusion()).await - } - - async fn put_retro(&self, retro: &Retro) -> Result<()> { - self.inner.db.put_json(keys::retro(), retro).await - } - - async fn get_retro(&self) -> Result> { - self.inner.db.get_json(keys::retro()).await - } - - async fn put_graph(&self, dot_source: &str) -> Result<()> { - self.inner.db.put_text(keys::graph(), dot_source).await - } - - async fn get_graph(&self) -> Result> { - self.inner.db.get_text(keys::graph()).await - } - - async fn put_sandbox(&self, record: &SandboxRecord) -> Result<()> { - self.inner.db.put_json(keys::sandbox(), record).await - } - - async fn get_sandbox(&self) -> Result> { - self.inner.db.get_json(keys::sandbox()).await - } - - async fn put_node_prompt(&self, node: &NodeVisitRef<'_>, prompt: &str) -> Result<()> { +impl SlateRunStore { + pub async fn put_node_prompt(&self, node: &NodeVisitRef<'_>, prompt: &str) -> Result<()> { self.inner .db .put_text(&keys::node_prompt(node), prompt) .await } - async fn put_node_response(&self, node: &NodeVisitRef<'_>, response: &str) -> Result<()> { + pub async fn put_node_response(&self, node: &NodeVisitRef<'_>, response: &str) -> Result<()> { self.inner .db .put_text(&keys::node_response(node), response) .await } - async fn put_node_status( + pub async fn put_node_status( &self, node: &NodeVisitRef<'_>, status: &NodeStatusRecord, @@ -391,7 +227,7 @@ impl RunStore for SlateRunStore { .await } - async fn put_node_outcome( + pub async fn put_node_outcome( &self, node: &NodeVisitRef<'_>, outcome: &NodeOutcomeRecord, @@ -402,7 +238,7 @@ impl RunStore for SlateRunStore { .await } - async fn put_node_provider_used( + pub async fn put_node_provider_used( &self, node: &NodeVisitRef<'_>, provider_used: &serde_json::Value, @@ -413,11 +249,11 @@ impl RunStore for SlateRunStore { .await } - async fn put_node_diff(&self, node: &NodeVisitRef<'_>, diff: &str) -> Result<()> { + pub async fn put_node_diff(&self, node: &NodeVisitRef<'_>, diff: &str) -> Result<()> { self.inner.db.put_text(&keys::node_diff(node), diff).await } - async fn put_node_script_invocation( + pub async fn put_node_script_invocation( &self, node: &NodeVisitRef<'_>, invocation: &serde_json::Value, @@ -428,7 +264,7 @@ impl RunStore for SlateRunStore { .await } - async fn put_node_script_timing( + pub async fn put_node_script_timing( &self, node: &NodeVisitRef<'_>, timing: &serde_json::Value, @@ -439,7 +275,7 @@ impl RunStore for SlateRunStore { .await } - async fn put_node_parallel_results( + pub async fn put_node_parallel_results( &self, node: &NodeVisitRef<'_>, results: &serde_json::Value, @@ -450,19 +286,19 @@ impl RunStore for SlateRunStore { .await } - async fn put_node_stdout(&self, node: &NodeVisitRef<'_>, log: &str) -> Result<()> { + pub async fn put_node_stdout(&self, node: &NodeVisitRef<'_>, log: &str) -> Result<()> { self.inner.db.put_text(&keys::node_stdout(node), log).await } - async fn put_node_stderr(&self, node: &NodeVisitRef<'_>, log: &str) -> Result<()> { + pub async fn put_node_stderr(&self, node: &NodeVisitRef<'_>, log: &str) -> Result<()> { self.inner.db.put_text(&keys::node_stderr(node), log).await } - async fn get_node(&self, node: &NodeVisitRef<'_>) -> Result { + pub async fn get_node(&self, node: &NodeVisitRef<'_>) -> Result { self.build_node_snapshot(node).await } - async fn list_node_visits(&self, node_id: &str) -> Result> { + pub async fn list_node_visits(&self, node_id: &str) -> Result> { let prefix = format!("nodes/{node_id}/visit-"); let mut iter = self.inner.db.scan_prefix(prefix.as_bytes()).await?; let mut visits = BTreeSet::new(); @@ -477,7 +313,7 @@ impl RunStore for SlateRunStore { Ok(visits.into_iter().collect()) } - async fn list_node_ids(&self) -> Result> { + pub async fn list_node_ids(&self) -> Result> { let mut iter = self.inner.db.scan_prefix(b"nodes/").await?; let mut node_ids = BTreeSet::new(); while let Some(entry) = iter.next().await? { @@ -502,40 +338,13 @@ impl RunStore for SlateRunStore { Ok(node_ids.into_iter().collect()) } - async fn put_final_patch(&self, patch: &str) -> Result<()> { - self.inner.db.put_text(keys::final_patch(), patch).await - } - - async fn get_final_patch(&self) -> Result> { - self.inner.db.get_text(keys::final_patch()).await - } - - async fn put_pull_request(&self, record: &PullRequestRecord) -> Result<()> { - self.inner.db.put_json(keys::pull_request(), record).await - } - - async fn get_pull_request(&self) -> Result> { - self.inner.db.get_json(keys::pull_request()).await - } - - async fn reset_for_rewind(&self) -> Result<()> { + pub async fn reset_for_rewind(&self) -> Result<()> { let db = self.inner.db.writer()?; - for key in [ - keys::status(), - keys::checkpoint(), - keys::conclusion(), - keys::retro(), - keys::sandbox(), - keys::final_patch(), - keys::pull_request(), - keys::retro_prompt(), - keys::retro_response(), - ] { + for key in [keys::retro_prompt(), keys::retro_response()] { db.delete(key).await?; } for prefix in [ b"nodes/".as_slice(), - keys::CHECKPOINTS_PREFIX.as_bytes(), keys::ARTIFACT_VALUES_PREFIX.as_bytes(), keys::ARTIFACT_NODES_PREFIX.as_bytes(), ] { @@ -544,7 +353,7 @@ impl RunStore for SlateRunStore { Ok(()) } - async fn append_event(&self, payload: &EventPayload) -> Result { + pub async fn append_event(&self, payload: &EventPayload) -> Result { payload.validate(&self.inner.run_id)?; let seq = self.inner.event_seq.fetch_add(1, Ordering::SeqCst); self.inner @@ -557,15 +366,15 @@ impl RunStore for SlateRunStore { Ok(seq) } - async fn list_events(&self) -> Result> { + pub async fn list_events(&self) -> Result> { self.inner.db.list_events_from(1).await } - async fn list_events_from(&self, seq: u32) -> Result> { + pub async fn list_events_from(&self, seq: u32) -> Result> { self.inner.db.list_events_from(seq).await } - async fn watch_events_from( + pub async fn watch_events_from( &self, seq: u32, ) -> Result> + Send>>> { @@ -603,55 +412,68 @@ impl RunStore for SlateRunStore { Ok(Box::pin(UnboundedReceiverStream::new(receiver))) } - async fn put_retro_prompt(&self, text: &str) -> Result<()> { + pub async fn put_retro_prompt(&self, text: &str) -> Result<()> { self.inner.db.put_text(keys::retro_prompt(), text).await } - async fn get_retro_prompt(&self) -> Result> { + pub async fn get_retro_prompt(&self) -> Result> { self.inner.db.get_text(keys::retro_prompt()).await } - async fn put_retro_response(&self, text: &str) -> Result<()> { + pub async fn put_retro_response(&self, text: &str) -> Result<()> { self.inner.db.put_text(keys::retro_response(), text).await } - async fn get_retro_response(&self) -> Result> { + pub async fn get_retro_response(&self) -> Result> { self.inner.db.get_text(keys::retro_response()).await } - async fn put_artifact_value(&self, artifact_id: &str, value: &serde_json::Value) -> Result<()> { + pub async fn put_artifact_value( + &self, + artifact_id: &str, + value: &serde_json::Value, + ) -> Result<()> { self.inner .db .put_json(&keys::artifact_value(artifact_id), value) .await } - async fn get_artifact_value(&self, artifact_id: &str) -> Result> { + pub async fn get_artifact_value(&self, artifact_id: &str) -> Result> { self.inner .db .get_json(&keys::artifact_value(artifact_id)) .await } - async fn list_artifact_values(&self) -> Result> { + pub async fn list_artifact_values(&self) -> Result> { self.inner.db.list_artifact_values().await } - async fn put_asset(&self, node: &NodeVisitRef<'_>, filename: &str, data: &[u8]) -> Result<()> { + pub async fn put_asset( + &self, + node: &NodeVisitRef<'_>, + filename: &str, + data: &[u8], + ) -> Result<()> { self.inner .db .put_bytes(&keys::node_asset(node, filename), data) .await } - async fn get_asset(&self, node: &NodeVisitRef<'_>, filename: &str) -> Result> { + pub async fn get_asset( + &self, + node: &NodeVisitRef<'_>, + filename: &str, + ) -> Result> { self.inner .db .get_bytes(&keys::node_asset(node, filename)) .await } - async fn list_assets(&self, node: &NodeVisitRef<'_>) -> Result> { + pub async fn list_assets(&self, node: &NodeVisitRef<'_>) -> Result> { let prefix = format!("{}/", keys::node_asset_prefix(node)); let mut iter = self.inner.db.scan_prefix(prefix.as_bytes()).await?; let mut assets = Vec::new(); @@ -665,64 +487,12 @@ impl RunStore for SlateRunStore { Ok(assets) } - async fn list_all_assets(&self) -> Result> { + pub async fn list_all_assets(&self) -> Result> { self.inner.db.list_all_assets().await } - async fn state(&self) -> Result { - let mut state = self.projected_state().await?; - state.merge_legacy( - self.get_snapshot_legacy().await?, - self.get_graph().await?, - self.get_retro_prompt().await?, - self.get_retro_response().await?, - self.list_checkpoints().await?, - ); - Ok(state) - } - - async fn get_snapshot(&self) -> Result> { - self.state().await.map(|state| state.to_snapshot()) - } -} - -impl SlateRunStore { - async fn get_snapshot_legacy(&self) -> Result> { - let Some(run) = self.get_run().await? else { - return Ok(None); - }; - - let mut iter = self.inner.db.scan_prefix(b"nodes/").await?; - let mut visits = BTreeSet::new(); - while let Some(entry) = iter.next().await? { - let key = key_to_string(&entry.key)?; - if let Some((node_id, visit, _)) = keys::parse_node_key(&key) { - visits.insert((node_id, visit)); - } - } - - let mut nodes = Vec::new(); - for (node_id, visit) in visits { - let node = NodeVisitRef { - node_id: &node_id, - visit, - }; - nodes.push(self.build_node_snapshot(&node).await?); - } - - Ok(Some(RunSnapshot { - run, - start: self.get_start().await?, - status: self.get_status().await?, - checkpoint: self.get_checkpoint().await?, - conclusion: self.get_conclusion().await?, - retro: self.get_retro().await?, - graph: self.get_graph().await?, - sandbox: self.get_sandbox().await?, - final_patch: self.get_final_patch().await?, - pull_request: self.get_pull_request().await?, - nodes, - })) + pub async fn state(&self) -> Result { + self.projected_state().await } } @@ -794,13 +564,6 @@ impl SlateRunDb { } } - async fn list_checkpoints(&self) -> Result> { - match self { - Self::Writer(db) => list_checkpoints(db).await, - Self::Reader(db) => list_checkpoints(db.as_ref()).await, - } - } - async fn list_artifact_values(&self) -> Result> { match self { Self::Writer(db) => list_artifact_values(db).await, @@ -910,23 +673,6 @@ where Ok(events) } -async fn list_checkpoints(db: &R) -> Result> -where - R: DbRead + Sync, -{ - let mut iter = db.scan_prefix(keys::CHECKPOINTS_PREFIX.as_bytes()).await?; - let mut checkpoints = Vec::new(); - while let Some(entry) = iter.next().await? { - let key = key_to_string(&entry.key)?; - let Some(seq) = keys::parse_checkpoint_seq(&key) else { - continue; - }; - checkpoints.push((seq, serde_json::from_slice(&entry.value)?)); - } - checkpoints.sort_by_key(|(seq, _)| *seq); - Ok(checkpoints) -} - async fn list_artifact_values(db: &R) -> Result> where R: DbRead + Sync, diff --git a/lib/crates/fabro-workflow/src/event.rs b/lib/crates/fabro-workflow/src/event.rs index dec103a97..fd9d4c620 100644 --- a/lib/crates/fabro-workflow/src/event.rs +++ b/lib/crates/fabro-workflow/src/event.rs @@ -5,7 +5,7 @@ use std::sync::atomic::{AtomicI64, Ordering}; use anyhow::{Context, Result}; use chrono::{SecondsFormat, Utc}; -use fabro_store::{EventPayload, NodeVisitRef, RunStore}; +use fabro_store::{EventPayload, NodeVisitRef, RunStoreHandle, SlateRunStore}; use fabro_types::RunId; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; @@ -1517,7 +1517,7 @@ pub fn event_payload_from_redacted_json(line: &str, run_id: &RunId) -> Result Result<()> { @@ -1562,7 +1562,7 @@ pub struct StoreProgressLogger { impl StoreProgressLogger { #[must_use] - pub fn new(run_store: Arc) -> Self { + pub fn new(run_store: RunStoreHandle) -> Self { let (tx, mut rx) = mpsc::unbounded_channel(); tokio::spawn(async move { @@ -1627,7 +1627,7 @@ impl StoreProgressLogger { } async fn project_provider_used_from_event_payload( - run_store: &dyn RunStore, + run_store: &SlateRunStore, payload: &EventPayload, ) -> Result<()> { let value = payload.as_value(); diff --git a/lib/crates/fabro-workflow/src/git.rs b/lib/crates/fabro-workflow/src/git.rs index 7502dae5b..0053a81b0 100644 --- a/lib/crates/fabro-workflow/src/git.rs +++ b/lib/crates/fabro-workflow/src/git.rs @@ -2,7 +2,7 @@ use std::path::Path; use std::process::Command; use fabro_checkpoint::git::Store; -use fabro_store::{NodeVisitRef, RunStore}; +use fabro_store::{NodeVisitRef, SlateRunStore}; use fabro_types::Settings; use crate::error::{FabroError, Result}; @@ -353,7 +353,7 @@ pub fn scan_node_files(run_dir: &Path) -> Vec<(String, Vec)> { result } -pub async fn scan_node_files_from_store(run_store: &dyn RunStore) -> Vec<(String, Vec)> { +pub async fn scan_node_files_from_store(run_store: &SlateRunStore) -> Vec<(String, Vec)> { let mut result = Vec::new(); let Ok(node_ids) = run_store.list_node_ids().await else { return result; @@ -441,10 +441,12 @@ fn node_file_path(node_id: &str, visit: u32, filename: &str) -> String { mod tests { use super::*; use chrono::Utc; - use fabro_graphviz::graph::Graph; - use fabro_store::{InMemoryStore, Store}; - use fabro_types::{NodeStatusRecord, RunRecord, StageStatus, fixtures}; + use fabro_store::SlateStore; + use fabro_types::{NodeStatusRecord, StageStatus, fixtures}; + use object_store::memory::InMemory; use std::fs; + use std::sync::Arc; + use std::time::Duration; /// Create a temporary git repo with an initial commit. fn init_repo(dir: &Path) { @@ -469,6 +471,14 @@ mod tests { .unwrap(); } + fn test_store() -> Arc { + Arc::new(SlateStore::new( + Arc::new(InMemory::new()), + "", + Duration::from_millis(1), + )) + } + #[test] fn ensure_clean_on_clean_repo() { let dir = tempfile::tempdir().unwrap(); @@ -583,25 +593,11 @@ mod tests { #[tokio::test] async fn scan_node_files_from_store_reconstructs_allowlisted_entries() { - let store = InMemoryStore::default(); - let created_at = Utc::now(); + let store = test_store(); let run = store - .create_run(&fixtures::RUN_1, created_at, None) + .create_run(&fixtures::RUN_1, chrono::Utc::now(), None) .await .unwrap(); - run.put_run(&RunRecord { - run_id: fixtures::RUN_1, - created_at, - settings: Settings::default(), - graph: Graph::new("test"), - workflow_slug: None, - working_directory: std::path::PathBuf::from("."), - host_repo_path: None, - base_branch: None, - labels: std::collections::HashMap::new(), - }) - .await - .unwrap(); let node = NodeVisitRef { node_id: "work", visit: 2, diff --git a/lib/crates/fabro-workflow/src/handler/agent.rs b/lib/crates/fabro-workflow/src/handler/agent.rs index e87cb21d2..f34f2d90a 100644 --- a/lib/crates/fabro-workflow/src/handler/agent.rs +++ b/lib/crates/fabro-workflow/src/handler/agent.rs @@ -373,30 +373,40 @@ mod tests { use super::*; use crate::event::EventEmitter; use fabro_graphviz::graph::AttrValue; - use fabro_store::{InMemoryStore, NodeVisitRef, RunStore, Store}; + use fabro_store::{NodeVisitRef, RunStoreHandle, SlateStore}; use fabro_types::fixtures; + use object_store::memory::InMemory; use std::sync::Arc; + use std::time::Duration; use tempfile::TempDir; fn make_services() -> EngineServices { EngineServices::test_default() } + fn test_store() -> Arc { + Arc::new(SlateStore::new( + Arc::new(InMemory::new()), + "", + Duration::from_millis(1), + )) + } + async fn make_services_with_run_store() -> ( EngineServices, - Arc, + RunStoreHandle, crate::event::StoreProgressLogger, ) { - let store = InMemoryStore::default(); + let store = test_store(); let run_store = store .create_run(&fixtures::RUN_1, chrono::Utc::now(), None) .await .unwrap(); let services = EngineServices { - run_store: Arc::clone(&run_store), + run_store: run_store.clone(), ..EngineServices::test_default() }; - let logger = crate::event::StoreProgressLogger::new(Arc::clone(&run_store)); + let logger = crate::event::StoreProgressLogger::new(run_store.clone()); logger.register(services.emitter.as_ref()); (services, run_store, logger) } diff --git a/lib/crates/fabro-workflow/src/handler/command.rs b/lib/crates/fabro-workflow/src/handler/command.rs index d548afa4a..832b6e113 100644 --- a/lib/crates/fabro-workflow/src/handler/command.rs +++ b/lib/crates/fabro-workflow/src/handler/command.rs @@ -1,12 +1,12 @@ use std::path::Path; -use async_trait::async_trait; use crate::context::Context; use crate::context::keys; use crate::error::FabroError; use crate::event::WorkflowRunEvent; use crate::outcome::{Outcome, OutcomeExt}; use crate::run_dir::{node_dir, visit_from_context}; +use async_trait::async_trait; use fabro_graphviz::graph::{Graph, Node}; use tokio::fs; @@ -199,8 +199,9 @@ mod tests { use super::*; use crate::outcome::StageStatus; use fabro_graphviz::graph::AttrValue; - use fabro_store::{InMemoryStore, NodeVisitRef, RunStore, Store}; + use fabro_store::{NodeVisitRef, RunStoreHandle, SlateStore}; use fabro_types::fixtures; + use object_store::memory::InMemory; use std::sync::Arc; use std::time::Duration; @@ -208,22 +209,30 @@ mod tests { EngineServices::test_default() } + fn test_store() -> Arc { + Arc::new(SlateStore::new( + Arc::new(InMemory::new()), + "", + Duration::from_millis(1), + )) + } + async fn make_services_with_run_store() -> ( EngineServices, - Arc, + RunStoreHandle, crate::event::StoreProgressLogger, ) { - let store = InMemoryStore::default(); + let store = test_store(); let run_store = store .create_run(&fixtures::RUN_1, chrono::Utc::now(), None) .await .unwrap(); let services = EngineServices { emitter: Arc::new(crate::event::EventEmitter::new(fixtures::RUN_1)), - run_store: Arc::clone(&run_store), + run_store: run_store.clone(), ..EngineServices::test_default() }; - let logger = crate::event::StoreProgressLogger::new(Arc::clone(&run_store)); + let logger = crate::event::StoreProgressLogger::new(run_store.clone()); logger.register(services.emitter.as_ref()); (services, run_store, logger) } @@ -585,10 +594,7 @@ mod tests { .unwrap(); logger.flush().await; - let snapshot = run_store - .state() - .await - .unwrap(); + let snapshot = run_store.state().await.unwrap(); let node = snapshot .node(&NodeVisitRef { node_id: "script_node", diff --git a/lib/crates/fabro-workflow/src/handler/fan_in.rs b/lib/crates/fabro-workflow/src/handler/fan_in.rs index 948cb3069..ea70c68d6 100644 --- a/lib/crates/fabro-workflow/src/handler/fan_in.rs +++ b/lib/crates/fabro-workflow/src/handler/fan_in.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use async_trait::async_trait; use fabro_agent::Sandbox; -use fabro_store::{NodeVisitRef, RunStore}; +use fabro_store::{NodeVisitRef, RunStoreHandle}; use crate::context::Context; use crate::context::keys; @@ -226,7 +226,7 @@ async fn llm_evaluate( node_id: &str, emitter: &Arc, sandbox: &Arc, - run_store: Arc, + run_store: RunStoreHandle, ) -> Result { let results_text = serde_json::to_string_pretty(results).unwrap_or_else(|_| results.to_string()); diff --git a/lib/crates/fabro-workflow/src/handler/manager_loop.rs b/lib/crates/fabro-workflow/src/handler/manager_loop.rs index 4c8763580..6646467d6 100644 --- a/lib/crates/fabro-workflow/src/handler/manager_loop.rs +++ b/lib/crates/fabro-workflow/src/handler/manager_loop.rs @@ -17,8 +17,9 @@ use crate::run_options::RunOptions; use async_trait::async_trait; use chrono::Utc; use fabro_graphviz::graph::{AttrValue, Graph, Node}; -use fabro_store::{InMemoryStore, Store}; +use fabro_store::SlateStore; use fabro_types::Settings; +use object_store::memory::InMemory; use tokio::time::{sleep, timeout}; use super::{EngineServices, Handler}; @@ -192,7 +193,12 @@ impl Handler for SubWorkflowHandler { let hook_runner = services.hook_runner.clone(); let env = services.env.clone(); let dry_run = services.dry_run; - let run_store = InMemoryStore::default() + let store = Arc::new(SlateStore::new( + Arc::new(InMemory::new()), + "", + Duration::from_millis(1), + )); + let run_store = store .create_run( &child_run_options.run_id, Utc::now(), diff --git a/lib/crates/fabro-workflow/src/handler/mod.rs b/lib/crates/fabro-workflow/src/handler/mod.rs index 291b9c93c..c4e851495 100644 --- a/lib/crates/fabro-workflow/src/handler/mod.rs +++ b/lib/crates/fabro-workflow/src/handler/mod.rs @@ -15,12 +15,16 @@ use std::any::Any; use std::collections::HashMap; use std::path::Path; use std::sync::Arc; +#[cfg(test)] +use std::time::Duration; use async_trait::async_trait; use fabro_agent::Sandbox; -use fabro_store::RunStore; +use fabro_store::RunStoreHandle; #[cfg(test)] -use fabro_store::Store; +use fabro_store::SlateStore; +#[cfg(test)] +use object_store::memory::InMemory; use crate::context::Context; use crate::error::FabroError; @@ -36,7 +40,7 @@ pub struct EngineServices { pub registry: Arc, pub emitter: Arc, pub sandbox: Arc, - pub run_store: Arc, + pub run_store: RunStoreHandle, /// Git state for the current run. Set via `set_git_state` at the start of /// `run_via_core` and read by parallel/fan-in handlers. pub(crate) git_state: std::sync::RwLock>>, @@ -71,6 +75,11 @@ impl EngineServices { /// Test-only default: empty registry, no hooks, local sandbox at cwd. #[cfg(test)] pub fn test_default() -> Self { + let store = Arc::new(SlateStore::new( + Arc::new(InMemory::new()), + "", + Duration::from_millis(1), + )); Self { registry: Arc::new(HandlerRegistry::new(Box::new(start::StartHandler))), emitter: Arc::new(EventEmitter::default()), @@ -78,10 +87,10 @@ impl EngineServices { std::env::current_dir().unwrap_or_else(|_| std::path::PathBuf::from(".")), )), run_store: futures::executor::block_on(async { - fabro_store::InMemoryStore::default() + store .create_run(&fabro_types::RunId::new(), chrono::Utc::now(), None) .await - .expect("in-memory test run store should initialize") + .expect("slate-backed test run store should initialize") }), git_state: std::sync::RwLock::new(None), hook_runner: None, diff --git a/lib/crates/fabro-workflow/src/handler/parallel.rs b/lib/crates/fabro-workflow/src/handler/parallel.rs index f44388402..1cc79a518 100644 --- a/lib/crates/fabro-workflow/src/handler/parallel.rs +++ b/lib/crates/fabro-workflow/src/handler/parallel.rs @@ -594,14 +594,24 @@ fn find_join_node(results: &[BranchResult], graph: &Graph) -> Option { mod tests { use super::*; use fabro_graphviz::graph::{AttrValue, Edge}; - use fabro_store::{InMemoryStore, RunStore, Store}; + use fabro_store::SlateStore; use fabro_types::fixtures; + use object_store::memory::InMemory; use std::sync::Arc; + use std::time::Duration; fn make_services() -> EngineServices { EngineServices::test_default() } + fn test_store() -> Arc { + Arc::new(SlateStore::new( + Arc::new(InMemory::new()), + "", + Duration::from_millis(1), + )) + } + fn test_context() -> Context { let context = Context::new(); context.set( @@ -680,13 +690,13 @@ mod tests { #[tokio::test] async fn parallel_handler_stores_results_in_run_store() { - let store = Arc::new(InMemoryStore::default()); + let store = test_store(); let run_store = store .create_run(&fixtures::RUN_1, chrono::Utc::now(), None) .await .unwrap(); let services = EngineServices { - run_store: Arc::clone(&run_store) as Arc, + run_store: run_store.clone(), ..EngineServices::test_default() }; let mut node = Node::new("par"); diff --git a/lib/crates/fabro-workflow/src/handler/prompt.rs b/lib/crates/fabro-workflow/src/handler/prompt.rs index 5c79bfe6f..8c78d6b03 100644 --- a/lib/crates/fabro-workflow/src/handler/prompt.rs +++ b/lib/crates/fabro-workflow/src/handler/prompt.rs @@ -2,7 +2,6 @@ use std::path::Path; use async_trait::async_trait; -use fabro_model::Provider; use crate::context::keys; use crate::context::{Context, WorkflowContext}; use crate::error::FabroError; @@ -10,6 +9,7 @@ use crate::event::WorkflowRunEvent; use crate::outcome::Outcome; use crate::run_dir::{node_dir, visit_from_context}; use fabro_graphviz::graph::{Graph, Node}; +use fabro_model::Provider; use tokio::fs; use super::agent::{ @@ -185,30 +185,40 @@ impl Handler for PromptHandler { mod tests { use super::*; use fabro_graphviz::graph::AttrValue; - use fabro_store::{InMemoryStore, NodeVisitRef, RunStore, Store}; + use fabro_store::{NodeVisitRef, RunStoreHandle, SlateStore}; use fabro_types::fixtures; + use object_store::memory::InMemory; use std::sync::Arc; + use std::time::Duration; use tempfile::TempDir; fn make_services() -> EngineServices { EngineServices::test_default() } + fn test_store() -> Arc { + Arc::new(SlateStore::new( + Arc::new(InMemory::new()), + "", + Duration::from_millis(1), + )) + } + async fn make_services_with_run_store() -> ( EngineServices, - Arc, + RunStoreHandle, crate::event::StoreProgressLogger, ) { - let store = InMemoryStore::default(); + let store = test_store(); let run_store = store .create_run(&fixtures::RUN_1, chrono::Utc::now(), None) .await .unwrap(); let services = EngineServices { - run_store: Arc::clone(&run_store), + run_store: run_store.clone(), ..EngineServices::test_default() }; - let logger = crate::event::StoreProgressLogger::new(Arc::clone(&run_store)); + let logger = crate::event::StoreProgressLogger::new(run_store.clone()); logger.register(services.emitter.as_ref()); (services, run_store, logger) } diff --git a/lib/crates/fabro-workflow/src/lifecycle/disk.rs b/lib/crates/fabro-workflow/src/lifecycle/disk.rs index 97b98236f..3062eb9d3 100644 --- a/lib/crates/fabro-workflow/src/lifecycle/disk.rs +++ b/lib/crates/fabro-workflow/src/lifecycle/disk.rs @@ -2,8 +2,8 @@ use std::path::PathBuf; use std::sync::Arc; use async_trait::async_trait; -use fabro_store::{NodeVisitRef, RunStore}; -use fabro_types::{NodeStatusRecord, RunId}; +use fabro_store::RunStoreHandle; +use fabro_types::RunId; use fabro_core::error::Result as CoreResult; use fabro_core::graph::NodeSpec; @@ -15,20 +15,18 @@ use super::circuit_breaker::CircuitBreakerLifecycle; use crate::event::{EventEmitter, RunNoticeLevel, WorkflowRunEvent, append_workflow_event}; use crate::graph::WorkflowGraph; use crate::graph::WorkflowNode; -use crate::outcome::{OutcomeExt, StageUsage}; -use crate::records::{Checkpoint, StartRecord}; +use crate::outcome::StageUsage; use crate::run_options::RunOptions; -use crate::run_status::RunStatus; use fabro_graphviz::graph::types::Graph as GvGraph; type WfRunState = RunState>; type WfNodeResult = NodeResult>; -/// Sub-lifecycle responsible for writing run state to disk (node status, checkpoints). +/// Sub-lifecycle responsible for emitting store-backed run lifecycle events. pub(crate) struct DiskLifecycle { pub run_dir: PathBuf, pub run_id: RunId, - pub run_store: Arc, + pub run_store: RunStoreHandle, pub graph: Arc, pub run_options: Arc, pub emitter: Arc, @@ -39,31 +37,6 @@ pub(crate) struct DiskLifecycle { #[async_trait] impl RunLifecycle for DiskLifecycle { async fn on_run_start(&self, _graph: &WorkflowGraph, _state: &WfRunState) -> CoreResult<()> { - let git_state = self.run_options.git.as_ref(); - let start_record = StartRecord { - run_id: self.run_id, - start_time: chrono::Utc::now(), - run_branch: git_state.and_then(|g| g.run_branch.clone()), - base_sha: git_state.and_then(|g| g.base_sha.clone()), - }; - if let Err(err) = self.run_store.put_start(&start_record).await { - self.emitter.emit(&WorkflowRunEvent::RunNotice { - level: RunNoticeLevel::Warn, - code: "start_store_save_failed".to_string(), - message: format!("failed to save start record to store: {err}"), - }); - } - if let Err(err) = self - .run_store - .put_status(&fabro_types::RunStatusRecord::new(RunStatus::Running, None)) - .await - { - self.emitter.emit(&WorkflowRunEvent::RunNotice { - level: RunNoticeLevel::Warn, - code: "status_store_save_failed".to_string(), - message: format!("failed to save running status to store: {err}"), - }); - } if let Err(err) = append_workflow_event( self.run_store.as_ref(), &self.run_id, @@ -82,35 +55,10 @@ impl RunLifecycle for DiskLifecycle { async fn after_node( &self, - node: &WorkflowNode, - result: &mut WfNodeResult, - state: &WfRunState, + _node: &WorkflowNode, + _result: &mut WfNodeResult, + _state: &WfRunState, ) -> CoreResult<()> { - let gv = node.inner(); - let visit = state.node_visits.get(gv.id.as_str()).copied().unwrap_or(1); - let node_status = NodeStatusRecord { - status: result.outcome.status.clone(), - notes: result.outcome.notes.clone(), - failure_reason: result.outcome.failure_reason().map(ToOwned::to_owned), - timestamp: chrono::Utc::now(), - }; - if let Err(err) = self - .run_store - .put_node_status( - &NodeVisitRef { - node_id: &gv.id, - visit: u32::try_from(visit).unwrap_or(u32::MAX), - }, - &node_status, - ) - .await - { - self.emitter.emit(&WorkflowRunEvent::RunNotice { - level: RunNoticeLevel::Warn, - code: "node_status_store_save_failed".to_string(), - message: format!("[node: {}] node status store save failed: {err}", node.id()), - }); - } Ok(()) } @@ -131,7 +79,7 @@ impl RunLifecycle for DiskLifecycle { let mut node_outcomes = state.node_outcomes.clone(); node_outcomes.insert(node.id().to_string(), result.outcome.clone()); - let checkpoint = Checkpoint { + let _checkpoint = fabro_types::Checkpoint { timestamp: chrono::Utc::now(), current_node: node.id().to_string(), completed_nodes: state.completed_nodes.clone(), @@ -144,21 +92,6 @@ impl RunLifecycle for DiskLifecycle { loop_failure_signatures: loop_sigs, restart_failure_signatures: restart_sigs, }; - if let Err(err) = self.run_store.put_checkpoint(&checkpoint).await { - self.emitter.emit(&WorkflowRunEvent::RunNotice { - level: RunNoticeLevel::Warn, - code: "checkpoint_store_save_failed".to_string(), - message: format!("[node: {}] checkpoint store save failed: {err}", node.id()), - }); - } - if let Err(err) = self.run_store.append_checkpoint(&checkpoint).await { - self.emitter.emit(&WorkflowRunEvent::RunNotice { - level: RunNoticeLevel::Warn, - code: "checkpoint_store_append_failed".to_string(), - message: format!("[node: {}] checkpoint append failed: {err}", node.id()), - }); - } - Ok(()) } } diff --git a/lib/crates/fabro-workflow/src/lifecycle/git.rs b/lib/crates/fabro-workflow/src/lifecycle/git.rs index 19f7afe26..24f7c346d 100644 --- a/lib/crates/fabro-workflow/src/lifecycle/git.rs +++ b/lib/crates/fabro-workflow/src/lifecycle/git.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; use std::sync::{Arc, Mutex}; use async_trait::async_trait; -use fabro_store::RunStore; +use fabro_store::RunStoreHandle; use fabro_types::RunId; use fabro_core::error::{CoreError, Result as CoreResult}; @@ -39,7 +39,7 @@ pub(crate) struct GitLifecycle { pub emitter: Arc, pub run_dir: PathBuf, pub run_id: RunId, - pub run_store: Arc, + pub run_store: RunStoreHandle, pub run_options: Arc, pub start_node_id: Option, // Cross-lifecycle data (shared with EventLifecycle) @@ -66,27 +66,19 @@ impl RunLifecycle for GitLifecycle { ) { let git_author = self.run_options.git_author(); let store = MetadataStore::new(repo_path, &git_author); - let run_json = self - .run_store - .get_run() - .await - .ok() - .flatten() - .and_then(|record| serde_json::to_vec_pretty(&record).ok()); - let start_json = self - .run_store - .get_start() - .await - .ok() - .flatten() - .and_then(|record| serde_json::to_vec_pretty(&record).ok()); - let sandbox_json = self - .run_store - .get_sandbox() - .await - .ok() - .flatten() - .and_then(|record| serde_json::to_vec_pretty(&record).ok()); + let state = self.run_store.state().await.ok(); + let run_json = state + .as_ref() + .and_then(|state| state.run.as_ref()) + .and_then(|record| serde_json::to_vec_pretty(record).ok()); + let start_json = state + .as_ref() + .and_then(|state| state.start.as_ref()) + .and_then(|record| serde_json::to_vec_pretty(record).ok()); + let sandbox_json = state + .as_ref() + .and_then(|state| state.sandbox.as_ref()) + .and_then(|record| serde_json::to_vec_pretty(record).ok()); let mut files: Vec<(&str, &[u8])> = Vec::new(); if let Some(ref data) = run_json { files.push(("run.json", data)); @@ -137,10 +129,10 @@ impl RunLifecycle for GitLifecycle { // Build checkpoint JSON for shadow branch if let Some(cp_json) = self .run_store - .get_checkpoint() + .state() .await .ok() - .flatten() + .and_then(|state| state.checkpoint) .and_then(|checkpoint| serde_json::to_vec_pretty(&checkpoint).ok()) { let mut extra_entries: Vec<(String, Vec)> = { @@ -205,31 +197,6 @@ impl RunLifecycle for GitLifecycle { diff: None, }; - match self.run_store.get_checkpoint().await { - Ok(Some(mut checkpoint)) => { - checkpoint.git_commit_sha = Some(sha.clone()); - if let Err(err) = self.run_store.put_checkpoint(&checkpoint).await { - self.emitter.emit(&WorkflowRunEvent::RunNotice { - level: RunNoticeLevel::Warn, - code: "checkpoint_store_resave_failed".to_string(), - message: format!( - "[node: {node_id}] checkpoint store re-save with SHA failed: {err}" - ), - }); - } - } - Ok(None) => {} - Err(err) => { - self.emitter.emit(&WorkflowRunEvent::RunNotice { - level: RunNoticeLevel::Warn, - code: "checkpoint_store_load_failed".to_string(), - message: format!( - "[node: {node_id}] checkpoint store load failed: {err}" - ), - }); - } - } - // Push run branch (skip in dry-run mode) if !self.run_options.dry_run_enabled() { if let Some(branch) = self @@ -277,7 +244,6 @@ impl RunLifecycle for GitLifecycle { } // Save diff.patch - let visit = state.node_visits.get(node_id).copied().unwrap_or(1); let prev = self .last_git_sha .lock() @@ -292,22 +258,6 @@ impl RunLifecycle for GitLifecycle { .unwrap_or_else(|| sha.clone()); match git_diff(&*self.sandbox, &prev).await { Ok(patch) if !patch.is_empty() => { - let node_ref = fabro_store::NodeVisitRef { - node_id, - visit: u32::try_from(visit).unwrap_or(u32::MAX), - }; - if let Err(err) = self.run_store.put_node_diff(&node_ref, &patch).await { - self.emitter.emit(&WorkflowRunEvent::RunNotice { - level: RunNoticeLevel::Warn, - code: "git_diff_store_failed".to_string(), - message: format!( - "[node: {node_id}] failed to persist diff in run store: {err}" - ), - }); - return Err(CoreError::Other(format!( - "failed to persist node diff for '{node_id}': {err}" - ))); - } git_result.diff = Some(patch); } Ok(_) => {} @@ -353,15 +303,6 @@ impl RunLifecycle for GitLifecycle { match git_diff(&*self.sandbox, &base_sha).await { Ok(patch) if !patch.is_empty() => { *self.final_patch.lock().unwrap() = Some(patch.clone()); - if let Err(err) = self.run_store.put_final_patch(&patch).await { - self.emitter.emit(&WorkflowRunEvent::RunNotice { - level: RunNoticeLevel::Warn, - code: "final_diff_store_failed".to_string(), - message: format!( - "failed to persist final diff in run store: {err}" - ), - }); - } } Ok(_) => { *self.final_patch.lock().unwrap() = None; diff --git a/lib/crates/fabro-workflow/src/lifecycle/mod.rs b/lib/crates/fabro-workflow/src/lifecycle/mod.rs index 699c33530..c9c146678 100644 --- a/lib/crates/fabro-workflow/src/lifecycle/mod.rs +++ b/lib/crates/fabro-workflow/src/lifecycle/mod.rs @@ -14,7 +14,7 @@ use std::sync::{Arc, Mutex}; use std::time::Instant; use async_trait::async_trait; -use fabro_store::RunStore; +use fabro_store::RunStoreHandle; use fabro_store::RuntimeState; use fabro_types::RunId; @@ -85,7 +85,7 @@ impl WorkflowLifecycle { sandbox: &Arc, graph: Arc, run_dir: &PathBuf, - run_store: Arc, + run_store: RunStoreHandle, run_options: &Arc, is_resume: bool, on_node: crate::OnNodeCallback, @@ -122,7 +122,7 @@ impl WorkflowLifecycle { run_start: Mutex::new(Instant::now()), restarted_from: Arc::clone(&restarted_from), base_branch: run_options.base_branch.clone(), - base_sha: run_options.display_base_sha.clone(), + base_sha: run_options.git.as_ref().and_then(|g| g.base_sha.clone()), run_branch: run_options.git.as_ref().and_then(|g| g.run_branch.clone()), worktree_dir: working_directory.clone(), goal: (!graph.goal().is_empty()).then(|| graph.goal().to_string()), @@ -146,7 +146,7 @@ impl WorkflowLifecycle { let disk = DiskLifecycle { run_dir: run_dir.clone(), run_id: run_options.run_id, - run_store: Arc::clone(&run_store), + run_store: run_store.clone(), graph: Arc::clone(&graph), run_options: Arc::clone(run_options), emitter: Arc::clone(emitter), diff --git a/lib/crates/fabro-workflow/src/operations/create.rs b/lib/crates/fabro-workflow/src/operations/create.rs index 1ed6527f2..02d4801d9 100644 --- a/lib/crates/fabro-workflow/src/operations/create.rs +++ b/lib/crates/fabro-workflow/src/operations/create.rs @@ -2,7 +2,7 @@ use chrono::{Local, Utc}; use fabro_graphviz::graph::{AttrValue, Graph}; use fabro_model::{Catalog, Provider}; use fabro_sandbox::SandboxProvider; -use fabro_store::Store; +use fabro_store::SlateStore; use fabro_types::{RunId, Settings}; use std::collections::BTreeMap; use std::collections::HashMap; @@ -13,7 +13,6 @@ use crate::pipeline::types::PersistOptions; use crate::pipeline::{self, Persisted, TransformOptions, Validated}; use crate::records::RunRecord; use crate::run_lookup::default_runs_base; -use crate::run_status::{RunStatus, RunStatusRecord}; use crate::transforms::{Transform, expand_vars}; use fabro_sandbox::daytona::detect_repo_info; @@ -56,7 +55,7 @@ struct PersistCreateOptions { } /// Resolve workflow inputs, normalize settings, and persist a run directory. -pub async fn create(store: &dyn Store, request: CreateRunInput) -> Result { +pub async fn create(store: &SlateStore, request: CreateRunInput) -> Result { let resolved = resolve_workflow(ResolveWorkflowInput { workflow: request.workflow, settings: request.settings, @@ -133,7 +132,7 @@ pub async fn create(store: &dyn Store, request: CreateRunInput) -> Result, @@ -152,18 +151,6 @@ async fn persist_created_run( .or_else(|_| Err(FabroError::engine(err.to_string())))?, }; - run_store.put_run(record).await.map_err(store_error)?; - if !workflow_source.is_empty() { - run_store - .put_graph(workflow_source) - .await - .map_err(store_error)?; - } - run_store - .put_status(&RunStatusRecord::new(RunStatus::Submitted, None)) - .await - .map_err(store_error)?; - let envelope = canonicalize_event_at( &record.run_id, &WorkflowRunEvent::RunCreated { @@ -411,15 +398,20 @@ pub(crate) fn make_run_dir(runs_base: &Path, run_id: &str, dry_run: bool) -> Pat mod tests { use super::*; use fabro_graphviz::graph::AttrValue; - use fabro_store::{InMemoryStore, SlateStore, Store}; + use fabro_store::{SlateStore, StoreHandle}; use fabro_types::fixtures; use object_store::local::LocalFileSystem; + use object_store::memory::InMemory; use std::sync::Arc; use std::time::Duration; use crate::operations::{ValidateInput, validate}; - fn memory_store() -> InMemoryStore { - InMemoryStore::default() + fn memory_store() -> StoreHandle { + Arc::new(SlateStore::new( + Arc::new(InMemory::new()), + "", + Duration::from_millis(1), + )) } fn validate_dot(dot_source: &str, settings: Settings) -> Validated { @@ -719,7 +711,7 @@ mod tests { ); let run_store = store.open_run(&fixtures::RUN_1).await.unwrap(); assert_eq!( - run_store.get_status().await.unwrap().unwrap().status, + run_store.state().await.unwrap().status.unwrap().status, crate::run_status::RunStatus::Submitted ); assert!(!created.run_dir.join("id.txt").exists()); @@ -816,7 +808,11 @@ mod tests { std::fs::create_dir_all(storage_dir.join("store")).unwrap(); let object_store = Arc::new(LocalFileSystem::new_with_prefix(storage_dir.join("store")).unwrap()); - let store = Arc::new(SlateStore::new(object_store, "", Duration::from_millis(1))); + let store = StoreHandle::from(Arc::new(SlateStore::new( + object_store, + "", + Duration::from_millis(1), + ))); let created = create( store.as_ref(), CreateRunInput { diff --git a/lib/crates/fabro-workflow/src/operations/rebuild_meta.rs b/lib/crates/fabro-workflow/src/operations/rebuild_meta.rs index 9f599285d..367ca6eb4 100644 --- a/lib/crates/fabro-workflow/src/operations/rebuild_meta.rs +++ b/lib/crates/fabro-workflow/src/operations/rebuild_meta.rs @@ -6,7 +6,7 @@ use anyhow::{Context, Result, bail}; use fabro_checkpoint::branch::BranchStore; use fabro_checkpoint::git::Store as GitStore; use fabro_store::{ - ListRunsQuery, NodeVisitRef, RunStore as DurableRunStore, Store as DurableStore, + ListRunsQuery, NodeVisitRef, RunStoreHandle as DurableRunStore, SlateStore as DurableStore, }; use fabro_types::RunId; use git2::{Repository, Signature}; @@ -18,7 +18,7 @@ use crate::records::Checkpoint; pub async fn rebuild_metadata_branch( git_store: &GitStore, - run_store: &dyn DurableRunStore, + run_store: &DurableRunStore, run_id: &RunId, ) -> Result<()> { let branch = MetadataStore::branch_name(&run_id.to_string()); @@ -26,10 +26,7 @@ pub async fn rebuild_metadata_branch( bail!("metadata branch already exists for run {run_id}"); } - let state = run_store - .state() - .await? - ; + let state = run_store.state().await?; let run_record = state .run .clone() @@ -157,7 +154,7 @@ pub async fn rebuild_metadata_branch( pub async fn build_timeline_or_rebuild( git_store: &GitStore, - run_store: Option<&dyn DurableRunStore>, + run_store: Option<&DurableRunStore>, run_id: &RunId, ) -> Result { let branch = MetadataStore::branch_name(&run_id.to_string()); @@ -178,7 +175,7 @@ pub async fn build_timeline_or_rebuild( pub async fn find_run_id_by_prefix_or_store( repo: &Repository, - fabro_store: &dyn DurableStore, + fabro_store: &DurableStore, prefix: &str, ) -> Result { if let Some(run_id) = find_run_id_by_prefix_in_refs(repo, prefix)? { @@ -334,19 +331,18 @@ fn resolve_prefix_matches(prefix: &str, matches: Vec) -> Result { #[cfg(test)] mod tests { + use chrono::{TimeZone, Utc}; + use fabro_graphviz::graph::Graph; + use fabro_store::{NodeVisitRef, SlateStore, StoreHandle}; + use fabro_types::{RunId, RunRecord, SandboxRecord, Settings, StartRecord, fixtures}; + use object_store::memory::InMemory; use std::collections::HashMap; use std::path::PathBuf; use std::sync::Arc; - - use chrono::{TimeZone, Utc}; - use fabro_graphviz::graph::Graph; - use fabro_store::{InMemoryStore, NodeVisitRef, Store as _}; - use fabro_types::{ - NodeStatusRecord, RunId, RunRecord, SandboxRecord, Settings, StageStatus, StartRecord, - fixtures, - }; + use std::time::Duration; use super::*; + use crate::event::{WorkflowRunEvent, append_workflow_event}; use crate::operations::test_support::{make_checkpoint_json, temp_repo, test_sig}; use crate::records::Checkpoint; @@ -362,6 +358,14 @@ mod tests { fixtures::RUN_1 } + fn memory_store() -> StoreHandle { + Arc::new(SlateStore::new( + Arc::new(InMemory::new()), + "", + Duration::from_millis(1), + )) + } + fn sample_run_record(run_id: RunId, host_repo_path: Option<&str>) -> RunRecord { RunRecord { run_id, @@ -422,26 +426,130 @@ mod tests { } } - fn sample_node_status() -> NodeStatusRecord { - NodeStatusRecord { - status: StageStatus::Success, - notes: Some("done".to_string()), - failure_reason: None, - timestamp: created_at(), - } - } - async fn create_run_store( - store: &InMemoryStore, + store: &SlateStore, run_id: RunId, host_repo_path: Option<&str>, - ) -> Arc { + ) -> DurableRunStore { let run_store = store.create_run(&run_id, created_at(), None).await.unwrap(); + let run_record = sample_run_record(run_id, host_repo_path); + append_workflow_event( + run_store.as_ref(), + &run_id, + &WorkflowRunEvent::RunCreated { + run_id, + settings: serde_json::to_value(&run_record.settings).unwrap(), + graph: serde_json::to_value(&run_record.graph).unwrap(), + workflow_source: None, + workflow_config: None, + labels: run_record.labels.clone().into_iter().collect(), + run_dir: String::new(), + working_directory: run_record.working_directory.display().to_string(), + host_repo_path: run_record.host_repo_path.clone(), + base_branch: run_record.base_branch.clone(), + workflow_slug: run_record.workflow_slug.clone(), + db_prefix: None, + }, + ) + .await + .unwrap(); run_store - .put_run(&sample_run_record(run_id, host_repo_path)) - .await - .unwrap(); - run_store + } + + async fn append_start_event(run_store: &DurableRunStore, run_id: RunId) { + let start = sample_start_record(run_id); + append_workflow_event( + run_store, + &run_id, + &WorkflowRunEvent::WorkflowRunStarted { + name: "test".to_string(), + run_id, + base_branch: None, + base_sha: start.base_sha, + run_branch: start.run_branch, + worktree_dir: None, + goal: None, + }, + ) + .await + .unwrap(); + } + + async fn append_sandbox_event(run_store: &DurableRunStore, run_id: RunId) { + let sandbox = sample_sandbox_record(); + append_workflow_event( + run_store, + &run_id, + &WorkflowRunEvent::SandboxInitialized { + provider: sandbox.provider, + working_directory: sandbox.working_directory, + identifier: sandbox.identifier, + host_working_directory: sandbox.host_working_directory, + container_mount_point: sandbox.container_mount_point, + }, + ) + .await + .unwrap(); + } + + async fn append_checkpoint_event( + run_store: &DurableRunStore, + run_id: RunId, + checkpoint: Checkpoint, + ) { + append_workflow_event( + run_store, + &run_id, + &WorkflowRunEvent::CheckpointCompleted { + node_id: checkpoint.current_node.clone(), + status: "success".to_string(), + current_node: checkpoint.current_node.clone(), + completed_nodes: checkpoint.completed_nodes.clone(), + node_retries: checkpoint.node_retries.clone().into_iter().collect(), + context_values: checkpoint.context_values.clone().into_iter().collect(), + node_outcomes: checkpoint.node_outcomes.clone().into_iter().collect(), + next_node_id: checkpoint.next_node_id.clone(), + git_commit_sha: checkpoint.git_commit_sha.clone(), + loop_failure_signatures: checkpoint + .loop_failure_signatures + .clone() + .into_iter() + .map(|(signature, count)| (signature.to_string(), count)) + .collect(), + restart_failure_signatures: checkpoint + .restart_failure_signatures + .clone() + .into_iter() + .map(|(signature, count)| (signature.to_string(), count)) + .collect(), + node_visits: checkpoint.node_visits.clone().into_iter().collect(), + diff: None, + }, + ) + .await + .unwrap(); + } + + async fn append_prompt_event( + run_store: &DurableRunStore, + run_id: RunId, + node: &NodeVisitRef<'_>, + text: &str, + ) { + append_workflow_event( + run_store, + &run_id, + &WorkflowRunEvent::Prompt { + stage: node.node_id.to_string(), + visit: node.visit, + text: text.to_string(), + mode: None, + provider: None, + model: None, + }, + ) + .await + .unwrap(); } fn seed_run_branch(git_store: &GitStore, run_id: RunId, nodes: &[&str]) -> Vec { @@ -472,46 +580,41 @@ mod tests { #[tokio::test] async fn rebuild_metadata_branch_round_trips_timeline() { let (_dir, git_store) = temp_repo(); - let durable_store = InMemoryStore::default(); + let durable_store = memory_store(); let run_store = create_run_store(&durable_store, test_run_id(), None).await; - run_store - .put_start(&sample_start_record(test_run_id())) - .await - .unwrap(); - run_store - .put_sandbox(&sample_sandbox_record()) - .await - .unwrap(); + append_start_event(&run_store, test_run_id()).await; + append_sandbox_event(&run_store, test_run_id()).await; - run_store - .append_checkpoint(&sample_checkpoint( - "start", - &["start"], - &[("start", 1)], - Some("aaa"), - )) - .await - .unwrap(); - run_store - .append_checkpoint(&sample_checkpoint( + append_checkpoint_event( + &run_store, + test_run_id(), + sample_checkpoint("start", &["start"], &[("start", 1)], Some("aaa")), + ) + .await; + append_checkpoint_event( + &run_store, + test_run_id(), + sample_checkpoint( "build", &["start", "build"], &[("start", 1), ("build", 1)], Some("bbb"), - )) - .await - .unwrap(); - run_store - .append_checkpoint(&sample_checkpoint( + ), + ) + .await; + append_checkpoint_event( + &run_store, + test_run_id(), + sample_checkpoint( "build", &["start", "build"], &[("start", 1), ("build", 2)], Some("ccc"), - )) - .await - .unwrap(); + ), + ) + .await; - rebuild_metadata_branch(&git_store, run_store.as_ref(), &test_run_id()) + rebuild_metadata_branch(&git_store, &run_store, &test_run_id()) .await .unwrap(); @@ -531,51 +634,35 @@ mod tests { #[tokio::test] async fn rebuild_metadata_branch_preserves_historical_node_visits() { let (_dir, git_store) = temp_repo(); - let durable_store = InMemoryStore::default(); + let durable_store = memory_store(); let run_store = create_run_store(&durable_store, test_run_id(), None).await; let build_v1 = NodeVisitRef { node_id: "build", visit: 1, }; - run_store - .put_node_prompt(&build_v1, "visit one") - .await - .unwrap(); - run_store - .put_node_status(&build_v1, &sample_node_status()) - .await - .unwrap(); + append_prompt_event(&run_store, test_run_id(), &build_v1, "visit one").await; let build_v2 = NodeVisitRef { node_id: "build", visit: 2, }; - run_store - .put_node_prompt(&build_v2, "visit two") - .await - .unwrap(); + append_prompt_event(&run_store, test_run_id(), &build_v2, "visit two").await; - run_store - .append_checkpoint(&sample_checkpoint( - "build", - &["build"], - &[("build", 1)], - Some("aaa"), - )) - .await - .unwrap(); - run_store - .append_checkpoint(&sample_checkpoint( - "build", - &["build"], - &[("build", 2)], - Some("bbb"), - )) - .await - .unwrap(); + append_checkpoint_event( + &run_store, + test_run_id(), + sample_checkpoint("build", &["build"], &[("build", 1)], Some("aaa")), + ) + .await; + append_checkpoint_event( + &run_store, + test_run_id(), + sample_checkpoint("build", &["build"], &[("build", 2)], Some("bbb")), + ) + .await; - rebuild_metadata_branch(&git_store, run_store.as_ref(), &test_run_id()) + rebuild_metadata_branch(&git_store, &run_store, &test_run_id()) .await .unwrap(); @@ -624,7 +711,7 @@ mod tests { #[tokio::test] async fn rebuild_metadata_branch_refuses_to_overwrite_existing_branch() { let (_dir, git_store) = temp_repo(); - let durable_store = InMemoryStore::default(); + let durable_store = memory_store(); let run_store = create_run_store(&durable_store, test_run_id(), None).await; let sig = test_sig(); @@ -633,7 +720,7 @@ mod tests { bs.ensure_branch().unwrap(); bs.write_entry("run.json", b"{}", "init run").unwrap(); - let err = rebuild_metadata_branch(&git_store, run_store.as_ref(), &test_run_id()) + let err = rebuild_metadata_branch(&git_store, &run_store, &test_run_id()) .await .unwrap_err(); assert!(err.to_string().contains("metadata branch already exists")); @@ -642,23 +729,19 @@ mod tests { #[tokio::test] async fn build_timeline_or_rebuild_rebuilds_missing_branch() { let (_dir, git_store) = temp_repo(); - let durable_store = InMemoryStore::default(); + let durable_store = memory_store(); let run_store = create_run_store(&durable_store, test_run_id(), None).await; - run_store - .append_checkpoint(&sample_checkpoint( - "start", - &["start"], - &[("start", 1)], - Some("aaa"), - )) + append_checkpoint_event( + &run_store, + test_run_id(), + sample_checkpoint("start", &["start"], &[("start", 1)], Some("aaa")), + ) + .await; + + let timeline = build_timeline_or_rebuild(&git_store, Some(&run_store), &test_run_id()) .await .unwrap(); - let timeline = - build_timeline_or_rebuild(&git_store, Some(run_store.as_ref()), &test_run_id()) - .await - .unwrap(); - assert_eq!(timeline.entries.len(), 1); assert_eq!(timeline.entries[0].node_name, "start"); } @@ -666,35 +749,36 @@ mod tests { #[tokio::test] async fn build_timeline_or_rebuild_preserves_existing_branch() { let (_dir, git_store) = temp_repo(); - let durable_store = InMemoryStore::default(); + let durable_store = memory_store(); let run_store = create_run_store(&durable_store, test_run_id(), None).await; - run_store - .append_checkpoint(&sample_checkpoint( - "start", - &["start"], - &[("start", 1)], - Some("aaa"), - )) - .await - .unwrap(); - run_store - .append_checkpoint(&sample_checkpoint( + append_checkpoint_event( + &run_store, + test_run_id(), + sample_checkpoint("start", &["start"], &[("start", 1)], Some("aaa")), + ) + .await; + append_checkpoint_event( + &run_store, + test_run_id(), + sample_checkpoint( "build", &["start", "build"], &[("start", 1), ("build", 1)], Some("bbb"), - )) - .await - .unwrap(); - run_store - .append_checkpoint(&sample_checkpoint( + ), + ) + .await; + append_checkpoint_event( + &run_store, + test_run_id(), + sample_checkpoint( "test", &["start", "build", "test"], &[("start", 1), ("build", 1), ("test", 1)], Some("ccc"), - )) - .await - .unwrap(); + ), + ) + .await; let sig = test_sig(); let branch = MetadataStore::branch_name(&test_run_id().to_string()); @@ -714,10 +798,9 @@ mod tests { ) .unwrap(); - let timeline = - build_timeline_or_rebuild(&git_store, Some(run_store.as_ref()), &test_run_id()) - .await - .unwrap(); + let timeline = build_timeline_or_rebuild(&git_store, Some(&run_store), &test_run_id()) + .await + .unwrap(); assert_eq!(timeline.entries.len(), 2); assert_eq!(timeline.entries[0].node_name, "start"); @@ -737,13 +820,13 @@ mod tests { #[tokio::test] async fn rebuild_metadata_branch_errors_when_run_record_is_missing() { let (_dir, git_store) = temp_repo(); - let durable_store = InMemoryStore::default(); + let durable_store = memory_store(); let run_store = durable_store .create_run(&test_run_id(), created_at(), None) .await .unwrap(); - let err = rebuild_metadata_branch(&git_store, run_store.as_ref(), &test_run_id()) + let err = rebuild_metadata_branch(&git_store, &run_store, &test_run_id()) .await .unwrap_err(); assert!(err.to_string().contains("run record not found")); @@ -752,7 +835,7 @@ mod tests { #[tokio::test] async fn find_run_id_by_prefix_or_store_falls_back_to_store() { let (dir, git_store) = temp_repo(); - let durable_store = InMemoryStore::default(); + let durable_store = memory_store(); let repo_path = dir.path().to_string_lossy().to_string(); let repo_run_id = parse_run_id("01ARZ3NDEKTSV4RRFFQ69G5FAV"); let _run_store = create_run_store(&durable_store, repo_run_id, Some(&repo_path)).await; @@ -769,7 +852,7 @@ mod tests { async fn find_run_id_by_prefix_or_store_excludes_other_repos() { let (_dir, git_store) = temp_repo(); let (other_dir, _other_git_store) = temp_repo(); - let durable_store = InMemoryStore::default(); + let durable_store = memory_store(); let other_repo_path = other_dir.path().to_string_lossy().to_string(); let other_run_id = parse_run_id("01ARZ3NDEKTSV4RRFFQ69G5FAV"); let _run_store = @@ -786,7 +869,7 @@ mod tests { #[tokio::test] async fn find_run_id_by_prefix_or_store_requires_exact_match_without_repo_path() { let (_dir, git_store) = temp_repo(); - let durable_store = InMemoryStore::default(); + let durable_store = memory_store(); let repo_run_id = parse_run_id("01ARZ3NDEKTSV4RRFFQ69G5FAV"); let _run_store = create_run_store(&durable_store, repo_run_id, None).await; let prefix = &repo_run_id.to_string()[..6]; @@ -809,7 +892,7 @@ mod tests { #[tokio::test] async fn exact_match_wins_over_prefix_ambiguity() { let (_dir, git_store) = temp_repo(); - let durable_store = InMemoryStore::default(); + let durable_store = memory_store(); let repo_path = git_store.repo_dir().to_string_lossy().to_string(); let exact_run_id = parse_run_id("01ARZ3NDEKTSV4RRFFQ69G5FAV"); let other_run_id = parse_run_id("01ARZ3NDEKTSV4RRFFQ69G5FAW"); @@ -853,31 +936,30 @@ mod tests { #[tokio::test] async fn rebuild_metadata_branch_persists_backfilled_run_shas_in_checkpoint_blobs() { let (_dir, git_store) = temp_repo(); - let durable_store = InMemoryStore::default(); + let durable_store = memory_store(); let run_store = create_run_store(&durable_store, test_run_id(), None).await; - run_store - .append_checkpoint(&sample_checkpoint( - "start", - &["start"], - &[("start", 1)], - None, - )) - .await - .unwrap(); - run_store - .append_checkpoint(&sample_checkpoint( + append_checkpoint_event( + &run_store, + test_run_id(), + sample_checkpoint("start", &["start"], &[("start", 1)], None), + ) + .await; + append_checkpoint_event( + &run_store, + test_run_id(), + sample_checkpoint( "build", &["start", "build"], &[("start", 1), ("build", 1)], None, - )) - .await - .unwrap(); + ), + ) + .await; let expected_shas = seed_run_branch(&git_store, test_run_id(), &["start", "build"]); - rebuild_metadata_branch(&git_store, run_store.as_ref(), &test_run_id()) + rebuild_metadata_branch(&git_store, &run_store, &test_run_id()) .await .unwrap(); @@ -921,7 +1003,7 @@ mod tests { #[tokio::test] async fn rebuild_metadata_branch_is_atomic_on_failure() { let (_dir, git_store) = temp_repo(); - let durable_store = InMemoryStore::default(); + let durable_store = memory_store(); let run_store = create_run_store(&durable_store, test_run_id(), None).await; let bad_node = "bad\0node"; @@ -929,21 +1011,15 @@ mod tests { node_id: bad_node, visit: 1, }; - run_store - .put_node_prompt(&bad_visit, "prompt") - .await - .unwrap(); - run_store - .append_checkpoint(&sample_checkpoint( - bad_node, - &[bad_node], - &[(bad_node, 1)], - None, - )) - .await - .unwrap(); + append_prompt_event(&run_store, test_run_id(), &bad_visit, "prompt").await; + append_checkpoint_event( + &run_store, + test_run_id(), + sample_checkpoint(bad_node, &[bad_node], &[(bad_node, 1)], None), + ) + .await; - let err = rebuild_metadata_branch(&git_store, run_store.as_ref(), &test_run_id()) + let err = rebuild_metadata_branch(&git_store, &run_store, &test_run_id()) .await .unwrap_err(); assert!(err.to_string().contains("nul") || err.to_string().contains("NUL")); diff --git a/lib/crates/fabro-workflow/src/operations/resume.rs b/lib/crates/fabro-workflow/src/operations/resume.rs index fcac5048f..663e170c0 100644 --- a/lib/crates/fabro-workflow/src/operations/resume.rs +++ b/lib/crates/fabro-workflow/src/operations/resume.rs @@ -5,7 +5,7 @@ use fabro_store::RuntimeState; use crate::error::FabroError; use crate::event::{WorkflowRunEvent, append_workflow_event}; use crate::outcome::StageStatus; -use crate::run_status::{self, RunStatus}; +use crate::run_status::RunStatus; use super::start::{StartServices, Started, execute_persisted_run}; @@ -40,14 +40,6 @@ pub async fn resume(run_dir: &Path, services: StartServices) -> Result, seed_context: Option, - run_store: Arc, + run_store: RunStoreHandle, git: Option, github_app: Option, worktree_mode: Option, @@ -67,7 +67,7 @@ pub struct StartServices { pub cancel_token: Option>, pub emitter: Arc, pub interviewer: Arc, - pub run_store: Arc, + pub run_store: RunStoreHandle, pub github_app: Option, pub on_node: crate::OnNodeCallback, pub registry_override: Option>, @@ -112,13 +112,28 @@ pub(super) async fn execute_persisted_run( ) -> Result { let cancel_token = services.cancel_token.clone(); let run_id = services.run_id; - let run_store = Arc::clone(&services.run_store); - if let Err(err) = run_store - .put_status(&run_status::RunStatusRecord::new( - RunStatus::Starting, - Some(StatusReason::SandboxInitializing), - )) - .await + let run_store = services.run_store.clone(); + if let Err(err) = run_store.state().await { + let error = FabroError::engine(err.to_string()); + let _ = persist_detached_failure( + run_id, + run_store.as_ref(), + run_dir, + "bootstrap", + StatusReason::BootstrapFailed, + &error, + ) + .await; + return Err(error); + } + if let Err(err) = append_workflow_event( + run_store.as_ref(), + &run_id, + &WorkflowRunEvent::RunStarting { + reason: Some(StatusReason::SandboxInitializing), + }, + ) + .await { let error = FabroError::engine(err.to_string()); let _ = persist_detached_failure( @@ -132,22 +147,9 @@ pub(super) async fn execute_persisted_run( .await; return Err(error); } - append_workflow_event( - run_store.as_ref(), - &run_id, - &WorkflowRunEvent::RunStarting { - reason: Some(StatusReason::SandboxInitializing), - }, - ) - .await - .map_err(|err| FabroError::engine(err.to_string()))?; - let mut bootstrap_guard = DetachedRunBootstrapGuard::arm( - run_id, - run_dir, - Arc::clone(&run_store), - cancel_token.clone(), - ); + let mut bootstrap_guard = + DetachedRunBootstrapGuard::arm(run_id, run_dir, run_store.clone(), cancel_token.clone()); let persisted = match Persisted::load_from_store(services.run_store.as_ref(), run_dir).await { Ok(persisted) => persisted, @@ -185,7 +187,7 @@ pub(super) async fn execute_persisted_run( bootstrap_guard.defuse(); let mut completion_guard = - DetachedRunCompletionGuard::arm(run_id, Arc::clone(&run_store), cancel_token); + DetachedRunCompletionGuard::arm(run_id, run_store.clone(), cancel_token); let run_start = Instant::now(); let started = Box::pin(session.run(persisted, checkpoint)).await; @@ -211,15 +213,15 @@ pub(super) async fn execute_persisted_run( async fn persist_terminal_engine_failure( run_id: RunId, - run_store: &dyn RunStore, + run_store: &SlateRunStore, _run_dir: &Path, error: &FabroError, duration: Duration, ) { let engine_result: Result = Err(error.clone()); - let (final_status, failure_reason, run_status, status_reason) = + let (final_status, failure_reason, _run_status, status_reason) = classify_engine_result(&engine_result); - let conclusion = build_conclusion_from_store( + let _conclusion = build_conclusion_from_store( run_store, final_status, failure_reason, @@ -227,15 +229,6 @@ async fn persist_terminal_engine_failure( None, ) .await; - if let Err(err) = run_store.put_conclusion(&conclusion).await { - tracing::warn!(error = %err, "Failed to save terminal engine failure conclusion to store"); - } - if let Err(err) = run_store - .put_status(&run_status::RunStatusRecord::new(run_status, status_reason)) - .await - { - tracing::warn!(error = %err, "Failed to save terminal engine failure status to store"); - } if let Err(err) = append_workflow_event( run_store, &run_id, @@ -257,18 +250,18 @@ impl RunSession { let record = persisted.run_record(); let mut settings = record.settings.clone(); let working_directory = record.working_directory.clone(); - let git = services + let state = services .run_store - .get_start() + .state() .await - .map_err(|err| FabroError::engine(err.to_string()))? - .and_then(|start| { - start.run_branch.as_ref().map(|_| GitCheckpointOptions { - base_sha: start.base_sha.clone(), - run_branch: start.run_branch.clone(), - meta_branch: Some(MetadataStore::branch_name(&record.run_id.to_string())), - }) - }); + .map_err(|err| FabroError::engine(err.to_string()))?; + let git = state.start.and_then(|start| { + start.run_branch.as_ref().map(|_| GitCheckpointOptions { + base_sha: start.base_sha.clone(), + run_branch: start.run_branch.clone(), + meta_branch: Some(MetadataStore::branch_name(&record.run_id.to_string())), + }) + }); if let Some(env) = settings .sandbox @@ -498,12 +491,12 @@ impl RunSession { }); } - let store_progress_logger = StoreProgressLogger::new(Arc::clone(&self.run_store)); + let store_progress_logger = StoreProgressLogger::new(self.run_store.clone()); store_progress_logger.register(self.emitter.as_ref()); let init_options = InitOptions { run_id: record.run_id, - run_store: Arc::clone(&self.run_store), + run_store: self.run_store.clone(), dry_run: run_options.dry_run_enabled(), emitter: self.emitter, sandbox: self.sandbox, @@ -545,7 +538,7 @@ impl RunSession { let retro_opts = RetroOptions { run_id: executed.run_options.run_id, - run_store: Arc::clone(&executed.run_store), + run_store: executed.run_store.clone(), workflow_name: executed.graph.name.clone(), goal: executed.graph.goal().to_string(), run_dir: executed.run_options.run_dir.clone(), @@ -566,7 +559,7 @@ impl RunSession { let finalize_opts = FinalizeOptions { run_dir: retroed.run_options.run_dir.clone(), run_id: retroed.run_options.run_id, - run_store: Arc::clone(&retroed.run_store), + run_store: retroed.run_store.clone(), workflow_name: retroed.graph.name.clone(), hook_runner: retroed.hook_runner.clone(), preserve_sandbox: self.preserve_sandbox, @@ -574,7 +567,7 @@ impl RunSession { }; let pr_opts = PullRequestOptions { run_dir: retroed.run_options.run_dir.clone(), - run_store: Arc::clone(&retroed.run_store), + run_store: retroed.run_store.clone(), pr_config: self.pr_config, github_app: self.pr_github_app, origin_url: self.pr_origin_url, @@ -599,7 +592,7 @@ impl RunSession { struct DetachedRunBootstrapGuard { run_id: RunId, - run_store: Arc, + run_store: RunStoreHandle, cancel_token: Option>, active: bool, } @@ -608,7 +601,7 @@ impl DetachedRunBootstrapGuard { fn arm( run_id: RunId, _run_dir: &Path, - run_store: Arc, + run_store: RunStoreHandle, cancel_token: Option>, ) -> Self { Self { @@ -637,15 +630,9 @@ impl Drop for DetachedRunBootstrapGuard { StatusReason::SandboxInitFailed }; let run_id = self.run_id; - let run_store = Arc::clone(&self.run_store); + let run_store = self.run_store.clone(); if let Ok(handle) = Handle::try_current() { handle.spawn(async move { - let _ = run_store - .put_status(&run_status::RunStatusRecord::new( - RunStatus::Failed, - Some(reason), - )) - .await; let _ = append_workflow_event( run_store.as_ref(), &run_id, @@ -667,7 +654,7 @@ const POSTRUN_ABORTED_MESSAGE: &str = "Run aborted before post-run finalization const POSTRUN_CANCELLED_MESSAGE: &str = "Run cancelled before post-run finalization completed."; struct DetachedRunCompletionGuard { - run_store: Arc, + run_store: RunStoreHandle, run_id: RunId, cancel_token: Option>, active: bool, @@ -676,7 +663,7 @@ struct DetachedRunCompletionGuard { impl DetachedRunCompletionGuard { fn arm( run_id: RunId, - run_store: Arc, + run_store: RunStoreHandle, cancel_token: Option>, ) -> Self { Self { @@ -740,16 +727,10 @@ impl Drop for DetachedRunCompletionGuard { Some((self.run_id, line)) } }; - let run_store = Arc::clone(&self.run_store); + let run_store = self.run_store.clone(); let run_id = self.run_id; if let Ok(handle) = Handle::try_current() { handle.spawn(async move { - let _ = run_store - .put_status(&run_status::RunStatusRecord::new( - RunStatus::Failed, - Some(reason), - )) - .await; let _ = append_workflow_event( run_store.as_ref(), &run_id, @@ -761,15 +742,6 @@ impl Drop for DetachedRunCompletionGuard { }, ) .await; - if let Err(err) = run_store - .put_conclusion(&build_failure_conclusion(message)) - .await - { - tracing::warn!( - error = %err, - "Failed to save post-run abort conclusion to store" - ); - } if let Some((run_id, line)) = serialized_notice.or_else(|| { let envelope = canonicalize_event( &run_id, @@ -802,7 +774,7 @@ impl Drop for DetachedRunCompletionGuard { async fn persist_detached_failure( run_id: RunId, - run_store: &dyn RunStore, + run_store: &SlateRunStore, run_dir: &Path, phase: &'static str, reason: StatusReason, @@ -830,19 +802,6 @@ async fn persist_detached_failure( ) .map_err(|err| FabroError::Io(err.to_string()))?; - let conclusion = build_failure_conclusion(&message); - if let Err(err) = run_store.put_conclusion(&conclusion).await { - tracing::warn!(error = %err, "Failed to save detached failure conclusion to store"); - } - if let Err(err) = run_store - .put_status(&run_status::RunStatusRecord::new( - RunStatus::Failed, - Some(reason), - )) - .await - { - tracing::warn!(error = %err, "Failed to save detached failure status to store"); - } if let Err(err) = append_workflow_event( run_store, &run_id, @@ -879,33 +838,17 @@ async fn persist_detached_failure( Ok(()) } -fn build_failure_conclusion(message: &str) -> Conclusion { - Conclusion { - timestamp: Utc::now(), - status: StageStatus::Fail, - duration_ms: 0, - failure_reason: Some(message.to_string()), - final_git_commit_sha: None, - stages: vec![], - total_cost: None, - total_retries: 0, - total_input_tokens: 0, - total_output_tokens: 0, - total_cache_read_tokens: 0, - total_cache_write_tokens: 0, - total_reasoning_tokens: 0, - has_pricing: false, - } -} - #[cfg(test)] mod tests { use std::collections::HashMap; + use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; + use std::time::Duration; use chrono::Utc; - use fabro_store::{InMemoryStore, Store}; + use fabro_store::{SlateStore, StoreHandle}; use fabro_types::{Settings, fixtures}; + use object_store::memory::InMemory; use super::*; use crate::context::Context; @@ -923,8 +866,16 @@ mod tests { start -> exit }"#; - async fn persisted_workflow(dot: &str, run_dir: &Path) -> (Persisted, InMemoryStore) { - let store = InMemoryStore::default(); + fn memory_store() -> StoreHandle { + Arc::new(SlateStore::new( + Arc::new(InMemory::new()), + "", + Duration::from_millis(1), + )) + } + + async fn persisted_workflow(dot: &str, run_dir: &Path) -> (Persisted, StoreHandle) { + let store = memory_store(); let created = crate::operations::create( &store, crate::operations::CreateRunInput { @@ -960,7 +911,7 @@ mod tests { } async fn test_start_services( - store: &InMemoryStore, + store: &SlateStore, _run_dir: &Path, emitter: Arc, registry: Arc, @@ -1047,7 +998,7 @@ mod tests { assert_eq!(started.finalized.conclusion.status, StageStatus::Success); let run_store = store.open_run(&fixtures::RUN_1).await.unwrap(); - assert!(run_store.get_conclusion().await.unwrap().is_some()); + assert!(run_store.state().await.unwrap().conclusion.is_some()); } #[tokio::test] @@ -1089,7 +1040,7 @@ mod tests { let (_persisted, store) = persisted_workflow(MINIMAL_DOT, &run_dir).await; let services = test_start_services(&store, &run_dir, emitter, registry).await; - // Write a checkpoint to the store (not disk) so start() sees it + // Seed an authoritative checkpoint event so start() sees it let checkpoint = Checkpoint::from_context( &Context::new(), "start", @@ -1101,11 +1052,41 @@ mod tests { HashMap::new(), HashMap::new(), ); - services - .run_store - .put_checkpoint(&checkpoint) - .await - .unwrap(); + append_workflow_event( + services.run_store.as_ref(), + &services.run_id, + &WorkflowRunEvent::CheckpointCompleted { + node_id: checkpoint.current_node.clone(), + status: checkpoint + .node_outcomes + .get(&checkpoint.current_node) + .map_or_else( + || "success".to_string(), + |outcome| outcome.status.to_string(), + ), + current_node: checkpoint.current_node.clone(), + completed_nodes: checkpoint.completed_nodes.clone(), + node_retries: checkpoint.node_retries.clone().into_iter().collect(), + context_values: checkpoint.context_values.clone().into_iter().collect(), + node_outcomes: checkpoint.node_outcomes.clone().into_iter().collect(), + next_node_id: checkpoint.next_node_id.clone(), + git_commit_sha: checkpoint.git_commit_sha.clone(), + loop_failure_signatures: checkpoint + .loop_failure_signatures + .iter() + .map(|(sig, count)| (sig.to_string(), *count)) + .collect(), + restart_failure_signatures: checkpoint + .restart_failure_signatures + .iter() + .map(|(sig, count)| (sig.to_string(), *count)) + .collect(), + node_visits: checkpoint.node_visits.clone().into_iter().collect(), + diff: None, + }, + ) + .await + .unwrap(); let result = start(&run_dir, services).await; diff --git a/lib/crates/fabro-workflow/src/pipeline/execute.rs b/lib/crates/fabro-workflow/src/pipeline/execute.rs index c1349ab56..4acd795d5 100644 --- a/lib/crates/fabro-workflow/src/pipeline/execute.rs +++ b/lib/crates/fabro-workflow/src/pipeline/execute.rs @@ -73,7 +73,7 @@ pub async fn execute(init: Initialized) -> Executed { registry, emitter: Arc::clone(&emitter), sandbox: Arc::clone(&sandbox), - run_store: Arc::clone(&run_store), + run_store: run_store.clone(), git_state: std::sync::RwLock::new(git_state), hook_runner: hook_runner.clone(), env, @@ -93,7 +93,7 @@ pub async fn execute(init: Initialized) -> Executed { &sandbox, graph_arc, &run_options.run_dir, - Arc::clone(&run_store), + run_store.clone(), &settings_arc, checkpoint.is_some(), on_node, diff --git a/lib/crates/fabro-workflow/src/pipeline/execute/tests.rs b/lib/crates/fabro-workflow/src/pipeline/execute/tests.rs index 74cbc4b6f..8b5d1c054 100644 --- a/lib/crates/fabro-workflow/src/pipeline/execute/tests.rs +++ b/lib/crates/fabro-workflow/src/pipeline/execute/tests.rs @@ -13,13 +13,14 @@ use fabro_graphviz::graph::{AttrValue, Edge, Graph, Node}; use fabro_hooks::HookSettings; use fabro_interview::AutoApproveInterviewer; use fabro_sandbox::SandboxSpec; -use fabro_store::InMemoryStore; +use fabro_store::{SlateStore, StoreHandle}; use fabro_types::{RunId, Settings, fixtures}; +use object_store::memory::InMemory; use super::*; use crate::context::{self, Context}; use crate::error::FabroError; -use crate::event::{EventEmitter, RunEventEnvelope}; +use crate::event::{EventEmitter, RunEventEnvelope, StoreProgressLogger}; use crate::handler::start::StartHandler; use crate::handler::{Handler as HandlerTrait, HandlerRegistry}; use crate::outcome::{Outcome, OutcomeExt, StageStatus}; @@ -156,8 +157,12 @@ fn test_lifecycle(setup_commands: Vec) -> LifecycleOptions { } } -async fn test_run_store(_run_dir: &Path, run_id: &RunId) -> Arc { - let store: &dyn fabro_store::Store = &InMemoryStore::default(); +async fn test_run_store(_run_dir: &Path, run_id: &RunId) -> fabro_store::RunStoreHandle { + let store: StoreHandle = Arc::new(SlateStore::new( + Arc::new(InMemory::new()), + "", + Duration::from_millis(1), + )); store .create_run(run_id, chrono::Utc::now(), None) .await @@ -175,13 +180,17 @@ async fn execute_test_run_with_options( ) -> Executed { let run_id_value = run_options.run_id; let git_options = run_options.git.clone(); + let run_store = test_run_store(&run_options.run_dir, &run_id_value).await; + let emitter = test_emitter_arc("test-run"); + let store_logger = StoreProgressLogger::new(run_store.clone()); + store_logger.register(&emitter); let initialized = initialize( persisted_workflow(graph, String::new(), &run_options.run_dir, run_id_value), InitOptions { run_id: run_id_value, - run_store: test_run_store(&run_options.run_dir, &run_id_value).await, + run_store, dry_run: false, - emitter: test_emitter_arc("test-run"), + emitter, sandbox: SandboxSpec::Local { working_directory: std::env::current_dir().unwrap(), }, @@ -217,7 +226,9 @@ async fn execute_test_run_with_options( .await .unwrap(); - execute(initialized).await + let executed = execute(initialized).await; + store_logger.flush().await; + executed } #[tokio::test] @@ -503,7 +514,15 @@ async fn execute_runs_simple_workflow() { async fn execute_saves_checkpoint() { let dir = tempfile::tempdir().unwrap(); let executed = execute_test_run(dir.path(), simple_graph(), "test-run").await; - assert!(executed.run_store.get_checkpoint().await.unwrap().is_some()); + assert!( + executed + .run_store + .state() + .await + .unwrap() + .checkpoint + .is_some() + ); } #[tokio::test] @@ -547,7 +566,13 @@ async fn execute_error_when_no_start_node() { async fn execute_mirrors_graph_goal_to_context() { let dir = tempfile::tempdir().unwrap(); let executed = execute_test_run(dir.path(), simple_graph(), "test-run").await; - let cp = executed.run_store.get_checkpoint().await.unwrap().unwrap(); + let cp = executed + .run_store + .state() + .await + .unwrap() + .checkpoint + .unwrap(); assert_eq!( cp.context_values.get(context::keys::GRAPH_GOAL), Some(&serde_json::json!("Run tests")) @@ -587,7 +612,13 @@ async fn execute_conditional_routing_uses_unconditional_success_path() { g.edges.push(Edge::new("path_b", "exit")); let executed = execute_test_run(dir.path(), g, "test-run").await; - let cp = executed.run_store.get_checkpoint().await.unwrap().unwrap(); + let cp = executed + .run_store + .state() + .await + .unwrap() + .checkpoint + .unwrap(); assert!(cp.completed_nodes.contains(&"path_b".to_string())); assert!(!cp.completed_nodes.contains(&"path_a".to_string())); } @@ -603,7 +634,8 @@ async fn execute_writes_start_json_and_node_status() { }); let executed = execute_test_run_with_options(run_options, simple_graph(), None).await; - let start = executed.run_store.get_start().await.unwrap().unwrap(); + let state = executed.run_store.state().await.unwrap(); + let start = state.start.as_ref().unwrap(); assert_eq!(start.run_id, test_run_id("test-run")); assert_eq!( start.run_branch.as_deref(), @@ -611,15 +643,13 @@ async fn execute_writes_start_json_and_node_status() { ); assert_eq!(start.base_sha.as_deref(), Some("abc123")); - let node = executed - .run_store - .get_node(&fabro_store::NodeVisitRef { + let node = state + .node(&fabro_store::NodeVisitRef { node_id: "start", visit: 1, }) - .await .unwrap(); - assert_eq!(node.status.unwrap().status, StageStatus::Success); + assert_eq!(node.status.as_ref().unwrap().status, StageStatus::Success); } #[tokio::test] @@ -668,15 +698,15 @@ async fn timeout_causes_fail_status_record() { Some(Arc::new(registry)), ) .await; - let status = executed - .run_store - .get_node(&fabro_store::NodeVisitRef { + let state = executed.run_store.state().await.unwrap(); + let status = state + .node(&fabro_store::NodeVisitRef { node_id: "work", visit: 1, }) - .await .unwrap() .status + .as_ref() .unwrap(); assert_eq!(status.status, StageStatus::Fail); } diff --git a/lib/crates/fabro-workflow/src/pipeline/finalize.rs b/lib/crates/fabro-workflow/src/pipeline/finalize.rs index 2a9d5d080..a65f51125 100644 --- a/lib/crates/fabro-workflow/src/pipeline/finalize.rs +++ b/lib/crates/fabro-workflow/src/pipeline/finalize.rs @@ -10,7 +10,7 @@ use crate::run_options::RunOptions; use crate::run_status::{RunStatus, StatusReason}; use crate::sandbox_git::git_push_host; use fabro_hooks::{HookContext, HookEvent, HookRunner}; -use fabro_store::RunStore; +use fabro_store::SlateRunStore; use super::types::{Concluded, FinalizeOptions, Retroed}; @@ -63,7 +63,7 @@ pub fn classify_engine_result( } pub(crate) async fn build_conclusion_from_store( - run_store: &dyn RunStore, + run_store: &SlateRunStore, status: StageStatus, failure_reason: Option, run_duration_ms: u64, @@ -181,7 +181,7 @@ pub fn persist_terminal_outcome( pub async fn write_finalize_commit( run_options: &RunOptions, _run_dir: &Path, - run_store: &dyn RunStore, + run_store: &SlateRunStore, ) { let (Some(meta_branch), Some(repo_path)) = ( run_options @@ -275,7 +275,7 @@ pub async fn finalize( retro: _, } = retroed; - let (final_status, failure_reason, run_status, status_reason) = + let (final_status, failure_reason, _run_status, _status_reason) = classify_engine_result(&outcome); let conclusion = build_conclusion_from_store( options.run_store.as_ref(), @@ -324,20 +324,6 @@ pub async fn finalize( ); } - if let Err(err) = options.run_store.put_conclusion(&conclusion).await { - tracing::warn!(error = %err, "Failed to save conclusion to store"); - } - if let Err(err) = options - .run_store - .put_status(&fabro_types::RunStatusRecord::new( - run_status, - status_reason, - )) - .await - { - tracing::warn!(error = %err, "Failed to save terminal status to store"); - } - Ok(Concluded { run_id: run_options.run_id, outcome, @@ -353,13 +339,16 @@ pub async fn finalize( mod tests { use std::collections::HashMap; use std::sync::Arc; + use std::time::Duration; use chrono::Utc; use fabro_graphviz::graph::Graph; - use fabro_store::{InMemoryStore, Store}; + use fabro_store::SlateStore; use fabro_types::{RunId, Settings, fixtures}; + use object_store::memory::InMemory; use super::*; + use crate::event::StoreProgressLogger; use crate::pipeline::types::Retroed; use crate::run_options::RunOptions; @@ -383,12 +372,20 @@ mod tests { } } + fn test_store() -> Arc { + Arc::new(SlateStore::new( + Arc::new(InMemory::new()), + "", + Duration::from_millis(1), + )) + } + #[tokio::test] async fn finalize_writes_conclusion_json() { let temp = tempfile::tempdir().unwrap(); let run_dir = temp.path().join("run"); std::fs::create_dir_all(&run_dir).unwrap(); - let inner_store = InMemoryStore::default() + let inner_store = test_store() .create_run( &test_run_id(), Utc::now(), @@ -396,14 +393,17 @@ mod tests { ) .await .unwrap(); - let run_store: Arc = inner_store; + let run_store = inner_store; + let emitter = Arc::new(EventEmitter::new(test_run_id())); + let store_logger = StoreProgressLogger::new(run_store.clone()); + store_logger.register(&emitter); let retroed = Retroed { graph: Graph::new("test"), outcome: Ok(Outcome::success()), run_options: test_run_options(&run_dir), - run_store: Arc::clone(&run_store), + run_store: run_store.clone(), hook_runner: None, - emitter: Arc::new(EventEmitter::default()), + emitter, sandbox: Arc::new(fabro_agent::LocalSandbox::new( std::env::current_dir().unwrap(), )), @@ -416,7 +416,7 @@ mod tests { &FinalizeOptions { run_dir: run_dir.clone(), run_id: test_run_id(), - run_store: Arc::clone(&run_store), + run_store: run_store.clone(), workflow_name: "test".to_string(), hook_runner: None, preserve_sandbox: true, @@ -425,8 +425,8 @@ mod tests { ) .await .unwrap(); + store_logger.flush().await; - assert!(run_store.get_conclusion().await.unwrap().is_some()); assert_eq!(concluded.conclusion.status, StageStatus::Success); } } diff --git a/lib/crates/fabro-workflow/src/pipeline/initialize.rs b/lib/crates/fabro-workflow/src/pipeline/initialize.rs index 8ba4955d4..9cb6fc46f 100644 --- a/lib/crates/fabro-workflow/src/pipeline/initialize.rs +++ b/lib/crates/fabro-workflow/src/pipeline/initialize.rs @@ -520,9 +520,6 @@ pub async fn initialize( host_working_directory: sandbox_record.host_working_directory.clone(), container_mount_point: sandbox_record.container_mount_point.clone(), }); - if let Err(err) = options.run_store.put_sandbox(&sandbox_record).await { - tracing::warn!(error = %err, "Failed to save sandbox record to store"); - } let env = build_sandbox_env( &options.sandbox_env, @@ -670,15 +667,18 @@ pub async fn initialize( mod tests { use std::collections::HashMap; use std::sync::Arc; + use std::time::Duration; use chrono::Utc; use fabro_graphviz::graph::{AttrValue, Edge, Graph, Node}; use fabro_interview::AutoApproveInterviewer; use fabro_sandbox::SandboxSpec; - use fabro_store::InMemoryStore; + use fabro_store::{SlateStore, StoreHandle}; use fabro_types::{RunId, Settings, fixtures}; + use object_store::memory::InMemory; use super::*; + use crate::event::StoreProgressLogger; use crate::pipeline::types::InitOptions; use crate::records::RunRecord; use crate::run_options::RunOptions; @@ -687,6 +687,14 @@ mod tests { fixtures::RUN_1 } + fn memory_store() -> StoreHandle { + Arc::new(SlateStore::new( + Arc::new(InMemory::new()), + "", + Duration::from_millis(1), + )) + } + fn simple_graph() -> (Graph, String) { let source = r"digraph test { start [shape=Mdiamond]; @@ -754,14 +762,14 @@ mod tests { std::fs::create_dir_all(&run_dir).unwrap(); let (graph, source) = simple_graph(); let persisted = test_persisted(graph, source.clone(), &run_dir); - let emitter = Arc::new(crate::event::EventEmitter::default()); + let emitter = Arc::new(crate::event::EventEmitter::new(test_run_id())); let initialized = initialize( persisted, InitOptions { run_id: test_run_id(), run_store: { - let store: &dyn fabro_store::Store = &InMemoryStore::default(); + let store = memory_store(); let inner = store .create_run( &test_run_id(), @@ -829,24 +837,29 @@ mod tests { std::fs::create_dir_all(&run_dir).unwrap(); let (graph, source) = simple_graph(); let persisted = test_persisted(graph, source, &run_dir); - let emitter = Arc::new(crate::event::EventEmitter::default()); + let emitter = Arc::new(crate::event::EventEmitter::new(test_run_id())); + let store = memory_store(); + let run_store = store + .create_run( + &test_run_id(), + chrono::Utc::now(), + Some(run_dir.to_string_lossy().as_ref()), + ) + .await + .unwrap(); + let store_logger = StoreProgressLogger::new(run_store.clone()); + let seen = Arc::new(std::sync::Mutex::new(Vec::new())); + emitter.on_event({ + let seen = Arc::clone(&seen); + move |event| seen.lock().unwrap().push(event.event.clone()) + }); + store_logger.register(&emitter); let initialized = initialize( persisted, InitOptions { run_id: test_run_id(), - run_store: { - let store: &dyn fabro_store::Store = &InMemoryStore::default(); - let inner = store - .create_run( - &test_run_id(), - chrono::Utc::now(), - Some(run_dir.to_string_lossy().as_ref()), - ) - .await - .unwrap(); - inner - }, + run_store, dry_run: false, emitter, sandbox: SandboxSpec::Local { @@ -883,8 +896,14 @@ mod tests { ) .await .unwrap(); + store_logger.flush().await; - assert!(initialized.run_store.get_sandbox().await.unwrap().is_some()); assert_eq!(initialized.run_options.run_dir, run_dir); + assert!( + seen.lock() + .unwrap() + .iter() + .any(|event| event == "sandbox.initialized") + ); } } diff --git a/lib/crates/fabro-workflow/src/pipeline/persist.rs b/lib/crates/fabro-workflow/src/pipeline/persist.rs index 2d4ff94c0..f82e3b8cd 100644 --- a/lib/crates/fabro-workflow/src/pipeline/persist.rs +++ b/lib/crates/fabro-workflow/src/pipeline/persist.rs @@ -1,6 +1,6 @@ use std::path::Path; -use fabro_store::RunStore; +use fabro_store::SlateRunStore; use crate::error::FabroError; @@ -26,7 +26,7 @@ pub(crate) fn persist( } pub(crate) async fn load_from_store( - run_store: &dyn RunStore, + run_store: &SlateRunStore, run_dir: &Path, ) -> Result { let state = run_store @@ -55,12 +55,24 @@ mod tests { use chrono::Utc; use fabro_graphviz::graph::{AttrValue, Edge, Graph, Node}; - use fabro_store::{InMemoryStore, Store}; + use fabro_store::{RunStoreHandle, SlateStore, StoreHandle}; use fabro_types::{Settings, fixtures}; + use object_store::memory::InMemory; + use std::sync::Arc; + use std::time::Duration; use super::*; + use crate::event::{WorkflowRunEvent, append_workflow_event}; use crate::records::RunRecord; + fn memory_store() -> StoreHandle { + Arc::new(SlateStore::new( + Arc::new(InMemory::new()), + "", + Duration::from_millis(1), + )) + } + fn graph_and_source() -> (Graph, String) { let source = r#"digraph test { graph [goal="Ship feature"]; @@ -130,8 +142,8 @@ mod tests { run_dir: &Path, record: &RunRecord, source: Option<&str>, - ) -> std::sync::Arc { - let store = InMemoryStore::default(); + ) -> RunStoreHandle { + let store = memory_store(); let run_store = store .create_run( &record.run_id, @@ -140,10 +152,26 @@ mod tests { ) .await .unwrap(); - run_store.put_run(record).await.unwrap(); - if let Some(source) = source { - run_store.put_graph(source).await.unwrap(); - } + append_workflow_event( + run_store.as_ref(), + &record.run_id, + &WorkflowRunEvent::RunCreated { + run_id: record.run_id, + settings: serde_json::to_value(&record.settings).unwrap(), + graph: serde_json::to_value(&record.graph).unwrap(), + workflow_source: source.map(ToOwned::to_owned), + workflow_config: None, + labels: record.labels.clone().into_iter().collect(), + run_dir: run_dir.to_string_lossy().to_string(), + working_directory: record.working_directory.display().to_string(), + host_repo_path: record.host_repo_path.clone(), + base_branch: record.base_branch.clone(), + workflow_slug: record.workflow_slug.clone(), + db_prefix: None, + }, + ) + .await + .unwrap(); run_store } @@ -214,10 +242,23 @@ mod tests { let run_store = seeded_store(&run_dir, &expected, Some(&source)).await; let loaded = load_from_store(run_store.as_ref(), &run_dir).await.unwrap(); - assert_eq!( - serde_json::to_value(loaded.run_record()).unwrap(), - serde_json::to_value(expected).unwrap() + let loaded_record = loaded.run_record(); + assert_eq!(loaded_record.run_id, expected.run_id); + assert!( + (loaded_record.created_at.timestamp_millis() - expected.created_at.timestamp_millis()) + .abs() + <= 1 ); + assert_eq!(loaded_record.settings, expected.settings); + assert_eq!( + serde_json::to_value(&loaded_record.graph).unwrap(), + serde_json::to_value(&expected.graph).unwrap() + ); + assert_eq!(loaded_record.workflow_slug, expected.workflow_slug); + assert_eq!(loaded_record.working_directory, expected.working_directory); + assert_eq!(loaded_record.host_repo_path, expected.host_repo_path); + assert_eq!(loaded_record.base_branch, expected.base_branch); + assert_eq!(loaded_record.labels, expected.labels); assert_eq!(loaded.source(), source); assert!(loaded.diagnostics().is_empty()); } diff --git a/lib/crates/fabro-workflow/src/pipeline/pull_request.rs b/lib/crates/fabro-workflow/src/pipeline/pull_request.rs index 130081daf..da9cb9b33 100644 --- a/lib/crates/fabro-workflow/src/pipeline/pull_request.rs +++ b/lib/crates/fabro-workflow/src/pipeline/pull_request.rs @@ -1,7 +1,7 @@ use std::path::Path; use fabro_config::run::MergeStrategy; -use fabro_store::RunStore; +use fabro_store::SlateRunStore; use fabro_types::PullRequestRecord; use tracing::{debug, info}; @@ -292,7 +292,7 @@ fn emit_run_notice( }); } -async fn load_pull_request_diff(run_store: &dyn RunStore, run_dir: &Path) -> String { +async fn load_pull_request_diff(run_store: &SlateRunStore, run_dir: &Path) -> String { let _ = run_dir; run_store .state() @@ -311,7 +311,7 @@ pub async fn build_pr_body( diff: &str, goal: &str, model: &str, - run_store: &dyn RunStore, + run_store: &SlateRunStore, run_dir: &Path, conclusion: Option<&Conclusion>, ) -> Result { @@ -340,7 +340,9 @@ pub async fn build_pr_body( .ok(); let retro = run_state.as_ref().and_then(|state| state.retro.clone()); let run_record = run_state.as_ref().and_then(|state| state.run.clone()); - let dot_source = run_state.as_ref().and_then(|state| state.graph_source.clone()); + let dot_source = run_state + .as_ref() + .and_then(|state| state.graph_source.clone()); // Build LLM prompt let system = if plan_text.is_some() { @@ -418,7 +420,7 @@ pub async fn maybe_open_pull_request( model: &str, draft: bool, auto_merge: Option, - run_store: &dyn RunStore, + run_store: &SlateRunStore, run_dir: &Path, conclusion: Option<&Conclusion>, ) -> Result, String> { @@ -489,11 +491,6 @@ pub async fn maybe_open_pull_request( title, }; - run_store - .put_pull_request(&record) - .await - .map_err(|err| format!("failed to persist pull request in run store: {err}"))?; - Ok(Some(record)) } @@ -522,7 +519,8 @@ pub async fn pull_request(concluded: Concluded, options: &PullRequestOptions) -> result.status, StageStatus::Success | StageStatus::PartialSuccess ) { - let diff = load_pull_request_diff(options.run_store.as_ref(), &options.run_dir).await; + let diff = + load_pull_request_diff(options.run_store.as_ref(), &options.run_dir).await; if let (Some(base_branch), Some(run_branch), Some(creds), Some(origin)) = ( &run_options.base_branch, pushed_branch.as_deref(), @@ -598,6 +596,7 @@ mod tests { use std::sync::{Arc, Once}; use super::*; + use crate::event::{WorkflowRunEvent, append_workflow_event}; use crate::records::StageSummary; use chrono::Utc; use fabro_graphviz::graph::Graph; @@ -609,9 +608,11 @@ mod tests { use fabro_retro::retro::{ AggregateStats, FrictionKind, FrictionPoint, OpenItem, OpenItemKind, StageRetro, }; - use fabro_store::{InMemoryStore, Store}; + use fabro_store::SlateStore; use fabro_types::{RunRecord, Settings, fixtures}; use futures::stream; + use object_store::memory::InMemory; + use std::time::Duration; struct MockProvider { response_text: String, @@ -684,6 +685,14 @@ mod tests { } } + fn test_store() -> Arc { + Arc::new(SlateStore::new( + Arc::new(InMemory::new()), + "", + Duration::from_millis(1), + )) + } + fn install_mock_llm() { static INIT: Once = Once::new(); @@ -1034,7 +1043,7 @@ mod tests { install_mock_llm(); let tmp = tempfile::tempdir().unwrap(); - let store = InMemoryStore::default(); + let store = test_store(); let run_store = store .create_run( &fixtures::RUN_1, @@ -1066,7 +1075,7 @@ mod tests { install_mock_llm(); let tmp = tempfile::tempdir().unwrap(); - let store = InMemoryStore::default(); + let store = test_store(); let created_at = Utc::now(); let run_store = store .create_run( @@ -1077,32 +1086,55 @@ mod tests { .await .unwrap(); - run_store - .put_run(&RunRecord { + let run_record = RunRecord { + run_id: fixtures::RUN_1, + created_at, + settings: Settings::default(), + graph: Graph::new("test"), + workflow_slug: Some("test".to_string()), + working_directory: PathBuf::from("/tmp/project"), + host_repo_path: Some("/tmp/project".to_string()), + base_branch: Some("main".to_string()), + labels: HashMap::new(), + }; + append_workflow_event( + run_store.as_ref(), + &fixtures::RUN_1, + &WorkflowRunEvent::RunCreated { run_id: fixtures::RUN_1, - created_at, - settings: Settings::default(), - graph: Graph::new("test"), - workflow_slug: Some("test".to_string()), - working_directory: PathBuf::from("/tmp/project"), - host_repo_path: Some("/tmp/project".to_string()), - base_branch: Some("main".to_string()), - labels: HashMap::new(), - }) - .await - .unwrap(); - run_store - .put_graph("digraph test { plan -> code }") - .await - .unwrap(); - run_store.put_retro(&make_test_retro()).await.unwrap(); + settings: serde_json::to_value(&run_record.settings).unwrap(), + graph: serde_json::to_value(&run_record.graph).unwrap(), + workflow_source: Some("digraph test { plan -> code }".to_string()), + workflow_config: None, + labels: run_record.labels.clone().into_iter().collect(), + run_dir: tmp.path().display().to_string(), + working_directory: run_record.working_directory.display().to_string(), + host_repo_path: run_record.host_repo_path.clone(), + base_branch: run_record.base_branch.clone(), + workflow_slug: run_record.workflow_slug.clone(), + db_prefix: None, + }, + ) + .await + .unwrap(); + append_workflow_event( + run_store.as_ref(), + &fixtures::RUN_1, + &WorkflowRunEvent::RetroCompleted { + duration_ms: 1, + response: Some(String::new()), + retro: Some(serde_json::to_value(make_test_retro()).unwrap()), + }, + ) + .await + .unwrap(); let conclusion = make_test_conclusion(); let body = build_pr_body( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "mock-model", - run_store.as_ref(), + run_store.as_ref(), tmp.path(), Some(&conclusion), ) @@ -1292,7 +1324,7 @@ mod tests { #[tokio::test] async fn empty_diff_returns_none() { let tmp = tempfile::tempdir().unwrap(); - let store = InMemoryStore::default(); + let store = test_store(); let run_store = store .create_run( &fixtures::RUN_1, @@ -1327,7 +1359,7 @@ mod tests { #[tokio::test] async fn load_pull_request_diff_uses_store_without_disk_patch() { let tmp = tempfile::tempdir().unwrap(); - let store = InMemoryStore::default(); + let store = test_store(); let created_at = Utc::now(); let run_store = store .create_run( @@ -1337,24 +1369,55 @@ mod tests { ) .await .unwrap(); - run_store - .put_run(&RunRecord { + let run_record = RunRecord { + run_id: fixtures::RUN_1, + created_at, + settings: Settings::default(), + graph: Graph::new("test"), + workflow_slug: None, + working_directory: tmp.path().to_path_buf(), + host_repo_path: None, + base_branch: None, + labels: std::collections::HashMap::new(), + }; + append_workflow_event( + run_store.as_ref(), + &fixtures::RUN_1, + &WorkflowRunEvent::RunCreated { run_id: fixtures::RUN_1, - created_at, - settings: Settings::default(), - graph: Graph::new("test"), - workflow_slug: None, - working_directory: tmp.path().to_path_buf(), + settings: serde_json::to_value(&run_record.settings).unwrap(), + graph: serde_json::to_value(&run_record.graph).unwrap(), + workflow_source: None, + workflow_config: None, + labels: run_record.labels.clone().into_iter().collect(), + run_dir: tmp.path().display().to_string(), + working_directory: tmp.path().display().to_string(), host_repo_path: None, base_branch: None, - labels: std::collections::HashMap::new(), - }) - .await - .unwrap(); - run_store - .put_final_patch("diff --git a/src/lib.rs b/src/lib.rs\n+fn from_store() {}\n") - .await - .unwrap(); + workflow_slug: None, + db_prefix: None, + }, + ) + .await + .unwrap(); + append_workflow_event( + run_store.as_ref(), + &fixtures::RUN_1, + &WorkflowRunEvent::WorkflowRunCompleted { + duration_ms: 1, + artifact_count: 0, + status: "success".to_string(), + reason: None, + total_cost: None, + final_git_commit_sha: None, + final_patch: Some( + "diff --git a/src/lib.rs b/src/lib.rs\n+fn from_store() {}\n".to_string(), + ), + usage: None, + }, + ) + .await + .unwrap(); let diff = load_pull_request_diff(run_store.as_ref(), tmp.path()).await; diff --git a/lib/crates/fabro-workflow/src/pipeline/retro.rs b/lib/crates/fabro-workflow/src/pipeline/retro.rs index ccde6ac56..dd971c589 100644 --- a/lib/crates/fabro-workflow/src/pipeline/retro.rs +++ b/lib/crates/fabro-workflow/src/pipeline/retro.rs @@ -8,8 +8,6 @@ use fabro_retro::retro_agent::{ use super::types::{Executed, RetroOptions, Retroed}; use crate::event::WorkflowRunEvent; -#[cfg(test)] -use crate::records::RunRecord; pub async fn run_retro(options: &RetroOptions, dry_run: bool) -> Option { let state = match options.run_store.state().await { @@ -56,10 +54,6 @@ pub async fn run_retro(options: &RetroOptions, dry_run: bool) -> Option { &stage_durations, ); - if let Err(err) = options.run_store.put_retro(&retro).await { - tracing::warn!(error = %err, "Failed to save initial retro to store"); - } - let retro_start = std::time::Instant::now(); let retro_prompt = build_retro_prompt(RETRO_DATA_DIR); if let Some(ref emitter) = options.emitter { @@ -115,9 +109,6 @@ pub async fn run_retro(options: &RetroOptions, dry_run: bool) -> Option { retro: serde_json::to_value(&retro).ok(), }); } - if let Err(err) = options.run_store.put_retro(&retro).await { - tracing::warn!(error = %err, "Failed to save retro with narrative to store"); - } } Err(e) => { if let Some(ref emitter) = options.emitter { @@ -178,17 +169,20 @@ pub async fn retro(executed: Executed, options: &RetroOptions) -> Retroed { mod tests { use std::collections::HashMap; use std::sync::{Arc, Mutex}; + use std::time::Duration; use chrono::Utc; use fabro_graphviz::graph::Graph; - use fabro_store::{InMemoryStore, Store}; + use fabro_store::SlateStore; use fabro_types::{RunId, Settings, fixtures}; + use object_store::memory::InMemory; use super::*; use crate::context::Context; use crate::event::EventEmitter; + use crate::event::{StoreProgressLogger, WorkflowRunEvent, append_workflow_event}; use crate::pipeline::types::Executed; - use crate::records::{Checkpoint, CheckpointExt}; + use crate::records::{Checkpoint, CheckpointExt, RunRecord}; use crate::run_options::RunOptions; fn test_run_id() -> RunId { @@ -214,12 +208,20 @@ mod tests { checkpoint } + fn test_store() -> Arc { + Arc::new(SlateStore::new( + Arc::new(InMemory::new()), + "", + Duration::from_millis(1), + )) + } + async fn test_run_store( run_dir: &std::path::Path, checkpoint: &Checkpoint, - ) -> Arc { + ) -> fabro_store::RunStoreHandle { let created_at = Utc::now(); - let inner = InMemoryStore::default() + let inner = test_store() .create_run( &test_run_id(), created_at, @@ -227,22 +229,69 @@ mod tests { ) .await .unwrap(); - let run_store: Arc = inner; - run_store - .put_run(&RunRecord { + let run_store = inner; + let run_record = RunRecord { + run_id: test_run_id(), + created_at, + settings: Settings::default(), + graph: Graph::new("test"), + workflow_slug: None, + working_directory: run_dir.to_path_buf(), + host_repo_path: None, + base_branch: None, + labels: std::collections::HashMap::new(), + }; + append_workflow_event( + run_store.as_ref(), + &test_run_id(), + &WorkflowRunEvent::RunCreated { run_id: test_run_id(), - created_at, - settings: Settings::default(), - graph: Graph::new("test"), - workflow_slug: None, - working_directory: run_dir.to_path_buf(), + settings: serde_json::to_value(&run_record.settings).unwrap(), + graph: serde_json::to_value(&run_record.graph).unwrap(), + workflow_source: None, + workflow_config: None, + labels: run_record.labels.clone().into_iter().collect(), + run_dir: run_dir.to_string_lossy().to_string(), + working_directory: run_dir.to_string_lossy().to_string(), host_repo_path: None, base_branch: None, - labels: std::collections::HashMap::new(), - }) - .await - .unwrap(); - run_store.put_checkpoint(checkpoint).await.unwrap(); + workflow_slug: None, + db_prefix: None, + }, + ) + .await + .unwrap(); + append_workflow_event( + run_store.as_ref(), + &test_run_id(), + &WorkflowRunEvent::CheckpointCompleted { + node_id: checkpoint.current_node.clone(), + status: "success".to_string(), + current_node: checkpoint.current_node.clone(), + completed_nodes: checkpoint.completed_nodes.clone(), + node_retries: checkpoint.node_retries.clone().into_iter().collect(), + context_values: checkpoint.context_values.clone().into_iter().collect(), + node_outcomes: checkpoint.node_outcomes.clone().into_iter().collect(), + next_node_id: checkpoint.next_node_id.clone(), + git_commit_sha: checkpoint.git_commit_sha.clone(), + loop_failure_signatures: checkpoint + .loop_failure_signatures + .clone() + .into_iter() + .map(|(signature, count)| (signature.to_string(), count)) + .collect(), + restart_failure_signatures: checkpoint + .restart_failure_signatures + .clone() + .into_iter() + .map(|(signature, count)| (signature.to_string(), count)) + .collect(), + node_visits: checkpoint.node_visits.clone().into_iter().collect(), + diff: None, + }, + ) + .await + .unwrap(); run_store } @@ -270,7 +319,9 @@ mod tests { let checkpoint = build_checkpoint(); let run_store = test_run_store(&run_dir, &checkpoint).await; - let emitter = Arc::new(EventEmitter::default()); + let emitter = Arc::new(EventEmitter::new(test_run_id())); + let store_logger = StoreProgressLogger::new(run_store.clone()); + store_logger.register(&emitter); let sandbox: Arc = Arc::new(fabro_agent::LocalSandbox::new( std::env::current_dir().unwrap(), )); @@ -278,7 +329,7 @@ mod tests { graph: Graph::new("test"), outcome: Ok(crate::outcome::Outcome::success()), run_options: test_run_options(&run_dir), - run_store: Arc::clone(&run_store), + run_store: run_store.clone(), hook_runner: None, emitter: Arc::clone(&emitter), sandbox: Arc::clone(&sandbox), @@ -308,8 +359,8 @@ mod tests { }, ) .await; + store_logger.flush().await; - assert!(retroed.run_store.get_retro().await.unwrap().is_some()); assert!(retroed.retro.is_some()); } diff --git a/lib/crates/fabro-workflow/src/pipeline/types.rs b/lib/crates/fabro-workflow/src/pipeline/types.rs index 23b02b45a..f9f05a277 100644 --- a/lib/crates/fabro-workflow/src/pipeline/types.rs +++ b/lib/crates/fabro-workflow/src/pipeline/types.rs @@ -11,7 +11,7 @@ use fabro_llm::Provider; use fabro_mcp::config::McpServerSettings; use fabro_model::FallbackTarget; use fabro_sandbox::SandboxSpec; -use fabro_store::RunStore; +use fabro_store::{RunStoreHandle, SlateRunStore}; use fabro_types::RunId; use fabro_validate::Diagnostic; @@ -195,7 +195,7 @@ impl Persisted { } pub async fn load_from_store( - run_store: &dyn RunStore, + run_store: &SlateRunStore, run_dir: &Path, ) -> Result { super::persist::load_from_store(run_store, run_dir).await @@ -227,7 +227,7 @@ pub struct DevcontainerSpec { pub struct InitOptions { pub run_id: RunId, - pub run_store: Arc, + pub run_store: RunStoreHandle, pub dry_run: bool, pub emitter: Arc, pub sandbox: SandboxSpec, @@ -251,7 +251,7 @@ pub struct Initialized { pub graph: Graph, pub source: String, pub run_options: RunOptions, - pub run_store: Arc, + pub run_store: RunStoreHandle, pub(crate) checkpoint: Option, pub(crate) seed_context: Option, pub emitter: Arc, @@ -272,7 +272,7 @@ pub struct Executed { pub graph: Graph, pub outcome: Result, pub run_options: RunOptions, - pub run_store: Arc, + pub run_store: RunStoreHandle, pub hook_runner: Option>, pub emitter: Arc, pub sandbox: Arc, @@ -289,7 +289,7 @@ pub struct Retroed { pub graph: Graph, pub outcome: Result, pub run_options: RunOptions, - pub run_store: Arc, + pub run_store: RunStoreHandle, pub hook_runner: Option>, pub emitter: Arc, pub sandbox: Arc, @@ -328,7 +328,7 @@ pub struct TransformOptions { /// Options for the RETRO phase. pub struct RetroOptions { pub run_id: RunId, - pub run_store: Arc, + pub run_store: RunStoreHandle, pub workflow_name: String, pub goal: String, pub run_dir: PathBuf, @@ -346,7 +346,7 @@ pub struct RetroOptions { pub struct FinalizeOptions { pub run_dir: PathBuf, pub run_id: RunId, - pub run_store: Arc, + pub run_store: RunStoreHandle, pub workflow_name: String, pub hook_runner: Option>, pub preserve_sandbox: bool, @@ -356,7 +356,7 @@ pub struct FinalizeOptions { /// Options for the PULL_REQUEST phase. pub struct PullRequestOptions { pub run_dir: PathBuf, - pub run_store: Arc, + pub run_store: RunStoreHandle, pub pr_config: Option, pub github_app: Option, pub origin_url: Option, diff --git a/lib/crates/fabro-workflow/src/run_lookup.rs b/lib/crates/fabro-workflow/src/run_lookup.rs index 50af6a59b..c7ff56fa4 100644 --- a/lib/crates/fabro-workflow/src/run_lookup.rs +++ b/lib/crates/fabro-workflow/src/run_lookup.rs @@ -3,7 +3,7 @@ use std::path::{Path, PathBuf}; use anyhow::{Context, Result, bail}; use chrono::{DateTime, Utc}; -use fabro_store::{ListRunsQuery, Store}; +use fabro_store::{ListRunsQuery, SlateStore}; use fabro_types::RunId; use serde::Serialize; @@ -170,7 +170,7 @@ fn scan_runs_inner(base: &Path, include_status: bool) -> Result> { Ok(runs) } -pub async fn scan_runs_combined(store: &dyn Store, base: &Path) -> Result> { +pub async fn scan_runs_combined(store: &SlateStore, base: &Path) -> Result> { let mut runs_by_id: HashMap = HashMap::new(); if let Ok(store_runs) = store.list_runs(&ListRunsQuery::default()).await { @@ -373,7 +373,7 @@ pub fn resolve_run(base: &Path, identifier: &str) -> Result { } pub async fn resolve_run_combined( - store: &dyn Store, + store: &SlateStore, base: &Path, identifier: &str, ) -> Result { @@ -438,15 +438,27 @@ fn run_id_matches(run_id: RunId, prefix: &str) -> bool { mod tests { use std::collections::HashMap; use std::path::PathBuf; + use std::sync::Arc; + use std::time::Duration; use chrono::Utc; use fabro_graphviz::graph::Graph; - use fabro_store::{InMemoryStore, Store}; - use fabro_types::{RunStatus, RunStatusRecord, Settings, fixtures}; + use fabro_store::{SlateStore, StoreHandle}; + use fabro_types::{RunStatus, Settings, fixtures}; + use object_store::memory::InMemory; use super::scan_runs_combined; + use crate::event::{WorkflowRunEvent, append_workflow_event}; use crate::records::{RunRecord, RunRecordExt}; + fn memory_store() -> StoreHandle { + Arc::new(SlateStore::new( + Arc::new(InMemory::new()), + "", + Duration::from_millis(1), + )) + } + fn sample_run_record() -> RunRecord { RunRecord { run_id: fixtures::RUN_1, @@ -471,7 +483,7 @@ mod tests { run_record.save(&run_dir).unwrap(); std::fs::write(run_dir.join("id.txt"), format!("{}\n", fixtures::RUN_1)).unwrap(); - let store = InMemoryStore::default(); + let store = memory_store(); let run_dir_string = run_dir.to_string_lossy().to_string(); let run_store = store .create_run( @@ -481,11 +493,33 @@ mod tests { ) .await .unwrap(); - run_store.put_run(&run_record).await.unwrap(); - run_store - .put_status(&RunStatusRecord::new(RunStatus::Submitted, None)) - .await - .unwrap(); + append_workflow_event( + run_store.as_ref(), + &fixtures::RUN_1, + &WorkflowRunEvent::RunCreated { + run_id: fixtures::RUN_1, + settings: serde_json::to_value(&run_record.settings).unwrap(), + graph: serde_json::to_value(&run_record.graph).unwrap(), + workflow_source: None, + workflow_config: None, + labels: run_record.labels.clone().into_iter().collect(), + run_dir: run_dir_string.clone(), + working_directory: run_record.working_directory.display().to_string(), + host_repo_path: run_record.host_repo_path.clone(), + base_branch: run_record.base_branch.clone(), + workflow_slug: run_record.workflow_slug.clone(), + db_prefix: None, + }, + ) + .await + .unwrap(); + append_workflow_event( + run_store.as_ref(), + &fixtures::RUN_1, + &WorkflowRunEvent::RunSubmitted { reason: None }, + ) + .await + .unwrap(); let runs = scan_runs_combined(&store, temp.path()).await.unwrap(); let run = runs diff --git a/lib/crates/fabro-workflow/src/test_support.rs b/lib/crates/fabro-workflow/src/test_support.rs index e33c71a2a..e69225de8 100644 --- a/lib/crates/fabro-workflow/src/test_support.rs +++ b/lib/crates/fabro-workflow/src/test_support.rs @@ -1,15 +1,16 @@ -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::path::PathBuf; use std::sync::Arc; +use std::time::Duration; use chrono::Utc; use fabro_agent::Sandbox; use fabro_graphviz::graph::Graph as GvGraph; -use fabro_store::{InMemoryStore, RunStore, Store}; -use fabro_types::run::RunRecord; +use fabro_store::{SlateRunStore, SlateStore}; +use object_store::memory::InMemory; use crate::error::Result; -use crate::event::EventEmitter; +use crate::event::{EventEmitter, WorkflowRunEvent, append_workflow_event}; use crate::git::scan_node_files_from_store; use crate::handler::HandlerRegistry; use crate::outcome::Outcome; @@ -41,32 +42,50 @@ async fn initialized( ) -> Initialized { std::fs::create_dir_all(&run_options.run_dir).expect("failed to create run dir"); let created_at = Utc::now(); - let inner_store = InMemoryStore::default() + let store = Arc::new(SlateStore::new( + Arc::new(InMemory::new()), + "", + Duration::from_millis(1), + )); + let inner_store = store .create_run( &run_options.run_id, created_at, Some(run_options.run_dir.to_string_lossy().as_ref()), ) .await - .expect("failed to create in-memory run store"); + .expect("failed to create slate-backed test run store"); let run_store = inner_store; - run_store - .put_run(&RunRecord { + append_workflow_event( + run_store.as_ref(), + &run_options.run_id, + &WorkflowRunEvent::RunCreated { run_id: run_options.run_id, - created_at, - settings: run_options.settings.clone(), - graph: graph.clone(), - workflow_slug: run_options.workflow_slug.clone(), - working_directory: PathBuf::from(sandbox.working_directory()), + settings: serde_json::to_value(&run_options.settings) + .expect("failed to serialize settings"), + graph: serde_json::to_value(graph).expect("failed to serialize graph"), + workflow_source: None, + workflow_config: None, + labels: run_options + .labels + .clone() + .into_iter() + .collect::>(), + run_dir: run_options.run_dir.display().to_string(), + working_directory: PathBuf::from(sandbox.working_directory()) + .display() + .to_string(), host_repo_path: run_options .host_repo_path .as_ref() .map(|path| path.display().to_string()), base_branch: run_options.base_branch.clone(), - labels: run_options.labels.clone(), - }) - .await - .expect("failed to seed run record in run store"); + workflow_slug: run_options.workflow_slug.clone(), + db_prefix: None, + }, + ) + .await + .expect("failed to seed run.created event in run store"); let emitter = bound_emitter(run_options.run_id, &emitter); Initialized { graph: graph.clone(), @@ -166,12 +185,17 @@ pub async fn run_graph_from_checkpoint( executed.outcome } -async fn persist_run_artifacts_for_tests(run_store: &dyn RunStore, run_dir: &std::path::Path) { - if let Ok(Some(checkpoint)) = run_store.get_checkpoint().await { +async fn persist_run_artifacts_for_tests(run_store: &SlateRunStore, run_dir: &std::path::Path) { + let state: fabro_store::RunState = match run_store.state().await { + Ok(state) => state, + Err(_) => return, + }; + + if let Some(checkpoint) = state.checkpoint.as_ref() { let _ = checkpoint.save(&run_dir.join("checkpoint.json")); } - if let Ok(Some(final_patch)) = run_store.get_final_patch().await { + if let Some(final_patch) = state.final_patch.as_ref() { let _ = std::fs::write(run_dir.join("final.patch"), final_patch); } From ac8ae9ba6a102a98e9cda24152381939e8033298 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 3 Apr 2026 06:44:08 -0700 Subject: [PATCH 3/3] plan --- ...-001-feat-server-daemon-management-plan.md | 446 ++++++++++++++++++ 1 file changed, 446 insertions(+) create mode 100644 docs/plans/2026-04-02-001-feat-server-daemon-management-plan.md diff --git a/docs/plans/2026-04-02-001-feat-server-daemon-management-plan.md b/docs/plans/2026-04-02-001-feat-server-daemon-management-plan.md new file mode 100644 index 000000000..469c39b49 --- /dev/null +++ b/docs/plans/2026-04-02-001-feat-server-daemon-management-plan.md @@ -0,0 +1,446 @@ +--- +title: "feat: Add server daemon management with Unix socket support" +type: feat +status: active +date: 2026-04-02 +deepened: 2026-04-02 +--- + +# feat: Add server daemon management with Unix socket support + +## Overview + +Transform `fabro server` from a foreground-only TCP server into a proper daemon with background/foreground modes, stop/status lifecycle commands, Unix socket binding, and flock-based locking to prevent thundering herd when multiple CLI invocations auto-start the server. + +## Problem Frame + +The fabro server currently runs only in the foreground on a TCP port. Users must manually manage the process lifecycle. There is no way to check if a server is running, stop it gracefully, or prevent duplicate instances. When the CLI eventually auto-starts the server on demand, concurrent CLI invocations could race to start multiple servers simultaneously. + +## Requirements Trace + +- R1. `fabro server start` launches as a background daemon by default +- R2. `fabro server start --foreground` retains current blocking behavior +- R3. `fabro server stop` sends SIGTERM, waits, escalates to SIGKILL +- R4. `fabro server status` reports running/stopped with PID, bind address, uptime +- R5. A JSON server record tracks PID and metadata; stale records are auto-cleaned +- R6. `--bind` replaces `--host`/`--port`, supporting both Unix sockets and TCP addresses +- R7. Default bind is `{storage_dir}/fabro.sock` (Unix socket) +- R8. flock-based locking prevents concurrent start attempts (thundering herd) +- R9. Only one server instance can run at a time per storage directory +- R10. TLS is not supported on Unix sockets (only on TCP) + +## Scope Boundaries + +- No systemd/launchd integration (out of scope) +- No log rotation or `server logs` subcommand beyond basic prev-file rotation (future work) +- Breaking change: `--host` and `--port` are removed, replaced by `--bind` +- Client-side Unix socket connectivity (TypeScript Axios client, CLI-to-server calls) is tracked as a follow-up concern -- this plan covers the server side only + +## Context & Research + +### Relevant Code and Patterns + +- `lib/crates/fabro-cli/src/commands/run/launcher.rs` -- JSON-based launcher records with PID tracking, stale detection via `process_alive()` + `ps` command-line matching, lazy cleanup on read +- `lib/crates/fabro-cli/src/commands/run/start.rs` -- Self-re-exec pattern: spawns `fabro __detached` with `pre_exec_setsid`, stdout/stderr to log file, writes record after spawn, checks `try_wait()` for immediate failure, forwards `--storage-dir` to child +- `lib/crates/fabro-cli/src/commands/run/detached.rs` -- `scopeguard::guard(launcher_path, remove_launcher_record)` for guaranteed cleanup on exit; `title_init()`/`title_set()` for proctitle; receives record path via `--launcher-path` arg +- `lib/crates/fabro-proc/src/` -- `process_alive`, `sigterm`, `sigkill`, `pre_exec_setsid`, `title_init`/`title_set`. All functions are thin libc wrappers -- no retry loops or higher-level logic +- `lib/crates/fabro-server/src/serve.rs` -- Current `serve_command()` with `ServeArgs`, config polling, webhook manager lifecycle. No graceful shutdown wired. TLS path uses manual `loop { listener.accept() }` via `hyper_util`, not `axum::serve` +- `lib/crates/fabro-cli/src/args.rs:650` -- `#[command(name = "__detached", hide = true)]` hidden subcommand pattern with its own args struct +- `lib/crates/fabro-cli/src/args.rs:972-984` -- `ServerNamespace`/`ServerCommand` enum +- `lib/crates/fabro-cli/src/main.rs:108-195` -- Config log level extraction and command dispatch for server + +### Institutional Learnings + +No `docs/solutions/` directory exists. Patterns are embedded in the launcher record system. + +## Key Technical Decisions + +- **Hidden subcommand for daemon child, not a flag**: The daemon spawns `fabro server __serve --record-path --bind ...` as a detached child. `__serve` is a hidden `ServerCommand` variant with its own args struct, matching the `__detached` pattern in `RunCommands`. This keeps `ServeArgs` focused on server runtime concerns and process lifecycle in `fabro-cli`. + +- **Process lifecycle stays in fabro-cli**: `fabro-server` has no dependency on `fabro-proc` and should not gain one. A `foreground.rs` in `fabro-cli/src/commands/server/` wraps `serve_command()` with record writes, scopeguard cleanup, and proctitle management -- same boundary as `detached.rs` wrapping workflow operations. + +- **Record ownership: parent writes, child cleans up**: In daemon mode, the parent writes `server.json` after spawn (and cleans up on failure). The child receives the record path via `--record-path` and sets up a `scopeguard` to remove it on exit. This matches the launcher record ownership model exactly. + +- **Server record, not bare PID file**: A JSON `ServerRecord` (pid, bind: Bind, log_path, started_at) stored at `{storage_dir}/server.json`, following the `LauncherRecord` pattern. The `bind` field uses the `Bind` enum (not a raw string) so consumers get type-safe access without re-parsing. Enables `status` to report rich info and supports PID-command-line validation against PID reuse. + +- **flock on a separate lock file**: `{storage_dir}/server.lock` is acquired with `LOCK_EX | LOCK_NB` before any start attempt. Losers of the race block on the lock (with timeout), then discover the server already running. The lock file is separate from `server.json` so the record can be atomically rewritten without interfering with the lock. `flock()` auto-releases on process crash. + +- **Keep fabro-proc thin**: Only `try_flock_exclusive(file) -> io::Result` goes in `fabro-proc` (thin libc wrapper). The timeout/retry loop lives in the caller (`server/start.rs`), consistent with every other function in `fabro-proc` being a single-syscall wrapper. + +- **`Bind` enum defined in fabro-server**: A `Bind` enum (`Bind::Unix(PathBuf) | Bind::Tcp(SocketAddr)`) lives in `fabro-server` since it's a server concern ("what am I binding to?"). Used in both `ServeArgs` resolution and `ServerRecord`. Serializes cleanly via serde tagged enum: `{"unix": "/path"}` or `{"tcp": "127.0.0.1:3000"}`. Consumers (stop, status) can match on the variant directly instead of re-parsing a string -- e.g., `stop` knows to clean up the socket file when `bind` is `Bind::Unix`. + +- **Bind address parsing**: `parse_bind(s: &str) -> Result` in `fabro-server`. If the value contains `/`, it's a Unix socket path. Otherwise it's `host:port` TCP. Default: `{storage_dir}/fabro.sock`. Unix socket paths are validated against the 104-byte limit on macOS (108 on Linux). + +- **Graceful shutdown via SIGTERM handler**: Wire `tokio::signal::unix::signal(SignalKind::terminate())` into `axum::serve().with_graceful_shutdown()`. The foreground mode also handles SIGINT (ctrl-c). **Known limitation:** The TLS codepath uses a manual `loop { listener.accept() }` via `hyper_util` and cannot use `with_graceful_shutdown`. Under TLS, `server stop` will rely on SIGTERM causing process exit (default behavior) but may need SIGKILL escalation. This is acceptable for now. + +- **Process title includes bind address**: `fabro: server {bind}` to support PID-command-line validation and disambiguate if the single-instance invariant ever relaxes. + +- **No TLS on Unix sockets**: When bind is a Unix socket, skip the TLS codepath entirely. TLS only applies to TCP binds. + +- **Log rotation on start**: Rename existing `server.log` to `server.log.prev` before starting, so crash diagnostics from the previous run are preserved. + +## Open Questions + +### Resolved During Planning + +- **Where does flock live?** Only `try_flock_exclusive` in `fabro-proc` (thin wrapper). Timeout loop in caller. +- **Default bind address?** `{storage_dir}/fabro.sock` (Unix socket). Resolved storage dir is used so it respects `--storage-dir` and config overrides. +- **How does the parent know the daemon is ready?** Poll-connect to the socket/port with a short timeout (up to 5s). Same approach as pg_ctl. +- **What happens to TLS + Unix socket?** Not supported. If `--bind` is a socket path and TLS is configured, emit a warning and skip TLS. +- **Hidden flag or hidden subcommand?** Hidden subcommand (`__serve`), matching the `__detached` pattern. Not a flag on `ServeArgs`. +- **Where does process lifecycle code live?** In `fabro-cli/src/commands/server/`, not in `fabro-server`. Same boundary as `detached.rs`. +- **Client connectivity with Unix socket default?** Tracked as follow-up -- this plan covers server-side only. The `server.json` record stores the bind address so clients can discover it. + +### Deferred to Implementation + +- Exact readiness polling interval and timeout values (start with 50ms interval, 5s timeout) +- Whether `server stop` should print tail of server.log on timeout before SIGKILL +- Exact chmod permissions on the Unix socket file (start with default, tighten if needed) + +## High-Level Technical Design + +> *This illustrates the intended approach and is directional guidance for review, not implementation specification. The implementing agent should treat it as context, not code to reproduce.* + +``` + fabro server start + | + acquire flock(server.lock) + / \ + (got lock) (blocked) + | | + read server.json wait for lock (retry loop in start.rs) + / \ | + (live pid) (none/stale) (got lock) + | | | + "already rotate server.log -> server.log.prev + running" spawn: fabro server __serve --record-path ... --bind ... + exit 1 pre_exec_setsid, stdout/stderr -> server.log + | + parent: write server.json with child PID + parent: poll-connect to bind address + / \ + (connected) (timeout) + | | + exit 0 print error + log tail + clean up server.json + exit 1 + + fabro server __serve --record-path --bind ... + | + title_init() + title_set("fabro: server ") + scopeguard(record_path, remove_server_record) + register SIGTERM/SIGINT shutdown + | + bind listener (Unix or TCP) + axum::serve(...).with_graceful_shutdown(...) + | + (on shutdown signal) + scopeguard fires: remove server.json + remove socket file (if Unix) +``` + +```mermaid +graph TB + U1[Unit 1: flock in fabro-proc] + U2[Unit 2: ServerRecord] + U3[Unit 3: Bind enum + --bind + Unix socket] + U4[Unit 4: daemon spawn + __serve] + U5[Unit 5: server stop] + U6[Unit 6: server status] + U7[Unit 7: main.rs dispatch] + + U1 --> U4 + U3 --> U2 + U2 --> U4 + U2 --> U5 + U2 --> U6 + U3 --> U4 + U4 --> U7 + U5 --> U7 + U6 --> U7 +``` + +Units 1 and 3 can run in parallel. Unit 2 depends on Unit 3 (for the `Bind` enum type). Units 5 and 6 can run in parallel after Unit 2. Unit 7 ties everything together. + +## Implementation Units + +- [ ] **Unit 1: Add flock wrapper to fabro-proc** + + **Goal:** Provide `try_flock_exclusive` in `fabro-proc` as a thin libc wrapper for advisory file locking. + + **Requirements:** R8 + + **Dependencies:** None + + **Files:** + - Create: `lib/crates/fabro-proc/src/flock.rs` + - Modify: `lib/crates/fabro-proc/src/lib.rs` + - Test: `lib/crates/fabro-proc/src/flock.rs` (inline tests) + + **Approach:** + - `try_flock_exclusive(file: &File) -> io::Result` -- `libc::flock(fd, LOCK_EX | LOCK_NB)`, returns `Ok(false)` on `EWOULDBLOCK` + - `flock_unlock(file: &File) -> io::Result<()>` -- `libc::flock(fd, LOCK_UN)` for explicit unlock + - Unix-only (`#[cfg(unix)]`), consistent with existing `fabro-proc` gating + - No retry loops or timeout logic -- keep this crate at the syscall-wrapper level + + **Patterns to follow:** + - `lib/crates/fabro-proc/src/signal.rs` -- same style: thin wrapper over libc, `#[cfg(unix)]` gating, pub functions re-exported from `lib.rs` + - `lib/crates/fabro-proc/src/pre_exec.rs` -- unsafe block style and safety comments + + **Test scenarios:** + - Happy path: acquire lock on a temp file, confirm returns true + - Happy path: release lock (drop file), re-acquire succeeds + - Edge case: try_flock_exclusive on already-locked file (held by another fd) returns Ok(false) + + **Verification:** + - `cargo nextest run -p fabro-proc` passes + - `cargo clippy -p fabro-proc -- -D warnings` clean + +- [ ] **Unit 2: Add ServerRecord and lifecycle helpers** + + **Goal:** Create a `ServerRecord` struct with read/write/remove/is_running helpers, mirroring `LauncherRecord`. + + **Requirements:** R5, R9 + + **Dependencies:** Unit 3 (for `Bind` enum type) + + **Files:** + - Create: `lib/crates/fabro-cli/src/commands/server/record.rs` + - Create: `lib/crates/fabro-cli/src/commands/server/mod.rs` + - Modify: `lib/crates/fabro-cli/src/commands/mod.rs` (add `pub(crate) mod server;` gated with `#[cfg(feature = "server")]`) + + **Approach:** + - `ServerRecord { pid: u32, bind: Bind, log_path: PathBuf, started_at: DateTime }` where `Bind` is the enum from `fabro-server` (Unit 3). The `bind` field serializes as a tagged enum (`{"unix": "/path"}` or `{"tcp": "127.0.0.1:3000"}`), giving consumers type-safe access -- e.g., `stop` matches on `Bind::Unix` to know it should remove the socket file + - Paths: `server_record_path(storage_dir) -> {storage_dir}/server.json`, `server_lock_path(storage_dir) -> {storage_dir}/server.lock`, `server_log_path(storage_dir) -> {storage_dir}/server.log` + - `server_record_is_running(record) -> bool` -- `process_alive(pid)` + `ps` command-line match for "fabro" and "server" + - `active_server_record(storage_dir) -> Option` -- read, check liveness, lazy-clean stale + - Path helpers only -- lock acquisition logic lives in Unit 4 + + **Patterns to follow:** + - `lib/crates/fabro-cli/src/commands/run/launcher.rs` -- identical record lifecycle pattern: `write_*`, `read_*`, `remove_*`, `*_is_running`, `active_*` + + **Test scenarios:** + - Happy path: write record, read it back, fields match + - Happy path: active_server_record returns None when no record file exists + - Edge case: active_server_record returns None and removes file when PID is dead (use pid u32::MAX) + - Edge case: active_server_record returns None and removes file when process doesn't match command-line check + + **Verification:** + - `cargo nextest run -p fabro-cli` for the new module's tests pass + +- [ ] **Unit 3: Add Bind enum, replace --host/--port with --bind, add Unix socket listener** + + **Goal:** Define the `Bind` enum as the shared type for bind addresses. Change `ServeArgs` to use `--bind` instead of `--host`/`--port`. Support both Unix socket and TCP binding in `serve_command`. Wire graceful shutdown. + + **Requirements:** R6, R7, R10 + + **Dependencies:** None (parallel with Unit 1) + + **Files:** + - Create: `lib/crates/fabro-server/src/bind.rs` (Bind enum + parse_bind + Display impl) + - Modify: `lib/crates/fabro-server/src/lib.rs` (pub mod bind) + - Modify: `lib/crates/fabro-server/src/serve.rs` + - Modify: `lib/crates/fabro-server/Cargo.toml` (ensure tokio `net` feature includes unix support) + - Modify: `lib/crates/fabro-cli/src/main.rs` (update dispatch if ServeArgs shape changes) + - Test: `lib/crates/fabro-server/tests/it/api.rs` (update any tests using --host/--port) + - Test: `lib/crates/fabro-server/src/bind.rs` (inline tests for parse_bind) + + **Approach:** + - Define `Bind` enum in `bind.rs`: `Bind::Unix(PathBuf) | Bind::Tcp(SocketAddr)` with `Serialize`/`Deserialize` (serde tagged enum), `Display`, `Clone`, `Debug`, `PartialEq`. This type is used by both `serve_command` and `ServerRecord` (Unit 2) + - `parse_bind(bind: &str) -> Result`. Contains `/` -> Unix socket; otherwise `host:port` parsed as `SocketAddr`. Validate Unix socket path length (104 bytes on macOS, 108 on Linux) + - `Bind::display()` shows the address for human-readable output (used in proctitle, status, logs) + - Replace `--host` and `--port` on `ServeArgs` with `--bind` (Option, no default in clap -- default computed at runtime from resolved storage dir using the socket path) + - In `serve_command`, branch on `Bind` variant: `UnixListener::bind` vs `TcpListener::bind` + - For Unix sockets: remove stale socket file before bind, skip TLS codepath (log warning if TLS configured) + - Wire `axum::serve(listener, router).with_graceful_shutdown(shutdown_signal())` for the non-TLS path, where `shutdown_signal` awaits SIGTERM or SIGINT. **Note:** The TLS path (`serve_tls`) uses a manual accept loop and cannot use `with_graceful_shutdown` -- document as known limitation, SIGTERM will still cause process exit + - Derive `Clone` on `ServeArgs` to simplify the config polling task (currently manually clones each field) + + **Patterns to follow:** + - Current `serve.rs` listener binding and axum::serve pattern + - Axum 0.8 supports `UnixListener` directly via `axum::serve` + + **Test scenarios:** + - Happy path: parse_bind with "127.0.0.1:3000" returns Tcp variant + - Happy path: parse_bind with "/tmp/fabro.sock" returns Unix variant + - Edge case: parse_bind with invalid address returns error + - Edge case: parse_bind with path exceeding 104 bytes returns error on macOS + - Happy path: server binds to Unix socket and accepts HTTP requests over it + - Happy path: server binds to TCP address (existing behavior preserved) + - Edge case: stale socket file is removed before binding + - Integration: graceful shutdown on SIGTERM -- server stops accepting connections and exits cleanly + + **Verification:** + - `cargo nextest run -p fabro-server` passes + - Existing server tests still pass (adapted for --bind) + +- [ ] **Unit 4: Add daemon spawn, __serve hidden subcommand, and foreground wrapper** + + **Goal:** Make `server start` launch a background daemon by default. `--foreground` retains current behavior. Both modes write/clean server records. Daemon mode uses flock to prevent thundering herd. `__serve` is the hidden subcommand the daemon child runs. + + **Requirements:** R1, R2, R8, R9 + + **Dependencies:** Units 1, 2, 3 + + **Files:** + - Create: `lib/crates/fabro-cli/src/commands/server/start.rs` (daemon spawn logic + flock retry loop) + - Create: `lib/crates/fabro-cli/src/commands/server/foreground.rs` (wraps `serve_command` with record lifecycle, scopeguard, proctitle) + - Modify: `lib/crates/fabro-cli/src/args.rs` (add `__Serve` hidden variant to `ServerCommand` with its own args struct; add `--foreground` flag to `Start` variant args) + - Modify: `lib/crates/fabro-cli/src/commands/server/mod.rs` (dispatch) + + **Approach:** + - **`ServerCommand::__Serve(ServeChildArgs)`**: Hidden subcommand with `--record-path`, `--bind`, plus forwarded args (`--model`, `--provider`, `--dry-run`, `--sandbox`, `--max-concurrent-runs`, `--config`, `--storage-dir`). Dispatches to `foreground.rs`. + - **`foreground.rs`**: `title_init()` + `title_set("fabro: server {bind}")`, `scopeguard::guard(record_path, remove_server_record)`, then calls `serve_command()`. Mirrors `detached.rs` wrapping workflow operations. + - **`start.rs` daemon path**: Open `server.lock`, retry `try_flock_exclusive` in a loop (50ms intervals, 5s timeout); check `active_server_record`; if running, print "already running" and exit 1; rotate `server.log` to `server.log.prev`; spawn self with `fabro server __serve --record-path --bind ...` using `pre_exec_setsid`, stdout/stderr to `server.log`, stdin null, `env_remove("FABRO_JSON")`; write `server.json` with child PID; check `try_wait()` for immediate failure; poll-connect to bind address (50ms intervals, 5s timeout); on success print "server started (pid N) on ", exit 0; on failure print error + tail of log, clean up, exit 1 + - **`start.rs` foreground path** (`--foreground`): Write `server.json`, register scopeguard for cleanup, then call `serve_command()` directly (no re-exec) + - **Forward all relevant args to child**: `--storage-dir`, `--config`, `--model`, `--provider`, `--dry-run`, `--sandbox`, `--max-concurrent-runs` + + **Patterns to follow:** + - `lib/crates/fabro-cli/src/commands/run/start.rs` -- self-re-exec with `pre_exec_setsid`, log redirect, record write, `try_wait` check, `--storage-dir` forwarding, `env_remove("FABRO_JSON")` + - `lib/crates/fabro-cli/src/commands/run/detached.rs` -- `scopeguard` cleanup, `title_init`/`title_set`, receives record path via arg + - `lib/crates/fabro-cli/src/args.rs:650` -- `#[command(name = "__detached", hide = true)]` pattern + + **Test scenarios:** + - Happy path: `server start` spawns daemon, writes server.json, exits 0 + - Happy path: `server start --foreground` runs in foreground, writes server.json, cleans up on exit + - Edge case: `server start` when already running prints "already running" and exits 1 + - Edge case: `server start` with stale server.json (dead PID) cleans up and starts fresh + - Happy path: flock prevents concurrent start -- second caller waits and finds server running + - Edge case: daemon fails to start (bad bind address) -- parent reports error, cleans up record + - Edge case: daemon child exits immediately -- parent detects via try_wait, reports error + - Integration: after daemon start, server.json contains correct PID and bind address + - Integration: server.log.prev contains previous log content after restart + + **Verification:** + - `cargo nextest run -p fabro-cli` passes + - Manual: `fabro server start` starts daemon, `fabro server start` again says "already running" + +- [ ] **Unit 5: Add server stop subcommand** + + **Goal:** `fabro server stop` sends SIGTERM, waits for graceful exit, escalates to SIGKILL, cleans up. + + **Requirements:** R3 + + **Dependencies:** Unit 2 + + **Files:** + - Create: `lib/crates/fabro-cli/src/commands/server/stop.rs` + - Modify: `lib/crates/fabro-cli/src/args.rs` (add `Stop` variant to `ServerCommand` with `StopArgs { timeout }`) + - Modify: `lib/crates/fabro-cli/src/commands/server/mod.rs` + + **Approach:** + - Read `active_server_record` -- if None, print "not running", exit 1 + - `fabro_proc::sigterm(pid)` + - Poll `process_alive(pid)` at 100ms intervals up to `--timeout` (default 10s) + - If still alive after timeout, `fabro_proc::sigkill(pid)` + - Remove `server.json` and socket file (match on `record.bind` -- if `Bind::Unix(path)`, remove the socket file) + - Print "server stopped" + + **Patterns to follow:** + - `fabro_proc::sigterm`/`sigkill`/`process_alive` for signal management + - `launcher.rs::remove_launcher_record` for cleanup + + **Test scenarios:** + - Happy path: stop a running server -- sends SIGTERM, process exits, record cleaned up + - Edge case: stop when not running -- prints "not running", exits 1 + - Edge case: stop with stale record (dead PID) -- cleans up record, prints "not running", exits 1 + - Edge case: process doesn't exit within timeout -- escalates to SIGKILL + - Happy path: Unix socket file is removed after stop + + **Verification:** + - `cargo nextest run -p fabro-cli` passes + - Manual: `fabro server start && fabro server stop` completes cleanly + +- [ ] **Unit 6: Add server status subcommand** + + **Goal:** `fabro server status` reports running/stopped state with metadata. Supports `--json`. + + **Requirements:** R4 + + **Dependencies:** Unit 2 + + **Files:** + - Create: `lib/crates/fabro-cli/src/commands/server/status.rs` + - Modify: `lib/crates/fabro-cli/src/args.rs` (add `Status` variant to `ServerCommand` with `StatusArgs { json }`) + - Modify: `lib/crates/fabro-cli/src/commands/server/mod.rs` + + **Approach:** + - Read `active_server_record` -- if None, print "not running", exit 1 + - Compute uptime from `started_at` + - Human output: "running (pid N) on , started X ago" + - `--json`: `{ "status": "running", "pid": N, "bind": "...", "started_at": "...", "uptime_seconds": N }` + - Exit code: 0 = running, 1 = not running (same as `pg_ctl status`) + + **Patterns to follow:** + - Other CLI commands that support `--json` output (check `globals.json` usage pattern) + + **Test scenarios:** + - Happy path: status when running -- prints info, exits 0 + - Happy path: status --json when running -- outputs valid JSON with expected fields + - Edge case: status when not running -- prints "not running", exits 1 + - Edge case: status with stale record -- cleans up, prints "not running", exits 1 + + **Verification:** + - `cargo nextest run -p fabro-cli` passes + +- [ ] **Unit 7: Update main.rs dispatch and config loading for new server subcommands** + + **Goal:** Wire all server subcommands (start, stop, status, __serve) into CLI dispatch. Fix config log level extraction to handle new ServerCommand variants. + + **Requirements:** R1, R2, R3, R4 + + **Dependencies:** Units 4, 5, 6 + + **Files:** + - Modify: `lib/crates/fabro-cli/src/main.rs` + - Modify: `lib/crates/fabro-cli/src/args.rs` (update `Commands::name()` match arm for all new variants) + + **Approach:** + - Change `let ServerCommand::Start(args) = ns.command;` to `match ns.command { Start(..) => ..., Stop(..) => ..., Status(..) => ..., __Serve(..) => ... }` in both the config log level block and the dispatch block + - `__Serve` and `Start` (when in foreground/daemon mode) load server settings for log level; `Stop` and `Status` load user settings + - Update `Commands::name()` to return `"server start"`, `"server stop"`, `"server status"`, `"server __serve"` for telemetry + - Log prefix: `"server"` for `Start`/`__Serve`, `"cli"` for `Stop`/`Status` + - `#[cfg(feature = "server")]` gating on all new paths + + **Patterns to follow:** + - Existing dispatch pattern in `main.rs` for other namespace commands (e.g., `Commands::RunCmd`) + - Feature gating on all server references + + **Test scenarios:** + - Happy path: `fabro server stop --help` prints help text + - Happy path: `fabro server status --help` prints help text + - Happy path: `fabro server start --help` still works with new --bind flag and --foreground flag + - Happy path: telemetry name returns correct values for each subcommand + + **Verification:** + - `cargo nextest run -p fabro-cli` passes (including existing server tests) + - `cargo clippy --workspace -- -D warnings` clean + +## System-Wide Impact + +- **Interaction graph:** `serve_command()` gains a graceful shutdown signal handler. A new `foreground.rs` wrapper in `fabro-cli` manages server record lifecycle around `serve_command()`. Config polling task and webhook manager lifecycle preserved unchanged. Webhook manager shutdown (line 274-277 of `serve.rs`) becomes reachable for the first time via graceful shutdown -- the existing code is correct but the implementing agent should not add a scopeguard that drops the tokio runtime before async shutdown runs. +- **Error propagation:** Daemon start failures surface to the parent via poll-connect timeout + log tail. Stop failures surface via exit code. +- **State lifecycle risks:** Stale `server.json` after crash -- mitigated by PID liveness + command-line check on every read, with lazy cleanup (same proven pattern as launcher records). Stale socket file -- removed before bind attempt. +- **API surface parity:** The `--bind` change is breaking for anyone using `--host`/`--port`. No API endpoint changes. Client-side connectivity to Unix sockets (TypeScript Axios client, CLI HTTP calls) is not addressed in this plan and needs follow-up. +- **Integration coverage:** Unit tests can verify record lifecycle and parse_bind. Integration tests should cover the full start/status/stop cycle with a real server process. +- **Unchanged invariants:** All HTTP routes, auth, config reloading, webhook manager, and SSE streaming behavior are unchanged. The server's runtime behavior is identical once it's listening. + +## Risks & Dependencies + +| Risk | Mitigation | +|------|------------| +| PID reuse after crash leads to signaling wrong process | Two-phase check: `process_alive` + `ps` command-line matching for "fabro" and "server" with bind address in proctitle, same proven pattern as launcher records | +| flock not supported on all filesystems (e.g., NFS) | Storage dir is local by convention (~/.fabro). Document that network filesystems are unsupported for storage | +| Unix socket path exceeds 104-byte limit on macOS | `parse_bind` validates path length at parse time with a clear error message | +| Axum 0.8 UnixListener support | Axum 0.8 is already the workspace version; `serve()` accepts `UnixListener` natively | +| Breaking --host/--port removal | Acceptable per scope decision. Users see a clear clap error pointing to --bind | +| Race between parent writing server.json and child exiting | Check `try_wait()` immediately after spawn (same pattern as `start.rs`); if child already exited, clean up and report error | +| TLS path lacks graceful shutdown | Documented as known limitation. SIGTERM still causes process exit; stop command escalates to SIGKILL after timeout. TLS + daemon is an uncommon combination | +| Client-side code assumes TCP | Tracked as follow-up. Server record stores bind address for client discovery | + +## Sources & References + +- Related code: `lib/crates/fabro-cli/src/commands/run/launcher.rs`, `lib/crates/fabro-cli/src/commands/run/start.rs`, `lib/crates/fabro-cli/src/commands/run/detached.rs` +- Related code: `lib/crates/fabro-proc/src/signal.rs`, `lib/crates/fabro-proc/src/pre_exec.rs` +- Related code: `lib/crates/fabro-server/src/serve.rs` +- Related code: `lib/crates/fabro-cli/src/args.rs:650` (`__detached` hidden subcommand pattern), `lib/crates/fabro-cli/src/args.rs:972-984` (`ServerCommand`) +- Related code: `lib/crates/fabro-cli/src/main.rs:108-195`