From 3e8b2ebfcc1ced2812911b17ebfded67b7c052b1 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 18 Sep 2026 12:32:06 -0400 Subject: [PATCH] Run the server's lifecycle over platform records instead of run_events Step 4 of the legacy executor deletion, first commit of several: step 4 spans commits because the legacy event log and its consumers cannot go in one compiling change. This commit moves every writer off `run_events`; the reducer, `EventBody`, the Slate bridge and the API's event types still exist for the readers the next commits port or delete. Writers: - The server records a run's lifecycle (submitted, runnable, starting, running, blocked, paused, control requests and effects, the terminal status), its title, parent link, archive state, notices and pull request state as platform records (`fabro_store::platform_records`), through the new `server::run_records` module. Every append wakes the projector and waits for its pass, so the read that follows a write holds the record. - Pull request creation is recorded as `pull_request.requested`, `pull_request.created`, `pull_request.failed`, `pull_request.linked` and `pull_request.unlinked`; the projection folds them into the run's pull request and creation state. - Answers to questions are recorded as `interview.answered` with the answering principal and the answer text; the interview adapter no longer posts legacy `interview.*` events (`QuestionSink` is now an optional observer). - The worker (`fabro run __run-worker`) records its lifecycle, notices and pause state over `HttpPlatformRecords`; `HttpRunStore` for the legacy event log and the worker's `run_store` are gone. - `persist_created_run` appends `run.created` and `run.submitted`. Readers: - A stream follower (`server::stream_follower`) follows each live run's stream (Petri events and platform records), folds lifecycle records into the in-memory run state, forwards items to the global attach broadcast, and syncs blocked and paused from the projection. - Slack posts questions from the projection's pending interviews, finishes them on `interview.answered` or `question_expired`, and sends lifecycle notifications with `notification.sent` dedupe. - `GET /runs/{id}/events` and the attach endpoints serve only the run stream; the per-event, per-stage and `POST /runs/{id}/events` endpoints and their tests are deleted. - `Database::load_run_projection` reads the Petri projection only. Deleted with the writers: - The SQLite blob and run-history activation migrations and their legacy Slate imports (`legacy_blob_import`, `legacy_run_history_import`, the activation backup): a greenfield server has no Slate history to import, and the run-history verification refused to start a server whose runs have no legacy events. - `fabro-workflow`'s `operations::archive` and `operations::run_store`. - The server's legacy-event unit tests and the CLI's `HttpRunStore` tests. The in-process answer transport is now set after the starting and running records land, not gated on the live status still being `Starting` (the records already moved it). The manifest validation test for a `run.agent.mcps.` catalog reference now expects `unsupported.workflow_toml.run.agent.mcps.reference`: Petri's Fabro frontend has no server catalog to resolve it against. Legacy readers still fail their tests until the next commits: the reducer and Slate tests in fabro-store, the fabro-workflow create tests that read the run back through the legacy store, the CLI tests seeded through `POST /runs/{id}/events`, the CLI's legacy attach and render paths, the sessions API, the OpenAPI conformance test, and the web fixtures. Co-Authored-By: Claude Fable 5.1 --- docs/internal/migrations-strategy.md | 2 +- .../src/commands/run/petri_worker.rs | 163 +- lib/apps/fabro-cli/src/commands/run/runner.rs | 272 +- lib/apps/fabro-cli/tests/it/scenario/petri.rs | 13 +- .../2026082301_sqlite_blob_activation.rs | 746 -- ...026082801_sqlite_run_history_activation.rs | 603 -- .../migrations/sqlite_activation_backup.rs | 186 - lib/apps/fabro-server/src/lib.rs | 1 - lib/apps/fabro-server/src/migrations.rs | 9 - lib/apps/fabro-server/src/petri_runs.rs | 60 +- lib/apps/fabro-server/src/serve.rs | 21 +- lib/apps/fabro-server/src/server.rs | 1021 ++- .../src/server/automation_scheduler.rs | 34 +- .../src/server/handler/artifacts.rs | 34 +- .../fabro-server/src/server/handler/events.rs | 922 +- .../src/server/handler/lifecycle.rs | 263 +- .../src/server/handler/pull_requests.rs | 67 +- .../fabro-server/src/server/handler/runs.rs | 103 +- .../fabro-server/src/server/handler/system.rs | 22 +- .../fabro-server/src/server/petri_runs.rs | 145 +- .../src/server/pull_request_supervisor.rs | 91 +- .../src/server/resource_sampler.rs | 4 - .../fabro-server/src/server/run_records.rs | 96 + .../src/server/stream_follower.rs | 164 + lib/apps/fabro-server/src/server/tests.rs | 7425 +---------------- lib/apps/fabro-server/tests/it/api/events.rs | 187 - .../fabro-server/tests/it/api/mcp_servers.rs | 20 +- lib/apps/fabro-server/tests/it/api/mod.rs | 1 - .../tests/it/scenario/petri_stream.rs | 14 - lib/components/fabro-petri/src/interview.rs | 126 +- lib/components/fabro-petri/src/projection.rs | 84 +- lib/components/fabro-petri/tests/interview.rs | 2 +- .../fabro-petri/tests/projection.rs | 6 +- .../fabro-petri/tests/support/mod.rs | 2 +- lib/components/fabro-store/src/keys.rs | 37 +- .../fabro-store/src/legacy_blob_import.rs | 2007 ----- .../src/legacy_run_history_import.rs | 2234 ----- lib/components/fabro-store/src/lib.rs | 11 - .../fabro-store/src/platform_records.rs | 114 +- .../fabro-store/src/run_summary_store.rs | 137 +- lib/components/fabro-store/src/slate/mod.rs | 86 +- .../fabro-workflow/src/operations/archive.rs | 470 -- .../fabro-workflow/src/operations/create.rs | 71 +- .../fabro-workflow/src/operations/mod.rs | 27 +- .../src/operations/run_store.rs | 11 - .../fabro-workflow/src/pull_request.rs | 43 +- 46 files changed, 1557 insertions(+), 16600 deletions(-) delete mode 100644 lib/apps/fabro-server/migrations/2026082301_sqlite_blob_activation.rs delete mode 100644 lib/apps/fabro-server/migrations/2026082801_sqlite_run_history_activation.rs delete mode 100644 lib/apps/fabro-server/migrations/sqlite_activation_backup.rs delete mode 100644 lib/apps/fabro-server/src/migrations.rs create mode 100644 lib/apps/fabro-server/src/server/run_records.rs create mode 100644 lib/apps/fabro-server/src/server/stream_follower.rs delete mode 100644 lib/apps/fabro-server/tests/it/api/events.rs delete mode 100644 lib/components/fabro-store/src/legacy_blob_import.rs delete mode 100644 lib/components/fabro-store/src/legacy_run_history_import.rs delete mode 100644 lib/components/fabro-workflow/src/operations/archive.rs delete mode 100644 lib/components/fabro-workflow/src/operations/run_store.rs diff --git a/docs/internal/migrations-strategy.md b/docs/internal/migrations-strategy.md index 026abe844..8c6c0ffea 100644 --- a/docs/internal/migrations-strategy.md +++ b/docs/internal/migrations-strategy.md @@ -20,7 +20,7 @@ The crate-local `src/migrations.rs` module is the registry. It imports numbered Examples: - `fabro-config` owns settings-file migrations. -- `fabro-server` owns server startup activation migrations for SQLite blob storage and run history. +- `fabro-db` owns the SQL schema migrations under `lib/foundation/fabro-db/migrations/`. Keep migration APIs `pub(crate)` unless another crate genuinely orchestrates the migration. diff --git a/lib/apps/fabro-cli/src/commands/run/petri_worker.rs b/lib/apps/fabro-cli/src/commands/run/petri_worker.rs index 6917b899f..f215143a4 100644 --- a/lib/apps/fabro-cli/src/commands/run/petri_worker.rs +++ b/lib/apps/fabro-cli/src/commands/run/petri_worker.rs @@ -12,9 +12,9 @@ //! every lease the run takes over the API names it. `--mode start` loads //! the admitted graphs through the client's blob read and runs them; //! `--mode resume` continues the run from its records. Either way the -//! worker appends the lifecycle events Fabro's read side needs -//! (`run.starting`, `run.running`, then `run.completed` or `run.failed`) -//! through the client, as the legacy worker does. +//! worker records the lifecycle transitions Fabro's read side needs +//! (`starting`, `running`, then `succeeded` or `failed`) as platform +//! records through the client. //! //! The server's controls arrive over the control channel and go to Petri //! through [`PetriControls`]: cancel (and `SIGTERM`/`SIGINT`) fires one @@ -24,15 +24,15 @@ //! through the API continues; pause and unpause hold and release admission //! through the run's [`RunControls`]; a steer goes to the run's one live //! agent stage, or is refused with a `run.notice` record saying why. The -//! paused state is mirrored to Fabro's lifecycle as the legacy worker -//! reported it: a `run.paused` lifecycle event when admission is held and -//! `run.unpaused` when it is released, so the server's live status and the -//! projection agree with Petri's own `run.paused` and `run.unpaused` -//! records. A resumed run that was paused when its worker died comes back -//! paused, and the mirror reports that too. The interrupt and pair -//! controls have no Petri adapter yet and are ignored with a warning. A -//! control channel that is lost for good cancels the run the same way, and -//! the worker exits with that loss as its error once the run has settled. +//! paused state is mirrored to Fabro's lifecycle: a `paused` lifecycle +//! record when admission is held and `unpaused` when it is released, so +//! the server's live status and the projection agree with Petri's own +//! `run.paused` and `run.unpaused` records. A resumed run that was paused when +//! its worker died comes back paused, and the mirror reports that too. The +//! interrupt and pair controls have no Petri adapter yet and are ignored with a +//! warning. A control channel that is lost for good cancels the run the same +//! way, and the worker exits with that loss as its error once the run has +//! settled. //! //! Fabro's hooks ride the run with their platform records over the same //! client: the checkpoint commit in the run's workspace, on the host or @@ -66,20 +66,21 @@ use fabro_petri::blobs::ClientBlobs; use fabro_petri::controls::RunControls; use fabro_petri::engine::{self, Conclusion, Execution, RunRequest}; use fabro_petri::hooks::HooksSpec; -use fabro_petri::interview::{Approval, EventSinkQuestions, FabroInterviewer}; +use fabro_petri::interview::{Approval, FabroInterviewer}; use fabro_petri::petri::OwnerId; -use fabro_petri::platform_records::HttpPlatformRecords; +use fabro_petri::platform_records::{HttpPlatformRecords, PlatformRecords}; use fabro_petri::runtime::{self, RuntimeSpec}; use fabro_petri::secrets::VaultSecrets; use fabro_petri::{HttpRunStore, admission}; use fabro_static::EnvVars; use fabro_store::RunProjection; +use fabro_store::platform_records::{ + PlatformRecord, RunLifecycleKind, RunLifecycleRecord, RunNoticeRecord, +}; use fabro_types::settings::run::{ApprovalMode, RunMode}; -use fabro_types::{FailureReason, RunId, RunNoticeLevel, RunTiming, StageOutcome, SuccessReason}; +use fabro_types::{FailureReason, RunId, RunNoticeLevel, RunStatus, SuccessReason}; use fabro_vault::Vault; use fabro_workflow::Error as WorkflowError; -use fabro_workflow::event::{self as workflow_event, Event, RunEventSink}; -use fabro_workflow::runtime_store::RunStoreHandle; use fabro_workflow::services::FabroRunToolServices; use tokio::sync::RwLock as AsyncRwLock; use tokio::task::JoinHandle; @@ -95,9 +96,6 @@ pub(super) struct PetriWorker<'a> { pub(super) run_id: RunId, pub(super) target: ServerTarget, pub(super) client: Client, - /// The legacy run store over the same client, which carries the - /// lifecycle events to the server with its retries. - pub(super) run_store: RunStoreHandle, pub(super) run_state: RunProjection, pub(super) storage_dir: &'a Path, pub(super) run_dir: PathBuf, @@ -129,12 +127,15 @@ pub(super) async fn execute(worker: PetriWorker<'_>) -> Result<()> { let cancel_token = CancellationToken::new(); runner::install_signal_handlers(cancel_token.clone())?; let interviewer = Arc::new(ControlInterviewer::new()); - let sink = RunEventSink::map( - runner::stamp_system_worker, - RunEventSink::backend(worker.run_store.clone()), - ); + // Fabro's own records of the run, over the client. + let records: Arc = + Arc::new(HttpPlatformRecords::new(worker.client.clone_for_reuse())); let controls = RunControls::new(); - let petri_controls = Arc::new(PetriControls::new(run_id, controls.clone(), sink.clone())); + let petri_controls = Arc::new(PetriControls::new( + run_id, + controls.clone(), + Arc::clone(&records), + )); let mut control_manager = runner::spawn_worker_control_manager( worker.target.clone(), run_id, @@ -149,8 +150,7 @@ pub(super) async fn execute(worker: PetriWorker<'_>) -> Result<()> { } else { Approval::Prompt }; - let questions = Arc::new(EventSinkQuestions::new(sink.clone(), run_id)); - let petri_interviewer = FabroInterviewer::new(interviewer, questions, approval); + let petri_interviewer = FabroInterviewer::new(interviewer, approval); let observers = vec![petri_interviewer.observer()]; let vault = runner::load_worker_vault(worker.storage_dir).await?; @@ -181,16 +181,16 @@ pub(super) async fn execute(worker: PetriWorker<'_>) -> Result<()> { }; let started = Instant::now(); - for event in [Event::RunStarting, Event::RunRunning] { - workflow_event::append_event_to_sink(&sink, &run_id, &event).await?; + for transition in [ + (RunLifecycleKind::Starting, RunStatus::Starting), + (RunLifecycleKind::Running, RunStatus::Running), + ] { + lifecycle(&records, run_id, transition.0, transition.1, None).await?; } runner::set_worker_title(&run_id, WorkerTitlePhase::Running); - let hooks = HooksSpec::for_run( - Arc::new(HttpPlatformRecords::new(worker.client.clone_for_reuse())), - &worker.run_state.spec.settings.run, - ) - .with_test_gates(test_checkpoint_gates()); + let hooks = HooksSpec::for_run(Arc::clone(&records), &worker.run_state.spec.settings.run) + .with_test_gates(test_checkpoint_gates()); let request = RunRequest { run_id: run_id.to_string(), run_dir: worker.run_dir.join("petri"), @@ -216,7 +216,7 @@ pub(super) async fn execute(worker: PetriWorker<'_>) -> Result<()> { ))), hooks: Some(hooks), }; - let paused_mirror = mirror_paused_state(run_id, &controls, sink.clone()); + let paused_mirror = mirror_paused_state(run_id, &controls, Arc::clone(&records)); let run = Box::pin(engine::run(request)); tokio::pin!(run); let mut control_lost = None; @@ -235,33 +235,31 @@ pub(super) async fn execute(worker: PetriWorker<'_>) -> Result<()> { control_manager.finish(); paused_mirror.abort(); - let timing = RunTiming { - wall_time_ms: u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX), - ..RunTiming::default() - }; - let (event, phase, failure) = match engine::conclusion(&result) { + info!( + run_id = %run_id, + elapsed_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX), + "Petri run ended" + ); + let (record, phase, failure) = match engine::conclusion(&result) { Conclusion::Succeeded => { info!(run_id = %run_id, "Petri run completed"); ( - Event::WorkflowRunCompleted { - timing, - artifact_count: 0, - status: StageOutcome::Succeeded.to_string(), - reason: SuccessReason::Completed, - final_git_commit_sha: None, - final_patch: None, - diff_summary: None, - usage: None, - }, + ( + RunLifecycleKind::Succeeded, + RunStatus::Succeeded { + reason: SuccessReason::Completed, + }, + None, + ), WorkerTitlePhase::Succeeded, None, ) } Conclusion::Failed { reason, message } => { info!(run_id = %run_id, error = %message, "Petri run did not succeed"); - let error = match reason { - FailureReason::Cancelled => WorkflowError::Cancelled, - _ => WorkflowError::engine(message.clone()), + let detail = match reason { + FailureReason::Cancelled => WorkflowError::Cancelled.to_string(), + _ => message.clone(), }; let phase = if reason == FailureReason::Cancelled { WorkerTitlePhase::Cancelled @@ -269,15 +267,17 @@ pub(super) async fn execute(worker: PetriWorker<'_>) -> Result<()> { WorkerTitlePhase::Failed }; ( - Event::workflow_run_failed_from_error( - &error, timing, reason, None, None, None, None, + ( + RunLifecycleKind::Failed, + RunStatus::Failed { reason }, + Some(detail), ), phase, Some(message), ) } }; - workflow_event::append_event_to_sink(&sink, &run_id, &event).await?; + lifecycle(&records, run_id, record.0, record.1, record.2).await?; runner::set_worker_title(&run_id, phase); if let Some(lost) = control_lost { return Err(lost); @@ -295,15 +295,19 @@ pub(super) struct PetriControls { run_id: RunId, controls: RunControls, /// Where a refused steer's notice goes. - sink: RunEventSink, + records: Arc, } impl PetriControls { - pub(super) fn new(run_id: RunId, controls: RunControls, sink: RunEventSink) -> Self { + pub(super) fn new( + run_id: RunId, + controls: RunControls, + records: Arc, + ) -> Self { Self { run_id, controls, - sink, + records, } } @@ -352,15 +356,12 @@ impl PetriControls { /// A `run.notice` record on the run, so a refused control is visible in /// the run's stream and not only in the worker's log. async fn notice(&self, code: &str, message: String) { - let event = Event::RunNotice { + let record = PlatformRecord::RunNotice(RunNoticeRecord { level: RunNoticeLevel::Warn, code: code.to_string(), message, - exec_output_tail: None, - }; - if let Err(error) = - workflow_event::append_event_to_sink(&self.sink, &self.run_id, &event).await - { + }); + if let Err(error) = self.records.append(&self.run_id, &record, None).await { warn!(run_id = %self.run_id, error = %error, "the control notice was not recorded"); } } @@ -382,14 +383,31 @@ fn control_name(message: &WorkerControlMessage) -> &'static str { } } -/// Mirror the run's paused state to Fabro's lifecycle: `run.paused` when +/// One lifecycle transition of the run, recorded through the client. +async fn lifecycle( + records: &Arc, + run_id: RunId, + transition: RunLifecycleKind, + status: RunStatus, + reason: Option, +) -> Result<()> { + let mut record = RunLifecycleRecord::new(transition).with_status(status); + record.reason = reason; + records + .append(&run_id, &PlatformRecord::RunLifecycle(record), None) + .await + .with_context(|| format!("recording the run's {transition} transition"))?; + Ok(()) +} + +/// Mirror the run's paused state to Fabro's lifecycle: `paused` when /// admission is held (a pause, or a resume that came back paused) and -/// `run.unpaused` when it is released, each once per change, with the +/// `unpaused` when it is released, each once per change, with the /// worker's title alongside. Aborted with the run. fn mirror_paused_state( run_id: RunId, controls: &RunControls, - sink: RunEventSink, + records: Arc, ) -> JoinHandle<()> { let mut changes = controls.paused_changes(); tokio::spawn(async move { @@ -400,12 +418,13 @@ fn mirror_paused_state( continue; } last = paused; - let (event, phase) = if paused { - (Event::RunPaused, WorkerTitlePhase::Paused) + let (transition, phase) = if paused { + (RunLifecycleKind::Paused, WorkerTitlePhase::Paused) } else { - (Event::RunUnpaused, WorkerTitlePhase::Running) + (RunLifecycleKind::Unpaused, WorkerTitlePhase::Running) }; - if let Err(error) = workflow_event::append_event_to_sink(&sink, &run_id, &event).await { + let record = PlatformRecord::RunLifecycle(RunLifecycleRecord::new(transition)); + if let Err(error) = records.append(&run_id, &record, None).await { warn!(run_id = %run_id, error = %error, "the paused state was not reported"); } runner::set_worker_title(&run_id, phase); diff --git a/lib/apps/fabro-cli/src/commands/run/runner.rs b/lib/apps/fabro-cli/src/commands/run/runner.rs index d3b4f1092..fa6c32d69 100644 --- a/lib/apps/fabro-cli/src/commands/run/runner.rs +++ b/lib/apps/fabro-cli/src/commands/run/runner.rs @@ -4,7 +4,6 @@ use std::sync::Arc; use std::time::Duration; use anyhow::{Context, Result, anyhow}; -use async_trait::async_trait; use fabro_client::ServerTarget; use fabro_config::Storage; use fabro_interview::{ @@ -14,11 +13,9 @@ use fabro_interview::{ WorkerControlMessage, }; use fabro_manifest::SuppliedWorkflowVersionPackager; -use fabro_store::{EventEnvelope, RunProjection, RunProjectionReducer}; use fabro_tool::fabro_client::ClientBackend; -use fabro_types::{BlobHash, Principal, RunEvent, RunId}; +use fabro_types::RunId; use fabro_vault::{SecretStore, Vault}; -use fabro_workflow::runtime_store::{RunStoreBackend, RunStoreHandle}; use fabro_workflow::services::FabroRunToolServices; use futures::{SinkExt, StreamExt}; use jsonwebtoken::dangerous::insecure_decode; @@ -29,7 +26,7 @@ use tokio::net::TcpStream; use tokio::net::UnixStream; #[cfg(unix)] use tokio::signal::unix::{SignalKind, signal}; -use tokio::sync::{Mutex, RwLock as AsyncRwLock, oneshot}; +use tokio::sync::{RwLock as AsyncRwLock, oneshot}; use tokio::task::JoinHandle; use tokio::time::{self, Instant, MissedTickBehavior}; use tokio_tungstenite::tungstenite::client::IntoClientRequest; @@ -43,12 +40,6 @@ use super::petri_worker::{self, PetriControls, PetriWorker}; use crate::args::RunWorkerMode; use crate::server_client; -const RUN_STORE_RETRY_DELAYS: [Duration; 3] = [ - Duration::from_millis(50), - Duration::from_millis(100), - Duration::from_millis(250), -]; - #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum WorkerTitlePhase { Start, @@ -74,16 +65,14 @@ pub(crate) async fn execute( let target = server.parse::()?; let client = server_client::connect_server_target_with_bearer(&target, worker_token).await?; - let run_store = HttpRunStore::connect(run_id, client.clone_for_reuse()).await?; - let run_state = run_store - .state() + let run_state = client + .get_run_state(&run_id) .await .with_context(|| format!("failed to load run state for {run_id}"))?; Box::pin(petri_worker::execute(PetriWorker { run_id, target, client, - run_store, run_state, storage_dir: &storage_dir, run_dir, @@ -663,150 +652,6 @@ async fn apply_worker_control_message( } } -#[derive(Clone)] -struct HttpRunStore { - run_id: RunId, - client: server_client::Client, - state: Arc>, - events: Arc>>>, -} - -impl HttpRunStore { - async fn connect(run_id: RunId, client: server_client::Client) -> Result { - let state = client - .get_run_state(&run_id) - .await - .with_context(|| format!("failed to fetch run state for {run_id}"))?; - Ok(RunStoreHandle::new(Arc::new(Self { - run_id, - client, - state: Arc::new(Mutex::new(state)), - events: Arc::new(Mutex::new(None)), - }))) - } - - async fn with_retries(&self, operation: &'static str, mut op: F) -> Result - where - F: FnMut() -> Fut, - Fut: std::future::Future>, - { - let mut last_error = None; - for attempt in 0..=RUN_STORE_RETRY_DELAYS.len() { - match op().await { - Ok(value) => return Ok(value), - Err(err) => last_error = Some(err), - } - if let Some(delay) = RUN_STORE_RETRY_DELAYS.get(attempt) { - time::sleep(*delay).await; - } - } - Err(last_error - .unwrap_or_else(|| anyhow!("run store operation failed")) - .context(format!( - "worker lost canonical run store during {operation}" - ))) - } - - async fn refresh_state_from_server(&self) -> Result { - self.with_retries("refresh state", || { - let client = self.client.clone_for_reuse(); - let run_id = self.run_id; - async move { client.get_run_state(&run_id).await } - }) - .await - } - - async fn apply_acknowledged_event(&self, seq: u32, event: &RunEvent) -> Result<()> { - let envelope = EventEnvelope { - seq, - event: event.clone(), - }; - - { - let mut state = self.state.lock().await; - if let Err(err) = state.apply_event(&envelope) { - tracing::warn!(run_id = %self.run_id, error = %err, "failed to apply acknowledged event to local run-state mirror; refreshing from server"); - drop(state); - let refreshed = self.refresh_state_from_server().await?; - *self.state.lock().await = refreshed; - } - } - - let mut events = self.events.lock().await; - if let Some(cached) = events.as_mut() { - cached.push(envelope); - } - - Ok(()) - } -} - -#[async_trait] -impl RunStoreBackend for HttpRunStore { - async fn load_state(&self) -> Result { - Ok(self.state.lock().await.clone()) - } - - async fn list_events(&self) -> Result> { - let mut cached = self.events.lock().await; - if let Some(events) = cached.as_ref() { - return Ok(events.clone()); - } - - let events = self - .with_retries("list run events", || { - let client = self.client.clone_for_reuse(); - let run_id = self.run_id; - async move { client.list_run_events(&run_id, None, None).await } - }) - .await?; - *cached = Some(events.clone()); - Ok(events) - } - - async fn append_run_event(&self, event: &RunEvent) -> Result<()> { - let seq = Box::pin(self.with_retries("append run event", || { - let client = self.client.clone_for_reuse(); - let run_id = self.run_id; - let event = event.clone(); - async move { client.append_run_event(&run_id, &event).await } - })) - .await?; - // Both the sandbox lifecycle and the lithos event shapes grew this - // future past clippy's stack budget; box it once at the call. - Box::pin(self.apply_acknowledged_event(seq, event)).await - } - - async fn write_blob(&self, data: &[u8]) -> Result { - self.with_retries("write run blob", || { - let client = self.client.clone_for_reuse(); - let run_id = self.run_id; - let data = data.to_vec(); - async move { client.write_run_blob(&run_id, &data).await } - }) - .await - } - - async fn read_blob(&self, blob_hash: &BlobHash) -> Result> { - self.with_retries("read run blob", || { - let client = self.client.clone_for_reuse(); - let run_id = self.run_id; - let blob_hash = *blob_hash; - async move { client.read_run_blob(&run_id, &blob_hash).await } - }) - .await - } - - async fn read_run_log(&self) -> Result>> { - self.with_retries("get run logs", || { - let client = self.client.clone_for_reuse(); - let run_id = self.run_id; - async move { client.get_run_logs(&run_id).await } - }) - .await - } -} - pub(super) fn set_worker_title(run_id: &RunId, phase: WorkerTitlePhase) { fabro_proc::title_set(&worker_title(run_id, phase)); } @@ -832,15 +677,6 @@ fn worker_title(run_id: &RunId, phase: WorkerTitlePhase) -> String { format!("fabro {short_id} {phase}") } -pub(super) fn stamp_system_worker(mut event: RunEvent) -> RunEvent { - if event.actor.is_none() { - event.actor = Some(Principal::Worker { - run_id: event.run_id, - }); - } - event -} - /// `SIGTERM` and `SIGINT` cancel the run, the way the server's cancel does. pub(super) fn install_signal_handlers(cancel_token: CancellationToken) -> Result<()> { #[cfg(unix)] @@ -873,16 +709,13 @@ mod tests { use std::sync::Arc; use std::time::Duration; - use chrono::Utc; use fabro_client::ServerTarget; use fabro_config::Storage; use fabro_interview::{ AnswerValue, ControlInterviewer, Interviewer, Question, WorkerControlEnvelope, }; - use fabro_types::run_event::RunStatusTransitionProps; - use fabro_types::{AuthMethod, EventBody, IdpIdentity, Principal, QuestionType, fixtures}; + use fabro_types::{QuestionType, fixtures}; use fabro_vault::{SecretType, Vault}; - use fabro_workflow::event::RunEventSink; use tokio::time; use tokio_tungstenite::tungstenite::protocol::{Message as TestWebSocketMessage, Role}; use tokio_util::sync::CancellationToken; @@ -893,19 +726,17 @@ mod tests { WorkerControls, WorkerTitlePhase, apply_worker_control_delivery_frame, apply_worker_control_message, build_worker_control_stream_request, connect_worker_control_stream, handle_worker_control_socket, initial_worker_title_phase, - load_worker_vault, next_worker_control_reconnect_backoff, stamp_system_worker, - worker_title, + load_worker_vault, next_worker_control_reconnect_backoff, worker_title, }; use crate::args::RunWorkerMode; - /// A run's controls over a sink that keeps nothing: what the channel + /// A run's controls over records kept in memory: what the channel /// tests drive. fn test_controls() -> WorkerControls { - let sink = RunEventSink::callback(|_event| async move { Ok(()) }); Arc::new(PetriControls::new( fixtures::RUN_1, fabro_petri::controls::RunControls::new(), - sink, + Arc::new(fabro_petri::test_support::MemoryPlatformRecords::new()), )) } @@ -952,32 +783,6 @@ mod tests { .expect("test worker token should encode") } - fn test_user_principal(login: &str) -> Principal { - Principal::user( - IdpIdentity::new("https://github.com", "12345").unwrap(), - login.to_string(), - AuthMethod::Github, - ) - } - - fn running_event(actor: Option) -> fabro_types::RunEvent { - fabro_types::RunEvent { - id: "evt_1".to_string(), - ts: Utc::now(), - run_id: fixtures::RUN_1, - node_id: None, - node_label: None, - stage_id: None, - parallel_group_id: None, - parallel_branch_id: None, - session_id: None, - parent_session_id: None, - tool_call_id: None, - actor, - body: EventBody::RunRunning(RunStatusTransitionProps::default()), - } - } - #[test] fn worker_title_uses_short_run_id_and_phase() { let short_id: String = fixtures::RUN_1.to_string().chars().take(12).collect(); @@ -1003,67 +808,6 @@ mod tests { ); } - #[test] - fn stamp_system_worker_fills_missing_actor_only() { - let stamped = stamp_system_worker(running_event(None)); - - assert_eq!( - stamped.actor, - Some(Principal::Worker { - run_id: fixtures::RUN_1, - }) - ); - - let existing_actor = test_user_principal("octocat"); - let stamped = stamp_system_worker(running_event(Some(existing_actor.clone()))); - assert_eq!(stamped.actor, Some(existing_actor)); - } - - #[tokio::test] - async fn worker_event_stamp_applies_to_all_fanout_sinks() { - let first = Arc::new(tokio::sync::Mutex::new(Vec::new())); - let second = Arc::new(tokio::sync::Mutex::new(Vec::new())); - let first_events = Arc::clone(&first); - let second_events = Arc::clone(&second); - let sink = RunEventSink::map( - stamp_system_worker, - RunEventSink::fanout(vec![ - RunEventSink::callback(move |event| { - let first_events = Arc::clone(&first_events); - async move { - first_events.lock().await.push(event); - Ok(()) - } - }), - RunEventSink::callback(move |event| { - let second_events = Arc::clone(&second_events); - async move { - second_events.lock().await.push(event); - Ok(()) - } - }), - ]), - ); - let event = running_event(None); - - sink.write_run_event(&event).await.unwrap(); - - let first = first.lock().await; - let second = second.lock().await; - assert_eq!( - first[0].actor, - Some(Principal::Worker { - run_id: fixtures::RUN_1, - }) - ); - assert_eq!( - second[0].actor, - Some(Principal::Worker { - run_id: fixtures::RUN_1, - }) - ); - } - #[tokio::test] async fn worker_control_routes_answer_by_question_id() { let interviewer = Arc::new(ControlInterviewer::new()); diff --git a/lib/apps/fabro-cli/tests/it/scenario/petri.rs b/lib/apps/fabro-cli/tests/it/scenario/petri.rs index 865ea8c78..cac1da09b 100644 --- a/lib/apps/fabro-cli/tests/it/scenario/petri.rs +++ b/lib/apps/fabro-cli/tests/it/scenario/petri.rs @@ -169,7 +169,8 @@ impl RunningServer { .stdout(self.stderr_log()) .stderr(self.stderr_log()); let mut child = cmd.spawn().expect("the server spawns"); - wait_for_http_ready(&self.api_base_url, &mut child).await; + let log_path = self.storage_dir.with_file_name("server.stderr.log"); + wait_for_http_ready(&self.api_base_url, &mut child, &log_path).await; self.child = Some(child); } @@ -370,7 +371,7 @@ fn reserve_port() -> u16 { .port() } -async fn wait_for_http_ready(base_url: &str, child: &mut Child) { +async fn wait_for_http_ready(base_url: &str, child: &mut Child, log_path: &Path) { let client = fabro_test::test_http_client(); let deadline = Instant::now() + Duration::from_secs(10); loop { @@ -378,7 +379,13 @@ async fn wait_for_http_ready(base_url: &str, child: &mut Child) { Ok(response) if response.status().is_success() => return, Ok(_) | Err(_) if Instant::now() < deadline => { if let Some(status) = child.try_wait().expect("the server polls") { - panic!("the server exited before it was ready with status {status}"); + let log = std::fs::read_to_string(log_path).unwrap_or_default(); + let tail = log.lines().rev().take(20).collect::>(); + panic!( + "the server exited before it was ready with status {status}; its log ends \ + with:\n{}", + tail.into_iter().rev().collect::>().join("\n") + ); } tokio::time::sleep(Duration::from_millis(25)).await; } diff --git a/lib/apps/fabro-server/migrations/2026082301_sqlite_blob_activation.rs b/lib/apps/fabro-server/migrations/2026082301_sqlite_blob_activation.rs deleted file mode 100644 index 5bc18a47b..000000000 --- a/lib/apps/fabro-server/migrations/2026082301_sqlite_blob_activation.rs +++ /dev/null @@ -1,746 +0,0 @@ -//! Fail-closed activation of SQLite blob storage. -//! -//! This compatibility bridge remains until at least 30 calendar days after -//! the first successful production activation, and until the cold-start, -//! warm-restart, production-observation, and backup-integrity evidence is -//! complete and Scott explicitly approves its removal. The date is an -//! eligibility floor, never an automatic deletion trigger. - -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use std::time::Duration; - -use object_store::ObjectStore; -use tokio::fs; -use tracing::{debug, info, warn}; - -use crate::migrations::sqlite_activation_backup::{self, BackupError}; -use crate::migrations::sqlite_run_history_activation::BACKUP_SUFFIX as RUN_HISTORY_BACKUP_SUFFIX; -use crate::server::resource_sampler; - -/// Earliest date this bridge becomes eligible for removal, assuming the first -/// production activation happens no earlier than this change ships. Removal -/// additionally requires the evidence and explicit approval described in the -/// module docs; the date alone never triggers deletion. -pub(crate) const REMOVAL_DEADLINE: &str = "2026-09-22"; - -const DISK_HEADROOM_BYTES: u64 = 64 * 1024 * 1024; -const BACKUP_SUFFIX: &str = ".pre-blob-activation.bak"; - -pub(crate) struct ActivatedBlobStorage { - pub(crate) store: Arc, - pub(crate) run_history_identity: fabro_store::LegacyRunHistorySourceIdentity, -} - -impl std::fmt::Debug for ActivatedBlobStorage { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("ActivatedBlobStorage") - .field("run_history_identity", &self.run_history_identity) - .finish_non_exhaustive() - } -} - -#[derive(Debug, thiserror::Error)] -pub(crate) enum BlobActivationError { - #[error("canonicalizing the SQLite database path {path}")] - Canonicalize { - path: PathBuf, - #[source] - source: std::io::Error, - }, - #[error("inventorying the legacy blob source")] - Inventory(#[source] fabro_store::LegacyBlobInventoryError), - #[error("identifying legacy run history before importing blobs")] - RunHistorySourceIdentity(#[source] fabro_store::LegacyRunHistorySourceIdentityError), - #[error( - "activation backup is missing at {path} while {existing_rows} of {legacy_rows} legacy blob rows are already present in SQLite" - )] - MissingBackupAfterImport { - path: PathBuf, - legacy_rows: u64, - existing_rows: u64, - }, - #[error("reading SQLite file metadata at {path}")] - SqliteMetadata { - path: PathBuf, - #[source] - source: std::io::Error, - }, - #[error("the blob activation disk requirement overflowed")] - DiskRequirementOverflow, - #[error( - "insufficient disk space for blob activation: {available_bytes} bytes available, {required_bytes} required" - )] - InsufficientDisk { - required_bytes: u64, - available_bytes: u64, - }, - #[error(transparent)] - Backup(#[from] BackupError), - #[error("importing legacy blobs into SQLite")] - Import(#[source] Box), - #[error("verifying legacy and SQLite blobs")] - Verification(#[source] Box), - #[error("running the live SQLite integrity check")] - LiveIntegrity(#[source] sqlx::Error), - #[error("the live SQLite integrity check did not return exactly one ok result")] - LiveIntegrityFailed, - #[error("running the final SQLite WAL truncate checkpoint")] - FinalCheckpoint(#[source] sqlx::Error), -} - -pub(crate) async fn activate_blob_storage( - database: &fabro_db::Database, - sqlite_path: &Path, - object_store: Arc, - slatedb_prefix: String, - flush_interval: Duration, - cache_path: Option, -) -> Result { - let canonical_path = fs::canonicalize(sqlite_path).await.map_err(|source| { - BlobActivationError::Canonicalize { - path: sqlite_path.to_path_buf(), - source, - } - })?; - let backup_path = fabro_db::append_to_path(&canonical_path, BACKUP_SUFFIX); - info!( - database_path = %canonical_path.display(), - backup_path = %backup_path.display(), - "Starting SQLite blob storage activation" - ); - - let blob_store = Arc::new(fabro_store::BlobStore::new(database.clone_pool())); - let run_summary_store = Arc::new(fabro_store::RunSummaryStore::new(database.clone_pool())); - let store = Arc::new(fabro_store::Database::new( - object_store, - slatedb_prefix, - flush_interval, - cache_path, - Arc::clone(&blob_store), - run_summary_store, - )); - - let inventory = store - .legacy_blob_inventory(database.pool()) - .await - .map_err(BlobActivationError::Inventory)?; - let run_history_identity = store - .legacy_run_history_source_identity() - .await - .map_err(BlobActivationError::RunHistorySourceIdentity)?; - let backup_exists = sqlite_activation_backup::backup_exists(&backup_path).await?; - if backup_exists { - sqlite_activation_backup::validate_backup(&backup_path).await?; - } - if !backup_exists && inventory.pending_rows < inventory.rows { - return Err(BlobActivationError::MissingBackupAfterImport { - path: backup_path, - legacy_rows: inventory.rows, - existing_rows: inventory.rows - inventory.pending_rows, - }); - } - let backup_required = inventory.rows > 0 && !backup_exists; - let run_history_backup_path = - fabro_db::append_to_path(&canonical_path, RUN_HISTORY_BACKUP_SUFFIX); - let run_history_backup_exists = - sqlite_activation_backup::backup_exists(&run_history_backup_path).await?; - if run_history_backup_exists { - sqlite_activation_backup::validate_backup(&run_history_backup_path).await?; - } - let run_history_backup_required = - run_history_identity.events != 0 && !run_history_backup_exists; - // The resource sampler treats a path with no matching mount as an - // unsupported-but-benign condition (tmpfs or squashfs roots, network - // filesystems, an unreadable mount table), so the preflight does too: - // skipping the capacity check must not block a boot the import itself - // could complete. - if let Some(available_free_bytes) = resource_sampler::available_space_for_path(&canonical_path) - { - let sqlite_bytes = if backup_required || run_history_backup_required { - sqlite_file_set_bytes(&canonical_path).await? - } else { - 0 - }; - let backup_reserve = if backup_required { sqlite_bytes } else { 0 }; - let run_history_backup_reserve = if run_history_backup_required { - projected_sqlite_bytes(sqlite_bytes, inventory.pending_bytes)? - } else { - 0 - }; - // Only the rows the import still has to copy need new space; rows - // already present in SQLite cost nothing on a warm restart. Reserve - // the projected post-import database size as well when run-history - // activation will immediately take its own full SQLite snapshot. - let required_free_bytes = compute_disk_preflight( - inventory.pending_bytes, - backup_reserve, - run_history_backup_reserve, - available_free_bytes, - )?; - debug!( - legacy_rows = inventory.rows, - legacy_bytes = inventory.bytes, - pending_rows = inventory.pending_rows, - pending_bytes = inventory.pending_bytes, - backup_required, - backup_reserve, - run_history_events = run_history_identity.events, - run_history_backup_required, - run_history_backup_reserve, - required_free_bytes, - available_free_bytes, - "Checked SQLite blob activation disk capacity" - ); - } else { - warn!( - database_path = %canonical_path.display(), - "No filesystem mount matched the SQLite database path; skipping the blob activation disk preflight" - ); - } - - let retained_backup = if backup_exists { - Some(backup_path) - } else if backup_required { - sqlite_activation_backup::create_backup(database.pool(), &backup_path).await?; - Some(backup_path) - } else { - None - }; - - let import = store - .import_legacy_blobs_into(database.pool()) - .await - .map_err(|source| BlobActivationError::Import(Box::new(source)))?; - // The import pass already validates every legacy digest and byte-compares - // every already-present row on each boot, so the independent verification - // sweep only needs to double-check boots that actually inserted rows. - let verification = if import.imported_rows > 0 { - Some( - store - .verify_legacy_blobs_in(database.pool()) - .await - .map_err(|source| BlobActivationError::Verification(Box::new(source)))?, - ) - } else { - None - }; - validate_live_integrity(database.pool()).await?; - final_truncate_checkpoint(database.pool()).await?; - - info!( - legacy_rows = inventory.rows, - legacy_bytes = inventory.bytes, - imported_rows = import.imported_rows, - existing_rows = import.existing_rows, - matched_rows = verification.as_ref().map(|report| report.matched_rows), - target_rows = verification.as_ref().map(|report| report.target_rows), - passive_checkpoints = import.passive_checkpoints, - backup_required, - backup_path = ?retained_backup, - run_history_backup_required, - removal_deadline = REMOVAL_DEADLINE, - "Activated SQLite blob storage" - ); - Ok(ActivatedBlobStorage { - store, - run_history_identity, - }) -} - -/// Fail-closed disk capacity check; returns the required free bytes. -fn compute_disk_preflight( - pending_bytes: u64, - backup_reserve: u64, - run_history_backup_reserve: u64, - available_free_bytes: u64, -) -> Result { - let import_reserve = blob_import_reserve(pending_bytes)?; - let required_free_bytes = backup_reserve - .checked_add(import_reserve) - .and_then(|value| value.checked_add(run_history_backup_reserve)) - .and_then(|value| value.checked_add(DISK_HEADROOM_BYTES)) - .ok_or(BlobActivationError::DiskRequirementOverflow)?; - if available_free_bytes < required_free_bytes { - return Err(BlobActivationError::InsufficientDisk { - required_bytes: required_free_bytes, - available_bytes: available_free_bytes, - }); - } - Ok(required_free_bytes) -} - -fn projected_sqlite_bytes( - sqlite_bytes: u64, - pending_bytes: u64, -) -> Result { - sqlite_bytes - .checked_add(blob_import_reserve(pending_bytes)?) - .ok_or(BlobActivationError::DiskRequirementOverflow) -} - -fn blob_import_reserve(pending_bytes: u64) -> Result { - let half = pending_bytes - .checked_add(1) - .ok_or(BlobActivationError::DiskRequirementOverflow)? - / 2; - pending_bytes - .checked_add(half) - .ok_or(BlobActivationError::DiskRequirementOverflow) -} - -async fn sqlite_file_set_bytes(path: &Path) -> Result { - let mut total = required_file_bytes(path).await?; - for suffix in ["-wal", "-shm"] { - let sibling = fabro_db::append_to_path(path, suffix); - let bytes = optional_file_bytes(&sibling).await?; - total = total - .checked_add(bytes) - .ok_or(BlobActivationError::DiskRequirementOverflow)?; - } - Ok(total) -} - -async fn required_file_bytes(path: &Path) -> Result { - fs::metadata(path) - .await - .map(|metadata| metadata.len()) - .map_err(|source| BlobActivationError::SqliteMetadata { - path: path.to_path_buf(), - source, - }) -} - -async fn optional_file_bytes(path: &Path) -> Result { - match fs::metadata(path).await { - Ok(metadata) => Ok(metadata.len()), - Err(source) if source.kind() == std::io::ErrorKind::NotFound => Ok(0), - Err(source) => Err(BlobActivationError::SqliteMetadata { - path: path.to_path_buf(), - source, - }), - } -} - -async fn validate_live_integrity(pool: &sqlx::SqlitePool) -> Result<(), BlobActivationError> { - let ok = sqlite_activation_backup::integrity_check_is_ok(pool) - .await - .map_err(BlobActivationError::LiveIntegrity)?; - if !ok { - return Err(BlobActivationError::LiveIntegrityFailed); - } - Ok(()) -} - -async fn final_truncate_checkpoint(pool: &sqlx::SqlitePool) -> Result<(), BlobActivationError> { - let (busy, _, _): (i64, i64, i64) = sqlx::query_as("PRAGMA wal_checkpoint(TRUNCATE)") - .fetch_one(pool) - .await - .map_err(BlobActivationError::FinalCheckpoint)?; - if busy != 0 { - // A concurrent reader (a backup tool, a replication agent, an - // operator shell) can keep the WAL from truncating. An untruncated - // WAL threatens no data integrity, so it must not block startup; a - // later checkpoint truncates once the reader is gone. - warn!("The final SQLite WAL truncate checkpoint could not complete; continuing startup"); - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - use std::time::Duration; - - use fabro_db::append_to_path; - use object_store::ObjectStore; - use object_store::memory::InMemory; - use tokio::fs; - - use super::{ - BACKUP_SUFFIX, BlobActivationError, DISK_HEADROOM_BYTES, activate_blob_storage, - compute_disk_preflight, final_truncate_checkpoint, projected_sqlite_bytes, - sqlite_file_set_bytes, - }; - use crate::migrations::sqlite_activation_backup::{self, BackupError, create_backup}; - - type TestResult = Result>; - - #[test] - fn disk_preflight_passes_at_equality_and_fails_one_byte_below() { - let pending_bytes = 3; - let backup_reserve = 10; - let run_history_backup_reserve = 20; - let required = - backup_reserve + pending_bytes + 2 + run_history_backup_reserve + DISK_HEADROOM_BYTES; - - let required_free_bytes = compute_disk_preflight( - pending_bytes, - backup_reserve, - run_history_backup_reserve, - required, - ) - .expect("exact equality must pass"); - assert_eq!(required_free_bytes, required); - - let error = compute_disk_preflight( - pending_bytes, - backup_reserve, - run_history_backup_reserve, - required - 1, - ) - .expect_err("one byte below must fail"); - assert!(matches!( - error, - BlobActivationError::InsufficientDisk { .. } - )); - } - - #[test] - fn disk_preflight_requires_only_headroom_without_a_backup_reserve() { - let required_free_bytes = - compute_disk_preflight(2, 0, 0, u64::MAX).expect("available capacity should pass"); - assert_eq!(required_free_bytes, 3 + DISK_HEADROOM_BYTES); - } - - #[test] - fn disk_preflight_reserves_the_projected_post_import_database() { - let projected = projected_sqlite_bytes(10, 3).expect("the projection should fit"); - assert_eq!(projected, 15); - - let required = compute_disk_preflight(3, 0, projected, u64::MAX) - .expect("available capacity should pass"); - assert_eq!(required, 20 + DISK_HEADROOM_BYTES); - } - - #[test] - fn disk_preflight_fails_closed_on_overflow() { - let error = compute_disk_preflight(u64::MAX, 1, 0, u64::MAX) - .expect_err("overflow must fail closed"); - assert!(matches!( - error, - BlobActivationError::DiskRequirementOverflow - )); - } - - #[tokio::test] - async fn disk_preflight_counts_the_sqlite_file_set_for_a_required_backup() -> TestResult<()> { - let directory = tempfile::tempdir()?; - let sqlite_path = directory.path().join("fabro.sqlite3"); - fs::write(&sqlite_path, [0_u8; 3]).await?; - fs::write(append_to_path(&sqlite_path, "-wal"), [0_u8; 5]).await?; - fs::write(append_to_path(&sqlite_path, "-shm"), [0_u8; 7]).await?; - - assert_eq!(sqlite_file_set_bytes(&sqlite_path).await?, 15); - Ok(()) - } - - #[tokio::test] - async fn backup_is_private_integrity_clean_and_does_not_create_journal_siblings() - -> TestResult<()> { - let directory = tempfile::tempdir()?; - let sqlite_path = directory.path().join("fabro.sqlite3"); - let database = fabro_db::Database::connect(&sqlite_path).await?; - database.migrate().await?; - let backup_path = append_to_path(&sqlite_path, BACKUP_SUFFIX); - - sqlite_activation_backup::create_backup(database.pool(), &backup_path).await?; - sqlite_activation_backup::validate_backup(&backup_path).await?; - - assert!(backup_path.is_file()); - assert!(!append_to_path(&backup_path, "-wal").exists()); - assert!(!append_to_path(&backup_path, "-shm").exists()); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt as _; - assert_eq!( - std::fs::metadata(&backup_path)?.permissions().mode() & 0o077, - 0 - ); - } - Ok(()) - } - - #[tokio::test] - async fn backup_publication_never_overwrites_an_existing_valid_backup() -> TestResult<()> { - let directory = tempfile::tempdir()?; - let sqlite_path = directory.path().join("fabro.sqlite3"); - let database = fabro_db::Database::connect(&sqlite_path).await?; - database.migrate().await?; - let backup_path = append_to_path(&sqlite_path, BACKUP_SUFFIX); - sqlite_activation_backup::create_backup(database.pool(), &backup_path).await?; - let original = fs::read(&backup_path).await?; - - sqlx::query("INSERT INTO blobs (hash, data) VALUES (?, ?)") - .bind(fabro_types::BlobHash::new(b"later").to_string()) - .bind(b"later".as_slice()) - .execute(database.pool()) - .await?; - sqlite_activation_backup::create_backup(database.pool(), &backup_path).await?; - - assert_eq!(fs::read(&backup_path).await?, original); - Ok(()) - } - - #[tokio::test] - async fn failed_backup_copy_never_publishes_a_destination() -> TestResult<()> { - let directory = tempfile::tempdir()?; - let sqlite_path = directory.path().join("fabro.sqlite3"); - let database = fabro_db::Database::connect(&sqlite_path).await?; - database.migrate().await?; - let backup_path = append_to_path(&sqlite_path, BACKUP_SUFFIX); - database.pool().close().await; - - let error = create_backup(database.pool(), &backup_path) - .await - .expect_err("a closed pool must fail backup creation"); - - assert!(matches!( - error, - BackupError::Stage(fabro_db::SnapshotStagingError::Write { .. }) - )); - assert!(!backup_path.exists()); - Ok(()) - } - - #[tokio::test] - async fn cold_activation_and_warm_restart_share_verified_sqlite_blobs() -> TestResult<()> { - let directory = tempfile::tempdir()?; - let sqlite_path = directory.path().join("fabro.sqlite3"); - let database = fabro_db::Database::connect(&sqlite_path).await?; - database.migrate().await?; - let object_store: Arc = Arc::new(InMemory::new()); - let source = fabro_store::test_support::test_database( - Arc::clone(&object_store), - "activation-test", - Duration::from_millis(1), - None, - ); - let legacy_bytes = b"legacy-blob"; - let legacy_hash = fabro_store::test_support::put_legacy_blob(&source, legacy_bytes).await?; - drop(source); - - let activation = activate_blob_storage( - &database, - &sqlite_path, - Arc::clone(&object_store), - "activation-test".to_string(), - Duration::from_millis(1), - None, - ) - .await?; - let store = activation.store; - assert_eq!( - store.blobs().read(&legacy_hash).await?.as_deref(), - Some(legacy_bytes.as_slice()) - ); - - let backup_path = append_to_path(&sqlite_path, BACKUP_SUFFIX); - let original_backup = fs::read(&backup_path).await?; - let run_id = fabro_types::RunId::new(); - let writer = store.create_run(&run_id).await?; - let reader = store.open_run_reader(&run_id).await?; - let sqlite_only_bytes = b"written-after-activation"; - let sqlite_only_hash = writer.write_blob(sqlite_only_bytes).await?; - assert_eq!( - reader.read_blob(&sqlite_only_hash).await?.as_deref(), - Some(sqlite_only_bytes.as_slice()) - ); - drop(reader); - drop(writer); - drop(store); - - let warm = activate_blob_storage( - &database, - &sqlite_path, - object_store, - "activation-test".to_string(), - Duration::from_millis(1), - None, - ) - .await?; - assert_eq!(fs::read(&backup_path).await?, original_backup); - assert_eq!( - warm.store.blobs().read(&sqlite_only_hash).await?.as_deref(), - Some(sqlite_only_bytes.as_slice()) - ); - Ok(()) - } - - #[tokio::test] - async fn missing_backup_after_prior_import_fails_closed() -> TestResult<()> { - let directory = tempfile::tempdir()?; - let sqlite_path = directory.path().join("fabro.sqlite3"); - let database = fabro_db::Database::connect(&sqlite_path).await?; - database.migrate().await?; - let object_store: Arc = Arc::new(InMemory::new()); - let source = fabro_store::test_support::test_database( - Arc::clone(&object_store), - "missing-backup-test", - Duration::from_millis(1), - None, - ); - let bytes = b"already-imported"; - let hash = fabro_store::test_support::put_legacy_blob(&source, bytes).await?; - sqlx::query("INSERT INTO blobs (hash, data) VALUES (?, ?)") - .bind(hash.to_string()) - .bind(bytes.as_slice()) - .execute(database.pool()) - .await?; - drop(source); - - let error = activate_blob_storage( - &database, - &sqlite_path, - object_store, - "missing-backup-test".to_string(), - Duration::from_millis(1), - None, - ) - .await - .expect_err("startup must not move the pre-activation rollback boundary"); - - assert!(matches!( - error, - BlobActivationError::MissingBackupAfterImport { - legacy_rows: 1, - existing_rows: 1, - .. - } - )); - assert!(!append_to_path(&sqlite_path, BACKUP_SUFFIX).exists()); - Ok(()) - } - - #[tokio::test] - async fn empty_inventory_skips_backup_and_serves_existing_sqlite_rows() -> TestResult<()> { - let directory = tempfile::tempdir()?; - let sqlite_path = directory.path().join("fabro.sqlite3"); - let database = fabro_db::Database::connect(&sqlite_path).await?; - database.migrate().await?; - let bytes = b"sqlite-only"; - let hash = fabro_types::BlobHash::new(bytes); - sqlx::query("INSERT INTO blobs (hash, data) VALUES (?, ?)") - .bind(hash.to_string()) - .bind(bytes.as_slice()) - .execute(database.pool()) - .await?; - - let activated = activate_blob_storage( - &database, - &sqlite_path, - Arc::new(InMemory::new()), - "empty-activation-test".to_string(), - Duration::from_millis(1), - None, - ) - .await?; - - assert!(!append_to_path(&sqlite_path, BACKUP_SUFFIX).exists()); - assert_eq!( - activated.store.blobs().read(&hash).await?.as_deref(), - Some(bytes.as_slice()) - ); - Ok(()) - } - - #[tokio::test] - async fn busy_final_checkpoint_warns_and_does_not_fail_startup() -> TestResult<()> { - use sqlx::Connection as _; - use sqlx::sqlite::{ - SqliteConnectOptions, SqliteConnection, SqliteJournalMode, SqlitePoolOptions, - }; - - let directory = tempfile::tempdir()?; - let sqlite_path = directory.path().join("fabro.sqlite3"); - let database = fabro_db::Database::connect(&sqlite_path).await?; - database.migrate().await?; - sqlx::query("INSERT INTO blobs (hash, data) VALUES (?, ?)") - .bind(fabro_types::BlobHash::new(b"wal-content").to_string()) - .bind(b"wal-content".as_slice()) - .execute(database.pool()) - .await?; - - // A reader holding an open snapshot models a backup tool or operator - // shell that outlives the checkpoint's busy timeout. - let reader_options = SqliteConnectOptions::new() - .filename(&sqlite_path) - .read_only(true) - .create_if_missing(false); - let mut reader = SqliteConnection::connect_with(&reader_options).await?; - sqlx::query("BEGIN").execute(&mut reader).await?; - sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM blobs") - .fetch_one(&mut reader) - .await?; - - // A short busy timeout keeps the blocked truncate from stalling the - // test for the production pool's full five seconds. - let checkpoint_options = SqliteConnectOptions::new() - .filename(&sqlite_path) - .journal_mode(SqliteJournalMode::Wal) - .busy_timeout(Duration::from_millis(50)) - .create_if_missing(false); - let checkpoint_pool = SqlitePoolOptions::new() - .max_connections(1) - .connect_with(checkpoint_options) - .await?; - - final_truncate_checkpoint(&checkpoint_pool).await?; - - // The reader really did block the truncate: the WAL was not reset. - let wal_bytes = fs::metadata(append_to_path(&sqlite_path, "-wal")) - .await? - .len(); - assert!(wal_bytes > 0, "the WAL should remain untruncated"); - drop(reader); - Ok(()) - } - - #[tokio::test] - async fn invalid_retained_backup_fails_before_importing() -> TestResult<()> { - let directory = tempfile::tempdir()?; - let sqlite_path = directory.path().join("fabro.sqlite3"); - let database = fabro_db::Database::connect(&sqlite_path).await?; - database.migrate().await?; - let object_store: Arc = Arc::new(InMemory::new()); - let source = fabro_store::test_support::test_database( - Arc::clone(&object_store), - "invalid-backup-test", - Duration::from_millis(1), - None, - ); - fabro_store::test_support::put_legacy_blob(&source, b"must-not-import").await?; - drop(source); - - let backup_path = append_to_path(&sqlite_path, BACKUP_SUFFIX); - fs::write(&backup_path, b"not a database").await?; - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt as _; - fs::set_permissions(&backup_path, std::fs::Permissions::from_mode(0o600)).await?; - } - - let error = activate_blob_storage( - &database, - &sqlite_path, - object_store, - "invalid-backup-test".to_string(), - Duration::from_millis(1), - None, - ) - .await - .expect_err("an invalid retained backup must fail closed"); - assert!(matches!( - error, - BlobActivationError::Backup( - BackupError::Integrity { .. } | BackupError::IntegrityFailed { .. } - ) - )); - let destination_rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM blobs") - .fetch_one(database.pool()) - .await?; - assert_eq!(destination_rows, 0); - Ok(()) - } -} diff --git a/lib/apps/fabro-server/migrations/2026082801_sqlite_run_history_activation.rs b/lib/apps/fabro-server/migrations/2026082801_sqlite_run_history_activation.rs deleted file mode 100644 index faf49f52d..000000000 --- a/lib/apps/fabro-server/migrations/2026082801_sqlite_run_history_activation.rs +++ /dev/null @@ -1,603 +0,0 @@ -//! Fail-closed activation of SQLite run history. -//! -//! This compatibility bridge remains for at least 30 days after the persisted -//! first-success timestamp, and until cold-start, warm-restart, production -//! observation, rollback-backup, deletion, and concurrent-reader evidence has -//! been accepted and Scott explicitly approves removal. The computed date is -//! an eligibility floor, never an automatic deletion trigger. - -use std::path::{Path, PathBuf}; - -use chrono::{DateTime, Duration, Utc}; -use tokio::fs; -use tracing::{info, warn}; - -use crate::migrations::sqlite_activation_backup::{self, BackupError}; - -pub(crate) const BACKUP_SUFFIX: &str = ".pre-run-history-activation.bak"; -const REMOVAL_WINDOW: Duration = Duration::days(30); - -#[derive(Clone, Debug, Eq, PartialEq)] -struct ActivationRecord { - source_fingerprint: Vec, - source_runs: u64, - source_events: u64, - activated_at_ms: i64, -} - -#[derive(Debug, thiserror::Error)] -pub(crate) enum RunHistoryActivationError { - #[error("canonicalizing the SQLite database path {path}")] - Canonicalize { - path: PathBuf, - #[source] - source: std::io::Error, - }, - #[error("reading the SQLite run-history activation state")] - ActivationState(#[source] sqlx::Error), - #[error("the persisted run-history activation marker does not match the legacy source")] - MarkerMismatch, - #[error("the persisted run-history activation marker contains an invalid count or timestamp")] - InvalidMarker, - #[error( - "SQLite contains {target_runs} run rows and {target_events} run events, but the legacy run-history source is empty and no activation marker exists" - )] - EmptySourceWithTarget { - target_runs: u64, - target_events: u64, - }, - #[error( - "run-history activation backup is missing at {path} after SQLite import progress was recorded" - )] - MissingBackupAfterProgress { path: PathBuf }, - #[error(transparent)] - Backup(#[from] BackupError), - #[error("importing legacy run history into SQLite")] - Import(#[source] Box), - #[error("verifying legacy and SQLite run history")] - Verification(#[source] Box), - #[error("running the live SQLite integrity check")] - LiveIntegrity(#[source] sqlx::Error), - #[error("the live SQLite integrity check failed")] - LiveIntegrityFailed, - #[error("persisting the SQLite run-history activation marker")] - PersistMarker(#[source] sqlx::Error), - #[error("the run-history activation timestamp is outside the supported range")] - InvalidActivationTimestamp, - #[error("running the final SQLite WAL truncate checkpoint")] - FinalCheckpoint(#[source] sqlx::Error), - #[error("a run-history activation count exceeds SQLite's integer range")] - CountOverflow, -} - -pub(crate) async fn activate_run_history( - database: &fabro_db::Database, - sqlite_path: &Path, - store: &fabro_store::Database, - identity: &fabro_store::LegacyRunHistorySourceIdentity, -) -> Result<(), RunHistoryActivationError> { - let canonical_path = fs::canonicalize(sqlite_path).await.map_err(|source| { - RunHistoryActivationError::Canonicalize { - path: sqlite_path.to_path_buf(), - source, - } - })?; - let backup_path = fabro_db::append_to_path(&canonical_path, BACKUP_SUFFIX); - info!( - database_path = %canonical_path.display(), - backup_path = %backup_path.display(), - "Starting SQLite run-history activation" - ); - - let marker = read_activation_record(database.pool()).await?; - let (target_runs, target_events) = target_counts(database.pool()).await?; - - if let Some(record) = &marker { - verify_marker(record, identity)?; - } else if identity.events == 0 && (target_runs != 0 || target_events != 0) { - return Err(RunHistoryActivationError::EmptySourceWithTarget { - target_runs, - target_events, - }); - } - - let backup_present = sqlite_activation_backup::backup_exists(&backup_path).await?; - if backup_present { - sqlite_activation_backup::validate_backup(&backup_path).await?; - } - let import_progress = target_events != 0 || marker.is_some(); - if identity.events != 0 && import_progress && !backup_present { - return Err(RunHistoryActivationError::MissingBackupAfterProgress { path: backup_path }); - } - let backup_required = identity.events != 0 && !backup_present; - if backup_required { - sqlite_activation_backup::create_backup(database.pool(), &backup_path).await?; - } - - let import = store - .import_legacy_run_history_into(database.pool()) - .await - .map_err(|source| RunHistoryActivationError::Import(Box::new(source)))?; - let verification = store - .verify_legacy_run_history_in(database.pool()) - .await - .map_err(|source| RunHistoryActivationError::Verification(Box::new(source)))?; - validate_live_integrity(database.pool()).await?; - - let activated_at_ms = marker.as_ref().map_or_else( - || Utc::now().timestamp_millis(), - |record| record.activated_at_ms, - ); - persist_activation_record(database.pool(), identity, activated_at_ms).await?; - final_truncate_checkpoint(database.pool()).await?; - - let activated_at = DateTime::::from_timestamp_millis(activated_at_ms) - .ok_or(RunHistoryActivationError::InvalidActivationTimestamp)?; - let removal_eligible_at = activated_at + REMOVAL_WINDOW; - info!( - source_runs = identity.runs, - source_events = identity.events, - imported_runs = import.imported_runs, - imported_events = import.imported_events, - existing_runs = import.verified_existing_runs, - existing_events = import.verified_existing_events, - tombstoned_source_runs = verification.tombstoned_source_runs, - tombstoned_source_events = verification.tombstoned_source_events, - target_runs = verification.target_runs, - target_events = verification.target_events, - sql_only_runs = verification.sql_only_runs, - sql_only_events = verification.sql_only_events, - backup_required, - backup_path = %backup_path.display(), - activated_at = %activated_at, - removal_eligible_at = %removal_eligible_at, - "Activated SQLite run history" - ); - Ok(()) -} - -async fn read_activation_record( - pool: &sqlx::SqlitePool, -) -> Result, RunHistoryActivationError> { - let row = sqlx::query_as::<_, (Vec, i64, i64, i64)>( - r" -SELECT source_fingerprint, source_runs, source_events, activated_at_ms -FROM legacy_run_history_activation -WHERE singleton = 1 -", - ) - .fetch_optional(pool) - .await - .map_err(RunHistoryActivationError::ActivationState)?; - row.map( - |(source_fingerprint, source_runs, source_events, activated_at_ms)| { - Ok(ActivationRecord { - source_fingerprint, - source_runs: u64::try_from(source_runs) - .map_err(|_| RunHistoryActivationError::InvalidMarker)?, - source_events: u64::try_from(source_events) - .map_err(|_| RunHistoryActivationError::InvalidMarker)?, - activated_at_ms, - }) - }, - ) - .transpose() -} - -fn verify_marker( - marker: &ActivationRecord, - identity: &fabro_store::LegacyRunHistorySourceIdentity, -) -> Result<(), RunHistoryActivationError> { - if marker.activated_at_ms < 0 { - return Err(RunHistoryActivationError::InvalidMarker); - } - if marker.source_fingerprint.as_slice() != identity.fingerprint() - || marker.source_runs != identity.runs - || marker.source_events != identity.events - { - return Err(RunHistoryActivationError::MarkerMismatch); - } - Ok(()) -} - -async fn target_counts(pool: &sqlx::SqlitePool) -> Result<(u64, u64), RunHistoryActivationError> { - let (runs, events): (i64, i64) = - sqlx::query_as("SELECT (SELECT COUNT(*) FROM runs), (SELECT COUNT(*) FROM run_events)") - .fetch_one(pool) - .await - .map_err(RunHistoryActivationError::ActivationState)?; - Ok(( - u64::try_from(runs).map_err(|_| RunHistoryActivationError::CountOverflow)?, - u64::try_from(events).map_err(|_| RunHistoryActivationError::CountOverflow)?, - )) -} - -async fn persist_activation_record( - pool: &sqlx::SqlitePool, - identity: &fabro_store::LegacyRunHistorySourceIdentity, - activated_at_ms: i64, -) -> Result<(), RunHistoryActivationError> { - let source_runs = - i64::try_from(identity.runs).map_err(|_| RunHistoryActivationError::CountOverflow)?; - let source_events = - i64::try_from(identity.events).map_err(|_| RunHistoryActivationError::CountOverflow)?; - // A pre-existing marker was already verified against `identity` above, so - // leaving it untouched on conflict keeps the original activation time. - sqlx::query( - r" -INSERT INTO legacy_run_history_activation ( - singleton, source_fingerprint, source_runs, source_events, activated_at_ms -) VALUES (1, ?, ?, ?, ?) -ON CONFLICT(singleton) DO NOTHING -", - ) - .bind(identity.fingerprint().as_slice()) - .bind(source_runs) - .bind(source_events) - .bind(activated_at_ms) - .execute(pool) - .await - .map_err(RunHistoryActivationError::PersistMarker)?; - Ok(()) -} - -async fn validate_live_integrity(pool: &sqlx::SqlitePool) -> Result<(), RunHistoryActivationError> { - let ok = sqlite_activation_backup::integrity_check_is_ok(pool) - .await - .map_err(RunHistoryActivationError::LiveIntegrity)?; - if !ok { - return Err(RunHistoryActivationError::LiveIntegrityFailed); - } - Ok(()) -} - -async fn final_truncate_checkpoint( - pool: &sqlx::SqlitePool, -) -> Result<(), RunHistoryActivationError> { - let (busy, _, _): (i64, i64, i64) = sqlx::query_as("PRAGMA wal_checkpoint(TRUNCATE)") - .fetch_one(pool) - .await - .map_err(RunHistoryActivationError::FinalCheckpoint)?; - if busy != 0 { - // A concurrent reader can keep the WAL from truncating, but all - // activation data is already committed and remains durable in that - // WAL. A later checkpoint can truncate it after the reader exits. - warn!( - "The final SQLite run-history WAL truncate checkpoint could not complete; continuing startup" - ); - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use std::path::PathBuf; - use std::sync::Arc; - use std::time::Duration as StdDuration; - - use chrono::{TimeZone as _, Utc}; - use fabro_types::{Graph, RunId, WorkflowSettings, test_support}; - use object_store::memory::InMemory; - use sqlx::Connection as _; - use tokio::fs; - use ulid::Ulid; - - use super::{ - BACKUP_SUFFIX, RunHistoryActivationError, activate_run_history, final_truncate_checkpoint, - read_activation_record, - }; - - type TestResult = Result>; - - struct TestContext { - _directory: tempfile::TempDir, - sqlite_path: PathBuf, - database: fabro_db::Database, - store: Arc, - } - - impl TestContext { - async fn new(prefix: &str) -> TestResult { - let directory = tempfile::tempdir()?; - let sqlite_path = directory.path().join("fabro.sqlite3"); - let database = fabro_db::Database::connect(&sqlite_path).await?; - database.migrate().await?; - let store = Arc::new(fabro_store::Database::new( - Arc::new(InMemory::new()), - prefix, - StdDuration::from_millis(1), - None, - Arc::new(fabro_store::BlobStore::new(database.clone_pool())), - Arc::new(fabro_store::RunSummaryStore::new(database.clone_pool())), - )); - Ok(Self { - _directory: directory, - sqlite_path, - database, - store, - }) - } - - async fn put_event( - &self, - run_id: &RunId, - seq: u32, - event: &str, - properties: serde_json::Value, - ) -> TestResult<()> { - let payload = serde_json::json!({ - "id": format!("evt-{seq}-{event}"), - "ts": Utc - .timestamp_millis_opt(1_788_000_000_000 + i64::from(seq)) - .single() - .unwrap() - .to_rfc3339(), - "run_id": run_id.to_string(), - "event": event, - "properties": properties, - }); - fabro_store::test_support::put_legacy_run_event(&self.store, run_id, seq, &payload) - .await?; - Ok(()) - } - - async fn put_created(&self, run_id: &RunId) -> TestResult<()> { - self.put_event( - run_id, - 1, - "run.created", - serde_json::json!({ - "title": "Activation test", - "settings": WorkflowSettings::default(), - "graph": Graph::new("test"), - "workflow_slug": "test-workflow", - "labels": {}, - "provenance": test_support::test_run_provenance(), - }), - ) - .await - } - - async fn source_identity(&self) -> TestResult { - Ok(self.store.legacy_run_history_source_identity().await?) - } - - fn backup_path(&self) -> PathBuf { - fabro_db::append_to_path(&self.sqlite_path, BACKUP_SUFFIX) - } - } - - fn run_id() -> RunId { - RunId::from(Ulid::from_parts(1_788_000_000_000, 1)) - } - - #[tokio::test] - async fn cold_activation_imports_and_warm_restart_preserves_marker() -> TestResult<()> { - let context = TestContext::new("cold-and-warm-run-activation").await?; - let run_id = run_id(); - context.put_created(&run_id).await?; - context - .put_event(&run_id, 2, "run.submitted", serde_json::json!({})) - .await?; - - let identity = context.source_identity().await?; - activate_run_history( - &context.database, - &context.sqlite_path, - &context.store, - &identity, - ) - .await?; - let first_marker = read_activation_record(context.database.pool()) - .await? - .unwrap(); - assert_eq!(first_marker.source_runs, 1); - assert_eq!(first_marker.source_events, 2); - assert!(context.backup_path().is_file()); - let backup_options = sqlx::sqlite::SqliteConnectOptions::new() - .filename(context.backup_path()) - .read_only(true) - .create_if_missing(false); - let mut backup = sqlx::SqliteConnection::connect_with(&backup_options).await?; - assert_eq!( - sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM run_events") - .fetch_one(&mut backup) - .await?, - 0, - "the retained backup must capture the exact pre-import boundary" - ); - assert_eq!( - sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM run_events") - .fetch_one(context.database.pool()) - .await?, - 2 - ); - - activate_run_history( - &context.database, - &context.sqlite_path, - &context.store, - &identity, - ) - .await?; - assert_eq!( - read_activation_record(context.database.pool()).await?, - Some(first_marker) - ); - Ok(()) - } - - #[tokio::test] - async fn busy_final_checkpoint_warns_and_does_not_fail_activation() -> TestResult<()> { - use sqlx::sqlite::{SqliteJournalMode, SqlitePoolOptions}; - - let context = TestContext::new("busy-final-run-checkpoint").await?; - sqlx::query("INSERT INTO blobs (hash, data) VALUES (?, ?)") - .bind(fabro_types::BlobHash::new(b"wal-content").to_string()) - .bind(b"wal-content".as_slice()) - .execute(context.database.pool()) - .await?; - - let reader_options = sqlx::sqlite::SqliteConnectOptions::new() - .filename(&context.sqlite_path) - .read_only(true) - .create_if_missing(false); - let mut reader = sqlx::SqliteConnection::connect_with(&reader_options).await?; - sqlx::query("BEGIN").execute(&mut reader).await?; - sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM blobs") - .fetch_one(&mut reader) - .await?; - - let checkpoint_options = sqlx::sqlite::SqliteConnectOptions::new() - .filename(&context.sqlite_path) - .journal_mode(SqliteJournalMode::Wal) - .busy_timeout(StdDuration::from_millis(50)) - .create_if_missing(false); - let checkpoint_pool = SqlitePoolOptions::new() - .max_connections(1) - .connect_with(checkpoint_options) - .await?; - - final_truncate_checkpoint(&checkpoint_pool).await?; - - let wal_bytes = fs::metadata(fabro_db::append_to_path(&context.sqlite_path, "-wal")) - .await? - .len(); - assert!(wal_bytes > 0, "the WAL should remain untruncated"); - drop(reader); - Ok(()) - } - - #[tokio::test] - async fn changed_legacy_source_fails_before_mutating_sqlite() -> TestResult<()> { - let context = TestContext::new("changed-run-activation-source").await?; - let run_id = run_id(); - context.put_created(&run_id).await?; - let original_identity = context.source_identity().await?; - activate_run_history( - &context.database, - &context.sqlite_path, - &context.store, - &original_identity, - ) - .await?; - - context - .put_event(&run_id, 2, "run.submitted", serde_json::json!({})) - .await?; - let changed_identity = context.source_identity().await?; - let error = activate_run_history( - &context.database, - &context.sqlite_path, - &context.store, - &changed_identity, - ) - .await - .expect_err("the source identity must remain stable after activation"); - assert!(matches!(error, RunHistoryActivationError::MarkerMismatch)); - assert_eq!( - sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM run_events") - .fetch_one(context.database.pool()) - .await?, - 1 - ); - Ok(()) - } - - #[tokio::test] - async fn empty_source_with_unmarked_target_fails_closed() -> TestResult<()> { - let context = TestContext::new("empty-source-with-target").await?; - let run_id = run_id(); - context.put_created(&run_id).await?; - let identity = context.source_identity().await?; - activate_run_history( - &context.database, - &context.sqlite_path, - &context.store, - &identity, - ) - .await?; - sqlx::query("DELETE FROM legacy_run_history_activation") - .execute(context.database.pool()) - .await?; - - let empty_store = Arc::new(fabro_store::Database::new( - Arc::new(InMemory::new()), - "empty-source", - StdDuration::from_millis(1), - None, - Arc::new(fabro_store::BlobStore::new(context.database.clone_pool())), - Arc::new(fabro_store::RunSummaryStore::new( - context.database.clone_pool(), - )), - )); - let empty_identity = empty_store.legacy_run_history_source_identity().await?; - let error = activate_run_history( - &context.database, - &context.sqlite_path, - &empty_store, - &empty_identity, - ) - .await - .expect_err("unmarked SQLite rows cannot be adopted from an empty source"); - assert!(matches!( - error, - RunHistoryActivationError::EmptySourceWithTarget { .. } - )); - Ok(()) - } - - #[tokio::test] - async fn missing_backup_after_import_progress_fails_closed() -> TestResult<()> { - let context = TestContext::new("missing-run-activation-backup").await?; - let run_id = run_id(); - context.put_created(&run_id).await?; - context - .store - .import_legacy_run_history_into(context.database.pool()) - .await?; - - let identity = context.source_identity().await?; - let error = activate_run_history( - &context.database, - &context.sqlite_path, - &context.store, - &identity, - ) - .await - .expect_err("partial import progress requires the retained backup"); - assert!(matches!( - error, - RunHistoryActivationError::MissingBackupAfterProgress { .. } - )); - Ok(()) - } - - #[tokio::test] - async fn empty_source_and_target_need_no_backup_on_cold_or_warm_start() -> TestResult<()> { - let context = TestContext::new("empty-run-activation").await?; - let identity = context.source_identity().await?; - activate_run_history( - &context.database, - &context.sqlite_path, - &context.store, - &identity, - ) - .await?; - let marker = read_activation_record(context.database.pool()) - .await? - .unwrap(); - assert_eq!((marker.source_runs, marker.source_events), (0, 0)); - assert!(!context.backup_path().exists()); - - activate_run_history( - &context.database, - &context.sqlite_path, - &context.store, - &identity, - ) - .await?; - assert!(!context.backup_path().exists()); - Ok(()) - } -} diff --git a/lib/apps/fabro-server/migrations/sqlite_activation_backup.rs b/lib/apps/fabro-server/migrations/sqlite_activation_backup.rs deleted file mode 100644 index 32f540a12..000000000 --- a/lib/apps/fabro-server/migrations/sqlite_activation_backup.rs +++ /dev/null @@ -1,186 +0,0 @@ -//! Pre-activation backup and integrity helpers shared by the SQLite -//! activation bridges. -//! -//! Every activation snapshots the live database to a private, integrity -//! checked backup file before importing legacy data, and re-validates any -//! backup it finds on a later start. This module owns that mechanism so the -//! blob and run-history bridges cannot drift apart. - -use std::path::{Path, PathBuf}; - -use futures_util::TryStreamExt as _; -use sqlx::Connection as _; -use sqlx::sqlite::{SqliteConnectOptions, SqliteConnection}; -use tokio::fs; -use tokio::task::{JoinError, spawn_blocking}; -use tracing::debug; - -const STAGING_SUFFIX: &str = ".tmp"; - -#[derive(Debug, thiserror::Error)] -pub(crate) enum BackupError { - #[error("reading activation backup metadata at {path}")] - Metadata { - path: PathBuf, - #[source] - source: std::io::Error, - }, - #[error("activation backup is not a regular file at {path}")] - NotRegular { path: PathBuf }, - #[error("activation backup permissions are not private at {path}")] - NotPrivate { path: PathBuf }, - #[error("opening or checking activation backup integrity at {path}")] - Integrity { - path: PathBuf, - #[source] - source: sqlx::Error, - }, - #[error("activation backup integrity check did not return exactly one ok result at {path}")] - IntegrityFailed { path: PathBuf }, - #[error("staging the pre-activation SQLite backup")] - Stage(#[source] fabro_db::SnapshotStagingError), - #[error("joining the activation backup publication task")] - JoinPublication(#[source] JoinError), - #[error("publishing the activation backup at {path} without overwriting")] - Publish { - path: PathBuf, - #[source] - source: std::io::Error, - }, -} - -pub(crate) async fn backup_exists(path: &Path) -> Result { - match fs::metadata(path).await { - Ok(_) => Ok(true), - Err(source) if source.kind() == std::io::ErrorKind::NotFound => Ok(false), - Err(source) => Err(BackupError::Metadata { - path: path.to_path_buf(), - source, - }), - } -} - -/// Snapshots `pool` to `backup_path` without overwriting an existing file. -/// -/// The snapshot is staged beside the target, validated, and then published -/// with an atomic no-clobber rename. A backup that another process published -/// concurrently is validated in place instead. -pub(crate) async fn create_backup( - pool: &sqlx::SqlitePool, - backup_path: &Path, -) -> Result<(), BackupError> { - let staging_path = fabro_db::append_to_path(backup_path, STAGING_SUFFIX); - fabro_db::write_snapshot_to_staging(pool, &staging_path) - .await - .map_err(BackupError::Stage)?; - validate_backup(&staging_path).await?; - - let publish_staging = staging_path.clone(); - let publish_backup = backup_path.to_path_buf(); - let already_exists = spawn_blocking(move || { - let staging = tempfile::TempPath::try_from_path(publish_staging)?; - match staging.persist_noclobber(&publish_backup) { - Ok(()) => { - // Make the rename's directory entry durable: the retained - // backup is the documented rollback artifact, so it must not - // vanish in a crash after the import has already committed. - fabro_db::sync_parent_directory(&publish_backup)?; - Ok(false) - } - Err(error) if error.error.kind() == std::io::ErrorKind::AlreadyExists => Ok(true), - Err(error) => Err(error.error), - } - }) - .await - .map_err(BackupError::JoinPublication)? - .map_err(|source| BackupError::Publish { - path: backup_path.to_path_buf(), - source, - })?; - - // The staging copy was validated just before the atomic rename, so only a - // concurrently published file still needs its own validation. - if already_exists { - debug!( - backup_path = %backup_path.display(), - "Reusing concurrently published SQLite activation backup" - ); - validate_backup(backup_path).await?; - } - Ok(()) -} - -/// Requires `path` to be a private regular file holding a SQLite database -/// whose `PRAGMA integrity_check` passes. -pub(crate) async fn validate_backup(path: &Path) -> Result<(), BackupError> { - let metadata = fs::symlink_metadata(path) - .await - .map_err(|source| BackupError::Metadata { - path: path.to_path_buf(), - source, - })?; - if !metadata.is_file() { - return Err(BackupError::NotRegular { - path: path.to_path_buf(), - }); - } - validate_private_permissions(path, &metadata)?; - - let options = SqliteConnectOptions::new() - .filename(path) - .read_only(true) - .immutable(true) - .create_if_missing(false); - let mut connection = SqliteConnection::connect_with(&options) - .await - .map_err(|source| BackupError::Integrity { - path: path.to_path_buf(), - source, - })?; - let ok = integrity_check_is_ok(&mut connection) - .await - .map_err(|source| BackupError::Integrity { - path: path.to_path_buf(), - source, - })?; - if !ok { - return Err(BackupError::IntegrityFailed { - path: path.to_path_buf(), - }); - } - Ok(()) -} - -/// Returns whether `PRAGMA integrity_check` reports exactly one `ok` row. -pub(crate) async fn integrity_check_is_ok<'a, E>(executor: E) -> Result -where - E: sqlx::Executor<'a, Database = sqlx::Sqlite>, -{ - let mut rows = sqlx::query_scalar::<_, String>("PRAGMA integrity_check").fetch(executor); - let first = rows.try_next().await?; - let second = rows.try_next().await?; - Ok(first.as_deref() == Some("ok") && second.is_none()) -} - -#[cfg(unix)] -fn validate_private_permissions( - path: &Path, - metadata: &std::fs::Metadata, -) -> Result<(), BackupError> { - use std::os::unix::fs::PermissionsExt as _; - - if metadata.permissions().mode() & 0o077 != 0 { - return Err(BackupError::NotPrivate { - path: path.to_path_buf(), - }); - } - Ok(()) -} - -#[cfg(not(unix))] -fn validate_private_permissions( - _path: &Path, - _metadata: &std::fs::Metadata, -) -> Result<(), BackupError> { - Ok(()) -} diff --git a/lib/apps/fabro-server/src/lib.rs b/lib/apps/fabro-server/src/lib.rs index 4dd63b123..ee3c1e02c 100644 --- a/lib/apps/fabro-server/src/lib.rs +++ b/lib/apps/fabro-server/src/lib.rs @@ -31,7 +31,6 @@ pub mod install; mod interp; pub mod jwt_auth; pub mod manifest_validation; -mod migrations; mod petri_check; mod petri_runs; mod principal_middleware; diff --git a/lib/apps/fabro-server/src/migrations.rs b/lib/apps/fabro-server/src/migrations.rs deleted file mode 100644 index 68c4f9bd5..000000000 --- a/lib/apps/fabro-server/src/migrations.rs +++ /dev/null @@ -1,9 +0,0 @@ -#[path = "../migrations/sqlite_activation_backup.rs"] -mod sqlite_activation_backup; -#[path = "../migrations/2026082301_sqlite_blob_activation.rs"] -mod sqlite_blob_activation; -#[path = "../migrations/2026082801_sqlite_run_history_activation.rs"] -mod sqlite_run_history_activation; - -pub(crate) use sqlite_blob_activation::activate_blob_storage; -pub(crate) use sqlite_run_history_activation::activate_run_history; diff --git a/lib/apps/fabro-server/src/petri_runs.rs b/lib/apps/fabro-server/src/petri_runs.rs index f61b3da13..c640aa6f6 100644 --- a/lib/apps/fabro-server/src/petri_runs.rs +++ b/lib/apps/fabro-server/src/petri_runs.rs @@ -164,8 +164,8 @@ mod tests { use fabro_config::daemon::ServerDaemon; use fabro_petri::petri::RunStore as _; use fabro_static::EnvVars; + use fabro_store::platform_records::{PlatformRecord, RunLifecycleKind, RunLifecycleRecord}; use fabro_types::{RunId, RunStatus, WorkflowPath, WorkflowVersion}; - use fabro_workflow::event::{Event, append_event}; use serde_json::json; use tokio::io::AsyncRead; use tokio::sync::Notify; @@ -173,7 +173,9 @@ mod tests { use tower::ServiceExt as _; use super::*; - use crate::server::{AppState, reconcile_incomplete_runs_on_startup, spawn_scheduler}; + use crate::server::{ + AppState, reconcile_incomplete_runs_on_startup, run_records, spawn_scheduler, + }; use crate::test_support::{ TestAppStateBuilder, build_test_router, test_register_workflow_version, test_secret_store_path, test_store_bundle, @@ -414,16 +416,17 @@ mod tests { // The worker took the run as far as running and holds its lease; // then the server died, so nothing released it. - let run_store = before - .stores - .runs - .open_run(&run_id) + for (transition, status) in [ + (RunLifecycleKind::Starting, RunStatus::Starting), + (RunLifecycleKind::Running, RunStatus::Running), + ] { + run_records::lifecycle( + &before, + run_id, + RunLifecycleRecord::new(transition).with_status(status), + ) .await - .expect("the run opens"); - for event in [Event::RunStarting, Event::RunRunning] { - append_event(&run_store, &run_id, &event) - .await - .expect("the lifecycle event appends"); + .expect("the lifecycle record appends"); } let held = before .petri_runs @@ -465,30 +468,33 @@ mod tests { None, "the previous worker's lease is released" ); - let reader = after - .stores - .runs - .open_run_reader(&run_id) + let run_state = run_records::projection(&after, run_id) .await - .expect("the run opens for reading"); - let run_state = reader.state().await.expect("the run state loads"); + .expect("the run state loads") + .expect("the run projects"); assert_eq!(run_state.status, RunStatus::Runnable); - let names = reader - .list_events() + let transitions = after + .stores + .run_summaries + .platform_records() + .read(&run_id) .await - .expect("the history lists") + .expect("the records list") .into_iter() - .map(|envelope| envelope.event.event_name().to_string()) + .filter_map(|stored| match stored.record { + PlatformRecord::RunLifecycle(record) => Some(record.transition), + _ => None, + }) .collect::>(); assert_eq!( - &names[names.len() - 4..], + &transitions[transitions.len() - 4..], [ - "run.starting", - "run.running", - "run.start_requested", - "run.runnable" + RunLifecycleKind::Starting, + RunLifecycleKind::Running, + RunLifecycleKind::StartRequested, + RunLifecycleKind::Runnable ], - "{names:?}" + "{transitions:?}" ); write_test_server_record(&after); diff --git a/lib/apps/fabro-server/src/serve.rs b/lib/apps/fabro-server/src/serve.rs index 61e77586c..89ab0f070 100644 --- a/lib/apps/fabro-server/src/serve.rs +++ b/lib/apps/fabro-server/src/serve.rs @@ -40,7 +40,7 @@ use crate::server::{ }; use crate::server_secrets::{ServerSecrets, process_env_snapshot}; use crate::startup::{resolve_startup, validate_startup_configuration}; -use crate::{migrations, static_files}; +use crate::static_files; pub const DEFAULT_TCP_PORT: u16 = 32276; type EnvLookup = Arc Option + Send + Sync>; @@ -748,25 +748,14 @@ where } else { None }; - let blob_activation = migrations::activate_blob_storage( - &database, - &sqlite_path, + let store = Arc::new(fabro_store::Database::new( object_store, slatedb_prefix, flush_interval, cache_path, - ) - .await - .context("activating SQLite blob storage")?; - migrations::activate_run_history( - &database, - &sqlite_path, - &blob_activation.store, - &blob_activation.run_history_identity, - ) - .await - .context("activating SQLite run history")?; - let store = blob_activation.store; + Arc::new(fabro_store::BlobStore::new(database.clone_pool())), + Arc::new(fabro_store::RunSummaryStore::new(database.clone_pool())), + )); // Refresh tokens now live in SQLite. Nothing reads the old records and no // reaper collects them any more, so clear them out once rather than // leaving them in the object store forever. Pending authorization codes diff --git a/lib/apps/fabro-server/src/server.rs b/lib/apps/fabro-server/src/server.rs index 5ea62daba..b31a0b153 100644 --- a/lib/apps/fabro-server/src/server.rs +++ b/lib/apps/fabro-server/src/server.rs @@ -75,10 +75,14 @@ use fabro_slack::payload::SlackAnswerSubmission; use fabro_slack::threads::ThreadRegistry; use fabro_slack::{blocks as slack_blocks, connection as slack_connection}; use fabro_static::EnvVars; +use fabro_store::platform_records::{ + InterviewAnsweredRecord, NotificationSentRecord, PlatformRecord, PlatformRecordKind, + RunLifecycleKind, RunLifecycleRecord, +}; use fabro_store::{ - ArtifactKey, ArtifactStore, AuthCodeStore, AuthSessionStore, Database, EventEnvelope, - EventPayload, KeyedMutex, NodeArtifact, PendingInterviewRecord, RunSessionRecordStore, - RunSummaryStore, StageArtifactEntry, StageId, + ArtifactKey, ArtifactStore, AuthCodeStore, AuthSessionStore, Database, KeyedMutex, + NodeArtifact, PendingInterviewRecord, RunSessionRecordStore, RunSummaryStore, + StageArtifactEntry, StageId, }; #[cfg(test)] use fabro_types::BlockedReason; @@ -88,10 +92,10 @@ use fabro_types::settings::server::{ GithubIntegrationSettings, GithubIntegrationStrategy, LogDestination, }; use fabro_types::{ - AskFabro, AskFabroUnavailableReason, BlobHash, EventBody, InterviewQuestionRecord, ModelRef, + AskFabro, AskFabroUnavailableReason, BlobHash, InterviewQuestionRecord, ModelRef, ModelTestMode, PendingReason, Principal, PullRequestLink, QuestionType, RunControlAction, - RunEvent, RunId, RunRunnableSource, RunStatusKind, SandboxProviderKind, ServerSettings, - SessionCapability, + RunId, RunRunnableSource, RunStatusKind, RunStreamItem, RunStreamItemKind, SandboxProviderKind, + ServerSettings, }; use fabro_util::error::{ SharedError, collect_causes, render_compact_with_causes, render_with_causes, @@ -99,8 +103,6 @@ use fabro_util::error::{ use fabro_util::version::FABRO_VERSION; use fabro_variable::{Error as VariableError, VariableStore}; use fabro_vault::{SecretStore, SecretStoreError, SecretType, Vault}; -use fabro_workflow::event::{self as workflow_event}; -use fabro_workflow::records::Checkpoint; use fabro_workflow::run_lookup::{ RunInfo, StatusFilter, filter_runs, scan_runs_with_summaries, scratch_base, }; @@ -142,8 +144,8 @@ use crate::jwt_auth::{self, AuthMode}; use crate::petri_runs::PetriRuns; use crate::principal_middleware::{ AuthContextSlot, RequestAuth, RequestAuthContext, RequireRunBlob, RequireRunManagementTarget, - RequireRunScoped, RequireRunStageScoped, RequireStageArtifact, RequireWorkerRunScoped, - RequireWorkerRunSegment, RequiredUser, principal_middleware, + RequireRunScoped, RequireStageArtifact, RequireWorkerRunScoped, RequireWorkerRunSegment, + RequiredUser, principal_middleware, }; use crate::request_id::{self, RequestId}; use crate::run_files::{FilesInFlight, new_files_in_flight}; @@ -163,12 +165,12 @@ mod handler; pub(crate) mod petri_runs; mod pull_request_supervisor; pub(crate) mod resource_sampler; +pub(crate) mod run_records; mod session_runtime; +pub(crate) mod stream_follower; pub(crate) use automation_scheduler::spawn_automation_scheduler; pub(crate) use handler::events::EventListParams; -#[cfg(test)] -pub(in crate::server) use handler::events::filtered_global_events; pub(crate) use handler::graph::render_graph_bytes; #[cfg(test)] pub(in crate::server) use handler::graph::{ @@ -264,8 +266,6 @@ struct ManagedRun { /// Stage IDs of currently running agent sessions that have no live /// steering capability, keyed to the session id that owns the marker. active_non_steerable_stages: HashMap, - event_tx: Option>, - checkpoint: Option, cancel_tx: Option>, cancel_token: Option, worker_ref: Option, @@ -442,16 +442,13 @@ struct LoadedPendingInterview { #[derive(Debug, Clone)] struct SlackLifecycleDetails { - kind: slack_blocks::RunLifecycleKind, - started_event_name: Option, - result: Option, - duration_ms: Option, -} - -#[derive(Debug, Clone, Default)] -struct PriorSlackLifecycleEventDetails { - started_event_name: Option, - pull_request: Option, + kind: slack_blocks::RunLifecycleKind, + /// The legacy name of the lifecycle event, which the notification + /// routes in the run's settings name: `run.started`, `run.completed`, + /// `run.failed`. + event_name: &'static str, + result: Option, + duration_ms: Option, } #[derive(Debug, Clone)] @@ -461,6 +458,14 @@ struct SlackLifecyclePullRequest { url: Option, } +/// A question posted to Slack: the message, and the question's text for +/// the update that closes it. +#[derive(Debug, Clone)] +struct SlackPostedQuestion { + message: SlackPostedMessage, + text: String, +} + #[derive(Debug, Clone)] struct SlackConnectionRuntimeState { status: IntegrationConnectionState, @@ -489,7 +494,7 @@ struct SlackService { client: SlackClient, app_token: String, default_channel: Option, - posted_messages: Arc>>, + posted_messages: Arc>>, thread_registry: Arc, connection: Arc>, } @@ -544,118 +549,156 @@ impl SlackService { }) } - async fn handle_event( - &self, - state: &AppState, - envelope: &EventEnvelope, - run_web_url: Option<&str>, - ) { - let event = &envelope.event; - match &event.body { - EventBody::InterviewStarted(props) => { - if props.question_id.is_empty() { - return; - } - let Some(default_channel) = self.default_channel.as_deref() else { - return; - }; - let key = (event.run_id, props.question_id.clone()); - if self - .posted_messages - .lock() - .expect("slack posted messages lock poisoned") - .contains_key(&key) - { - return; - } - - let question = runtime_question_from_interview_record(&InterviewQuestionRecord { - id: props.question_id.clone(), - text: props.question.clone(), - stage: props.stage.clone(), - question_type: props.question_type.parse().unwrap_or_default(), - options: props.options.clone(), - allow_freeform: props.allow_freeform, - timeout_seconds: props.timeout_seconds, - context_display: props.context_display.clone(), - review_target: props.review_target.clone(), - }); - let blocks = slack_blocks::question_to_blocks( - &event.run_id.to_string(), - &props.question_id, - &question, - run_web_url, - ); - - if let Ok(posted) = self - .client - .post_message(default_channel, &blocks, None) - .await - { - if question.allow_freeform || question.question_type == QuestionType::Freeform { - self.thread_registry.register( - &posted.ts, - &event.run_id.to_string(), - &props.question_id, - ); + /// What the run's stream says since the last look: a question asked, + /// answered or expired, and the lifecycle transitions the notification + /// routes name. + async fn observe(&self, state: &AppState, run_id: RunId, items: &[RunStreamItem]) { + let run_web_url = state.run_web_url(&run_id); + for item in items { + match item.kind { + RunStreamItemKind::Platform => { + let Some(record) = platform_record_of(item) else { + continue; + }; + match record { + PlatformRecord::InterviewAnswered(answered) => { + self.finish_interview( + run_id, + &answered.question, + answered.text.as_deref().unwrap_or_default(), + answered.answer.as_deref().unwrap_or("Answered"), + ) + .await; + } + PlatformRecord::RunLifecycle(lifecycle) => { + if let Some(details) = slack_lifecycle_details(&lifecycle) { + self.handle_lifecycle( + state, + run_id, + &details, + run_web_url.as_deref(), + ) + .await; + } + } + _ => {} + } + } + RunStreamItemKind::Petri => { + let Some(parsed) = petri_parsed(item) else { + continue; + }; + match parsed.get("kind").and_then(serde_json::Value::as_str) { + Some("question") => { + if let Some(question_id) = parsed + .get("question") + .and_then(|question| question.get("id")) + .and_then(serde_json::Value::as_str) + { + self.post_question( + state, + run_id, + question_id, + run_web_url.as_deref(), + ) + .await; + } + } + Some("question_expired") => { + if let Some(question_id) = + parsed.get("question").and_then(serde_json::Value::as_str) + { + let text = self.posted_question_text(run_id, question_id); + self.finish_interview(run_id, question_id, &text, "Timed out") + .await; + } + } + _ => {} } - self.posted_messages - .lock() - .expect("slack posted messages lock poisoned") - .insert(key, posted); } } - EventBody::InterviewCompleted(props) => { - self.finish_interview( - event.run_id, - &props.question_id, - &props.question, - &props.answer, - ) - .await; - } - EventBody::InterviewTimeout(props) => { - self.finish_interview( - event.run_id, - &props.question_id, - &props.question, - "Timed out", - ) - .await; - } - EventBody::InterviewInterrupted(props) => { - self.finish_interview( - event.run_id, - &props.question_id, - &props.question, - "Interrupted", - ) - .await; - } - EventBody::RunStarted(_) | EventBody::RunCompleted(_) | EventBody::RunFailed(_) => { - self.handle_lifecycle_event(state, envelope, run_web_url) - .await; - } - _ => {} } } - async fn handle_lifecycle_event( + /// Post a pending question to the default channel, once. + async fn post_question( &self, state: &AppState, - envelope: &EventEnvelope, + run_id: RunId, + question_id: &str, run_web_url: Option<&str>, ) { - let event = &envelope.event; - let Some(details) = slack_lifecycle_details(event) else { + let Some(default_channel) = self.default_channel.as_deref() else { return; }; - let event_name = event.body.event_name(); - let projection = match state.stores.runs.load_run_projection(&event.run_id).await { + let key = (run_id, question_id.to_string()); + if self + .posted_messages + .lock() + .expect("slack posted messages lock poisoned") + .contains_key(&key) + { + return; + } + let projection = match run_records::projection(state, run_id).await { + Ok(Some(projection)) => projection, + Ok(None) => return, + Err(err) => { + warn!(run_id = %run_id, error = %err, "Skipping Slack question: the run's projection could not be loaded"); + return; + } + }; + let Some(pending) = projection.pending_interviews.get(question_id) else { + return; + }; + let question = runtime_question_from_interview_record(&pending.question); + let blocks = slack_blocks::question_to_blocks( + &run_id.to_string(), + question_id, + &question, + run_web_url, + ); + if let Ok(posted) = self + .client + .post_message(default_channel, &blocks, None) + .await + { + if question.allow_freeform || question.question_type == QuestionType::Freeform { + self.thread_registry + .register(&posted.ts, &run_id.to_string(), question_id); + } + self.posted_messages + .lock() + .expect("slack posted messages lock poisoned") + .insert(key, SlackPostedQuestion { + message: posted, + text: pending.question.text.clone(), + }); + } + } + + fn posted_question_text(&self, run_id: RunId, question_id: &str) -> String { + self.posted_messages + .lock() + .expect("slack posted messages lock poisoned") + .get(&(run_id, question_id.to_string())) + .map(|posted| posted.text.clone()) + .unwrap_or_default() + } + + async fn handle_lifecycle( + &self, + state: &AppState, + run_id: RunId, + details: &SlackLifecycleDetails, + run_web_url: Option<&str>, + ) { + let event_name = details.event_name; + let projection = match run_records::projection(state, run_id).await { Ok(Some(projection)) => projection, Ok(None) => { warn!( - run_id = %event.run_id, + run_id = %run_id, event = event_name, "Skipping Slack lifecycle notification because run projection is missing" ); @@ -663,7 +706,7 @@ impl SlackService { } Err(err) => { warn!( - run_id = %event.run_id, + run_id = %run_id, event = event_name, error = %err, "Skipping Slack lifecycle notification because run projection could not be loaded" @@ -690,28 +733,34 @@ impl SlackService { } routes.sort_by_key(|(route_name, _)| *route_name); - // Only completed/failed events need to recover prior PR details (a - // run.started event cannot have a prior PullRequestCreated). - let prior = if matches!(details.kind, slack_blocks::RunLifecycleKind::Started) { - PriorSlackLifecycleEventDetails::default() - } else { - load_prior_slack_lifecycle_event_details(state, event.run_id, envelope.seq).await + // A notification is sent once per route and event: the `notification.sent` + // record is the memory that survives a restart. + let sent = match state + .stores + .run_summaries + .platform_records() + .read_kind(&run_id, PlatformRecordKind::NotificationSent) + .await + { + Ok(records) => records + .into_iter() + .filter_map(|stored| match stored.record { + PlatformRecord::NotificationSent(record) => Some((record.route, record.event)), + _ => None, + }) + .collect::>(), + Err(err) => { + warn!(run_id = %run_id, error = %err, "Skipping Slack lifecycle notification: sent notifications could not be read"); + return; + } }; - let workflow_label = slack_lifecycle_workflow_label( - projection.as_ref(), - details - .started_event_name - .as_deref() - .or(prior.started_event_name.as_deref()), - event_name, - ); - let pull_request = prior.pull_request.or_else(|| { - projection - .pull_request - .as_ref() - .map(slack_lifecycle_pull_request_from_link) - }); - let run_id = event.run_id.to_string(); + + let workflow_label = slack_lifecycle_workflow_label(projection.as_ref(), None, event_name); + let pull_request = projection + .pull_request + .as_ref() + .map(slack_lifecycle_pull_request_from_link); + let run_id_text = run_id.to_string(); let run_url = run_web_url.or(projection.web_url.as_deref()); let pull_request_blocks = pull_request @@ -723,7 +772,7 @@ impl SlackService { }); let blocks = slack_blocks::run_lifecycle_blocks(details.kind, &slack_blocks::RunLifecycleBlocks { - run_id: &run_id, + run_id: &run_id_text, run_url, workflow_label: &workflow_label, result: details.result.as_deref(), @@ -733,17 +782,36 @@ impl SlackService { let blocks = &blocks; let posts = routes.into_iter().filter_map(|(route_name, route)| { + if sent.contains(&(route_name.clone(), event_name.to_string())) { + return None; + } let channel = - resolve_slack_lifecycle_route_channel(event.run_id, route_name, route, event_name)?; + resolve_slack_lifecycle_route_channel(run_id, route_name, route, event_name)?; Some(async move { - if let Err(err) = self.client.post_message(&channel, blocks, None).await { - warn!( - run_id = %event.run_id, - event = event_name, - notification_route = route_name.as_str(), - error = %err, - "Failed to post Slack lifecycle notification" - ); + match self.client.post_message(&channel, blocks, None).await { + Ok(posted) => { + let record = PlatformRecord::NotificationSent(NotificationSentRecord { + route: route_name.clone(), + event: event_name.to_string(), + channel: Some(posted.channel_id.clone()), + thread: None, + message_id: Some(posted.ts.clone()), + question: None, + operation: None, + }); + if let Err(err) = run_records::append(state, run_id, record).await { + warn!(run_id = %run_id, error = %err, "the Slack notification was sent but not recorded"); + } + } + Err(err) => { + warn!( + run_id = %run_id, + event = event_name, + notification_route = route_name.as_str(), + error = %err, + "Failed to post Slack lifecycle notification" + ); + } } }) }); @@ -766,12 +834,17 @@ impl SlackService { let Some(posted) = posted else { return; }; + let question_text = if question_text.is_empty() { + posted.text.as_str() + } else { + question_text + }; - self.thread_registry.remove(&posted.ts); + self.thread_registry.remove(&posted.message.ts); let blocks = slack_blocks::answered_blocks(question_text, answer_text); let _ = self .client - .update_message(&posted.channel_id, &posted.ts, &blocks) + .update_message(&posted.message.channel_id, &posted.message.ts, &blocks) .await; } @@ -789,101 +862,60 @@ impl SlackService { } } -fn slack_lifecycle_details(event: &RunEvent) -> Option { - match &event.body { - EventBody::RunStarted(props) => Some(SlackLifecycleDetails { - kind: slack_blocks::RunLifecycleKind::Started, - started_event_name: Some(props.name.clone()), - result: None, - duration_ms: None, +fn slack_lifecycle_details(record: &RunLifecycleRecord) -> Option { + match record.transition { + RunLifecycleKind::Running => Some(SlackLifecycleDetails { + kind: slack_blocks::RunLifecycleKind::Started, + event_name: "run.started", + result: None, + duration_ms: None, }), - EventBody::RunCompleted(props) => Some(SlackLifecycleDetails { - kind: slack_blocks::RunLifecycleKind::Completed, - started_event_name: None, - result: Some(slack_lifecycle_completed_result( - &props.status, - props.reason, - )), - duration_ms: Some(props.timing.wall_time_ms), + RunLifecycleKind::Succeeded => Some(SlackLifecycleDetails { + kind: slack_blocks::RunLifecycleKind::Completed, + event_name: "run.completed", + result: Some(match record.status { + Some(RunStatus::Succeeded { reason }) => reason.to_string(), + _ => "completed".to_string(), + }), + duration_ms: None, }), - EventBody::RunFailed(props) => Some(SlackLifecycleDetails { - kind: slack_blocks::RunLifecycleKind::Failed, - started_event_name: None, - result: Some(slack_lifecycle_failed_result(&props.failure)), - duration_ms: Some(props.timing.wall_time_ms), + RunLifecycleKind::Failed | RunLifecycleKind::Dead => Some(SlackLifecycleDetails { + kind: slack_blocks::RunLifecycleKind::Failed, + event_name: "run.failed", + result: Some(slack_lifecycle_failed_result(record)), + duration_ms: None, }), _ => None, } } -fn slack_lifecycle_completed_result(status: &str, reason: SuccessReason) -> String { - let status = status.trim(); - let reason = reason.to_string(); - if status.is_empty() || status == reason { - reason - } else { - format!("{status} — {reason}") +fn slack_lifecycle_failed_result(record: &RunLifecycleRecord) -> String { + let reason = match record.status { + Some(RunStatus::Failed { reason }) => reason.to_string(), + Some(RunStatus::Dead) => "dead".to_string(), + _ => "failed".to_string(), + }; + match record.reason.as_deref().map(str::trim) { + Some(message) if !message.is_empty() => format!("{reason} — {message}"), + _ => reason, } } -fn slack_lifecycle_failed_result(failure: &fabro_types::RunFailure) -> String { - let reason = failure.reason.to_string(); - let message = failure.detail.message.trim(); - if message.is_empty() { - reason - } else { - format!("{reason} — {message}") +/// The platform record a stream item carries, when it carries one. +fn platform_record_of(item: &RunStreamItem) -> Option { + if item.kind != RunStreamItemKind::Platform { + return None; } + serde_json::from_value(item.item.get("record")?.clone()).ok() } -async fn load_prior_slack_lifecycle_event_details( - state: &AppState, - run_id: RunId, - before_seq: u32, -) -> PriorSlackLifecycleEventDetails { - let run_store = match state.stores.runs.open_run_reader(&run_id).await { - Ok(run_store) => run_store, - Err(err) => { - warn!( - run_id = %run_id, - error = %err, - "Unable to inspect prior run events for Slack lifecycle notification" - ); - return PriorSlackLifecycleEventDetails::default(); - } - }; - let events = match run_store.list_events().await { - Ok(events) => events, - Err(err) => { - warn!( - run_id = %run_id, - error = %err, - "Unable to load prior run events for Slack lifecycle notification" - ); - return PriorSlackLifecycleEventDetails::default(); - } - }; - - let mut details = PriorSlackLifecycleEventDetails::default(); - for envelope in events { - if envelope.seq >= before_seq { - break; - } - match envelope.event.body { - EventBody::RunStarted(props) if !props.name.trim().is_empty() => { - details.started_event_name = Some(props.name); - } - EventBody::PullRequestCreated(props) => { - details.pull_request = Some(SlackLifecyclePullRequest { - number: props.pr_number, - title: Some(props.title), - url: Some(props.pr_url), - }); - } - _ => {} - } +/// What a Petri event of the stream parsed out of a step's progress: a +/// question, an expiry, a note. +fn petri_parsed(item: &RunStreamItem) -> Option<&serde_json::Value> { + if item.kind != RunStreamItemKind::Petri { + return None; } - details + item.item.get("derived")?.get("parsed") } fn slack_lifecycle_workflow_label( @@ -981,11 +1013,13 @@ pub struct AppState { pub(crate) petri_runs: PetriRuns, /// The projector of Petri runs: signalled after each committed record. pub(crate) petri_projector: Arc, + /// The server's reader of every run's stream, into the live state. + pub(crate) stream_follower: Arc, scheduler_notify: Notify, automation_scheduler_notify: Notify, pull_request_scheduler_notify: Notify, pull_request_creation_queue: Mutex, - global_event_tx: broadcast::Sender, + global_event_tx: broadcast::Sender, /// Per-run coalescing registry for `GET /runs/{id}/files`. Concurrent /// callers for the same run share one materialization; different runs /// proceed in parallel. See `crate::run_files` for semantics. @@ -1439,12 +1473,7 @@ impl AppState { &self, run_id: &RunId, ) -> Result, ApiError> { - self.stores - .runs - .load_run_projection(run_id) - .await - .map_err(|err| ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()))? - .ok_or_else(|| ApiError::not_found("Run not found.")) + run_records::require_projection(self, *run_id).await } pub(crate) fn session_runtimes(&self) -> &SessionRuntimeManager { @@ -1644,14 +1673,13 @@ fn start_optional_slack_service(state: &Arc) { let mut rx = event_state.global_event_tx.subscribe(); loop { match rx.recv().await { - Ok(envelope) => { - // Resolve the run's web URL once per event so the Slack - // message can deep-link back to Fabro. Returns None when - // the web UI is disabled or `server.web.url` is unset, in - // which case `question_to_blocks` simply omits the link. - let run_web_url = event_state.run_web_url(&envelope.event.run_id); + Ok(item) => { event_service - .handle_event(event_state.as_ref(), &envelope, run_web_url.as_deref()) + .observe( + event_state.as_ref(), + item.run_id, + std::slice::from_ref(&item), + ) .await; } Err(RecvError::Lagged(_)) => {} @@ -2515,6 +2543,7 @@ pub(crate) fn build_app_state(config: AppStateConfig) -> anyhow::Result Result { - let Ok(run_store) = state.stores.runs.open_run(&id).await else { - return Ok(SandboxDeleteOutcome::Absent); - }; - let projection = match run_store.state().await { - Ok(projection) => projection, + let projection = match run_records::projection(state, id).await { + Ok(Some(projection)) => projection, + Ok(None) => return Ok(SandboxDeleteOutcome::Absent), Err(err) if force => { tracing::warn!( run_id = %id, - error = %render_with_causes(&err.to_string(), &collect_causes(&err)), + error = %format!("{err:#}"), "Skipping sandbox provider delete because run projection cannot be loaded" ); return Ok(SandboxDeleteOutcome::Cleaned); @@ -2683,9 +2710,13 @@ async fn delete_run_sandbox_resource( let delete_started = matches!(projection.status, RunStatus::Removing); let can_mark_removing = projection.status.can_transition_to(RunStatus::Removing); if !delete_started && can_mark_removing { - workflow_event::append_event(&run_store, &id, &workflow_event::Event::RunRemoving) - .await - .map_err(|err| ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()))?; + run_records::lifecycle( + state, + id, + run_records::transition(RunLifecycleKind::Removing, RunStatus::Removing), + ) + .await + .map_err(|err| ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()))?; } let preserve = projection @@ -2945,7 +2976,6 @@ fn clear_live_run_state(run: &mut ManagedRun) { run.accepted_questions.clear(); run.active_steerable_stages.clear(); run.active_non_steerable_stages.clear(); - run.event_tx = None; run.cancel_tx = None; run.cancel_token = None; run.worker_ref = None; @@ -2959,22 +2989,32 @@ fn cleanup_worker_control_bus_for_run(state: &AppState, run_id: RunId) { }); } -fn reconcile_live_interview_state_for_event(run: &mut ManagedRun, event: &RunEvent) { - match &event.body { - EventBody::InterviewCompleted(props) => { - run.accepted_questions.remove(&props.question_id); +/// A question the run's stream closed (answered, or expired) no longer holds +/// an accepted-answer claim; a terminal run holds none. +fn reconcile_live_interview_state(run: &mut ManagedRun, item: &RunStreamItem) { + match platform_record_of(item) { + Some(PlatformRecord::InterviewAnswered(answered)) => { + run.accepted_questions.remove(&answered.question); } - EventBody::InterviewTimeout(props) => { - run.accepted_questions.remove(&props.question_id); - } - EventBody::InterviewInterrupted(props) => { - run.accepted_questions.remove(&props.question_id); - } - EventBody::RunCompleted(_) | EventBody::RunFailed(_) => { + Some(PlatformRecord::RunLifecycle(record)) + if matches!( + record.transition, + RunLifecycleKind::Succeeded | RunLifecycleKind::Failed | RunLifecycleKind::Dead + ) => + { run.accepted_questions.clear(); } _ => {} } + if let Some(question_id) = petri_parsed(item) + .filter(|parsed| { + parsed.get("kind").and_then(serde_json::Value::as_str) == Some("question_expired") + }) + .and_then(|parsed| parsed.get("question")) + .and_then(serde_json::Value::as_str) + { + run.accepted_questions.remove(question_id); + } } fn claim_run_answer_transport( @@ -3055,12 +3095,13 @@ pub(crate) async fn reconcile_incomplete_runs_on_startup( let mut reconciled = 0usize; for summary in summaries { - let run_store = state.stores.runs.open_run(&summary.id).await?; + let Some(run_state) = run_records::projection(state, summary.id).await? else { + continue; + }; // A run continues from its records in a new worker, unless a cancel // was pending or the run was being removed: those end failed. if petri_run_resumes_on_restart(&summary) { - let run_state = run_store.state().await?; - petri_runs::reconcile_on_startup(state, summary.id, &run_store, &run_state).await?; + petri_runs::reconcile_on_startup(state, summary.id, &run_state).await?; reconciled += 1; continue; } @@ -3068,16 +3109,12 @@ pub(crate) async fn reconcile_incomplete_runs_on_startup( summary.lifecycle.pending_control, "Fabro server restarted before the run reached a terminal state.".to_string(), ); - let failure_event = workflow_event::Event::workflow_run_failed_from_error( - &error, - fabro_types::RunTiming::default(), - reason, - None, - None, - None, - None, - ); - workflow_event::append_event(&run_store, &summary.id, &failure_event).await?; + run_records::lifecycle( + state, + summary.id, + run_records::failed(reason, error.to_string()), + ) + .await?; reconciled += 1; } @@ -3124,8 +3161,9 @@ async fn persist_shutdown_run_failures( .collect::>(); for run_id in run_ids { - let run_store = state.stores.runs.open_run(&run_id).await?; - let run_state = run_store.state().await?; + let Some(run_state) = run_records::projection(state, run_id).await? else { + continue; + }; if run_state.status.is_terminal() { continue; } @@ -3134,16 +3172,12 @@ async fn persist_shutdown_run_failures( run_state.pending_control, "Fabro server shut down before the run reached a terminal state.".to_string(), ); - let failure_event = workflow_event::Event::workflow_run_failed_from_error( - &error, - fabro_types::RunTiming::default(), - reason, - None, - None, - None, - None, - ); - workflow_event::append_event(&run_store, &run_id, &failure_event).await?; + run_records::lifecycle( + state, + run_id, + run_records::failed(reason, error.to_string()), + ) + .await?; } Ok(()) @@ -3220,22 +3254,22 @@ async fn alive_refs(state: &AppState, refs: &[WorkerRef]) -> Vec { } async fn persist_cancelled_run_status(state: &AppState, run_id: RunId) -> anyhow::Result<()> { - let run_store = state.stores.runs.open_run(&run_id).await?; - let run_state = run_store.state().await?; + let Some(run_state) = run_records::projection(state, run_id).await? else { + anyhow::bail!("run {run_id} not found"); + }; if run_state.status.is_terminal() { return Ok(()); } - - let failure_event = workflow_event::Event::workflow_run_failed_from_error( - &WorkflowError::Cancelled, - fabro_types::RunTiming::default(), - FailureReason::Cancelled, - None, - None, - None, - None, - ); - workflow_event::append_event(&run_store, &run_id, &failure_event).await + run_records::lifecycle( + state, + run_id, + run_records::failed( + FailureReason::Cancelled, + WorkflowError::Cancelled.to_string(), + ), + ) + .await + .map(|_| ()) } /// Reject the run before execution if its effective sandbox provider is @@ -3262,52 +3296,16 @@ async fn fail_run_before_execution( reason: FailureReason, message: String, ) { - match state.stores.runs.open_run(&run_id).await { - Ok(run_store) => { - let failure_event = workflow_event::Event::workflow_run_failed_from_error( - &WorkflowError::engine(message.clone()), - fabro_types::RunTiming::default(), - reason, - None, - None, - None, - None, - ); - if let Err(err) = - workflow_event::append_event(&run_store, &run_id, &failure_event).await - { - error!(run_id = %run_id, error = %err, "Failed to persist run failure status"); - } - } - Err(err) => { - error!(run_id = %run_id, error = %err, "Failed to open run store while persisting run failure"); - } + if let Err(err) = + run_records::lifecycle(state, run_id, run_records::failed(reason, message.clone())).await + { + error!(run_id = %run_id, error = %err, "Failed to persist run failure status"); } fail_managed_run(state, run_id, reason, message); state.scheduler_notify.notify_one(); } -async fn forward_run_events_to_global( - state: Arc, - run_id: RunId, - mut run_events: broadcast::Receiver, -) { - loop { - match run_events.recv().await { - Ok(event) => { - let mut runs = state.runs.lock().expect("runs lock poisoned"); - if let Some(managed_run) = runs.get_mut(&run_id) { - reconcile_live_interview_state_for_event(managed_run, &event.event); - } - let _ = state.global_event_tx.send(event); - } - Err(RecvError::Lagged(_)) => {} - Err(RecvError::Closed) => break, - } - } -} - fn managed_run( dot_source: String, status: RunStatus, @@ -3324,8 +3322,6 @@ fn managed_run( accepted_questions: HashSet::new(), active_steerable_stages: HashMap::new(), active_non_steerable_stages: HashMap::new(), - event_tx: None, - checkpoint: None, cancel_tx: None, cancel_token: None, worker_ref: None, @@ -3373,50 +3369,42 @@ fn fail_managed_run(state: &Arc, run_id: RunId, reason: FailureReason, cleanup_worker_control_bus_for_run(state.as_ref(), run_id); } -fn update_live_run_from_event(state: &AppState, run_id: RunId, event: &RunEvent) { - use fabro_types::EventBody; - +/// Fold one lifecycle record of the run's stream into the in-memory run: +/// the status the scheduler and the control handlers read. A `runnable` +/// record is not folded: scheduling is owned by the start and approve +/// handlers, which set the live status and notify the scheduler themselves. +fn apply_lifecycle_to_managed_run(state: &AppState, run_id: RunId, record: &RunLifecycleRecord) { let mut runs = state.runs.lock().expect("runs lock poisoned"); let Some(managed_run) = runs.get_mut(&run_id) else { return; }; - - if matches!(&event.body, EventBody::RunRunnable(_)) { - // Scheduling is owned by the start/approve lifecycle handlers, which - // set the live status and notify the scheduler explicitly. Direct - // event ingestion still records durable history, but must not make - // externally injected events schedulable. - return; - } - - match &event.body { - EventBody::RunSubmitted(_) => managed_run.status = RunStatus::Submitted, - EventBody::RunPending(props) => { - managed_run.status = RunStatus::Pending { - reason: props.reason, - }; + match record.transition { + RunLifecycleKind::Submitted => managed_run.status = RunStatus::Submitted, + RunLifecycleKind::Pending => { + if let Some(status) = record.status { + managed_run.status = status; + } } - EventBody::RunStarting(_) => managed_run.status = RunStatus::Starting, - EventBody::RunRunning(_) => managed_run.status = RunStatus::Running, - EventBody::RunBlocked(props) => { + RunLifecycleKind::Starting => managed_run.status = RunStatus::Starting, + RunLifecycleKind::Running => managed_run.status = RunStatus::Running, + RunLifecycleKind::Blocked => { + let Some(RunStatus::Blocked { blocked_reason }) = record.status else { + return; + }; managed_run.status = match managed_run.status { RunStatus::Paused { .. } => RunStatus::Paused { - prior_block: Some(props.blocked_reason), - }, - _ => RunStatus::Blocked { - blocked_reason: props.blocked_reason, + prior_block: Some(blocked_reason), }, + _ => RunStatus::Blocked { blocked_reason }, }; } - EventBody::RunUnblocked(_) => { + RunLifecycleKind::Unblocked => { managed_run.status = match managed_run.status { - RunStatus::Paused { - prior_block: Some(_) | None, - } => RunStatus::Paused { prior_block: None }, + RunStatus::Paused { .. } => RunStatus::Paused { prior_block: None }, _ => RunStatus::Running, }; } - EventBody::RunPaused(_) => { + RunLifecycleKind::Paused => { let prior_block = match managed_run.status { RunStatus::Blocked { blocked_reason } => Some(blocked_reason), RunStatus::Paused { prior_block } => prior_block, @@ -3424,7 +3412,7 @@ fn update_live_run_from_event(state: &AppState, run_id: RunId, event: &RunEvent) }; managed_run.status = RunStatus::Paused { prior_block }; } - EventBody::RunUnpaused(_) => { + RunLifecycleKind::Unpaused => { managed_run.status = match managed_run.status { RunStatus::Paused { prior_block: Some(blocked_reason), @@ -3432,86 +3420,32 @@ fn update_live_run_from_event(state: &AppState, run_id: RunId, event: &RunEvent) _ => RunStatus::Running, }; } - EventBody::RunRemoving(_) => managed_run.status = RunStatus::Removing, - EventBody::RunCompleted(_) => { - let EventBody::RunCompleted(props) = &event.body else { - unreachable!( - "outer match arm already verified event.body is EventBody::RunCompleted" - ) - }; - managed_run.status = RunStatus::Succeeded { - reason: props.reason, - }; + RunLifecycleKind::Removing => managed_run.status = RunStatus::Removing, + RunLifecycleKind::Succeeded => { + managed_run.status = record.status.unwrap_or(RunStatus::Succeeded { + reason: SuccessReason::Completed, + }); managed_run.error = None; managed_run.active_steerable_stages.clear(); managed_run.active_non_steerable_stages.clear(); cleanup_worker_control_bus_for_run(state, run_id); } - EventBody::RunFailed(props) => { - managed_run.status = RunStatus::Failed { - reason: props.failure.reason, - }; - managed_run.error = Some(render_compact_with_causes( - &props.failure.detail.message, - &props.failure.detail.causes, - )); + RunLifecycleKind::Failed | RunLifecycleKind::Dead => { + managed_run.status = record.status.unwrap_or(RunStatus::Failed { + reason: FailureReason::WorkflowError, + }); + managed_run.error.clone_from(&record.reason); managed_run.active_steerable_stages.clear(); managed_run.active_non_steerable_stages.clear(); cleanup_worker_control_bus_for_run(state, run_id); } - // Track active agent sessions by steerability. Activated/deactivated - // are leased by session id so stale deactivations cannot clear a newer - // binding for the same stage. - EventBody::AgentSessionActivated(props) => { - if let (Some(stage_id), Some(session_id)) = - (event.stage_id.as_ref(), event.session_id.as_ref()) - { - if props.capabilities.contains(&SessionCapability::Steer) { - managed_run - .active_steerable_stages - .insert(stage_id.clone(), session_id.clone()); - managed_run.active_non_steerable_stages.remove(stage_id); - } else { - managed_run - .active_non_steerable_stages - .insert(stage_id.clone(), session_id.clone()); - managed_run.active_steerable_stages.remove(stage_id); - } - } - } - EventBody::AgentSessionDeactivated(_) => { - if let (Some(stage_id), Some(session_id)) = - (event.stage_id.as_ref(), event.session_id.as_ref()) - { - if managed_run - .active_steerable_stages - .get(stage_id) - .is_some_and(|current| current == session_id) - { - managed_run.active_steerable_stages.remove(stage_id); - } - if managed_run - .active_non_steerable_stages - .get(stage_id) - .is_some_and(|current| current == session_id) - { - managed_run.active_non_steerable_stages.remove(stage_id); - } - } - } - // ACP sessions are steerable via `agent.session.activated`; terminal - // ACP events and stage lifecycle events are still backstops for cleanup. - EventBody::AgentAcpCompleted(_) - | EventBody::AgentAcpCancelled(_) - | EventBody::AgentAcpTimedOut(_) - | EventBody::StageCompleted(_) - | EventBody::StageFailed(_) => { - if let Some(stage_id) = &event.stage_id { - managed_run.active_steerable_stages.remove(stage_id); - managed_run.active_non_steerable_stages.remove(stage_id); - } - } - _ => {} + RunLifecycleKind::Runnable + | RunLifecycleKind::StartRequested + | RunLifecycleKind::Approved + | RunLifecycleKind::Denied + | RunLifecycleKind::CancelRequested + | RunLifecycleKind::PauseRequested + | RunLifecycleKind::UnpauseRequested => {} } } @@ -3528,15 +3462,10 @@ async fn drain_worker_stderr( Ok(()) } -async fn fail_worker_launch( - state: &Arc, - run_store: &fabro_store::RunDatabase, - run_id: RunId, - err: anyhow::Error, -) { +async fn fail_worker_launch(state: &Arc, run_id: RunId, err: anyhow::Error) { tracing::error!(run_id = %run_id, error = %err, "Failed to spawn worker"); - let pending_control = match run_store.state().await { - Ok(run_state) => run_state.pending_control, + let pending_control = match run_records::projection(state, run_id).await { + Ok(run_state) => run_state.and_then(|run_state| run_state.pending_control), Err(state_err) => { tracing::warn!( run_id = %run_id, @@ -3558,57 +3487,45 @@ async fn fail_worker_launch( } else { launch_message }; - let failure_event = workflow_event::Event::workflow_run_failed_from_error( - &error, - fabro_types::RunTiming::default(), - reason, - None, - None, - None, - None, - ); - let _ = workflow_event::append_event(run_store, &run_id, &failure_event).await; + let _ = run_records::lifecycle( + state, + run_id, + run_records::failed(reason, error.to_string()), + ) + .await; fail_managed_run(state, run_id, reason, message); state.scheduler_notify.notify_one(); } -async fn append_worker_exit_failure( - run_store: &fabro_store::RunDatabase, - run_id: RunId, - worker_exit: &WorkerExit, -) { - let state = match run_store.state().await { - Ok(state) => state, +/// A worker that exited without recording the run's end left it failed. +async fn append_worker_exit_failure(state: &AppState, run_id: RunId, worker_exit: &WorkerExit) { + let run_state = match run_records::projection(state, run_id).await { + Ok(Some(run_state)) => run_state, + Ok(None) => return, Err(err) => { tracing::warn!(run_id = %run_id, error = %err, "Failed to load run state after worker exit"); return; } }; - - let terminal = state.status.is_terminal(); - if terminal { + if run_state.status.is_terminal() { return; } let (error, reason) = failure_for_incomplete_run( - state.pending_control, + run_state.pending_control, format!( "Worker exited before emitting a terminal run event: {}", worker_exit.detail ), ); - let failure_event = workflow_event::Event::workflow_run_failed_from_error( - &error, - fabro_types::RunTiming::default(), - reason, - None, - None, - None, - None, - ); - - if let Err(err) = workflow_event::append_event(run_store, &run_id, &failure_event).await { - tracing::warn!(run_id = %run_id, error = %err, "Failed to append worker exit failure"); + if let Err(err) = run_records::lifecycle( + state, + run_id, + run_records::failed(reason, error.to_string()), + ) + .await + { + tracing::warn!(run_id = %run_id, error = %err, "Failed to record the worker exit failure"); } } @@ -3829,7 +3746,21 @@ async fn deliver_answer_to_run( } }; + let answered = InterviewAnsweredRecord { + question: qid.to_string(), + principal: Some(submission.actor.clone()), + channel: None, + text: None, + answer: Some(answer_text(&submission.answer)), + }; if let Ok(()) = transport.submit(qid, submission).await { + // The answer reached the run; who gave it, and what, is Fabro's + // record beside the answer Petri records. + if let Err(err) = + run_records::append(state, run_id, PlatformRecord::InterviewAnswered(answered)).await + { + warn!(run_id = %run_id, question = qid, error = %err, "the answer was delivered but not recorded"); + } Ok(()) } else { release_run_answer_claim(state, run_id, qid); @@ -3841,6 +3772,22 @@ async fn deliver_answer_to_run( } } +/// An answer as text, for the record and the readers that show it. +fn answer_text(answer: &fabro_interview::Answer) -> String { + use fabro_interview::AnswerValue; + match &answer.value { + AnswerValue::Yes => "yes".to_string(), + AnswerValue::No => "no".to_string(), + AnswerValue::Cancelled => "cancelled".to_string(), + AnswerValue::Interrupted => "interrupted".to_string(), + AnswerValue::Skipped => "skipped".to_string(), + AnswerValue::Timeout => "timed out".to_string(), + AnswerValue::Selected(key) => key.clone(), + AnswerValue::MultiSelected(keys) => keys.join(", "), + AnswerValue::Text(text) => text.clone(), + } +} + #[allow( clippy::result_large_err, reason = "Answer request parsing returns HTTP 400 responses directly." @@ -3912,28 +3859,20 @@ async fn execute_run_subprocess(state: Arc, run_id: RunId) { (run_dir, managed_run.execution_mode) }; - let run_store = match state.stores.runs.open_run(&run_id).await { - Ok(run_store) => run_store, - Err(err) => { - tracing::error!(run_id = %run_id, error = %err, "Failed to open run store"); + stream_follower::follow_run(&state, run_id).await; + let run_state = match run_records::projection(&state, run_id).await { + Ok(Some(run_state)) => run_state, + Ok(None) => { + tracing::error!(run_id = %run_id, "Run not found at launch"); fail_managed_run( &state, run_id, FailureReason::WorkflowError, - format!("Failed to open run store: {err}"), + "Run not found at launch".to_string(), ); state.scheduler_notify.notify_one(); return; } - }; - tokio::spawn(forward_run_events_to_global( - Arc::clone(&state), - run_id, - run_store.subscribe(), - )); - - let run_state = match run_store.state().await { - Ok(run_state) => run_state, Err(err) => { tracing::error!(run_id = %run_id, error = %err, "Failed to load run state"); fail_managed_run( @@ -4010,7 +3949,7 @@ async fn execute_run_subprocess(state: Arc, run_id: RunId) { let started_worker = match launch_result { Ok(worker) => worker, Err(err) => { - fail_worker_launch(&state, &run_store, run_id, err).await; + fail_worker_launch(&state, run_id, err).await; return; } }; @@ -4036,16 +3975,12 @@ async fn execute_run_subprocess(state: Arc, run_id: RunId) { tracing::error!(run_id = %run_id, error = %err, "Failed while waiting on worker"); let message = format!("Worker wait failed: {err}"); state.worker_runtime.force_stop(&worker_ref).await; - let failure_event = workflow_event::Event::workflow_run_failed_from_error( - &WorkflowError::engine_with_source("Worker wait failed", err), - fabro_types::RunTiming::default(), - FailureReason::Terminated, - None, - None, - None, - None, - ); - let _ = workflow_event::append_event(&run_store, &run_id, &failure_event).await; + let _ = run_records::lifecycle( + &state, + run_id, + run_records::failed(FailureReason::Terminated, message.clone()), + ) + .await; fail_managed_run(&state, run_id, FailureReason::Terminated, message); state.scheduler_notify.notify_one(); return; @@ -4080,10 +4015,21 @@ async fn execute_run_subprocess(state: Arc, run_id: RunId) { // API drop here, so its lease never outlives it. state.petri_runs.worker_exited(run_id); state.petri_projector.signal(run_id); - append_worker_exit_failure(&run_store, run_id, &worker_exit).await; + append_worker_exit_failure(&state, run_id, &worker_exit).await; - let final_state = match run_store.state().await { - Ok(state) => state, + let final_state = match run_records::projection(&state, run_id).await { + Ok(Some(final_state)) => final_state, + Ok(None) => { + tracing::warn!(run_id = %run_id, "The run's final state is missing from the store"); + fail_managed_run( + &state, + run_id, + FailureReason::WorkflowError, + "The run's final state is missing from the store".to_string(), + ); + state.scheduler_notify.notify_one(); + return; + } Err(err) => { tracing::warn!(run_id = %run_id, error = %err, "Failed to load final run state from store"); fail_managed_run( @@ -4117,7 +4063,6 @@ async fn execute_run_subprocess(state: Arc, run_id: RunId) { }) }) .or_else(|| managed_run.error.clone()); - managed_run.checkpoint = final_state.current_checkpoint().cloned(); managed_run.run_dir = Some(run_dir); clear_live_run_state(managed_run); } @@ -4127,6 +4072,7 @@ async fn execute_run_subprocess(state: Arc, run_id: RunId) { /// Background task that promotes runnable runs when capacity is available. pub fn spawn_scheduler(state: Arc) { + stream_follower::spawn_stream_follower(Arc::clone(&state)); tokio::spawn(async move { loop { tokio::select! { @@ -4178,21 +4124,26 @@ async fn append_control_request( action: RunControlAction, actor: Option, ) -> anyhow::Result<()> { - let run_store = state.stores.runs.open_run(&run_id).await?; - let event = match action { - RunControlAction::Cancel => workflow_event::Event::RunCancelRequested { actor }, - RunControlAction::Pause => workflow_event::Event::RunPauseRequested { actor }, - RunControlAction::Unpause => workflow_event::Event::RunUnpauseRequested { actor }, + let _ = actor; + let kind = match action { + RunControlAction::Cancel => RunLifecycleKind::CancelRequested, + RunControlAction::Pause => RunLifecycleKind::PauseRequested, + RunControlAction::Unpause => RunLifecycleKind::UnpauseRequested, }; if action == RunControlAction::Cancel { - workflow_event::append_event_if(&run_store, &run_id, &event, |projection| { - projection.pending_control != Some(RunControlAction::Cancel) - }) + // A cancel already pending is not asked for twice. + let pending = run_records::projection(state, run_id) + .await? + .and_then(|projection| projection.pending_control); + if pending == Some(RunControlAction::Cancel) { + return Ok(()); + } + } + let mut record = RunLifecycleRecord::new(kind); + record.action = Some(action); + run_records::lifecycle(state, run_id, record) .await .map(|_| ()) - } else { - workflow_event::append_event(&run_store, &run_id, &event).await - } } /// Returns a 409 response with an actionable "unarchive first" message if the diff --git a/lib/apps/fabro-server/src/server/automation_scheduler.rs b/lib/apps/fabro-server/src/server/automation_scheduler.rs index 16938446e..59842d390 100644 --- a/lib/apps/fabro-server/src/server/automation_scheduler.rs +++ b/lib/apps/fabro-server/src/server/automation_scheduler.rs @@ -664,8 +664,10 @@ mod tests { assert_eq!(automation_ref.name.as_deref(), Some("Nightly")); assert_eq!(automation_ref.trigger_id.as_deref(), Some("schedule")); let run_id = runs[0].id; - let run_store = state.stores.runs.open_run_reader(&run_id).await.unwrap(); - let projection = run_store.state().await.unwrap(); + let projection = super::super::run_records::projection(&state, run_id) + .await + .unwrap() + .expect("the run projects"); assert!(projection.spec.workflow_version_id.is_some()); assert_eq!( projection.spec.target, @@ -676,16 +678,24 @@ mod tests { sha: Some("0123456789abcdef0123456789abcdef01234567".to_string()), })) ); - assert_eq!( - run_store - .list_events() - .await - .unwrap() - .iter() - .filter(|event| event.event.event_name() == "run.start_requested") - .count(), - 1 - ); + let start_requests = state + .stores + .run_summaries + .platform_records() + .read(&run_id) + .await + .unwrap() + .into_iter() + .filter(|stored| { + matches!( + &stored.record, + fabro_store::platform_records::PlatformRecord::RunLifecycle(record) + if record.transition + == fabro_store::platform_records::RunLifecycleKind::StartRequested + ) + }) + .count(); + assert_eq!(start_requests, 1); assert!(matches!( state .runs diff --git a/lib/apps/fabro-server/src/server/handler/artifacts.rs b/lib/apps/fabro-server/src/server/handler/artifacts.rs index d597e05be..3de18a632 100644 --- a/lib/apps/fabro-server/src/server/handler/artifacts.rs +++ b/lib/apps/fabro-server/src/server/handler/artifacts.rs @@ -103,14 +103,14 @@ async fn write_run_blob( if let Some(response) = reject_if_archived(state.as_ref(), &id).await { return response; } - match state.stores.runs.open_run(&id).await { - Ok(run_store) => match run_store.write_blob(&body).await { - Ok(blob_hash) => Json(WriteBlobResponse { hash: blob_hash }).into_response(), - Err(err) => { - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() - } - }, - Err(_) => ApiError::not_found("Run not found.").into_response(), + if let Err(err) = state.load_run_projection(&id).await { + return err.into_response(); + } + match state.store_ref().blobs().write(&body).await { + Ok(blob_hash) => Json(WriteBlobResponse { hash: blob_hash }).into_response(), + Err(err) => { + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() + } } } @@ -118,15 +118,15 @@ async fn read_run_blob( RequireRunBlob(id, blob_hash): RequireRunBlob, State(state): State>, ) -> Response { - match state.stores.runs.open_run_reader(&id).await { - Ok(run_store) => match run_store.read_blob(&blob_hash).await { - Ok(Some(bytes)) => octet_stream_response(bytes), - Ok(None) => ApiError::not_found("Blob not found.").into_response(), - Err(err) => { - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() - } - }, - Err(_) => ApiError::not_found("Run not found.").into_response(), + if let Err(err) = state.load_run_projection(&id).await { + return err.into_response(); + } + match state.store_ref().blobs().read(&blob_hash).await { + Ok(Some(bytes)) => octet_stream_response(bytes), + Ok(None) => ApiError::not_found("Blob not found.").into_response(), + Err(err) => { + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() + } } } diff --git a/lib/apps/fabro-server/src/server/handler/events.rs b/lib/apps/fabro-server/src/server/handler/events.rs index 655462aad..9b0cb4972 100644 --- a/lib/apps/fabro-server/src/server/handler/events.rs +++ b/lib/apps/fabro-server/src/server/handler/events.rs @@ -1,52 +1,32 @@ +//! A run's stream: Petri's events and Fabro's platform records, one +//! `stream_seq` each, as the projector commits them. `GET /runs/{id}/events` +//! pages it by `after`, `GET /runs/{id}/attach` follows it live, and +//! `GET /attach` follows every run's stream at once. + use std::sync::Arc; use std::time::Duration; -use axum::extract::DefaultBodyLimit; use fabro_api::types::PaginatedRunStreamList; use fabro_petri::petri::EVENT_CONTRACT_VERSION; -use fabro_types::run_event::MAX_RUN_EVENT_BODY_BYTES; -use fabro_types::{ - RunEventDetailContent, RunEventDetailContentKind, RunEventDetailEnvelope, - RunEventDetailResponse, RunStreamItem, -}; -use fabro_workflow::event::build_redacted_event_payload; +use fabro_types::RunStreamItem; use tokio::sync::broadcast::error::RecvError; use tokio::time::{self, Instant}; use super::super::{ - ApiError, AppState, AppendEventResponse, BroadcastStream, Event, EventBody, EventEnvelope, - EventPayload, HashSet, IntoResponse, Json, KeepAlive, PaginatedEventList, PaginationMeta, Path, - Query, RequireRunManagementTarget, RequireRunScoped, RequireRunStageScoped, RequiredUser, - Response, Router, RunEvent, RunId, Sse, State, StatusCode, StreamExt, UnboundedReceiverStream, - broadcast, get, mpsc, parse_run_id_path, parse_stage_id_path, redact_jsonl_line, - reject_if_archived, update_live_run_from_event, + ApiError, AppState, BroadcastStream, Event, HashSet, IntoResponse, Json, KeepAlive, + PaginationMeta, Path, Query, RequireRunManagementTarget, RequiredUser, Response, Router, RunId, + Sse, State, StatusCode, StreamExt, UnboundedReceiverStream, broadcast, get, mpsc, + parse_run_id_path, redact_jsonl_line, }; pub(super) fn routes() -> Router> { Router::new() .route("/attach", get(attach_events)) - .route( - "/runs/{id}/events", - get(list_run_events) - .post(append_run_event) - .layer(DefaultBodyLimit::max(MAX_RUN_EVENT_BODY_BYTES)), - ) - .route("/runs/{id}/events/{seq}", get(get_run_event_detail)) - .route( - "/runs/{id}/stages/{stageId}/events", - get(list_run_stage_events), - ) + .route("/runs/{id}/events", get(list_run_events)) .route("/runs/{id}/attach", get(attach_run_events)) } -#[derive(Clone, Copy, Default, PartialEq, Eq, serde::Deserialize)] -#[serde(rename_all = "snake_case")] -enum EventSequenceOrder { - #[default] - Asc, - Desc, -} - +/// Query parameters shared by the paged event listings: a page size. #[derive(serde::Deserialize)] pub(crate) struct EventListParams { #[serde(default)] @@ -65,86 +45,28 @@ impl EventListParams { } } -/// Query parameters for `/runs/{id}/events` only. Descending pagination via -/// `before_seq` + `order` is not part of the shared `EventListParams` -/// contract used by the session, stage, and pair transcript endpoints. +/// Query parameters for `/runs/{id}/events`: the stream cursor and a page +/// size. #[derive(serde::Deserialize)] struct RunEventListParams { #[serde(default)] - since_seq: Option, + limit: Option, + /// The run stream cursor: the last `stream_seq` seen. #[serde(default)] - before_seq: Option, - #[serde(default)] - order: Option, - #[serde(default)] - limit: Option, - /// The run stream cursor of a Petri run: the last `stream_seq` seen. - #[serde(default)] - after: Option, + after: Option, } impl RunEventListParams { - fn since_seq(&self) -> u32 { - self.since_seq.unwrap_or(1).max(1) - } - - fn order(&self) -> EventSequenceOrder { - self.order.unwrap_or_default() - } - fn limit(&self) -> usize { self.limit.unwrap_or(100).clamp(1, 1000) } - - fn cursor_error(&self) -> Option<&'static str> { - if self.after.is_some() && (self.since_seq.is_some() || self.before_seq.is_some()) { - return Some( - "after is the run stream cursor and cannot be combined with since_seq or before_seq.", - ); - } - match self.order() { - EventSequenceOrder::Asc if self.before_seq.is_some() => { - Some("before_seq requires order=desc.") - } - EventSequenceOrder::Desc if self.since_seq.is_some() => { - Some("since_seq cannot be combined with order=desc; use before_seq instead.") - } - _ => None, - } - } - - /// Why the parameters do not address a Petri run's stream, if they do - /// not: the legacy cursors have no meaning there. - fn stream_cursor_error(&self) -> Option<&'static str> { - if self.since_seq.is_some() || self.before_seq.is_some() || self.order.is_some() { - return Some( - "this run executes on Petri; its events are a run stream addressed by `after` \ - (the last stream_seq seen), not by since_seq, before_seq or order.", - ); - } - None - } } #[derive(serde::Deserialize)] struct AttachParams { + /// The run stream cursor: the last `stream_seq` seen. #[serde(default)] - since_seq: Option, - /// The run stream cursor of a Petri run: the last `stream_seq` seen. - #[serde(default)] - after: Option, -} - -#[derive(serde::Deserialize)] -struct EventDetailParams { - #[serde(default)] - max_content_length: Option, -} - -impl EventDetailParams { - fn max_content_length(&self) -> usize { - self.max_content_length.unwrap_or(20_000).clamp(1, 200_000) - } + after: Option, } #[derive(serde::Deserialize)] @@ -164,8 +86,8 @@ async fn attach_events( }; let stream = - filtered_global_events(state.global_event_tx.subscribe(), run_filter).filter_map(|event| { - sse_event_from_store(&event).map(Ok::) + filtered_global_events(state.global_event_tx.subscribe(), run_filter).filter_map(|item| { + sse_event_from_stream_item(&item).map(Ok::) }); let stream = futures_util::StreamExt::take_until(stream, state.shutdown_token().cancelled_owned()); @@ -176,11 +98,11 @@ async fn attach_events( } pub(in crate::server) fn filtered_global_events( - event_rx: broadcast::Receiver, + event_rx: broadcast::Receiver, run_filter: Option>, -) -> impl tokio_stream::Stream { +) -> impl tokio_stream::Stream { BroadcastStream::new(event_rx).filter_map(move |result| match result { - Ok(event) if event_matches_run_filter(&event, run_filter.as_ref()) => Some(event), + Ok(item) if item_matches_run_filter(&item, run_filter.as_ref()) => Some(item), Ok(_) | Err(_) => None, }) } @@ -209,139 +131,29 @@ fn parse_global_run_filter(raw: Option<&str>) -> Result>, } } -fn event_matches_run_filter(event: &EventEnvelope, run_filter: Option<&HashSet>) -> bool { +fn item_matches_run_filter(item: &RunStreamItem, run_filter: Option<&HashSet>) -> bool { let Some(run_filter) = run_filter else { return true; }; - run_filter.contains(&event.event.run_id) -} - -fn sse_event_from_store(event: &EventEnvelope) -> Option { - let data = serde_json::to_string(event).ok()?; - let data = redact_jsonl_line(&data); - Some(Event::default().data(data)) -} - -fn attach_event_is_terminal(event: &EventEnvelope) -> bool { - matches!( - &event.event.body, - EventBody::RunCompleted(_) | EventBody::RunFailed(_) - ) + run_filter.contains(&item.run_id) } fn run_projection_is_active(state: &fabro_store::RunProjection) -> bool { state.status.is_active() } -async fn append_run_event( - RequireRunScoped(id): RequireRunScoped, - State(state): State>, - Json(value): Json, -) -> Response { - if let Some(response) = reject_if_archived(state.as_ref(), &id).await { - return response; - } - let event = match RunEvent::from_value(value.clone()) { - Ok(event) => event, - Err(err) => { - return ApiError::bad_request(format!("Invalid run event: {err}")).into_response(); - } - }; - if event.run_id != id { - return ApiError::bad_request("Event run_id does not match path run ID.").into_response(); - } - if let Some(denied) = denied_dedicated_operation_event_name(&event.body) { - return ApiError::bad_request(format!( - "{denied} must be performed through its dedicated operation endpoint instead of injecting it via append_run_event" - )) - .into_response(); - } - let payload = match EventPayload::new(value, &id) { - Ok(payload) => payload, - Err(err) => return ApiError::bad_request(err.to_string()).into_response(), - }; - - match state.stores.runs.open_run(&id).await { - Ok(run_store) => match run_store.append_event(&payload).await { - Ok(seq) => { - update_live_run_from_event(&state, id, &event); - Json(AppendEventResponse { - seq: i64::from(seq), - }) - .into_response() - } - Err(err) => { - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() - } - }, - Err(_) => ApiError::not_found("Run not found.").into_response(), - } -} - async fn list_run_events( RequireRunManagementTarget(id, _actor): RequireRunManagementTarget, State(state): State>, Query(params): Query, ) -> Response { - if let Some(detail) = params.cursor_error() { - return ApiError::bad_request(detail).into_response(); - } - let limit = params.limit(); if let Err(response) = ensure_run_exists(&state, &id).await { return response; } - if let Some(detail) = params.stream_cursor_error() { - return ApiError::bad_request(detail).into_response(); - } list_run_stream(&state, id, params.after.unwrap_or(0), limit).await } -#[expect( - dead_code, - reason = "the legacy event list goes with the legacy events table" -)] -async fn list_run_events_legacy( - state: Arc, - id: RunId, - params: RunEventListParams, - limit: usize, -) -> Response { - match state.stores.runs.open_run_reader(&id).await { - Ok(run_store) => { - let events = match params.order() { - EventSequenceOrder::Asc => { - run_store - .list_events_from_with_limit(params.since_seq(), limit) - .await - } - EventSequenceOrder::Desc => { - run_store - .list_events_before_with_limit(params.before_seq, limit) - .await - } - }; - match events { - Ok(mut events) => { - let has_more = events.len() > limit; - events.truncate(limit); - Json(PaginatedEventList { - data: events, - meta: PaginationMeta { - has_more, - total: None, - }, - }) - .into_response() - } - Err(err) => ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(), - } - } - Err(_) => ApiError::not_found("Run not found.").into_response(), - } -} - /// The canonical 404 when there is no such run. async fn ensure_run_exists(state: &AppState, id: &RunId) -> Result<(), Response> { state @@ -506,696 +318,18 @@ async fn attach_run_stream(state: Arc, id: RunId, after: Option) .into_response() } -async fn list_run_stage_events( - RequireRunStageScoped(id, stage_id): RequireRunStageScoped, - State(state): State>, - Query(params): Query, -) -> Response { - let stage_id = match parse_stage_id_path(&stage_id) { - Ok(stage_id) => stage_id, - Err(response) => return response, - }; - let since_seq = params.since_seq(); - let limit = params.limit(); - match state.stores.runs.open_run_reader(&id).await { - Ok(run_store) => match run_store - .list_events_for_stage_from_with_limit(&stage_id, since_seq, limit) - .await - { - Ok(mut events) => { - let has_more = events.len() > limit; - events.truncate(limit); - Json(PaginatedEventList { - data: events, - meta: PaginationMeta { - has_more, - total: None, - }, - }) - .into_response() - } - Err(err) => { - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() - } - }, - Err(_) => ApiError::not_found("Run not found.").into_response(), - } -} - -async fn get_run_event_detail( - RequireRunScoped(id): RequireRunScoped, - State(state): State>, - Path((_id, seq)): Path<(String, u32)>, - Query(params): Query, -) -> Response { - let max_content_length = params.max_content_length(); - match state.stores.runs.open_run_reader(&id).await { - Ok(run_store) => match run_store.get_event(seq).await { - Ok(event) => { - let Some(envelope) = event else { - return ApiError::with_code( - StatusCode::NOT_FOUND, - "Event not found.", - "event_not_found", - ) - .into_response(); - }; - Json(detail_response(envelope, max_content_length)).into_response() - } - Err(err) => { - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() - } - }, - Err(_) => ApiError::not_found("Run not found.").into_response(), - } -} - -fn detail_response(envelope: EventEnvelope, max_content_length: usize) -> RunEventDetailResponse { - let raw_properties = event_properties(&envelope.event); - let redacted_properties = redacted_event_properties(&envelope.event); - let redacted = raw_properties != redacted_properties; - let mut properties = redacted_properties; - let event_name = envelope.event.event_name().to_string(); - let mut content = None; - let mut truncated = false; - - for (key, kind) in [ - ("text", RunEventDetailContentKind::Text), - ("output", RunEventDetailContentKind::ToolOutput), - ("arguments", RunEventDetailContentKind::ToolArguments), - ("error", RunEventDetailContentKind::Error), - ("details", RunEventDetailContentKind::Details), - ] { - if let Some(value) = properties.remove(key) { - let raw = match value { - serde_json::Value::String(value) => value, - other => serde_json::to_string(&other).unwrap_or_else(|_| String::new()), - }; - let (value, was_truncated) = truncate_content(raw, max_content_length); - truncated = truncated || was_truncated; - content = Some(RunEventDetailContent { kind, value }); - break; - } - } - - RunEventDetailResponse { - event: RunEventDetailEnvelope { - seq: envelope.seq, - id: envelope.event.id, - ts: envelope.event.ts, - run_id: envelope.event.run_id, - event: event_name, - actor: envelope.event.actor, - session_id: envelope.event.session_id, - node_id: envelope.event.node_id, - node_label: envelope.event.node_label, - stage_id: envelope.event.stage_id, - tool_call_id: envelope.event.tool_call_id, - }, - properties, - content, - truncated, - redacted, - max_content_length, - } -} - -fn event_properties(event: &RunEvent) -> serde_json::Map { - event - .properties() - .ok() - .and_then(|value| value.as_object().cloned()) - .unwrap_or_default() -} - -fn redacted_event_properties(event: &RunEvent) -> serde_json::Map { - build_redacted_event_payload(event, &event.run_id) - .ok() - .and_then(|payload| { - payload - .as_value() - .get("properties") - .and_then(serde_json::Value::as_object) - .cloned() - }) - .unwrap_or_else(|| event_properties(event)) -} - -fn truncate_content(value: String, max_content_length: usize) -> (String, bool) { - if value.len() <= max_content_length { - return (value, false); - } - let end = value.floor_char_boundary(max_content_length); - (value[..end].to_string(), true) -} - async fn attach_run_events( _auth: RequiredUser, State(state): State>, Path(id): Path, Query(params): Query, ) -> Response { - const ATTACH_REPLAY_BATCH_LIMIT: usize = 256; - let id = match parse_run_id_path(&id) { Ok(id) => id, Err(response) => return response, }; match ensure_run_exists(&state, &id).await { - Ok(()) => return attach_run_stream(state, id, params.after).await, - Err(response) => return response, - } - #[expect( - unreachable_code, - reason = "the legacy attach goes with the legacy events table" - )] - let Ok(run_store) = state.stores.runs.open_run_reader(&id).await else { - return ApiError::not_found("Run not found.").into_response(); - }; - let start_seq = match params.since_seq { - Some(seq) if seq >= 1 => seq, - Some(_) => 1, - None => match run_store.last_event_seq().await { - Ok(last_seq) => last_seq.map_or(1, |seq| seq.saturating_add(1)), - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }, - }; - let (sender, receiver) = mpsc::unbounded_channel(); - let shutdown = state.shutdown_token(); - tokio::spawn(async move { - let mut next_seq = start_seq; - - loop { - let Ok(replay_batch) = run_store - .list_events_from_with_limit(next_seq, ATTACH_REPLAY_BATCH_LIMIT) - .await - else { - return; - }; - let replay_has_more = replay_batch.len() > ATTACH_REPLAY_BATCH_LIMIT; - - for event in replay_batch.into_iter().take(ATTACH_REPLAY_BATCH_LIMIT) { - next_seq = event.seq.saturating_add(1); - let terminal = attach_event_is_terminal(&event); - if let Some(sse_event) = sse_event_from_store(&event) { - if sender - .send(Ok::(sse_event)) - .is_err() - { - return; - } - } - if terminal { - return; - } - } - - if replay_has_more { - continue; - } - - let Ok(state) = run_store.state().await else { - return; - }; - - if run_projection_is_active(&state) { - break; - } - - let Ok(tail_batch) = run_store - .list_events_from_with_limit(next_seq, ATTACH_REPLAY_BATCH_LIMIT) - .await - else { - return; - }; - let tail_has_more = tail_batch.len() > ATTACH_REPLAY_BATCH_LIMIT; - - for event in tail_batch.into_iter().take(ATTACH_REPLAY_BATCH_LIMIT) { - next_seq = event.seq.saturating_add(1); - let terminal = attach_event_is_terminal(&event); - if let Some(sse_event) = sse_event_from_store(&event) { - if sender - .send(Ok::(sse_event)) - .is_err() - { - return; - } - } - if terminal { - return; - } - } - - if tail_has_more { - continue; - } - - return; - } - - let Ok(mut live_stream) = run_store.watch_events_from(next_seq) else { - return; - }; - - loop { - tokio::select! { - biased; - () = shutdown.cancelled() => break, - next = live_stream.next() => { - let Some(result) = next else { - return; - }; - let Ok(event) = result else { - return; - }; - let terminal = attach_event_is_terminal(&event); - if let Some(sse_event) = sse_event_from_store(&event) { - if sender - .send(Ok::(sse_event)) - .is_err() - { - return; - } - } - if terminal { - return; - } - } - } - } - }); - - Sse::new(UnboundedReceiverStream::new(receiver)) - .keep_alive(KeepAlive::default()) - .into_response() -} - -/// Returns the wire event name if the given body has a dedicated operation -/// endpoint that clients must use instead of injecting via `append_run_event`. -/// These endpoints enforce authorization and status-transition preconditions -/// (e.g. "archive only from terminal") that a direct event append would -/// bypass. Other run-lifecycle events flow through this endpoint legitimately: -/// the worker subprocess emits state transitions during execution. -fn denied_dedicated_operation_event_name(body: &EventBody) -> Option<&str> { - match body { - EventBody::RunArchived(_) - | EventBody::RunUnarchived(_) - | EventBody::RunTitleUpdated(_) - | EventBody::RunCancelRequested(_) - | EventBody::RunPauseRequested(_) - | EventBody::RunUnpauseRequested(_) - | EventBody::PullRequestLinked(_) - | EventBody::PullRequestUnlinked(_) - | EventBody::RunSessionCreated(_) => Some(body.event_name()), - _ => None, - } -} - -#[cfg(test)] -mod stage_events_tests { - use std::time::Duration; - - use axum::body::{Body, to_bytes}; - use axum::http::{Request, StatusCode, header}; - use fabro_store::EventPayload; - use fabro_types::{Graph, PetriAdmission, RunId, WorkflowSettings, test_support}; - use fabro_workflow::event as workflow_event; - use http_body_util::BodyExt; - use serde_json::json; - use tokio::time::timeout; - use tower::ServiceExt; - - use crate::test_support::{build_test_router, test_app_state}; - - fn req_get(uri: &str) -> Request { - Request::builder() - .method("GET") - .uri(uri) - .body(Body::empty()) - .expect("stage events GET request should build") - } - - fn make_event(run_id: &RunId, idx: u32, node_id: Option<&str>) -> EventPayload { - make_event_with_stage_id(run_id, idx, node_id, None) - } - - async fn append_run_created(run_store: &fabro_store::RunDatabase, run_id: &RunId) { - workflow_event::append_event(run_store, run_id, &workflow_event::Event::RunCreated { - run_id: *run_id, - title: None, - settings: serde_json::to_value(WorkflowSettings::default()).unwrap(), - graph: serde_json::to_value(Graph::new("test")).unwrap(), - workflow_source: None, - labels: std::collections::BTreeMap::new(), - source_directory: None, - workflow_slug: None, - workflow_version_id: None, - target: None, - automation: None, - provenance: test_support::test_run_provenance(), - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, - admission: PetriAdmission::default(), - }) - .await - .expect("run.created should append"); - } - - fn make_event_with_stage_id( - run_id: &RunId, - idx: u32, - node_id: Option<&str>, - stage_id: Option<&str>, - ) -> EventPayload { - let mut value = json!({ - "id": format!("evt-{idx}"), - "ts": "2026-04-09T12:00:00Z", - "run_id": run_id.to_string(), - "event": "stage.prompt", - "properties": { - "visit": 1, - "text": format!("prompt {idx}"), - }, - }); - if let Some(node) = node_id { - value - .as_object_mut() - .unwrap() - .insert("node_id".into(), json!(node)); - } - if let Some(stage_id) = stage_id { - value - .as_object_mut() - .unwrap() - .insert("stage_id".into(), json!(stage_id)); - } - EventPayload::new(value, run_id).expect("event payload should validate") - } - - async fn body_json(response: axum::response::Response) -> serde_json::Value { - let bytes = to_bytes(response.into_body(), usize::MAX) - .await - .expect("response body should fit in memory"); - serde_json::from_slice(&bytes).expect("response body should be valid JSON") - } - - fn assert_event_stream_response(response: &axum::response::Response) { - assert_eq!(response.status(), StatusCode::OK); - let content_type = response - .headers() - .get(header::CONTENT_TYPE) - .expect("SSE response should set content-type") - .to_str() - .expect("content-type should be valid UTF-8"); - assert!( - content_type.contains("text/event-stream"), - "expected text/event-stream content-type, got {content_type:?}" - ); - } - - async fn assert_sse_body_is_live(body: &mut Body) { - let result = timeout(Duration::from_millis(100), body.frame()).await; - assert!( - result.is_err(), - "SSE body should remain open before shutdown cancellation" - ); - } - - async fn assert_sse_body_completes_after_shutdown(mut body: Body) { - timeout(Duration::from_secs(1), async { - while let Some(frame) = body.frame().await { - frame.expect("SSE body frame should be readable"); - } - }) - .await - .expect("SSE body should complete promptly after shutdown cancellation"); - } - - #[tokio::test] - async fn attach_events_ends_when_shutdown_fires() { - let state = test_app_state(); - let app = build_test_router(state.clone()); - - let response = app - .oneshot(req_get("/api/v1/attach")) - .await - .expect("attach request should complete"); - assert_event_stream_response(&response); - - let mut body = response.into_body(); - assert_sse_body_is_live(&mut body).await; - - state.shutdown_token().cancel(); - - assert_sse_body_completes_after_shutdown(body).await; - } - - #[tokio::test] - async fn attach_run_events_ends_when_shutdown_fires() { - let state = test_app_state(); - let app = build_test_router(state.clone()); - let run_id = RunId::new(); - let run_store = state - .store_ref() - .create_run(&run_id) - .await - .expect("test run should be creatable"); - append_run_created(&run_store, &run_id).await; - for event in [ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunRunnable { - source: fabro_types::RunRunnableSource::StartRequested, - actor: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - ] { - workflow_event::append_event(&run_store, &run_id, &event) - .await - .expect("run lifecycle event should append"); - } - - let response = app - .oneshot(req_get(&format!("/api/v1/runs/{run_id}/attach"))) - .await - .expect("run attach request should complete"); - assert_event_stream_response(&response); - - let mut body = response.into_body(); - assert_sse_body_is_live(&mut body).await; - - state.shutdown_token().cancel(); - - assert_sse_body_completes_after_shutdown(body).await; - } - - async fn seed_run_with_mixed_events() -> (RunId, axum::Router) { - let state = test_app_state(); - let app = build_test_router(state.clone()); - let run_id = RunId::new(); - let run_store = state - .store_ref() - .create_run(&run_id) - .await - .expect("test run should be creatable"); - append_run_created(&run_store, &run_id).await; - - // Seed 200 unrelated 'beta' events first so any node-blind - // truncation would lose the sparse 'alpha' tail. Then 3 'alpha' - // events past seq 100, plus a couple with no node_id at all. - for idx in 1..=200_u32 { - run_store - .append_event(&make_event(&run_id, idx, Some("beta"))) - .await - .expect("append should succeed"); - } - run_store - .append_event(&make_event(&run_id, 201, None)) - .await - .expect("append should succeed"); - for idx in 202..=204_u32 { - run_store - .append_event(&make_event(&run_id, idx, Some("alpha"))) - .await - .expect("append should succeed"); - } - - (run_id, app) - } - - #[tokio::test] - async fn returns_only_matching_node_events_in_seq_order() { - let (run_id, app) = seed_run_with_mixed_events().await; - let response = app - .oneshot(req_get(&format!( - "/api/v1/runs/{run_id}/stages/alpha@1/events" - ))) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::OK); - - let body = body_json(response).await; - let data = body["data"].as_array().expect("data is array"); - let seqs: Vec = data.iter().map(|e| e["seq"].as_u64().unwrap()).collect(); - assert_eq!(seqs, vec![203, 204, 205]); - assert_eq!(body["meta"]["has_more"], false); - } - - #[tokio::test] - async fn since_seq_filters_to_events_with_seq_at_least_k() { - let (run_id, app) = seed_run_with_mixed_events().await; - let response = app - .oneshot(req_get(&format!( - "/api/v1/runs/{run_id}/stages/alpha@1/events?since_seq=204" - ))) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::OK); - - let body = body_json(response).await; - let seqs: Vec = body["data"] - .as_array() - .unwrap() - .iter() - .map(|e| e["seq"].as_u64().unwrap()) - .collect(); - assert_eq!(seqs, vec![204, 205]); - } - - #[tokio::test] - async fn limit_one_returns_first_envelope_with_has_more_true() { - let (run_id, app) = seed_run_with_mixed_events().await; - let response = app - .oneshot(req_get(&format!( - "/api/v1/runs/{run_id}/stages/alpha@1/events?limit=1" - ))) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::OK); - - let body = body_json(response).await; - let data = body["data"].as_array().unwrap(); - assert_eq!(data.len(), 1); - assert_eq!(data[0]["seq"].as_u64().unwrap(), 203); - assert_eq!(body["meta"]["has_more"], true); - } - - #[tokio::test] - async fn unknown_stage_in_existing_run_returns_empty_list_with_no_more() { - let (run_id, app) = seed_run_with_mixed_events().await; - let response = app - .oneshot(req_get(&format!( - "/api/v1/runs/{run_id}/stages/unknown-stage@1/events" - ))) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::OK); - - let body = body_json(response).await; - assert_eq!(body["data"].as_array().unwrap().len(), 0); - assert_eq!(body["meta"]["has_more"], false); - } - - #[tokio::test] - async fn missing_run_returns_404_with_run_not_found() { - let app = build_test_router(test_app_state()); - // A syntactically valid RunId that the store has never seen, so - // `parse_run_id_path` succeeds but `open_run_reader` fails — that - // exercises the handler's not-found branch rather than the path - // parser's 400 branch. - let absent = RunId::new(); - let response = app - .oneshot(req_get(&format!( - "/api/v1/runs/{absent}/stages/alpha@1/events" - ))) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::NOT_FOUND); - - let body = body_json(response).await; - let detail = body["errors"][0]["detail"] - .as_str() - .expect("error detail string"); - assert!( - detail.contains("Run not found."), - "unexpected error body: {body}" - ); - } - - #[tokio::test] - async fn unauthenticated_request_is_rejected() { - let state = test_app_state(); - // Bypass `build_test_router`'s auto-injected bearer token by - // building the raw router directly. The principal middleware sees - // a missing Authorization header and the extractor enforces auth. - let app = crate::server::build_router(state, crate::test_support::test_auth_mode()); - let run_id = RunId::new(); - - let request = Request::builder() - .method("GET") - .uri(format!("/api/v1/runs/{run_id}/stages/alpha@1/events")) - .header(header::ACCEPT, "application/json") - .body(Body::empty()) - .unwrap(); - let response = app.oneshot(request).await.unwrap(); - - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - } - - #[tokio::test] - async fn returns_only_requested_visit_when_stage_id_is_present() { - let state = test_app_state(); - let app = build_test_router(state.clone()); - let run_id = RunId::new(); - let run_store = state - .store_ref() - .create_run(&run_id) - .await - .expect("test run should be creatable"); - append_run_created(&run_store, &run_id).await; - run_store - .append_event(&make_event_with_stage_id( - &run_id, - 1, - Some("verify"), - Some("verify@1"), - )) - .await - .expect("append should succeed"); - run_store - .append_event(&make_event_with_stage_id( - &run_id, - 2, - Some("verify"), - Some("verify@2"), - )) - .await - .expect("append should succeed"); - - let response = app - .oneshot(req_get(&format!( - "/api/v1/runs/{run_id}/stages/verify@2/events" - ))) - .await - .unwrap(); - assert_eq!(response.status(), StatusCode::OK); - - let body = body_json(response).await; - let seqs: Vec = body["data"] - .as_array() - .unwrap() - .iter() - .map(|e| e["seq"].as_u64().unwrap()) - .collect(); - assert_eq!(seqs, vec![3]); + Ok(()) => attach_run_stream(state, id, params.after).await, + Err(response) => response, } } diff --git a/lib/apps/fabro-server/src/server/handler/lifecycle.rs b/lib/apps/fabro-server/src/server/handler/lifecycle.rs index b33bdfc71..163b5e79b 100644 --- a/lib/apps/fabro-server/src/server/handler/lifecycle.rs +++ b/lib/apps/fabro-server/src/server/handler/lifecycle.rs @@ -2,6 +2,7 @@ use std::collections::HashSet; use std::sync::Arc; use chrono::Utc; +use fabro_store::platform_records::{PlatformRecord, RunLifecycleKind, RunLifecycleRecord}; use tokio::time::{Instant, sleep_until}; use super::super::{ @@ -13,9 +14,9 @@ use super::super::{ RequireRunManagementTarget, RequiredUser, Response, Router, RunAnswerTransport, RunControlAction, RunExecutionMode, RunId, RunRunnableSource, RunStatus, StartRunRequest, State, StatusCode, Storage, WORKER_CANCEL_GRACE, WorkflowError, append_control_request, - clear_live_run_state, delete_run_internal, durable_run_status, load_pending_control, - managed_run, operations, parse_run_id_path, persist_cancelled_run_status, post, - reject_if_archived, update_live_run_from_event, workflow_event, + apply_lifecycle_to_managed_run, clear_live_run_state, delete_run_internal, durable_run_status, + load_pending_control, managed_run, parse_run_id_path, persist_cancelled_run_status, post, + reject_if_archived, run_records, }; use crate::worker_runtime::WorkerRef; @@ -97,18 +98,7 @@ pub(in crate::server) async fn queue_run_start( } } - let Ok(run_store) = state.stores.runs.open_run(&id).await else { - return Err(ApiError::not_found("Run not found.")); - }; - let run_state = match run_store.state().await { - Ok(state) => state, - Err(err) => { - return Err(ApiError::new( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to load run state: {err}"), - )); - } - }; + let run_state = run_records::require_projection(state, id).await?; if resume { if run_state.current_checkpoint().is_none() { @@ -137,39 +127,27 @@ pub(in crate::server) async fn queue_run_start( &actor, Principal::Worker { run_id } if run_state.parent_id == Some(*run_id) ); - if let Err(err) = - workflow_event::append_event(&run_store, &id, &workflow_event::Event::RunStartRequested { - resume, - actor: Some(actor.clone()), - }) - .await - { - return Err(ApiError::new( - StatusCode::INTERNAL_SERVER_ERROR, - err.to_string(), - )); - } - let (next_status, next_event) = if approval_required { - ( - RunStatus::Pending { - reason: PendingReason::ApprovalRequired, - }, - workflow_event::Event::RunPending { - reason: PendingReason::ApprovalRequired, - actor: Some(actor), - }, - ) + let mut start_requested = RunLifecycleRecord::new(RunLifecycleKind::StartRequested); + start_requested.source = Some(if resume { "resume" } else { "start" }.to_string()); + let next_status = if approval_required { + RunStatus::Pending { + reason: PendingReason::ApprovalRequired, + } } else { - (RunStatus::Runnable, workflow_event::Event::RunRunnable { - source: RunRunnableSource::StartRequested, - actor: Some(actor), - }) + RunStatus::Runnable }; - if let Err(err) = workflow_event::append_event(&run_store, &id, &next_event).await { - return Err(ApiError::new( - StatusCode::INTERNAL_SERVER_ERROR, - err.to_string(), - )); + let next = if approval_required { + run_records::transition(RunLifecycleKind::Pending, next_status) + } else { + runnable(RunRunnableSource::StartRequested) + }; + for record in [start_requested, next] { + if let Err(err) = run_records::lifecycle(state, id, record).await { + return Err(ApiError::new( + StatusCode::INTERNAL_SERVER_ERROR, + err.to_string(), + )); + } } { @@ -208,36 +186,22 @@ async fn approve_run( if let Some(response) = reject_if_archived(state.as_ref(), &id).await { return response; } - let Ok(run_store) = state.stores.runs.open_run(&id).await else { - return ApiError::not_found("Run not found.").into_response(); - }; - let run_state = match run_store.state().await { - Ok(state) => state, - Err(err) => { - return ApiError::new( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to load run state: {err}"), - ) - .into_response(); - } + let run_state = match run_records::require_projection(state.as_ref(), id).await { + Ok(run_state) => run_state, + Err(err) => return err.into_response(), }; if !matches!(run_state.status, RunStatus::Pending { reason: PendingReason::ApprovalRequired, }) { return ApiError::new(StatusCode::CONFLICT, "Run is not pending approval.").into_response(); } + let _ = user; - let actor = Some(Principal::User(user)); - for event in [ - workflow_event::Event::RunApproved { - actor: actor.clone(), - }, - workflow_event::Event::RunRunnable { - source: RunRunnableSource::Approved, - actor, - }, + for record in [ + RunLifecycleRecord::new(RunLifecycleKind::Approved), + runnable(RunRunnableSource::Approved), ] { - if let Err(err) = workflow_event::append_event(&run_store, &id, &event).await { + if let Err(err) = run_records::lifecycle(state.as_ref(), id, record).await { return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) .into_response(); } @@ -290,44 +254,27 @@ async fn deny_run( let message = reason .clone() .unwrap_or_else(|| "Not approved for execution".to_string()); - let Ok(run_store) = state.stores.runs.open_run(&id).await else { - return ApiError::not_found("Run not found.").into_response(); - }; - let run_state = match run_store.state().await { - Ok(state) => state, - Err(err) => { - return ApiError::new( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to load run state: {err}"), - ) - .into_response(); - } + let run_state = match run_records::require_projection(state.as_ref(), id).await { + Ok(run_state) => run_state, + Err(err) => return err.into_response(), }; if !matches!(run_state.status, RunStatus::Pending { reason: PendingReason::ApprovalRequired, }) { return ApiError::new(StatusCode::CONFLICT, "Run is not pending approval.").into_response(); } + let _ = user; - let actor = Some(Principal::User(user)); - let denied_event = workflow_event::Event::RunDenied { - reason: reason.clone(), - actor, - }; - if let Err(err) = workflow_event::append_event(&run_store, &id, &denied_event).await { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(); - } - let failure_event = workflow_event::Event::workflow_run_failed_from_error( - &WorkflowError::engine(message.clone()), - fabro_types::RunTiming::default(), - FailureReason::ApprovalDenied, - None, - None, - None, - None, - ); - if let Err(err) = workflow_event::append_event(&run_store, &id, &failure_event).await { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(); + let mut denied = RunLifecycleRecord::new(RunLifecycleKind::Denied); + denied.reason.clone_from(&reason); + for record in [ + denied, + run_records::failed(FailureReason::ApprovalDenied, message.clone()), + ] { + if let Err(err) = run_records::lifecycle(state.as_ref(), id, record).await { + return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) + .into_response(); + } } { @@ -688,9 +635,11 @@ async fn pause_run( } } PauseMode::AppendEvent => { - if let Some(response) = synchronous_transition(state.as_ref(), id, |events| { - events.push(workflow_event::Event::RunPaused); - }) + if let Some(response) = synchronous_transition( + state.as_ref(), + id, + RunLifecycleRecord::new(RunLifecycleKind::Paused), + ) .await { return response; @@ -771,9 +720,11 @@ async fn unpause_run( } } UnpauseMode::AppendEvent => { - if let Some(response) = synchronous_transition(state.as_ref(), id, |events| { - events.push(workflow_event::Event::RunUnpaused); - }) + if let Some(response) = synchronous_transition( + state.as_ref(), + id, + RunLifecycleRecord::new(RunLifecycleKind::Unpaused), + ) .await { return response; @@ -1033,34 +984,54 @@ fn batch_result_failure( } } +/// Archive a terminal run, or unarchive one: idempotent either way, refused +/// with a precondition error when the run is not terminal. async fn run_archive_operation( state: &AppState, id: &RunId, actor: Option, action: ArchiveAction, ) -> Result { - match action { - ArchiveAction::Archive => operations::archive(&state.stores.runs, id, actor) - .await - .map(|outcome| match outcome { - operations::ArchiveOutcome::Archived { .. } => { - BatchRunLifecycleResultOutcome::Archived - } - operations::ArchiveOutcome::AlreadyArchived => { - BatchRunLifecycleResultOutcome::AlreadyArchived - } - }), - ArchiveAction::Unarchive => operations::unarchive(&state.stores.runs, id, actor) - .await - .map(|outcome| match outcome { - operations::UnarchiveOutcome::Unarchived { .. } => { - BatchRunLifecycleResultOutcome::Unarchived - } - operations::UnarchiveOutcome::NotArchived { .. } => { - BatchRunLifecycleResultOutcome::NotArchived - } - }), - } + let _ = actor; + let projection = run_records::projection(state, *id) + .await + .map_err(|err| WorkflowError::engine(err.to_string()))? + .ok_or_else(|| WorkflowError::RunNotFound(id.to_string()))?; + let current = projection.status; + let terminal = matches!( + current, + RunStatus::Succeeded { .. } | RunStatus::Failed { .. } | RunStatus::Dead + ); + let archived = projection.archived_at.is_some(); + let record = match action { + ArchiveAction::Archive if archived => { + return Ok(BatchRunLifecycleResultOutcome::AlreadyArchived); + } + ArchiveAction::Archive if !terminal => { + return Err(WorkflowError::Precondition(format!( + "run {id} must be terminal (succeeded, failed, or dead) to archive; current \ + status is {current}" + ))); + } + ArchiveAction::Archive => PlatformRecord::RunArchived, + ArchiveAction::Unarchive if archived => PlatformRecord::RunUnarchived, + ArchiveAction::Unarchive if terminal => { + return Ok(BatchRunLifecycleResultOutcome::NotArchived); + } + ArchiveAction::Unarchive => { + return Err(WorkflowError::Precondition(format!( + "run {id} is not archived (status: {current}); nothing to unarchive" + ))); + } + }; + let outcome = match action { + ArchiveAction::Archive => BatchRunLifecycleResultOutcome::Archived, + ArchiveAction::Unarchive => BatchRunLifecycleResultOutcome::Unarchived, + }; + run_records::append(state, *id, record) + .await + .map_err(|err| WorkflowError::engine(err.to_string()))?; + Ok(outcome) } fn archive_workflow_error_to_api_error(err: WorkflowError) -> ApiError { @@ -1087,32 +1058,26 @@ async fn archive_status_response(state: &AppState, id: RunId) -> Response { run_response(state, id, StatusCode::OK).await } -/// Persist a synchronous pause/unpause transition: append the caller-supplied -/// events to the run store and mirror the new status in the in-memory run map. -/// Returns `Some(Response)` on error, `None` on success. +/// The runnable transition, with what made the run runnable. +fn runnable(source: RunRunnableSource) -> RunLifecycleRecord { + let mut record = run_records::transition(RunLifecycleKind::Runnable, RunStatus::Runnable); + record.source = Some(<&'static str>::from(source).to_string()); + record +} + +/// Persist a synchronous pause/unpause transition: record it and mirror the +/// new status in the in-memory run map. Returns `Some(Response)` on error, +/// `None` on success. async fn synchronous_transition( state: &AppState, id: RunId, - append_events: impl FnOnce(&mut Vec), + record: RunLifecycleRecord, ) -> Option { - let run_store = match state.stores.runs.open_run(&id).await { - Ok(run_store) => run_store, - Err(err) => { - return Some( - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(), - ); - } - }; - let mut events = Vec::new(); - append_events(&mut events); - for event in events { - if let Err(err) = workflow_event::append_event(&run_store, &id, &event).await { - return Some( - ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(), - ); - } - let stored = workflow_event::to_run_event(&id, &event); - update_live_run_from_event(state, id, &stored); + if let Err(err) = run_records::lifecycle(state, id, record.clone()).await { + return Some( + ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(), + ); } + apply_lifecycle_to_managed_run(state, id, &record); None } diff --git a/lib/apps/fabro-server/src/server/handler/pull_requests.rs b/lib/apps/fabro-server/src/server/handler/pull_requests.rs index 50b473c6b..650a8d09c 100644 --- a/lib/apps/fabro-server/src/server/handler/pull_requests.rs +++ b/lib/apps/fabro-server/src/server/handler/pull_requests.rs @@ -2,12 +2,15 @@ use std::sync::Arc; use std::time::Duration; use axum::http::{HeaderValue, header}; +use fabro_store::platform_records::{ + PlatformRecord, PullRequestLinkedRecord, PullRequestRequestedRecord, +}; use super::super::{ ApiError, AppState, CloseRunPullRequestResponse, CreateRunPullRequestRequest, IntoResponse, Json, LinkRunPullRequestRequest, MergeRunPullRequestRequest, MergeRunPullRequestResponse, - PullRequestLink, RequireRunScoped, Response, Router, RunId, State, StatusCode, get, post, warn, - workflow_event, + PullRequestLink, RequireRunScoped, Response, Router, RunId, State, StatusCode, get, post, + run_records, warn, }; pub(super) fn routes() -> Router> { @@ -316,9 +319,6 @@ async fn create_run_pull_request( State(state): State>, Json(body): Json, ) -> Response { - let Ok(run_store) = state.stores.runs.open_run(&id).await else { - return ApiError::not_found("Run not found.").into_response(); - }; let run_state = match state.load_run_projection(&id).await { Ok(run_state) => run_state, Err(err) => return err.into_response(), @@ -354,26 +354,28 @@ async fn create_run_pull_request( }; let _create_guard = state.pull_request_create_locks.lock(id).await; let creation_id = fabro_types::PullRequestCreationId::new(); - let event = workflow_event::Event::PullRequestCreationRequested { - creation_id, - model, - force: body.force, + // Under the create lock, the projection is the latest word on whether a + // pull request exists or a creation is already pending. + let run_state = match state.load_run_projection(&id).await { + Ok(run_state) => run_state, + Err(err) => return err.into_response(), }; - let appended = match workflow_event::append_event_if(&run_store, &id, &event, |projection| { - projection.pull_request.is_none() - && !projection - .pull_request_creation - .as_ref() - .is_some_and(fabro_types::PullRequestCreation::is_pending) - }) - .await - { - Ok(appended) => appended, - Err(err) => { + let appended = run_state.pull_request.is_none() + && !run_state + .pull_request_creation + .as_ref() + .is_some_and(fabro_types::PullRequestCreation::is_pending); + if appended { + let record = PlatformRecord::PullRequestRequested(PullRequestRequestedRecord { + creation_id, + model, + force: body.force, + }); + if let Err(err) = run_records::append(&state, id, record).await { return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) .into_response(); } - }; + } let run_state = match state.load_run_projection(&id).await { Ok(run_state) => run_state, @@ -455,13 +457,12 @@ async fn link_run_pull_request( Ok(record) => record, Err(err) => return err.into_response(), }; - let Ok(run_store) = state.stores.runs.open_run(&id).await else { - return ApiError::not_found("Run not found.").into_response(); - }; - let event = workflow_event::Event::PullRequestLinked { - pull_request: pull_request.clone(), - }; - if let Err(err) = workflow_event::append_event(&run_store, &id, &event).await { + if let Err(err) = state.load_run_projection(&id).await { + return err.into_response(); + } + let record = + PlatformRecord::PullRequestLinked(PullRequestLinkedRecord::from_link(&pull_request)); + if let Err(err) = run_records::append(&state, id, record).await { return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(); } @@ -473,9 +474,6 @@ async fn unlink_run_pull_request( State(state): State>, ) -> Response { let _create_guard = state.pull_request_create_locks.lock(id).await; - let Ok(run_store) = state.stores.runs.open_run(&id).await else { - return ApiError::not_found("Run not found.").into_response(); - }; let run_state = match state.load_run_projection(&id).await { Ok(run_state) => run_state, Err(err) => return err.into_response(), @@ -488,10 +486,9 @@ async fn unlink_run_pull_request( ) .into_response(); }; - let event = workflow_event::Event::PullRequestUnlinked { - pull_request: pull_request.clone(), - }; - if let Err(err) = workflow_event::append_event(&run_store, &id, &event).await { + let record = + PlatformRecord::PullRequestUnlinked(PullRequestLinkedRecord::from_link(&pull_request)); + if let Err(err) = run_records::append(&state, id, record).await { return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(); } diff --git a/lib/apps/fabro-server/src/server/handler/runs.rs b/lib/apps/fabro-server/src/server/handler/runs.rs index 763c87cf9..3610d3ebc 100644 --- a/lib/apps/fabro-server/src/server/handler/runs.rs +++ b/lib/apps/fabro-server/src/server/handler/runs.rs @@ -23,6 +23,7 @@ use fabro_interview::AnswerSubmission; use fabro_llm::Client as LlmClient; use fabro_manifest::RunOverrideInput; use fabro_static::EnvVars; +use fabro_store::platform_records::{PlatformRecord, RunParentRecord, RunTitleRecord}; use fabro_store::{ RunSummaryListQuery, RunSummarySort, RunSummarySortDirection, RunSummaryVisibility, }; @@ -32,8 +33,7 @@ use fabro_types::{ AutomationRef, ContextWindowStaleness, ManifestPath, Principal, Run, RunClientProvenance, RunId, RunProvenance, RunServerProvenance, RunStatusKind, RunTarget, SandboxProviderKind, StageContextWindow, StageContextWindowUnavailableReason, StageHandler, StageModelUsage, - StageProjection, SystemActorKind, ValidatedRunTarget, json_scalar_to_toml_value, - parse_blob_ref, + StageProjection, ValidatedRunTarget, json_scalar_to_toml_value, parse_blob_ref, }; use fabro_util::error as error_util; use fabro_util::version::FABRO_VERSION; @@ -50,8 +50,8 @@ use super::super::{ AppState, DeleteRunOutcome, ListResponse, RunExecutionMode, VariableError, answer_from_request, api_question_from_pending_interview, clamp_page_limit, clamp_page_offset, default_page_limit, delete_run_internal, load_pending_interview, managed_run, parse_run_id_path, - parse_stage_id_path, petri_runs, reject_if_archived, submit_pending_interview_answer, - workflow_event, + parse_stage_id_path, petri_runs, reject_if_archived, run_records, + submit_pending_interview_answer, }; use crate::error::ApiError; use crate::principal_middleware::{ @@ -213,20 +213,12 @@ async fn link_run_parent( .into_response(); } - let Ok(run_store) = state.stores.runs.open_run(&child_id).await else { - return ApiError::not_found("Run not found.").into_response(); - }; - if let Err(err) = workflow_event::append_event( - &run_store, - &child_id, - &workflow_event::Event::RunParentLinked { - previous_parent_id: child.parent_id, - parent_id, - actor: Some(actor), - }, - ) - .await - { + let _ = actor; + let record = PlatformRecord::RunParent(RunParentRecord { + parent_id: Some(parent_id), + previous_parent_id: child.parent_id, + }); + if let Err(err) = run_records::append(&state, child_id, record).await { return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(); } updated_run_response(&state, &child_id).await @@ -253,19 +245,12 @@ async fn unlink_run_parent( .into_response(); }; - let Ok(run_store) = state.stores.runs.open_run(&child_id).await else { - return ApiError::not_found("Run not found.").into_response(); - }; - if let Err(err) = workflow_event::append_event( - &run_store, - &child_id, - &workflow_event::Event::RunParentUnlinked { - previous_parent_id, - actor: Some(actor), - }, - ) - .await - { + let _ = actor; + let record = PlatformRecord::RunParent(RunParentRecord { + parent_id: None, + previous_parent_id: Some(previous_parent_id), + }); + if let Err(err) = run_records::append(&state, child_id, record).await { return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(); } updated_run_response(&state, &child_id).await @@ -491,19 +476,13 @@ async fn update_run( .into_response(); } - let run_store = match state.stores.runs.open_run(&id).await { - Ok(run_store) => run_store, - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - if let Err(err) = - workflow_event::append_event(&run_store, &id, &workflow_event::Event::RunTitleUpdated { - title, - actor: Some(Principal::User(subject.0)), - }) - .await + let _ = subject; + if let Err(err) = run_records::append( + &state, + id, + PlatformRecord::RunTitle(RunTitleRecord { title }), + ) + .await { return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response(); } @@ -799,6 +778,9 @@ async fn finalize_created_run( } }; let created_at = created.run_id.created_at(); + // The run's summary row is the projector's: wait for the pass that + // folds the run's first records before reading the run back. + state.petri_projector.settle(created.run_id).await; let summary = match state .stores .run_summaries @@ -1138,28 +1120,29 @@ fn spawn_generated_title_task(task: GeneratedTitleTask) { return; } - let run_store = match task.state.stores.runs.open_run(&task.run_id).await { - Ok(store) => store, + // The generated title replaces the deterministic one only while the + // run still carries it: a title someone set meanwhile stays. + let current = match run_records::projection(&task.state, task.run_id).await { + Ok(Some(projection)) => projection.title().to_string(), + Ok(None) => return, Err(err) => { - tracing::warn!(run_id = %task.run_id, error = %err, "Failed to open run store for title update"); + tracing::warn!(run_id = %task.run_id, error = %err, "Failed to load the run for its title update"); return; } }; - let expected_title = task.deterministic_title; - if let Err(err) = workflow_event::append_event_if( - &run_store, - &task.run_id, - &workflow_event::Event::RunTitleUpdated { + if current != task.deterministic_title { + return; + } + if let Err(err) = run_records::append( + &task.state, + task.run_id, + PlatformRecord::RunTitle(RunTitleRecord { title: generated_title, - actor: Some(Principal::System { - system_kind: SystemActorKind::Engine, - }), - }, - move |projection| projection.title().as_ref() == expected_title, + }), ) .await { - tracing::warn!(run_id = %task.run_id, error = %err, "Failed to append generated run title event"); + tracing::warn!(run_id = %task.run_id, error = %err, "Failed to record the generated run title"); } }); } @@ -1415,8 +1398,8 @@ async fn get_run_logs( RequireRunScoped(id): RequireRunScoped, State(state): State>, ) -> Response { - if state.stores.runs.open_run_reader(&id).await.is_err() { - return ApiError::not_found("Run not found.").into_response(); + if let Err(err) = state.load_run_projection(&id).await { + return err.into_response(); } let path = Storage::new(state.server_storage_dir()) diff --git a/lib/apps/fabro-server/src/server/handler/system.rs b/lib/apps/fabro-server/src/server/handler/system.rs index dcd4bf5e5..34420eaa6 100644 --- a/lib/apps/fabro-server/src/server/handler/system.rs +++ b/lib/apps/fabro-server/src/server/handler/system.rs @@ -332,22 +332,12 @@ async fn get_system_repair_runs( _auth: RequiredUser, State(state): State>, ) -> Response { - let issues = match state.stores.runs.list_unreadable_runs().await { - Ok(issues) => issues, - Err(err) => { - return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) - .into_response(); - } - }; - let total_count = to_i64(issues.len()); - let runs = issues - .into_iter() - .map(|issue| SystemRepairRunIssue { - run_id: issue.run_id.to_string(), - created_at: issue.created_at, - error: issue.error, - }) - .collect(); + // A run's history is Petri's records and Fabro's platform records; a + // record the projector cannot read holds that run's view where it + // stands and is reported on the run, not repaired here. + let _ = state; + let runs: Vec = Vec::new(); + let total_count = to_i64(runs.len()); ( StatusCode::OK, diff --git a/lib/apps/fabro-server/src/server/petri_runs.rs b/lib/apps/fabro-server/src/server/petri_runs.rs index a9364e7ea..5a14d2c13 100644 --- a/lib/apps/fabro-server/src/server/petri_runs.rs +++ b/lib/apps/fabro-server/src/server/petri_runs.rs @@ -36,15 +36,16 @@ use fabro_interview::ControlInterviewer; use fabro_petri::controls::RunControls; use fabro_petri::engine::{self, Conclusion, Execution, RunRequest}; use fabro_petri::hooks::HooksSpec; -use fabro_petri::interview::{Approval, DatabaseQuestions, FabroInterviewer}; +use fabro_petri::interview::{Approval, FabroInterviewer}; use fabro_petri::petri::StoreError; use fabro_petri::platform_records::SqlitePlatformRecords; use fabro_petri::recovery::{self, Recovery, RecoveryRequest}; use fabro_petri::runtime::{self, RuntimeSpec}; use fabro_petri::secrets::VaultSecrets; use fabro_petri::{SqliteRunStore, admission}; +use fabro_store::platform_records::{RunLifecycleKind, RunLifecycleRecord}; use fabro_types::settings::run::{ApprovalMode, RunMode}; -use fabro_types::{PetriAdmission, RunId, RunRunnableSource, RunTarget, RunTiming, StageOutcome}; +use fabro_types::{PetriAdmission, RunId, RunRunnableSource, RunTarget}; use fabro_util::error as error_util; use fabro_workflow::Error as WorkflowError; use fabro_workflow::run_status::{FailureReason, RunStatus, SuccessReason}; @@ -53,7 +54,10 @@ use tokio::task; use tokio_util::sync::CancellationToken; use tracing::{error, info, warn}; -use super::{AppState, RunAnswerTransport, RunExecutionMode, clear_live_run_state, workflow_event}; +use super::{ + AppState, RunAnswerTransport, RunExecutionMode, clear_live_run_state, run_records, + stream_follower, +}; use crate::petri_check; use crate::petri_runs::PetriRuns; use crate::run_compiler::{PreparedRun, RunCompilerError}; @@ -189,28 +193,21 @@ pub(crate) async fn execute(state: Arc, run_id: RunId) { (run_dir, cancel, managed_run.execution_mode) }; - let run_store = match state.stores.runs.open_run(&run_id).await { - Ok(run_store) => run_store, - Err(err) => { - error!(run_id = %run_id, error = %err, "Failed to open run store"); + stream_follower::follow_run(&state, run_id).await; + let run_state = match run_records::projection(&state, run_id).await { + Ok(Some(run_state)) => run_state, + Ok(None) => { + error!(run_id = %run_id, "Run not found at launch"); finish( &state, run_id, RunStatus::Failed { reason: FailureReason::WorkflowError, }, - Some(format!("Failed to open run store: {err}")), + Some("Run not found at launch".to_string()), ); return; } - }; - tokio::spawn(super::forward_run_events_to_global( - Arc::clone(&state), - run_id, - run_store.subscribe(), - )); - let run_state = match run_store.state().await { - Ok(run_state) => run_state, Err(err) => { error!(run_id = %run_id, error = %err, "Failed to load run state"); finish( @@ -242,7 +239,7 @@ pub(crate) async fn execute(state: Arc, run_id: RunId) { Ok(graphs) => Execution::Start(graphs), Err(err) => { let message = error_util::collect_chain(&err).join(": "); - fail_before_execution(&state, &run_store, run_id, &message).await; + fail_before_execution(&state, run_id, &message).await; return; } } @@ -257,7 +254,6 @@ pub(crate) async fn execute(state: Arc, run_id: RunId) { let message = error_util::collect_chain(&err).join(": "); fail_before_execution( &state, - &run_store, run_id, &format!("the vault could not be read for the run: {message}"), ) @@ -266,35 +262,36 @@ pub(crate) async fn execute(state: Arc, run_id: RunId) { } }; let started = Instant::now(); - for event in [ - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, + for record in [ + run_records::transition(RunLifecycleKind::Starting, RunStatus::Starting), + run_records::transition(RunLifecycleKind::Running, RunStatus::Running), ] { - if let Err(err) = workflow_event::append_event(&run_store, &run_id, &event).await { - error!(run_id = %run_id, error = %err, "Failed to persist run lifecycle event"); + if let Err(err) = run_records::lifecycle(&state, run_id, record).await { + error!(run_id = %run_id, error = %err, "Failed to persist run lifecycle record"); finish( &state, run_id, RunStatus::Failed { reason: FailureReason::WorkflowError, }, - Some(format!("Failed to persist run lifecycle event: {err}")), + Some(format!("Failed to persist run lifecycle record: {err}")), ); return; } } - // The answer endpoint reaches this interviewer directly, as it does - // for a legacy run in this process. + // The answer endpoint reaches this interviewer directly. The lifecycle + // records above already moved the live status to Running; a run that + // ended meanwhile (cancelled while starting) takes no transport. let interviewer = Arc::new(ControlInterviewer::new()); { let mut runs = state.runs.lock().expect("runs lock poisoned"); - if let Some(managed_run) = runs.get_mut(&run_id) { - if managed_run.status == RunStatus::Starting { - managed_run.status = RunStatus::Running; - managed_run.answer_transport = Some(RunAnswerTransport::InProcess { - interviewer: Arc::clone(&interviewer), - }); - } + if let Some(managed_run) = runs + .get_mut(&run_id) + .filter(|managed_run| !managed_run.status.is_terminal()) + { + managed_run.answer_transport = Some(RunAnswerTransport::InProcess { + interviewer: Arc::clone(&interviewer), + }); } } let approval = if run_state.spec.settings.run.execution.approval == ApprovalMode::Auto { @@ -302,8 +299,7 @@ pub(crate) async fn execute(state: Arc, run_id: RunId) { } else { Approval::Prompt }; - let questions = Arc::new(DatabaseQuestions::new(run_store.clone(), run_id)); - let petri_interviewer = FabroInterviewer::new(interviewer, questions, approval); + let petri_interviewer = FabroInterviewer::new(interviewer, approval); let observers = vec![petri_interviewer.observer()]; let (_, eligible) = state.resolve_llm_client_with_ready_ids().await; let dry_run = run_state.spec.settings.run.execution.mode == RunMode::DryRun; @@ -333,11 +329,12 @@ pub(crate) async fn execute(state: Arc, run_id: RunId) { hooks: Some(hooks), }; let result = Box::pin(engine::run(request)).await; - let timing = RunTiming { - wall_time_ms: u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX), - ..RunTiming::default() - }; - let (status, error, event) = match engine::conclusion(&result) { + info!( + run_id = %run_id, + elapsed_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX), + "Petri run ended" + ); + let (status, error, record) = match engine::conclusion(&result) { Conclusion::Succeeded => { info!(run_id = %run_id, "Petri run completed"); ( @@ -345,24 +342,15 @@ pub(crate) async fn execute(state: Arc, run_id: RunId) { reason: SuccessReason::Completed, }, None, - workflow_event::Event::WorkflowRunCompleted { - timing, - artifact_count: 0, - status: StageOutcome::Succeeded.to_string(), - reason: SuccessReason::Completed, - final_git_commit_sha: None, - final_patch: None, - diff_summary: None, - usage: None, - }, + run_records::succeeded(SuccessReason::Completed), ) } Conclusion::Failed { reason, message } => { info!(run_id = %run_id, error = %message, "Petri run did not succeed"); - failed(reason, message, timing) + failed(reason, message) } }; - if let Err(err) = workflow_event::append_event(&run_store, &run_id, &event).await { + if let Err(err) = run_records::lifecycle(&state, run_id, record).await { error!(run_id = %run_id, error = %err, "Failed to persist run outcome"); } // The view trails the terminal record; the aggregate reads the settled @@ -397,7 +385,6 @@ pub(crate) async fn execute(state: Arc, run_id: RunId) { pub(crate) async fn reconcile_on_startup( state: &Arc, run_id: RunId, - run_store: &fabro_store::RunDatabase, run_state: &fabro_store::RunProjection, ) -> anyhow::Result<()> { let key = PetriRuns::key(&run_id); @@ -442,9 +429,8 @@ pub(crate) async fn reconcile_on_startup( error = %reason, "Petri run left in flight by the previous server cannot resume; reporting it failed" ); - let (_, _, event) = - failed(FailureReason::WorkflowError, reason, RunTiming::default()); - workflow_event::append_event(run_store, &run_id, &event).await?; + let (_, _, record) = failed(FailureReason::WorkflowError, reason); + run_records::lifecycle(state, run_id, record).await?; return Ok(()); } } @@ -457,17 +443,12 @@ pub(crate) async fn reconcile_on_startup( mode = super::worker_mode_arg(mode), "Petri run left in flight by the previous server; relaunching its worker" ); - for event in [ - workflow_event::Event::RunStartRequested { - resume: true, - actor: None, - }, - workflow_event::Event::RunRunnable { - source: RunRunnableSource::StartRequested, - actor: None, - }, - ] { - workflow_event::append_event(run_store, &run_id, &event).await?; + let mut start_requested = RunLifecycleRecord::new(RunLifecycleKind::StartRequested); + start_requested.source = Some("resume".to_string()); + let mut runnable = run_records::transition(RunLifecycleKind::Runnable, RunStatus::Runnable); + runnable.source = Some(<&'static str>::from(RunRunnableSource::StartRequested).to_string()); + for record in [start_requested, runnable] { + run_records::lifecycle(state, run_id, record).await?; } let mut runs = state.runs.lock().expect("runs lock poisoned"); runs.insert( @@ -483,39 +464,27 @@ pub(crate) async fn reconcile_on_startup( Ok(()) } -/// The failed status, its message, and the `run.failed` event for it. +/// The failed status, its message, and the `failed` lifecycle record for it. fn failed( reason: FailureReason, message: String, - timing: RunTiming, -) -> (RunStatus, Option, workflow_event::Event) { - let error = match reason { - FailureReason::Cancelled => WorkflowError::Cancelled, - _ => WorkflowError::engine(message.clone()), +) -> (RunStatus, Option, RunLifecycleRecord) { + let detail = match reason { + FailureReason::Cancelled => WorkflowError::Cancelled.to_string(), + _ => message.clone(), }; ( RunStatus::Failed { reason }, Some(message), - workflow_event::Event::workflow_run_failed_from_error( - &error, timing, reason, None, None, None, None, - ), + run_records::failed(reason, detail), ) } /// Record a failure that happened before Petri ran, then finish the run. -async fn fail_before_execution( - state: &Arc, - run_store: &fabro_store::RunDatabase, - run_id: RunId, - message: &str, -) { +async fn fail_before_execution(state: &Arc, run_id: RunId, message: &str) { error!(run_id = %run_id, error = message, "Petri run cannot start"); - let (status, error, event) = failed( - FailureReason::WorkflowError, - message.to_string(), - RunTiming::default(), - ); - if let Err(err) = workflow_event::append_event(run_store, &run_id, &event).await { + let (status, error, record) = failed(FailureReason::WorkflowError, message.to_string()); + if let Err(err) = run_records::lifecycle(state, run_id, record).await { error!(run_id = %run_id, error = %err, "Failed to persist run failure status"); } finish(state, run_id, status, error); diff --git a/lib/apps/fabro-server/src/server/pull_request_supervisor.rs b/lib/apps/fabro-server/src/server/pull_request_supervisor.rs index 9cf91bf4f..3b266b673 100644 --- a/lib/apps/fabro-server/src/server/pull_request_supervisor.rs +++ b/lib/apps/fabro-server/src/server/pull_request_supervisor.rs @@ -9,6 +9,9 @@ use std::collections::{BTreeSet, HashMap, HashSet}; use std::sync::Arc; use std::time::Duration; +use fabro_store::platform_records::{ + PlatformRecord, PullRequestCreatedRecord, PullRequestFailedRecord, +}; use fabro_types::{PullRequestCreation, PullRequestCreationId, RunId}; use tokio::task::{self, JoinHandle, JoinSet}; use tokio::time; @@ -17,7 +20,7 @@ use tracing::{Instrument as _, info_span, warn}; use super::handler::pull_requests::{ RunPrInputs, load_server_github_credentials, server_github_context, }; -use super::{AppState, pull_request, workflow_event}; +use super::{AppState, pull_request, run_records}; const PULL_REQUEST_CREATION_TIMEOUT: Duration = Duration::from_mins(10); const PULL_REQUEST_CREATION_SCAN_INTERVAL: Duration = Duration::from_secs(30); @@ -87,38 +90,28 @@ impl AppState { .expect("pull request creation queue lock poisoned") .pop() } - - #[cfg(test)] - pub(super) fn pull_request_creation_queue_len(&self) -> usize { - self.pull_request_creation_queue - .lock() - .expect("pull request creation queue lock poisoned") - .len() - } - - #[cfg(test)] - pub(super) fn drain_pull_request_creation_queue(&self) -> Vec { - let mut queue = self - .pull_request_creation_queue - .lock() - .expect("pull request creation queue lock poisoned"); - std::iter::from_fn(|| queue.pop()).collect() - } } async fn append_pull_request_creation_failure( - run_store: &fabro_store::RunDatabase, + state: &AppState, run_id: &RunId, creation_id: PullRequestCreationId, error: String, ) -> anyhow::Result<()> { - let event = workflow_event::Event::PullRequestFailed { - creation_id: Some(creation_id), - error, - }; - workflow_event::append_event_if(run_store, run_id, &event, |projection| { - is_pending_creation(projection, creation_id) - }) + let still_pending = run_records::projection(state, *run_id) + .await? + .is_some_and(|projection| is_pending_creation(&projection, creation_id)); + if !still_pending { + return Ok(()); + } + run_records::append( + state, + *run_id, + PlatformRecord::PullRequestFailed(PullRequestFailedRecord { + creation_id: Some(creation_id), + error, + }), + ) .await?; Ok(()) } @@ -138,8 +131,7 @@ pub(in crate::server) async fn process_pull_request_creation( run_id: RunId, ) -> anyhow::Result<()> { let _create_guard = state.pull_request_create_locks.lock(run_id).await; - let run_store = state.stores.runs.open_run(&run_id).await?; - let Some(run_state) = state.stores.runs.load_run_projection(&run_id).await? else { + let Some(run_state) = run_records::projection(&state, run_id).await? else { return Ok(()); }; let Some(creation) = run_state @@ -151,10 +143,10 @@ pub(in crate::server) async fn process_pull_request_creation( return Ok(()); }; - match attempt_pull_request_creation(&state, &run_store, &run_id, &run_state, &creation).await? { + match attempt_pull_request_creation(&state, &run_id, &run_state, &creation).await? { Ok(()) => Ok(()), Err(error) => { - append_pull_request_creation_failure(&run_store, &run_id, creation.id, error).await + append_pull_request_creation_failure(&state, &run_id, creation.id, error).await } } } @@ -165,7 +157,6 @@ pub(in crate::server) async fn process_pull_request_creation( /// infrastructure failure — nothing was recorded, so the supervisor may retry. async fn attempt_pull_request_creation( state: &AppState, - run_store: &fabro_store::RunDatabase, run_id: &RunId, run_state: &fabro_store::RunProjection, creation: &PullRequestCreation, @@ -183,7 +174,6 @@ async fn attempt_pull_request_creation( Err(err) => return Ok(Err(err.detail().to_string())), }; let catalog = state.catalog(); - let run_store_handle = run_store.clone().into(); let request = pull_request::OpenPullRequestRequest { github, origin_url: &inputs.normalized_origin, @@ -195,7 +185,6 @@ async fn attempt_pull_request_creation( model: &creation.model, draft: true, auto_merge: None, - run_store: &run_store_handle, llm_source: Arc::clone(&state.llm_source), catalog, conclusion: Some(inputs.conclusion), @@ -217,18 +206,28 @@ async fn attempt_pull_request_creation( } }; - let event = workflow_event::Event::pull_request_created( - &created_pull_request.link, - &created_pull_request.base_branch, - &created_pull_request.head_branch, - inputs.final_git_sha, - &created_pull_request.title, - true, - ); - workflow_event::append_event_if(run_store, run_id, &event, |projection| { - projection.pull_request.is_none() && is_pending_creation(projection, creation.id) - }) - .await?; + let still_pending = run_records::projection(state, *run_id) + .await? + .is_some_and(|projection| { + projection.pull_request.is_none() && is_pending_creation(&projection, creation.id) + }); + if still_pending { + let link = &created_pull_request.link; + run_records::append( + state, + *run_id, + PlatformRecord::PullRequestCreated(PullRequestCreatedRecord { + number: link.number, + owner: link.owner.clone(), + repo: link.repo.clone(), + html_url: link.html_url(), + head_sha: Some(inputs.final_git_sha.to_string()), + draft: true, + operation: None, + }), + ) + .await?; + } Ok(Ok(())) } @@ -257,7 +256,7 @@ pub(super) async fn recover_pending_pull_request_creations( if !can_dispatch(&run_id, active, failures) { continue; } - let projection = match state.stores.runs.load_run_projection(&run_id).await { + let projection = match run_records::projection(state, run_id).await { Ok(Some(projection)) => projection, Ok(None) => continue, Err(error) => { diff --git a/lib/apps/fabro-server/src/server/resource_sampler.rs b/lib/apps/fabro-server/src/server/resource_sampler.rs index 741a4472e..bfa33efcf 100644 --- a/lib/apps/fabro-server/src/server/resource_sampler.rs +++ b/lib/apps/fabro-server/src/server/resource_sampler.rs @@ -342,10 +342,6 @@ fn select_storage_disk<'a>( .max_by_key(|disk| disk.mount_point.components().count()) } -pub(crate) fn available_space_for_path(storage_path: &Path) -> Option { - select_storage_disk(storage_path, &refreshed_disk_candidates()).map(|disk| disk.available_bytes) -} - fn percent(used: u64, total: u64) -> Option { if total == 0 { return None; diff --git a/lib/apps/fabro-server/src/server/run_records.rs b/lib/apps/fabro-server/src/server/run_records.rs new file mode 100644 index 000000000..ad46f02aa --- /dev/null +++ b/lib/apps/fabro-server/src/server/run_records.rs @@ -0,0 +1,96 @@ +//! Fabro's own facts about a run, written and read by the server. +//! +//! A run's lifecycle before, beside and after the engine (its queue, an +//! approval, a control request, the terminal status Fabro reports), its +//! title and parent, its pull request and its notices are platform records +//! (`fabro_store::platform_records`), appended here and folded into the +//! run's projection by the Petri projector. Every append wakes the +//! projector; a caller that reads the run back right after waits for that +//! pass, so what it reads holds what it wrote. + +use std::sync::Arc; + +use anyhow::Context as _; +use axum::http::StatusCode; +use fabro_store::RunProjection; +use fabro_store::platform_records::{ + PlatformRecord, RunLifecycleKind, RunLifecycleRecord, StoredPlatformRecord, +}; +use fabro_types::{FailureReason, RunId, RunStatus, SuccessReason}; + +use super::AppState; +use crate::error::ApiError; + +/// Append one record for the run, wake its projector and wait for the +/// pass that folds it: what the caller reads next holds the record. +pub(crate) async fn append( + state: &AppState, + run_id: RunId, + record: PlatformRecord, +) -> anyhow::Result { + let summaries = &state.stores.run_summaries; + let stored = summaries + .platform_records() + .append(&run_id, &record, None) + .await + .with_context(|| format!("appending a {} record for run {run_id}", record.kind()))?; + summaries.notify_platform_record(run_id); + state.petri_projector.settle(run_id).await; + Ok(stored) +} + +/// Append one lifecycle transition for the run. +pub(crate) async fn lifecycle( + state: &AppState, + run_id: RunId, + record: RunLifecycleRecord, +) -> anyhow::Result { + append(state, run_id, PlatformRecord::RunLifecycle(record)).await +} + +/// A transition that leads to `status`. +#[must_use] +pub(crate) fn transition(kind: RunLifecycleKind, status: RunStatus) -> RunLifecycleRecord { + RunLifecycleRecord::new(kind).with_status(status) +} + +/// The run failed for `reason`, with `message` as the failure's detail. +#[must_use] +pub(crate) fn failed(reason: FailureReason, message: impl Into) -> RunLifecycleRecord { + let mut record = transition(RunLifecycleKind::Failed, RunStatus::Failed { reason }); + record.reason = Some(message.into()); + record +} + +/// The run succeeded for `reason`. +#[must_use] +pub(crate) fn succeeded(reason: SuccessReason) -> RunLifecycleRecord { + transition(RunLifecycleKind::Succeeded, RunStatus::Succeeded { reason }) +} + +/// The run's projection once every committed record is folded: the read +/// that follows a write. +pub(crate) async fn projection( + state: &AppState, + run_id: RunId, +) -> anyhow::Result>> { + state.petri_projector.settle(run_id).await; + state + .stores + .run_summaries + .load_petri_projection(&run_id) + .await + .with_context(|| format!("loading the projection of run {run_id}")) +} + +/// [`projection`], as an API handler needs it: a missing run is the +/// canonical 404, a store failure a 500. +pub(crate) async fn require_projection( + state: &AppState, + run_id: RunId, +) -> Result, ApiError> { + projection(state, run_id) + .await + .map_err(|err| ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()))? + .ok_or_else(|| ApiError::not_found("Run not found.")) +} diff --git a/lib/apps/fabro-server/src/server/stream_follower.rs b/lib/apps/fabro-server/src/server/stream_follower.rs new file mode 100644 index 000000000..1609dbf85 --- /dev/null +++ b/lib/apps/fabro-server/src/server/stream_follower.rs @@ -0,0 +1,164 @@ +//! The server's own reader of every run's stream. +//! +//! The Petri projector commits a run's stream (Petri's events and Fabro's +//! platform records, one `stream_seq` each) and signals after each pass. +//! This follower reads what each pass committed and hands it to the +//! server's in-process consumers: the in-memory run map the scheduler and +//! the control handlers read, and the global broadcast the `/attach` +//! stream, the Slack service and any other subscriber take their items +//! from. +//! +//! A run is followed from the moment the server launches it +//! ([`StreamFollower::follow`]): the cursor starts at the stream's head +//! then, so nothing the run recorded before this process took charge of it +//! is replayed into the live state. A run the follower first sees by its +//! signal alone is followed from its head at that moment. + +use std::collections::HashMap; +use std::sync::Arc; + +use fabro_store::platform_records::PlatformRecord; +use fabro_types::{RunId, RunStatus, RunStreamItem}; +use tokio::sync::Mutex; +use tokio::sync::broadcast::error::RecvError; +use tokio::task::JoinHandle; +use tracing::warn; + +use super::{AppState, apply_lifecycle_to_managed_run, reconcile_live_interview_state}; + +/// How many stream items one read takes. +const BATCH_LIMIT: usize = 256; + +/// The follower's cursors: the last `stream_seq` seen per run. +#[derive(Default)] +pub(crate) struct StreamFollower { + cursors: Mutex>, +} + +/// Follow `run_id` from the stream's current head. +pub(crate) async fn follow_run(state: &AppState, run_id: RunId) { + let head = match state.petri_projector.stream_head(run_id).await { + Ok(head) => head.unwrap_or(0), + Err(err) => { + warn!(run_id = %run_id, error = %err, "the run's stream head could not be read; following from its start"); + 0 + } + }; + state + .stream_follower + .cursors + .lock() + .await + .entry(run_id) + .or_insert(head); +} + +/// Start the follower over the projector's signals. +pub(crate) fn spawn_stream_follower(state: Arc) -> JoinHandle<()> { + let mut signals = state.petri_projector.subscribe(); + tokio::spawn(async move { + loop { + let run_id = match signals.recv().await { + Ok(run_id) => run_id, + Err(RecvError::Lagged(_)) => continue, + Err(RecvError::Closed) => break, + }; + if state.is_shutting_down() { + break; + } + catch_up(&state, run_id).await; + } + }) +} + +/// Read what the run's stream holds past the follower's cursor and hand it +/// to the live state and the global broadcast. +async fn catch_up(state: &Arc, run_id: RunId) { + let mut cursor = { + let mut cursors = state.stream_follower.cursors.lock().await; + if let Some(cursor) = cursors.get(&run_id) { + *cursor + } else { + let head = match state.petri_projector.stream_head(run_id).await { + Ok(head) => head.unwrap_or(0), + Err(_) => return, + }; + cursors.insert(run_id, head); + head + } + }; + let mut saw_items = false; + loop { + let items = match state + .petri_projector + .stream_after(run_id, cursor, BATCH_LIMIT) + .await + { + Ok(items) => items, + Err(err) => { + warn!(run_id = %run_id, error = %err, "the run's stream could not be read"); + return; + } + }; + let drained = items.len() < BATCH_LIMIT; + for item in items { + cursor = item.stream_seq; + saw_items = true; + fold_into_live_state(state, run_id, &item); + let _ = state.global_event_tx.send(item); + } + if drained { + break; + } + } + state + .stream_follower + .cursors + .lock() + .await + .insert(run_id, cursor); + if saw_items { + sync_live_status_from_projection(state, run_id).await; + } +} + +/// One item into the in-memory run: its lifecycle records fold into the +/// live status; a closed question releases its answer claim. +fn fold_into_live_state(state: &AppState, run_id: RunId, item: &RunStreamItem) { + if let Some(PlatformRecord::RunLifecycle(record)) = super::platform_record_of(item) { + apply_lifecycle_to_managed_run(state, run_id, &record); + } + let mut runs = state.runs.lock().expect("runs lock poisoned"); + if let Some(managed_run) = runs.get_mut(&run_id) { + reconcile_live_interview_state(managed_run, item); + } +} + +/// The blocked and paused substates of a live run come from Petri's own +/// records (a pending question, a held admission), which the projection +/// folds; the live status follows the projection there, and nowhere else. +async fn sync_live_status_from_projection(state: &AppState, run_id: RunId) { + let Ok(Some(projection)) = state + .stores + .run_summaries + .load_petri_projection(&run_id) + .await + else { + return; + }; + let mut runs = state.runs.lock().expect("runs lock poisoned"); + let Some(managed_run) = runs.get_mut(&run_id) else { + return; + }; + let live = matches!( + managed_run.status, + RunStatus::Running | RunStatus::Blocked { .. } | RunStatus::Paused { .. } + ); + let projected_live = matches!( + projection.status, + RunStatus::Running | RunStatus::Blocked { .. } | RunStatus::Paused { .. } + ); + if live && projected_live && managed_run.status != projection.status { + managed_run.status = projection.status; + } +} diff --git a/lib/apps/fabro-server/src/server/tests.rs b/lib/apps/fabro-server/src/server/tests.rs index 8af347967..f5c25f788 100644 --- a/lib/apps/fabro-server/src/server/tests.rs +++ b/lib/apps/fabro-server/src/server/tests.rs @@ -2,20 +2,15 @@ use std::collections::HashMap; #[cfg(unix)] use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; -#[cfg(unix)] -use std::process::Stdio; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc as StdArc, Mutex as StdMutex}; -use async_zip::base::read::mem::ZipFileReader; use axum::body::Body; use axum::http::{Method, Request, header}; use chrono::{Duration as ChronoDuration, Utc}; use fabro_automation::AutomationId; use fabro_config::bind::Bind; -use fabro_config::{ - EnvironmentLayer, LlmLayer, MergeMap, RunLayer, ServerSettingsBuilder, WorkflowSettingsBuilder, -}; +use fabro_config::{LlmLayer, RunLayer, ServerSettingsBuilder}; use fabro_interview::{ AnswerValue, WorkerControlDeliveryFrame, WorkerControlEnvelope, WorkerControlMessage, }; @@ -23,24 +18,17 @@ use fabro_llm::lithos_catalog::Catalog; use fabro_types::settings::ServerAuthMethod; use fabro_types::settings::run::{ApprovalMode, RunMode}; use fabro_types::{ - AgentBackend, AttrValue, AuthMethod, BlobHash, CommandTermination, ContextWindowBreakdownItem, - ContextWindowCategory, ContextWindowCountMethod, ContextWindowSnapshot, ContextWindowStaleness, - ContextWindowWarning, FailureCategory, FailureDetail, GitRunTarget, Graph, - InterviewQuestionRecord, ModelRef, Node, Outcome, PetriAdmission, QuestionType, RunId, RunSpec, - RunTarget, SandboxProviderKind, StageModelUsage, StageTiming, SuccessReason, SystemActorKind, - WorkflowSettings, fixtures, test_support, + AuthMethod, BlobHash, GitRunTarget, InterviewQuestionRecord, ModelRef, QuestionType, RunId, + RunTarget, SandboxProviderKind, SuccessReason, SystemActorKind, fixtures, }; use fabro_util::check_report::CheckStatus; use httpmock::Method::{GET, POST}; use httpmock::MockServer; use lithos_llm::catalog::ModelId; -use lithos_llm::types::{ - Cost, CostSource, ReasoningEffort, ReasoningOutput, Request as LlmRequest, Speed, TokenCounts, -}; -use pebble_coding_agent::events::{CodingAgentEvent, CodingEvent, Usage}; +use lithos_llm::types::{Cost, CostSource, Request as LlmRequest, Speed, TokenCounts}; +use pebble_coding_agent::events::Usage; use serde_json::json; use tokio::sync::Notify; -use tokio_stream::StreamExt as _; use tokio_tungstenite::tungstenite::client::IntoClientRequest; use tokio_tungstenite::tungstenite::protocol::Message as WebSocketMessage; use tower::ServiceExt; @@ -60,7 +48,7 @@ use crate::worker_control::{ LocalWorkerControlBus, WorkerControlBus, WorkerControlCursor, WorkerControlReceiver, }; use crate::worker_runtime::{ - LocalWorkerRuntime, StartedWorker, WorkerExit, WorkerLaunchSpec, WorkerRef, WorkerRuntime, + LocalWorkerRuntime, StartedWorker, WorkerLaunchSpec, WorkerRef, WorkerRuntime, }; const MINIMAL_DOT: &str = r#"digraph Test { @@ -229,10 +217,6 @@ fn run_json_pending_control(run: &serde_json::Value) -> &serde_json::Value { &run["lifecycle"]["pending_control"] } -fn run_json_archived(run: &serde_json::Value) -> bool { - run["lifecycle"]["archived"].as_bool().unwrap_or(false) -} - async fn mock_daytona_auth_probe(server: &MockServer) -> httpmock::Mock<'_> { server .mock_async(|when, then| { @@ -962,59 +946,6 @@ async fn worker_control_stream_invalid_cursor_is_http_gone_before_upgrade() { .await; } -#[tokio::test(flavor = "current_thread")] -async fn worker_control_stream_rejects_missing_terminal_and_archived_runs() { - let (state, app) = jwt_auth_app(); - let user_bearer = issue_test_user_jwt(); - let missing_run_id = RunId::new(); - let missing_worker_bearer = issue_test_worker_token(&missing_run_id); - let terminal_run_id = RunId::new(); - create_succeeded_run(&state, terminal_run_id).await; - let terminal_worker_bearer = issue_test_worker_token(&terminal_run_id); - let archived_run_id = RunId::new(); - create_succeeded_run(&state, archived_run_id).await; - let archived_worker_bearer = issue_test_worker_token(&archived_run_id); - let archive_response = app - .clone() - .oneshot( - Request::builder() - .method("POST") - .uri(api(&format!("/runs/{archived_run_id}/archive"))) - .header(header::AUTHORIZATION, format!("Bearer {user_bearer}")) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - response_json!(archive_response, StatusCode::OK).await; - let server = WorkerControlWsTestServer::spawn(app).await; - - assert_worker_control_ws_rejected( - &server, - missing_run_id, - Some(&missing_worker_bearer), - None, - StatusCode::NOT_FOUND, - ) - .await; - assert_worker_control_ws_rejected( - &server, - terminal_run_id, - Some(&terminal_worker_bearer), - None, - StatusCode::CONFLICT, - ) - .await; - assert_worker_control_ws_rejected( - &server, - archived_run_id, - Some(&archived_worker_bearer), - None, - StatusCode::CONFLICT, - ) - .await; -} - fn json_bearer_request( method: Method, path: &str, @@ -2781,96 +2712,6 @@ impl RecordingWorkerRuntime { } } -#[derive(Clone, Copy)] -enum PreStartWorkerOutcome { - LaunchFailure, - EarlyExit, -} - -/// Test worker runtime whose `start` fails before the worker reaches -/// `Starting`, either by refusing to launch or by exiting immediately. When -/// built with `held`, `start` blocks until `release_held_start` so a test can -/// act while the launch is in flight. -struct PreStartWorkerRuntime { - outcome: PreStartWorkerOutcome, - starts: AtomicUsize, - start_entered: Notify, - release_start: Option, -} - -impl PreStartWorkerRuntime { - fn new(outcome: PreStartWorkerOutcome) -> Self { - Self { - outcome, - starts: AtomicUsize::new(0), - start_entered: Notify::new(), - release_start: None, - } - } - - fn held(outcome: PreStartWorkerOutcome) -> Self { - Self { - release_start: Some(Notify::new()), - ..Self::new(outcome) - } - } - - fn start_count(&self) -> usize { - self.starts.load(Ordering::Relaxed) - } - - async fn wait_for_start(&self) { - wait_until( - &self.start_entered, - || self.start_count() > 0, - "test worker runtime should receive one start request", - ) - .await; - } - - fn release_held_start(&self) { - self.release_start - .as_ref() - .expect("runtime should have been built with a held start") - .notify_one(); - } -} - -#[async_trait::async_trait] -impl WorkerRuntime for PreStartWorkerRuntime { - async fn start(&self, _spec: WorkerLaunchSpec) -> anyhow::Result { - self.starts.fetch_add(1, Ordering::Relaxed); - self.start_entered.notify_waiters(); - if let Some(release_start) = &self.release_start { - release_start.notified().await; - } - - match self.outcome { - PreStartWorkerOutcome::LaunchFailure => { - anyhow::bail!("test worker launch failed") - } - PreStartWorkerOutcome::EarlyExit => Ok(StartedWorker { - worker_ref: test_worker_ref(u32::MAX), - stderr: Box::pin(tokio::io::empty()), - wait: Box::pin(async { - Ok(WorkerExit { - success: false, - detail: "test worker exited before starting".to_string(), - }) - }), - }), - } - } - - async fn request_stop(&self, _worker_ref: &WorkerRef) {} - - async fn force_stop(&self, _worker_ref: &WorkerRef) {} - - async fn is_alive(&self, _worker_ref: &WorkerRef) -> bool { - false - } -} - #[async_trait::async_trait] impl WorkerRuntime for RecordingWorkerRuntime { async fn start(&self, _spec: WorkerLaunchSpec) -> anyhow::Result { @@ -3076,50 +2917,6 @@ url = "http://127.0.0.1:32276" ); } -#[tokio::test] -async fn system_repair_runs_lists_sql_rows_without_readable_history() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - let run_store = state.stores.runs.create_run(&run_id).await.unwrap(); - append_default_run_created(&run_store, run_id).await; - state - .stores - .run_summaries - .test_delete_run_events(&run_id) - .await - .unwrap(); - - let response = app - .oneshot( - Request::builder() - .method(Method::GET) - .uri(api("/system/repair/runs")) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - - let body = response_json!(response, StatusCode::OK).await; - assert_eq!(body["total_count"], 1); - assert_eq!(body["runs"][0]["run_id"], run_id.to_string()); - let created_at = body["runs"][0]["created_at"] - .as_str() - .unwrap() - .parse::>() - .unwrap(); - assert_eq!(created_at, run_id.created_at()); - assert!( - body["runs"][0]["error"] - .as_str() - .unwrap() - .contains("head mismatch"), - "got: {}", - body["runs"][0]["error"] - ); -} - #[tokio::test] async fn create_run_response_omits_web_url_when_web_disabled() { let state = test_app_state_with_options( @@ -4837,30 +4634,6 @@ async fn wait_for_mock_hits(mock: &httpmock::Mock<'_>, expected: usize) { panic!("mock did not receive {expected} request(s)"); } -/// Poll `GET /runs/{id}/pull_request/creation` until the creation leaves -/// `pending`, returning the terminal creation body. -async fn wait_for_pull_request_creation(app: &Router, run_id: RunId) -> serde_json::Value { - for _ in 0..150 { - let response = app - .clone() - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/pull_request/creation"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - if body["status"] != "pending" { - return body; - } - tokio::time::sleep(std::time::Duration::from_millis(20)).await; - } - panic!("pull request creation for run {run_id} did not finish"); -} - async fn title_update_event_count(state: &AppState, run_id: RunId) -> usize { let run_store = state.stores.runs.open_run(&run_id).await.unwrap(); run_store @@ -5067,1453 +4840,6 @@ async fn create_and_start_run(app: &Router, dot_source: &str) -> String { run_id } -fn subprocess_pre_start_failure_state(runtime: StdArc) -> Arc { - let state = TestAppStateBuilder::new() - .vault_entries([(EnvVars::OPENAI_API_KEY, "test-openai-api-key")]) - .worker_runtime(runtime) - .build(); - write_test_server_record(&state.server_storage_dir()); - state -} - -fn run_failed_reasons(events: &[EventEnvelope]) -> Vec { - events - .iter() - .filter_map(|envelope| match &envelope.event.body { - EventBody::RunFailed(props) => Some(props.failure.reason), - _ => None, - }) - .collect() -} - -/// Asserts that a run which failed before its worker reached `Starting` -/// recorded exactly one `run.failed` event with `expected_reason` and that the -/// durable and in-memory statuses agree. Returns the run's events for further -/// inspection. -async fn assert_run_failed_before_start( - state: &Arc, - run_id: RunId, - expected_reason: FailureReason, -) -> Vec { - let run_store = state - .stores - .runs - .open_run_reader(&run_id) - .await - .expect("failed run should remain readable"); - let events = run_store - .list_events() - .await - .expect("failed run events should remain readable"); - assert_eq!(run_failed_reasons(&events), vec![expected_reason]); - - let expected_status = RunStatus::Failed { - reason: expected_reason, - }; - assert_eq!( - run_store - .state() - .await - .expect("failed run state should load") - .status, - expected_status - ); - assert_eq!( - state - .runs - .lock() - .expect("runs lock poisoned") - .get(&run_id) - .expect("managed run should remain present") - .status, - expected_status - ); - events -} - -async fn assert_subprocess_pre_start_failure( - outcome: PreStartWorkerOutcome, - expected_reason: FailureReason, -) { - let runtime = StdArc::new(PreStartWorkerRuntime::new(outcome)); - let state = subprocess_pre_start_failure_state(StdArc::clone(&runtime)); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = create_and_start_run(&app, MINIMAL_DOT) - .await - .parse::() - .expect("created run id should parse"); - let run_store = state - .stores - .runs - .open_run_reader(&run_id) - .await - .expect("created run should remain readable"); - - assert_eq!( - run_store - .state() - .await - .expect("runnable run state should load") - .status, - RunStatus::Runnable - ); - - execute_run(Arc::clone(&state), run_id).await; - - assert_eq!(runtime.start_count(), 1); - let events = assert_run_failed_before_start(&state, run_id, expected_reason).await; - let lifecycle_events = events - .iter() - .map(|envelope| envelope.event.event_name()) - .filter(|name| matches!(*name, "run.runnable" | "run.starting" | "run.failed")) - .collect::>(); - assert_eq!(lifecycle_events, vec!["run.runnable", "run.failed"]); - - let response = app - .clone() - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}"))) - .body(Body::empty()) - .expect("run request should build"), - ) - .await - .expect("run request should complete"); - let body = response_json!(response, StatusCode::OK).await; - assert_eq!(run_json_status(&body)["kind"], "failed"); - assert_eq!( - run_json_status(&body)["reason"], - expected_reason.to_string() - ); - - let response = app - .oneshot( - Request::builder() - .method("GET") - .uri(api("/runs")) - .body(Body::empty()) - .expect("run list request should build"), - ) - .await - .expect("run list request should complete"); - let body = response_json!(response, StatusCode::OK).await; - let run_id_string = run_id.to_string(); - let listed = body["data"] - .as_array() - .expect("run list data should be an array") - .iter() - .find(|run| run_json_id(run) == Some(run_id_string.as_str())) - .expect("failed run should remain listed"); - assert_eq!(run_json_status(listed)["kind"], "failed"); - assert_eq!( - run_json_status(listed)["reason"], - expected_reason.to_string() - ); -} - -#[tokio::test] -async fn subprocess_pre_start_failure_persists_launch_failure_from_runnable() { - assert_subprocess_pre_start_failure( - PreStartWorkerOutcome::LaunchFailure, - FailureReason::LaunchFailed, - ) - .await; -} - -#[tokio::test] -async fn subprocess_pre_start_failure_persists_early_worker_exit_from_runnable() { - assert_subprocess_pre_start_failure( - PreStartWorkerOutcome::EarlyExit, - FailureReason::Terminated, - ) - .await; -} - -#[tokio::test] -async fn subprocess_pre_start_failure_preserves_pending_cancellation() { - let runtime = StdArc::new(PreStartWorkerRuntime::held( - PreStartWorkerOutcome::LaunchFailure, - )); - let state = subprocess_pre_start_failure_state(StdArc::clone(&runtime)); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = create_and_start_run(&app, MINIMAL_DOT) - .await - .parse::() - .expect("created run id should parse"); - - let execution = tokio::spawn(execute_run(Arc::clone(&state), run_id)); - runtime.wait_for_start().await; - - let response = app - .oneshot( - Request::builder() - .method("POST") - .uri(api(&format!("/runs/{run_id}/cancel"))) - .body(Body::empty()) - .expect("cancel request should build"), - ) - .await - .expect("cancel request should complete"); - assert_status!(response, StatusCode::ACCEPTED).await; - - runtime.release_held_start(); - execution.await.expect("run execution task should complete"); - - assert_eq!(runtime.start_count(), 1); - assert_run_failed_before_start(&state, run_id, FailureReason::Cancelled).await; -} - -async fn create_durable_run_with_events( - state: &Arc, - run_id: RunId, - events: &[workflow_event::Event], -) { - let run_store = state.stores.runs.create_run(&run_id).await.unwrap(); - if !matches!( - events.first(), - Some(workflow_event::Event::RunCreated { .. }) - ) { - append_default_run_created(&run_store, run_id).await; - } - let needs_running = events.iter().any(|event| { - matches!( - event, - workflow_event::Event::WorkflowRunCompleted { .. } - | workflow_event::Event::WorkflowRunFailed { .. } - ) - }); - let has_starting = events - .iter() - .any(|event| matches!(event, workflow_event::Event::RunStarting)); - let has_runnable = events - .iter() - .any(|event| matches!(event, workflow_event::Event::RunRunnable { .. })); - let has_running = events - .iter() - .any(|event| matches!(event, workflow_event::Event::RunRunning)); - let mut inserted_runnable = has_runnable; - let mut inserted_starting = has_starting; - for event in events { - if !inserted_runnable - && matches!( - event, - workflow_event::Event::RunStarting - | workflow_event::Event::RunRunning - | workflow_event::Event::RunBlocked { .. } - | workflow_event::Event::RunPaused - | workflow_event::Event::WorkflowRunCompleted { .. } - | workflow_event::Event::WorkflowRunFailed { .. } - ) - { - workflow_event::append_event( - &run_store, - &run_id, - &workflow_event::Event::RunRunnable { - source: fabro_types::RunRunnableSource::StartRequested, - actor: None, - }, - ) - .await - .unwrap(); - inserted_runnable = true; - } - if !inserted_starting - && matches!( - event, - workflow_event::Event::RunRunning - | workflow_event::Event::RunBlocked { .. } - | workflow_event::Event::RunPaused - | workflow_event::Event::WorkflowRunCompleted { .. } - | workflow_event::Event::WorkflowRunFailed { .. } - ) - { - workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunStarting) - .await - .unwrap(); - inserted_starting = true; - } - if needs_running - && !has_running - && matches!( - event, - workflow_event::Event::WorkflowRunCompleted { .. } - | workflow_event::Event::WorkflowRunFailed { .. } - ) - { - workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunRunning) - .await - .unwrap(); - } - workflow_event::append_event(&run_store, &run_id, event) - .await - .unwrap(); - } -} - -fn stage_started_event(node_id: &str, handler_type: &str) -> workflow_event::Event { - workflow_event::Event::StageStarted { - graph_visit: None, - resumed_from_stage_id: None, - node_id: node_id.to_string(), - name: node_id.to_string(), - index: 1, - handler_type: handler_type.to_string(), - attempt: 1, - max_attempts: 1, - } -} - -fn command_started_event(node_id: &str) -> workflow_event::Event { - workflow_event::Event::CommandStarted { - node_id: node_id.to_string(), - script: "echo ok".to_string(), - command: "echo ok".to_string(), - language: "shell".to_string(), - timeout_ms: None, - } -} - -fn agent_session_activated_event(node_id: &str, visit: u32) -> workflow_event::Event { - workflow_event::Event::AgentSessionActivated { - node_id: node_id.to_string(), - visit, - session_id: "session-1".to_string(), - thread_id: None, - provider: Some("openai".to_string()), - model: Some("gpt-5.4".to_string()), - reasoning_effort: None, - speed: None, - permission_level: None, - capabilities: Vec::new(), - } -} - -fn stage_completed_event(node_id: &str) -> workflow_event::Event { - workflow_event::Event::StageCompleted { - node_id: node_id.to_string(), - name: node_id.to_string(), - index: 1, - timing: StageTiming::wall_only(42), - status: "succeeded".to_string(), - preferred_label: None, - suggested_next_ids: Vec::new(), - usage_by_model: Vec::new(), - usage: None, - failure: None, - notes: None, - files_touched: Vec::new(), - context_updates: None, - jump_to_node: None, - context_values: None, - node_visits: None, - loop_failure_signatures: None, - restart_failure_signatures: None, - response: None, - attempt: 1, - max_attempts: 1, - } -} - -fn agent_message_event( - stage: &str, - visit: u32, - session_id: &str, - text: &str, - context_window: Option, - reasoning: Option, -) -> workflow_event::Event { - workflow_event::Event::Agent { - stage: stage.to_string(), - visit, - event: CodingAgentEvent::new( - session_id, - CodingEvent::AssistantMessage { - text: text.to_string(), - model: "gpt-5.4".to_string(), - usage: Usage::default(), - tool_call_count: 0, - context_window, - reasoning, - }, - std::time::SystemTime::now(), - ), - } -} - -fn context_window_event( - stage: &str, - visit: u32, - context_window: ContextWindowSnapshot, -) -> workflow_event::Event { - agent_message_event( - stage, - visit, - "session-1", - "assistant response", - Some(context_window), - None, - ) -} - -fn context_window_snapshot( - input_tokens: u64, - warnings: Vec, -) -> ContextWindowSnapshot { - ContextWindowSnapshot { - provider: "openai".to_string(), - model: "gpt-5.4".to_string(), - context_window_tokens: 400_000, - input_tokens, - usage_percent: input_tokens as f64 * 100.0 / 400_000.0, - count_method: ContextWindowCountMethod::ResponseUsageScaledBreakdown, - staleness: ContextWindowStaleness::Live, - generated_at: std::time::SystemTime::now(), - event_seq: None, - breakdown: vec![ContextWindowBreakdownItem { - category: ContextWindowCategory::Conversation, - tokens: input_tokens, - usage_percent: input_tokens as f64 * 100.0 / 400_000.0, - }], - warnings, - } -} - -async fn append_default_run_created(run_store: &fabro_store::RunDatabase, run_id: RunId) { - workflow_event::append_event(run_store, &run_id, &workflow_event::Event::RunCreated { - run_id, - title: None, - settings: serde_json::to_value(WorkflowSettings::default()).unwrap(), - graph: serde_json::to_value(Graph::new("test")).unwrap(), - workflow_source: None, - labels: std::collections::BTreeMap::default(), - source_directory: None, - workflow_slug: None, - workflow_version_id: None, - target: None, - automation: None, - provenance: test_support::test_run_provenance(), - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, - admission: PetriAdmission::default(), - }) - .await - .unwrap(); -} - -fn workflow_settings_with_run_notifications( - run_toml: &str, - workflow_name: Option<&str>, -) -> WorkflowSettings { - let mut settings = WorkflowSettingsBuilder::new() - .server_manifest_defaults(RunLayer::default(), test_environment_defaults()) - .workflow_toml(run_toml) - .expect("run notification settings should parse") - .build() - .expect("run notification settings should resolve"); - settings.workflow.name = workflow_name.map(str::to_string); - settings -} - -fn test_environment_defaults() -> MergeMap { - MergeMap::from(HashMap::from([("default".to_string(), EnvironmentLayer { - provider: Some("local".to_string()), - ..EnvironmentLayer::default() - })])) -} - -async fn create_slack_notification_run( - state: &Arc, - run_id: RunId, - settings: WorkflowSettings, - graph_name: &str, - workflow_slug: Option<&str>, -) -> fabro_store::RunDatabase { - let run_store = state.stores.runs.create_run(&run_id).await.unwrap(); - workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunCreated { - run_id, - title: None, - settings: serde_json::to_value(settings).unwrap(), - graph: serde_json::to_value(Graph::new(graph_name)).unwrap(), - workflow_source: None, - labels: std::collections::BTreeMap::default(), - source_directory: None, - workflow_slug: workflow_slug.map(str::to_string), - workflow_version_id: None, - target: None, - automation: None, - provenance: test_support::test_run_provenance(), - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, - admission: PetriAdmission::default(), - }) - .await - .unwrap(); - run_store -} - -async fn append_slack_notification_event( - run_store: &fabro_store::RunDatabase, - run_id: RunId, - event: &workflow_event::Event, -) -> EventEnvelope { - workflow_event::append_event(run_store, &run_id, event) - .await - .unwrap(); - run_store - .list_events() - .await - .unwrap() - .last() - .expect("appended event should be present") - .clone() -} - -async fn mock_slack_post<'a>( - server: &'a MockServer, - body_includes: Vec, - ts: &'static str, -) -> httpmock::Mock<'a> { - server - .mock_async(move |when, then| { - let mut when = when - .method(POST) - .path("/chat.postMessage") - .header("authorization", "Bearer xoxb-test"); - for part in body_includes { - when = when.body_includes(part); - } - then.status(200) - .header("content-type", "application/json") - .json_body(json!({ - "ok": true, - "channel": "C123", - "ts": ts, - })); - }) - .await -} - -fn slack_lifecycle_service(base_url: String, default_channel: Option<&str>) -> SlackService { - SlackService { - client: fabro_slack::client::SlackClient::with_api_base_and_http( - "xoxb-test".to_string(), - base_url, - fabro_http::test_http_client().expect("test HTTP client should build"), - ), - app_token: "xapp-test".to_string(), - default_channel: default_channel.map(str::to_string), - posted_messages: StdArc::new(StdMutex::new(HashMap::new())), - thread_registry: StdArc::new(ThreadRegistry::new()), - connection: StdArc::new(StdMutex::new(SlackConnectionRuntimeState::default())), - } -} - -fn workflow_run_started_event(run_id: RunId) -> workflow_event::Event { - workflow_event::Event::WorkflowRunStarted { - name: "run.started event name".to_string(), - run_id, - base_branch: None, - base_sha: None, - run_branch: None, - worktree_dir: None, - goal: None, - } -} - -#[tokio::test] -async fn slack_lifecycle_run_started_posts_for_matching_enabled_route() { - let server = MockServer::start_async().await; - let post = mock_slack_post( - &server, - vec![ - r##""channel":"#deploys""##.to_string(), - "Fabro run started".to_string(), - "Deploy workflow".to_string(), - "Open in Fabro".to_string(), - ], - "100.1", - ) - .await; - let state = test_app_state(); - let service = slack_lifecycle_service(server.base_url(), None); - let run_id = fixtures::RUN_1; - let settings = workflow_settings_with_run_notifications( - r##" -[run.notifications.deploys] -enabled = true -provider = "slack" -events = ["run.started", "run.completed", "run.failed"] - -[run.notifications.deploys.slack] -channel = "#deploys" -"##, - Some("Deploy workflow"), - ); - let run_store = - create_slack_notification_run(&state, run_id, settings, "deploy-graph", Some("deploy")) - .await; - let envelope = - append_slack_notification_event(&run_store, run_id, &workflow_run_started_event(run_id)) - .await; - - service - .handle_event( - state.as_ref(), - &envelope, - Some("https://fabro.example/runs/run-1"), - ) - .await; - - post.assert_async().await; - assert!( - service - .posted_messages - .lock() - .expect("posted messages lock poisoned") - .is_empty(), - "lifecycle posts must not use interview message state" - ); -} - -#[tokio::test] -async fn slack_lifecycle_run_completed_posts_result_and_duration() { - let server = MockServer::start_async().await; - let post = mock_slack_post( - &server, - vec![ - r##""channel":"#deploys""##.to_string(), - "Fabro run completed".to_string(), - "succeeded — completed".to_string(), - "1m 5s".to_string(), - ], - "100.2", - ) - .await; - let state = test_app_state(); - let service = slack_lifecycle_service(server.base_url(), None); - let run_id = fixtures::RUN_1; - let settings = workflow_settings_with_run_notifications( - r##" -[run.notifications.deploys] -enabled = true -provider = "slack" -events = ["run.completed"] - -[run.notifications.deploys.slack] -channel = "#deploys" -"##, - Some("Deploy workflow"), - ); - let run_store = create_slack_notification_run(&state, run_id, settings, "deploy", None).await; - workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunRunnable { - source: fabro_types::RunRunnableSource::StartRequested, - actor: None, - }) - .await - .unwrap(); - workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunStarting) - .await - .unwrap(); - workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunRunning) - .await - .unwrap(); - let envelope = append_slack_notification_event( - &run_store, - run_id, - &workflow_event::Event::WorkflowRunCompleted { - timing: fabro_types::RunTiming::wall_only(65_432), - artifact_count: 0, - status: "succeeded".to_string(), - reason: SuccessReason::Completed, - final_git_commit_sha: None, - final_patch: None, - diff_summary: None, - usage: None, - }, - ) - .await; - - service.handle_event(state.as_ref(), &envelope, None).await; - - post.assert_async().await; -} - -#[tokio::test] -async fn slack_lifecycle_run_failed_posts_failure_result_message_and_duration() { - let server = MockServer::start_async().await; - let post = mock_slack_post( - &server, - vec![ - r##""channel":"#deploys""##.to_string(), - "Fabro run failed".to_string(), - "workflow_error — command <failed> & exited".to_string(), - "1.2s".to_string(), - ], - "100.3", - ) - .await; - let state = test_app_state(); - let service = slack_lifecycle_service(server.base_url(), None); - let run_id = fixtures::RUN_1; - let settings = workflow_settings_with_run_notifications( - r##" -[run.notifications.deploys] -enabled = true -provider = "slack" -events = ["run.failed"] - -[run.notifications.deploys.slack] -channel = "#deploys" -"##, - Some("Deploy workflow"), - ); - let run_store = create_slack_notification_run(&state, run_id, settings, "deploy", None).await; - workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunRunnable { - source: fabro_types::RunRunnableSource::StartRequested, - actor: None, - }) - .await - .unwrap(); - workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunStarting) - .await - .unwrap(); - workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunRunning) - .await - .unwrap(); - let envelope = append_slack_notification_event( - &run_store, - run_id, - &workflow_event::Event::WorkflowRunFailed { - failure: fabro_types::RunFailure { - reason: fabro_types::FailureReason::WorkflowError, - detail: FailureDetail::new( - "command & exited", - FailureCategory::Deterministic, - ), - }, - timing: fabro_types::RunTiming::wall_only(1_234), - final_git_commit_sha: None, - final_patch: None, - diff_summary: None, - usage: None, - }, - ) - .await; - - service.handle_event(state.as_ref(), &envelope, None).await; - - post.assert_async().await; -} - -#[tokio::test] -async fn slack_lifecycle_skips_non_matching_events_and_disabled_routes() { - let server = MockServer::start_async().await; - let unexpected = mock_slack_post(&server, Vec::new(), "100.4").await; - let state = test_app_state(); - let service = slack_lifecycle_service(server.base_url(), None); - let run_id = fixtures::RUN_1; - let settings = workflow_settings_with_run_notifications( - r##" -[run.notifications.disabled] -enabled = false -provider = "slack" -events = ["run.started"] - -[run.notifications.disabled.slack] -channel = "#deploys" - -[run.notifications.stage] -enabled = true -provider = "slack" -events = ["stage.completed"] - -[run.notifications.stage.slack] -channel = "#deploys" -"##, - Some("Deploy workflow"), - ); - let run_store = create_slack_notification_run(&state, run_id, settings, "deploy", None).await; - let envelope = - append_slack_notification_event(&run_store, run_id, &workflow_run_started_event(run_id)) - .await; - - service.handle_event(state.as_ref(), &envelope, None).await; - - unexpected.assert_calls_async(0).await; -} - -#[tokio::test] -async fn slack_lifecycle_missing_channel_is_skipped_without_blocking_other_routes() { - let server = MockServer::start_async().await; - let post = mock_slack_post( - &server, - vec![ - r##""channel":"#ops""##.to_string(), - "Fabro run started".to_string(), - ], - "100.5", - ) - .await; - let state = test_app_state(); - let service = slack_lifecycle_service(server.base_url(), None); - let run_id = fixtures::RUN_1; - let settings = workflow_settings_with_run_notifications( - r##" -[run.notifications.missing] -enabled = true -provider = "slack" -events = ["run.started"] - -[run.notifications.unresolved] -enabled = true -provider = "slack" -events = ["run.started"] - -[run.notifications.unresolved.slack] -channel = "{{ env.MISSING_SLACK_CHANNEL }}" - -[run.notifications.valid] -enabled = true -provider = "slack" -events = ["run.started"] - -[run.notifications.valid.slack] -channel = "#ops" -"##, - Some("Deploy workflow"), - ); - let run_store = create_slack_notification_run(&state, run_id, settings, "deploy", None).await; - let envelope = - append_slack_notification_event(&run_store, run_id, &workflow_run_started_event(run_id)) - .await; - - service.handle_event(state.as_ref(), &envelope, None).await; - - post.assert_async().await; -} - -#[tokio::test] -async fn slack_lifecycle_uses_prior_pull_request_created_details() { - let server = MockServer::start_async().await; - let post = mock_slack_post( - &server, - vec![ - "Fabro run completed".to_string(), - "https://github.com/fabro-sh/fabro/pull/42".to_string(), - "#42".to_string(), - "Ship <prod> & notify".to_string(), - ], - "100.6", - ) - .await; - let state = test_app_state(); - let service = slack_lifecycle_service(server.base_url(), None); - let run_id = fixtures::RUN_1; - let settings = workflow_settings_with_run_notifications( - r##" -[run.notifications.deploys] -enabled = true -provider = "slack" -events = ["run.completed"] - -[run.notifications.deploys.slack] -channel = "#deploys" -"##, - Some("Deploy workflow"), - ); - let run_store = create_slack_notification_run(&state, run_id, settings, "deploy", None).await; - workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunRunnable { - source: fabro_types::RunRunnableSource::StartRequested, - actor: None, - }) - .await - .unwrap(); - workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunStarting) - .await - .unwrap(); - workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunRunning) - .await - .unwrap(); - workflow_event::append_event( - &run_store, - &run_id, - &workflow_event::Event::PullRequestCreated { - pr_url: "https://github.com/fabro-sh/fabro/pull/42".to_string(), - pr_number: 42, - owner: "fabro-sh".to_string(), - repo: "fabro".to_string(), - base_branch: "main".to_string(), - head_branch: "fabro/run/test".to_string(), - head_sha: Some("final-sha".to_string()), - title: "Ship & notify".to_string(), - draft: false, - }, - ) - .await - .unwrap(); - let envelope = append_slack_notification_event( - &run_store, - run_id, - &workflow_event::Event::WorkflowRunCompleted { - timing: fabro_types::RunTiming::wall_only(1000), - artifact_count: 0, - status: "succeeded".to_string(), - reason: SuccessReason::Completed, - final_git_commit_sha: None, - final_patch: None, - diff_summary: None, - usage: None, - }, - ) - .await; - - service.handle_event(state.as_ref(), &envelope, None).await; - - post.assert_async().await; -} - -#[tokio::test] -async fn slack_interviews_keep_state_separate_from_lifecycle_notifications() { - let server = MockServer::start_async().await; - let interview_post = mock_slack_post( - &server, - vec![ - r##""channel":"#reviews""##.to_string(), - "Answer deploy question".to_string(), - ], - "200.1", - ) - .await; - let lifecycle_post = mock_slack_post( - &server, - vec![ - r##""channel":"#deploys""##.to_string(), - "Fabro run started".to_string(), - ], - "200.2", - ) - .await; - let state = test_app_state(); - let service = slack_lifecycle_service(server.base_url(), Some("#reviews")); - let run_id = fixtures::RUN_1; - let settings = workflow_settings_with_run_notifications( - r##" -[run.notifications.deploys] -enabled = true -provider = "slack" -events = ["run.started"] - -[run.notifications.deploys.slack] -channel = "#deploys" -"##, - Some("Deploy workflow"), - ); - let run_store = create_slack_notification_run(&state, run_id, settings, "deploy", None).await; - let lifecycle_envelope = - append_slack_notification_event(&run_store, run_id, &workflow_run_started_event(run_id)) - .await; - let interview_envelope = append_slack_notification_event( - &run_store, - run_id, - &workflow_event::Event::InterviewStarted { - question_id: "q-1".to_string(), - question: "Answer deploy question".to_string(), - stage: "review".to_string(), - question_type: "freeform".to_string(), - options: Vec::new(), - allow_freeform: true, - timeout_seconds: None, - context_display: None, - review_target: None, - }, - ) - .await; - - service - .handle_event(state.as_ref(), &lifecycle_envelope, None) - .await; - assert!( - service - .posted_messages - .lock() - .expect("posted messages lock poisoned") - .is_empty(), - "lifecycle notification should not record interview metadata" - ); - assert!( - service.thread_registry.resolve("200.2").is_none(), - "lifecycle notification should not register answer threads" - ); - - service - .handle_event(state.as_ref(), &interview_envelope, None) - .await; - - lifecycle_post.assert_async().await; - interview_post.assert_async().await; - assert!( - service - .posted_messages - .lock() - .expect("posted messages lock poisoned") - .contains_key(&(run_id, "q-1".to_string())), - "interview posts should retain interview state" - ); - assert!( - service.thread_registry.resolve("200.1").is_some(), - "freeform interview posts should register reply threads" - ); -} - -#[tokio::test] -async fn persist_cancelled_run_status_ignores_already_terminal_runs() { - let state = test_app_state(); - let run_id = fixtures::RUN_1; - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::WorkflowRunCompleted { - timing: fabro_types::RunTiming::wall_only(1000), - artifact_count: 0, - status: "succeeded".to_string(), - reason: SuccessReason::Completed, - final_git_commit_sha: None, - final_patch: None, - diff_summary: None, - usage: None, - }, - ]) - .await; - - persist_cancelled_run_status(state.as_ref(), run_id) - .await - .unwrap(); - - let run_store = state.stores.runs.open_run(&run_id).await.unwrap(); - let projection = run_store.state().await.unwrap(); - assert_eq!(projection.status, RunStatus::Succeeded { - reason: SuccessReason::Completed, - }); - assert!(!run_store.list_events().await.unwrap().iter().any(|event| { - matches!( - event.event.body, - EventBody::RunFailed(ref props) if props.failure.reason == FailureReason::Cancelled - ) - })); -} - -#[tokio::test] -async fn delete_terminal_managed_run_does_not_send_cancel_signal() { - let state = test_app_state(); - let run_id = fixtures::RUN_1; - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::WorkflowRunCompleted { - timing: fabro_types::RunTiming::wall_only(1000), - artifact_count: 0, - status: "succeeded".to_string(), - reason: SuccessReason::Completed, - final_git_commit_sha: None, - final_patch: None, - diff_summary: None, - usage: None, - }, - ]) - .await; - - let temp = tempfile::tempdir().unwrap(); - let run_dir = temp.path().join("run"); - std::fs::create_dir_all(&run_dir).unwrap(); - let cancel_token = CancellationToken::new(); - let mut run = managed_run( - MINIMAL_DOT.to_string(), - RunStatus::Running, - Utc::now(), - run_dir, - RunExecutionMode::Start, - ); - run.cancel_token = Some(cancel_token.clone()); - let (cancel_tx, _cancel_rx) = oneshot::channel(); - run.cancel_tx = Some(cancel_tx); - state - .runs - .lock() - .expect("runs lock poisoned") - .insert(run_id, run); - - delete_run_internal(state.as_ref(), run_id, true) - .await - .unwrap(); - - assert!(!cancel_token.is_cancelled()); -} - -/// Append a stage lifecycle event with an explicit `StageScope`, so the -/// stored envelope carries the full `stage_id` (`node_id@visit`). The bare -/// [`workflow_event::append_event`] helper only writes `node_id` because -/// stage lifecycle variants don't carry visit in their payload — production -/// always emits via `Emitter::emit_scoped`. -async fn append_scoped_stage_event( - state: &Arc, - run_id: RunId, - node_id: &str, - visit: u32, - event: &workflow_event::Event, -) { - let scope = fabro_workflow::event::StageScope { - node_id: node_id.to_string(), - visit, - parallel_group_id: None, - parallel_branch_id: None, - }; - append_event_with_scope(state, run_id, event, &scope).await; -} - -async fn append_event_with_scope( - state: &Arc, - run_id: RunId, - event: &workflow_event::Event, - scope: &fabro_workflow::event::StageScope, -) { - let stored = fabro_workflow::event::to_run_event_at(&run_id, event, Utc::now(), Some(scope)); - let payload = fabro_workflow::event::build_redacted_event_payload(&stored, &run_id).unwrap(); - let run_store = state.stores.runs.open_run(&run_id).await.unwrap(); - run_store.append_event(&payload).await.unwrap(); -} - -fn stage_status<'a>(body: &'a serde_json::Value, id: &str) -> &'a str { - body["data"] - .as_array() - .unwrap() - .iter() - .find(|stage| stage["id"] == id) - .and_then(|stage| stage["status"].as_str()) - .unwrap() -} - -#[tokio::test] -async fn list_run_stages_projects_retrying_until_completion() { - let state = test_app_state_with_isolated_storage(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - ]) - .await; - append_scoped_stage_event( - &state, - run_id, - "setup", - 1, - &workflow_event::Event::StageStarted { - graph_visit: None, - resumed_from_stage_id: None, - node_id: "setup".to_string(), - name: "Setup".to_string(), - index: 0, - handler_type: "command".to_string(), - attempt: 1, - max_attempts: 1, - }, - ) - .await; - append_scoped_stage_event( - &state, - run_id, - "setup", - 1, - &workflow_event::Event::StageCompleted { - node_id: "setup".to_string(), - name: "Setup".to_string(), - index: 0, - timing: fabro_types::StageTiming::wall_only(5), - status: "succeeded".to_string(), - preferred_label: None, - suggested_next_ids: Vec::new(), - usage_by_model: Vec::new(), - usage: None, - failure: None, - notes: None, - files_touched: Vec::new(), - context_updates: None, - jump_to_node: None, - context_values: None, - node_visits: None, - loop_failure_signatures: None, - restart_failure_signatures: None, - response: None, - attempt: 1, - max_attempts: 1, - }, - ) - .await; - append_scoped_stage_event( - &state, - run_id, - "work", - 1, - &workflow_event::Event::StageStarted { - graph_visit: None, - resumed_from_stage_id: None, - node_id: "work".to_string(), - name: "Work".to_string(), - index: 1, - handler_type: "command".to_string(), - attempt: 1, - max_attempts: 3, - }, - ) - .await; - append_scoped_stage_event( - &state, - run_id, - "work", - 1, - &workflow_event::Event::StageFailed { - node_id: "work".to_string(), - name: "Work".to_string(), - index: 1, - failure: FailureDetail::new("try again", FailureCategory::TransientInfra), - will_retry: true, - timing: fabro_types::StageTiming::wall_only(10), - usage_by_model: Vec::new(), - usage: None, - actor: None, - }, - ) - .await; - append_scoped_stage_event( - &state, - run_id, - "work", - 1, - &workflow_event::Event::StageRetrying { - node_id: "work".to_string(), - name: "Work".to_string(), - index: 1, - attempt: 2, - max_attempts: 3, - delay_ms: 100, - }, - ) - .await; - - let response = app - .clone() - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/stages"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - assert_eq!(stage_status(&body, "setup@1"), "succeeded"); - assert_eq!(stage_status(&body, "work@1"), "retrying"); - - append_scoped_stage_event( - &state, - run_id, - "work", - 1, - &workflow_event::Event::StageCompleted { - node_id: "work".to_string(), - name: "Work".to_string(), - index: 1, - timing: fabro_types::StageTiming::wall_only(25), - status: "partially_succeeded".to_string(), - preferred_label: None, - suggested_next_ids: Vec::new(), - usage_by_model: Vec::new(), - usage: None, - failure: None, - notes: None, - files_touched: Vec::new(), - context_updates: None, - jump_to_node: None, - context_values: None, - node_visits: None, - loop_failure_signatures: None, - restart_failure_signatures: None, - response: None, - attempt: 2, - max_attempts: 3, - }, - ) - .await; - - let response = app - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/stages"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - assert_eq!(stage_status(&body, "work@1"), "partially_succeeded"); -} - -#[tokio::test] -async fn list_run_stages_projects_running_stage_as_cancelled_after_cancelled_run_failure() { - let state = test_app_state_with_isolated_storage(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - ]) - .await; - append_scoped_stage_event( - &state, - run_id, - "work", - 1, - &workflow_event::Event::StageStarted { - graph_visit: None, - resumed_from_stage_id: None, - node_id: "work".to_string(), - name: "Work".to_string(), - index: 1, - handler_type: "agent".to_string(), - attempt: 1, - max_attempts: 1, - }, - ) - .await; - let run_store = state.stores.runs.open_run(&run_id).await.unwrap(); - workflow_event::append_event( - &run_store, - &run_id, - &workflow_event::Event::WorkflowRunFailed { - failure: fabro_types::RunFailure { - reason: fabro_types::FailureReason::Cancelled, - detail: FailureDetail::new("cancelled", FailureCategory::Canceled), - }, - timing: fabro_types::RunTiming::wall_only(100), - final_git_commit_sha: None, - final_patch: None, - diff_summary: None, - usage: None, - }, - ) - .await - .unwrap(); - - let response = app - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/stages"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - assert_eq!(stage_status(&body, "work@1"), "cancelled"); -} - -fn stage_entry<'a>(body: &'a serde_json::Value, id: &str) -> &'a serde_json::Value { - body["data"] - .as_array() - .unwrap() - .iter() - .find(|stage| stage["id"] == id) - .unwrap_or_else(|| panic!("stage {id} not found in {body:#?}")) -} - -#[tokio::test] -async fn list_run_stages_includes_stage_model_usage() { - let state = test_app_state_with_isolated_storage(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - ]) - .await; - append_scoped_stage_event( - &state, - run_id, - "prompt", - 1, - &workflow_event::Event::StageStarted { - graph_visit: None, - resumed_from_stage_id: None, - node_id: "prompt".to_string(), - name: "Prompt".to_string(), - index: 0, - handler_type: "prompt".to_string(), - attempt: 1, - max_attempts: 1, - }, - ) - .await; - append_scoped_stage_event( - &state, - run_id, - "prompt", - 1, - &workflow_event::Event::Prompt { - stage: "prompt".to_string(), - visit: 1, - text: "Summarize".to_string(), - mode: Some(StageModelUsage::MODE_PROMPT.to_string()), - provider: Some("openai".to_string()), - model: Some("gpt-5.5".to_string()), - reasoning_effort: Some(ReasoningEffort::High), - speed: Some(Speed::Fast), - }, - ) - .await; - - let response = app - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/stages"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - assert_eq!( - stage_entry(&body, "prompt@1")["provider_used"], - json!({ - "mode": "prompt", - "provider": "openai", - "model": "gpt-5.5", - "reasoning_effort": "high", - "speed": "fast" - }) - ); -} - fn test_priced_usage( model_id: &str, input_tokens: u64, @@ -6538,1008 +4864,6 @@ fn test_priced_usage( ) } -async fn create_priced_retry_run(state: &Arc, run_id: RunId) { - create_durable_run_with_events(state, run_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - ]) - .await; - - append_scoped_stage_event( - state, - run_id, - "verify", - 1, - &workflow_event::Event::StageFailed { - node_id: "verify".to_string(), - name: "Verify".to_string(), - index: 1, - failure: FailureDetail::new("try again", FailureCategory::TransientInfra), - will_retry: true, - timing: fabro_types::StageTiming::wall_only(1200), - usage_by_model: Vec::new(), - usage: Some(test_priced_usage("gpt-old", 100, 10)), - actor: None, - }, - ) - .await; - append_scoped_stage_event( - state, - run_id, - "verify", - 2, - &workflow_event::Event::StageCompleted { - node_id: "verify".to_string(), - name: "Verify".to_string(), - index: 1, - timing: fabro_types::StageTiming::wall_only(800), - status: "succeeded".to_string(), - preferred_label: None, - suggested_next_ids: Vec::new(), - usage_by_model: Vec::new(), - usage: Some(test_priced_usage("gpt-new", 200, 20)), - failure: None, - notes: None, - files_touched: Vec::new(), - context_updates: None, - jump_to_node: None, - context_values: None, - node_visits: None, - loop_failure_signatures: None, - restart_failure_signatures: None, - response: None, - attempt: 2, - max_attempts: 2, - }, - ) - .await; -} - -#[tokio::test] -async fn list_run_stages_distinguishes_visits() { - let state = test_app_state_with_isolated_storage(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - let mut graph = Graph::new("test"); - let mut verify = Node::new("verify"); - verify - .attrs - .insert("type".to_string(), AttrValue::String("command".to_string())); - graph.nodes.insert("verify".to_string(), verify); - - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::RunCreated { - run_id, - title: None, - settings: serde_json::to_value(fabro_types::WorkflowSettings::default()).unwrap(), - graph: serde_json::to_value(&graph).unwrap(), - workflow_source: None, - labels: std::collections::BTreeMap::default(), - source_directory: None, - workflow_slug: Some("test".to_string()), - workflow_version_id: None, - target: None, - automation: None, - provenance: test_support::test_run_provenance(), - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, - admission: PetriAdmission::default(), - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - ]) - .await; - - // First visit of `verify` — failed. - append_scoped_stage_event( - &state, - run_id, - "verify", - 1, - &workflow_event::Event::StageStarted { - graph_visit: None, - resumed_from_stage_id: None, - node_id: "verify".to_string(), - name: "Verify".to_string(), - index: 1, - handler_type: "command".to_string(), - attempt: 1, - max_attempts: 1, - }, - ) - .await; - append_scoped_stage_event( - &state, - run_id, - "verify", - 1, - &workflow_event::Event::StageCompleted { - node_id: "verify".to_string(), - name: "Verify".to_string(), - index: 1, - timing: fabro_types::StageTiming::wall_only(1500), - status: "failed".to_string(), - preferred_label: None, - suggested_next_ids: Vec::new(), - usage_by_model: Vec::new(), - usage: None, - failure: None, - notes: None, - files_touched: Vec::new(), - context_updates: None, - jump_to_node: None, - context_values: None, - node_visits: None, - loop_failure_signatures: None, - restart_failure_signatures: None, - response: None, - attempt: 1, - max_attempts: 1, - }, - ) - .await; - - // Second visit of `verify` — running. - append_scoped_stage_event( - &state, - run_id, - "verify", - 2, - &workflow_event::Event::StageStarted { - graph_visit: None, - resumed_from_stage_id: None, - node_id: "verify".to_string(), - name: "Verify".to_string(), - index: 1, - handler_type: "command".to_string(), - attempt: 1, - max_attempts: 1, - }, - ) - .await; - - let response = app - .clone() - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/stages"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - - let data = body["data"].as_array().unwrap(); - let verify_entries: Vec<_> = data.iter().filter(|s| s["node_id"] == "verify").collect(); - assert_eq!(verify_entries.len(), 2, "expected two verify visits"); - - let first = stage_entry(&body, "verify@1"); - assert_eq!(first["node_id"], "verify"); - assert_eq!(first["visit"], 1); - assert_eq!(first["handler"], "command"); - assert_eq!(first["status"], "failed"); - assert_eq!(first["wall_time_ms"], 1500); - - let second = stage_entry(&body, "verify@2"); - assert_eq!(second["node_id"], "verify"); - assert_eq!(second["visit"], 2); - assert_eq!(second["handler"], "command"); - assert_eq!(second["status"], "running"); - - // Old `dot_id` field must be gone. - assert!(first.get("dot_id").is_none(), "dot_id should be removed"); -} - -#[tokio::test] -async fn list_run_stages_exposes_execution_identity_for_resumed_stage() { - let state = test_app_state_with_isolated_storage(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - let mut graph = Graph::new("test"); - let mut work = Node::new("work"); - work.attrs - .insert("type".to_string(), AttrValue::String("agent".to_string())); - graph.nodes.insert("work".to_string(), work); - - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::RunCreated { - run_id, - title: None, - settings: serde_json::to_value(fabro_types::WorkflowSettings::default()).unwrap(), - graph: serde_json::to_value(&graph).unwrap(), - workflow_source: None, - labels: std::collections::BTreeMap::default(), - source_directory: None, - workflow_slug: Some("test".to_string()), - workflow_version_id: None, - target: None, - automation: None, - provenance: test_support::test_run_provenance(), - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, - admission: PetriAdmission::default(), - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - ]) - .await; - - // Legacy-shaped first execution without identity metadata. - append_scoped_stage_event( - &state, - run_id, - "work", - 1, - &workflow_event::Event::StageStarted { - graph_visit: None, - resumed_from_stage_id: None, - node_id: "work".to_string(), - name: "Work".to_string(), - index: 1, - handler_type: "agent".to_string(), - attempt: 1, - max_attempts: 1, - }, - ) - .await; - // Reexecution after cancel/resume: same graph visit, next ordinal. - append_scoped_stage_event( - &state, - run_id, - "work", - 2, - &workflow_event::Event::StageStarted { - graph_visit: Some(1), - resumed_from_stage_id: Some(fabro_types::StageId::new("work", 1)), - node_id: "work".to_string(), - name: "Work".to_string(), - index: 1, - handler_type: "agent".to_string(), - attempt: 1, - max_attempts: 1, - }, - ) - .await; - - let response = app - .clone() - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/stages"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - - let first = stage_entry(&body, "work@1"); - assert!( - first.get("graph_visit").is_none(), - "legacy stage should omit graph_visit" - ); - assert!( - first.get("resumed_from_stage_id").is_none(), - "legacy stage should omit resumed_from_stage_id" - ); - - let second = stage_entry(&body, "work@2"); - assert_eq!(second["visit"], 2); - assert_eq!(second["graph_visit"], 1); - assert_eq!(second["resumed_from_stage_id"], "work@1"); -} - -#[tokio::test] -async fn run_usage_includes_live_stage_timing_in_rows_and_totals() { - let state = test_app_state_with_isolated_storage(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - workflow_run_started_event(run_id), - ]) - .await; - append_scoped_stage_event( - &state, - run_id, - "work", - 1, - &stage_started_event("work", "command"), - ) - .await; - - tokio::time::sleep(std::time::Duration::from_millis(20)).await; - let response = app - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/usage"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - let stages = body["stages"].as_array().unwrap(); - - assert_eq!(stages.len(), 1); - let row_timing = &stages[0]["timing"]; - assert!(row_timing["active_time_ms"].as_u64().unwrap() > 0); - assert_eq!(row_timing["tool_time_ms"], row_timing["active_time_ms"]); - assert_eq!(&body["totals"]["timing"], row_timing); -} - -/// `checkpoint.completed_nodes` records every visit, so a looped node appears -/// once per re-entry. Usage must dedup so a retried node renders as one row -/// and `runtime_secs` is summed across all visits exactly once. -#[tokio::test] -async fn run_usage_dedups_retried_nodes_and_sums_their_durations() { - let state = test_app_state_with_isolated_storage(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - ]) - .await; - - // Visit 1 of `verify` — completed in 1.5s. - append_scoped_stage_event( - &state, - run_id, - "verify", - 1, - &workflow_event::Event::StageCompleted { - node_id: "verify".to_string(), - name: "Verify".to_string(), - index: 1, - timing: fabro_types::StageTiming::wall_only(1500), - status: "failed".to_string(), - preferred_label: None, - suggested_next_ids: Vec::new(), - usage_by_model: Vec::new(), - usage: None, - failure: None, - notes: None, - files_touched: Vec::new(), - context_updates: None, - jump_to_node: None, - context_values: None, - node_visits: None, - loop_failure_signatures: None, - restart_failure_signatures: None, - response: None, - attempt: 1, - max_attempts: 1, - }, - ) - .await; - - // Visit 2 of `verify` — completed in 0.8s. - append_scoped_stage_event( - &state, - run_id, - "verify", - 2, - &workflow_event::Event::StageCompleted { - node_id: "verify".to_string(), - name: "Verify".to_string(), - index: 1, - timing: fabro_types::StageTiming::wall_only(800), - status: "succeeded".to_string(), - preferred_label: None, - suggested_next_ids: Vec::new(), - usage_by_model: Vec::new(), - usage: None, - failure: None, - notes: None, - files_touched: Vec::new(), - context_updates: None, - jump_to_node: None, - context_values: None, - node_visits: None, - loop_failure_signatures: None, - restart_failure_signatures: None, - response: None, - attempt: 1, - max_attempts: 1, - }, - ) - .await; - - // Checkpoint records `verify` twice (once per visit) — this is what makes - // the dedup necessary. - let run_store = state.stores.runs.open_run(&run_id).await.unwrap(); - workflow_event::append_event( - &run_store, - &run_id, - &workflow_event::Event::CheckpointCompleted { - graph_visit: None, - resumed_from_stage_id: None, - node_id: "verify".to_string(), - status: "running".to_string(), - current_node: "verify".to_string(), - completed_nodes: vec!["verify".to_string(), "verify".to_string()], - node_retries: std::collections::BTreeMap::new(), - context_values: std::collections::BTreeMap::new(), - node_outcomes: std::collections::BTreeMap::from([( - "verify".to_string(), - Outcome::default(), - )]), - next_node_id: Some("done".to_string()), - git_commit_sha: None, - loop_failure_signatures: std::collections::BTreeMap::new(), - restart_failure_signatures: std::collections::BTreeMap::new(), - node_visits: std::collections::BTreeMap::from([("verify".to_string(), 2usize)]), - diff: None, - diff_summary: None, - }, - ) - .await - .unwrap(); - - let response = app - .clone() - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/usage"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - - let stages = body["stages"].as_array().unwrap(); - assert_eq!( - stages.len(), - 1, - "expected one row for the retried verify node" - ); - assert_eq!(stages[0]["stage"]["id"], "verify"); - // Duration on the row is the sum across visits (1.5s + 0.8s = 2.3s). - assert!( - stages[0]["timing"]["wall_time_ms"].as_u64().unwrap() == 2300, - "row runtime_secs should sum visits, got {}", - stages[0]["timing"]["wall_time_ms"] - ); - - // Totals must not double-count: a single 2.3s, not 4.6s. - assert!( - body["totals"]["timing"]["wall_time_ms"].as_u64().unwrap() == 2300, - "totals.runtime_secs should sum visits exactly once, got {}", - body["totals"]["timing"]["wall_time_ms"] - ); -} - -#[tokio::test] -async fn run_usage_sums_usage_across_retry_visits_and_uses_latest_model() { - let state = test_app_state_with_isolated_storage(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - - create_priced_retry_run(&state, run_id).await; - let success_usage = test_priced_usage("gpt-new", 200, 20); - let mut latest_outcome: Outcome> = Outcome::success(); - latest_outcome.usage = Some(success_usage); - latest_outcome.timing = Some(fabro_types::StageTiming::wall_only(800)); - let run_store = state.stores.runs.open_run(&run_id).await.unwrap(); - workflow_event::append_event( - &run_store, - &run_id, - &workflow_event::Event::CheckpointCompleted { - graph_visit: None, - resumed_from_stage_id: None, - node_id: "verify".to_string(), - status: "running".to_string(), - current_node: "verify".to_string(), - completed_nodes: vec!["verify".to_string(), "verify".to_string()], - node_retries: std::collections::BTreeMap::from([("verify".to_string(), 2)]), - context_values: std::collections::BTreeMap::new(), - node_outcomes: std::collections::BTreeMap::from([( - "verify".to_string(), - latest_outcome, - )]), - next_node_id: None, - git_commit_sha: None, - loop_failure_signatures: std::collections::BTreeMap::new(), - restart_failure_signatures: std::collections::BTreeMap::new(), - node_visits: std::collections::BTreeMap::from([("verify".to_string(), 2usize)]), - diff: None, - diff_summary: None, - }, - ) - .await - .unwrap(); - - let response = app - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/usage"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - - let stages = body["stages"].as_array().unwrap(); - assert_eq!(stages.len(), 1); - assert_eq!(stages[0]["stage"]["id"], "verify"); - assert_eq!(stages[0]["model"]["provider"], "openai"); - assert_eq!(stages[0]["model"]["model_id"], "gpt-new"); - assert_eq!(stages[0]["usage"]["tokens"]["input"], 300); - assert_eq!(stages[0]["usage"]["tokens"]["output"], 30); - assert_eq!(stages[0]["usage"]["cost"]["usd_micros"], 330); - assert!(stages[0]["timing"]["wall_time_ms"].as_u64().unwrap() == 2000); - - assert_eq!(body["totals"]["usage"]["tokens"]["input"], 300); - assert_eq!(body["totals"]["usage"]["tokens"]["output"], 30); - assert_eq!(body["totals"]["usage"]["cost"]["usd_micros"], 330); - assert!(body["totals"]["timing"]["wall_time_ms"].as_u64().unwrap() == 2000); - - let by_model = body["by_model"].as_array().unwrap(); - assert_eq!(by_model.len(), 2); - let old_model = by_model - .iter() - .find(|entry| entry["model"]["model_id"] == "gpt-old") - .unwrap(); - let new_model = by_model - .iter() - .find(|entry| entry["model"]["model_id"] == "gpt-new") - .unwrap(); - assert_eq!(old_model["model"]["provider"], "openai"); - assert_eq!(new_model["model"]["provider"], "openai"); - assert_eq!(old_model["stages"], 1); - assert_eq!(old_model["usage"]["tokens"]["input"], 100); - assert_eq!(new_model["stages"], 1); - assert_eq!(new_model["usage"]["tokens"]["input"], 200); -} - -/// The stage popover reads `usage` off the stages list, so it must be scoped -/// to one visit — unlike the Usage tab's rows, which sum every visit of a -/// node. -#[tokio::test] -async fn list_run_stages_reports_usage_per_visit() { - let state = test_app_state_with_isolated_storage(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - - create_priced_retry_run(&state, run_id).await; - - let response = app - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/stages"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - - let first = stage_entry(&body, "verify@1"); - assert_eq!(first["usage"]["tokens"]["input"], 100); - assert_eq!(first["usage"]["tokens"]["output"], 10); - assert_eq!(first["usage"]["cost"]["usd_micros"], 110); - - let second = stage_entry(&body, "verify@2"); - assert_eq!(second["usage"]["tokens"]["input"], 200); - assert_eq!(second["usage"]["tokens"]["output"], 20); - assert_eq!(second["usage"]["cost"]["usd_micros"], 220); -} - -#[tokio::test] -async fn list_run_stages_reports_zero_usage_for_a_stage_that_called_no_model() { - let state = test_app_state_with_isolated_storage(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - ]) - .await; - let started = stage_started_event("script", "command"); - append_scoped_stage_event(&state, run_id, "script", 1, &started).await; - - let response = app - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/stages"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - - let usage = &stage_entry(&body, "script@1")["usage"]; - assert_eq!(usage["tokens"]["input"], 0); - assert_eq!(usage["tokens"]["output"], 0); - // No model ran, so there is nothing to price — not a $0.00 cost. - assert!(usage.get("cost").is_none()); -} - -#[tokio::test] -async fn list_run_stages_shows_retrying_after_failed_event() { - let state = test_app_state_with_isolated_storage(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - ]) - .await; - - append_scoped_stage_event( - &state, - run_id, - "work", - 1, - &workflow_event::Event::StageStarted { - graph_visit: None, - resumed_from_stage_id: None, - node_id: "work".to_string(), - name: "Work".to_string(), - index: 0, - handler_type: "command".to_string(), - attempt: 1, - max_attempts: 3, - }, - ) - .await; - append_scoped_stage_event( - &state, - run_id, - "work", - 1, - &workflow_event::Event::StageFailed { - node_id: "work".to_string(), - name: "Work".to_string(), - index: 0, - failure: FailureDetail::new("flake", FailureCategory::TransientInfra), - will_retry: true, - timing: fabro_types::StageTiming::wall_only(5), - usage_by_model: Vec::new(), - usage: None, - actor: None, - }, - ) - .await; - append_scoped_stage_event( - &state, - run_id, - "work", - 1, - &workflow_event::Event::StageRetrying { - node_id: "work".to_string(), - name: "Work".to_string(), - index: 0, - attempt: 2, - max_attempts: 3, - delay_ms: 50, - }, - ) - .await; - - let response = app - .clone() - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/stages"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - assert_eq!(stage_status(&body, "work@1"), "retrying"); -} - -#[tokio::test] -async fn list_run_stages_shows_retrying_when_failed_will_retry() { - let state = test_app_state_with_isolated_storage(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - ]) - .await; - - append_scoped_stage_event( - &state, - run_id, - "work", - 1, - &workflow_event::Event::StageStarted { - graph_visit: None, - resumed_from_stage_id: None, - node_id: "work".to_string(), - name: "Work".to_string(), - index: 0, - handler_type: "command".to_string(), - attempt: 1, - max_attempts: 3, - }, - ) - .await; - // Only StageFailed, no StageRetrying yet — should still render retrying - // because props.will_retry is true. - append_scoped_stage_event( - &state, - run_id, - "work", - 1, - &workflow_event::Event::StageFailed { - node_id: "work".to_string(), - name: "Work".to_string(), - index: 0, - failure: FailureDetail::new("flake", FailureCategory::TransientInfra), - will_retry: true, - timing: fabro_types::StageTiming::wall_only(5), - usage_by_model: Vec::new(), - usage: None, - actor: None, - }, - ) - .await; - - let response = app - .clone() - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/stages"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - assert_eq!(stage_status(&body, "work@1"), "retrying"); -} - -#[tokio::test] -async fn run_usage_retried_node_then_succeeded_emits_one_row_with_final_attempt_duration() { - let state = test_app_state_with_isolated_storage(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - workflow_event::Event::StageStarted { - graph_visit: None, - resumed_from_stage_id: None, - node_id: "work".to_string(), - name: "Work".to_string(), - index: 0, - handler_type: "command".to_string(), - attempt: 1, - max_attempts: 3, - }, - workflow_event::Event::StageFailed { - node_id: "work".to_string(), - name: "Work".to_string(), - index: 0, - failure: FailureDetail::new("transient", FailureCategory::TransientInfra), - will_retry: true, - timing: fabro_types::StageTiming::wall_only(10), - usage_by_model: Vec::new(), - usage: None, - actor: None, - }, - workflow_event::Event::StageRetrying { - node_id: "work".to_string(), - name: "Work".to_string(), - index: 0, - attempt: 2, - max_attempts: 3, - delay_ms: 0, - }, - workflow_event::Event::StageStarted { - graph_visit: None, - resumed_from_stage_id: None, - node_id: "work".to_string(), - name: "Work".to_string(), - index: 0, - handler_type: "command".to_string(), - attempt: 2, - max_attempts: 3, - }, - workflow_event::Event::StageCompleted { - node_id: "work".to_string(), - name: "Work".to_string(), - index: 0, - timing: fabro_types::StageTiming::wall_only(25), - status: "succeeded".to_string(), - preferred_label: None, - suggested_next_ids: Vec::new(), - usage_by_model: Vec::new(), - usage: None, - failure: None, - notes: None, - files_touched: Vec::new(), - context_updates: None, - jump_to_node: None, - context_values: None, - node_visits: None, - loop_failure_signatures: None, - restart_failure_signatures: None, - response: None, - attempt: 2, - max_attempts: 3, - }, - ]) - .await; - - let response = app - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/usage"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - let stages = body["stages"].as_array().unwrap(); - assert_eq!(stages.len(), 1, "retry collapses to one row per node_id"); - let row = &stages[0]; - assert_eq!(row["stage"]["id"], "work"); - assert_eq!( - row["state"], "succeeded", - "final state mirrors the latest StageCompleted" - ); - let runtime = row["timing"]["wall_time_ms"].as_u64().unwrap(); - assert_eq!( - runtime, 25, - "runtime should equal final attempt's 25ms, got {runtime}" - ); -} - -fn revisit_test_started(node_id: &str) -> workflow_event::Event { - workflow_event::Event::StageStarted { - graph_visit: None, - resumed_from_stage_id: None, - node_id: node_id.to_string(), - name: node_id.to_string(), - index: 0, - handler_type: "command".to_string(), - attempt: 1, - max_attempts: 1, - } -} - -fn revisit_test_completed_with_visit( - node_id: &str, - duration_ms: u64, - visit: usize, -) -> workflow_event::Event { - let mut node_visits = std::collections::BTreeMap::new(); - node_visits.insert(node_id.to_string(), visit); - workflow_event::Event::StageCompleted { - node_id: node_id.to_string(), - name: node_id.to_string(), - index: 0, - timing: fabro_types::StageTiming::wall_only(duration_ms), - status: "succeeded".to_string(), - preferred_label: None, - suggested_next_ids: Vec::new(), - usage_by_model: Vec::new(), - usage: None, - failure: None, - notes: None, - files_touched: Vec::new(), - context_updates: None, - jump_to_node: None, - context_values: None, - node_visits: Some(node_visits), - loop_failure_signatures: None, - restart_failure_signatures: None, - response: None, - attempt: 1, - max_attempts: 1, - } -} - -#[tokio::test] -async fn run_usage_revisited_node_collapses_to_two_rows_with_summed_visit_duration() { - let state = test_app_state_with_isolated_storage(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - // A → B → A loop. Per-visit `node_visits` payload steers the reducer - // to attribute each StageCompleted to the right visit. - revisit_test_started("a"), - revisit_test_completed_with_visit("a", 1, 1), - revisit_test_started("b"), - revisit_test_completed_with_visit("b", 2, 1), - revisit_test_started("a"), - revisit_test_completed_with_visit("a", 99, 2), - ]) - .await; - - let response = app - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/usage"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - let stages = body["stages"].as_array().unwrap(); - assert_eq!(stages.len(), 2, "two distinct node_ids → two rows"); - assert_eq!( - stages[0]["stage"]["id"], "a", - "A appeared first → A's row first" - ); - assert_eq!(stages[1]["stage"]["id"], "b"); - let a_runtime = stages[0]["timing"]["wall_time_ms"].as_u64().unwrap(); - assert_eq!( - a_runtime, 100, - "A should sum both visit durations (1ms + 99ms), got {a_runtime}" - ); - let b_runtime = stages[1]["timing"]["wall_time_ms"].as_u64().unwrap(); - assert_eq!( - b_runtime, 2, - "B should carry its single visit's duration (2ms), got {b_runtime}" - ); -} - async fn append_raw_run_event( state: &Arc, run_id: RunId, @@ -7565,54 +4889,6 @@ async fn append_raw_run_event( run_store.append_event(&payload).await.unwrap(); } -async fn create_unreadable_durable_run(state: &Arc, run_id: RunId) { - let run_store = state.stores.runs.create_run(&run_id).await.unwrap(); - append_default_run_created(&run_store, run_id).await; - workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunRunnable { - source: fabro_types::RunRunnableSource::StartRequested, - actor: None, - }) - .await - .unwrap(); - workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunStarting) - .await - .unwrap(); - workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunRunning) - .await - .unwrap(); - let seq = run_store.last_event_seq().await.unwrap().unwrap() + 1; - let completed = workflow_event::to_run_event_at( - &run_id, - &workflow_event::Event::WorkflowRunCompleted { - timing: fabro_types::RunTiming::wall_only(1), - artifact_count: 0, - status: "legacy-status".to_string(), - reason: SuccessReason::Completed, - final_git_commit_sha: None, - final_patch: None, - diff_summary: None, - usage: None, - }, - "2026-05-05T20:46:33Z".parse().unwrap(), - None, - ); - let payload = workflow_event::build_redacted_event_payload(&completed, &run_id).unwrap(); - fabro_store::test_support::put_unvalidated_run_event( - &state.stores.runs, - &run_id, - seq, - payload.as_value(), - ) - .await - .unwrap(); - let unreadable = state.stores.runs.open_run_reader(&run_id).await; - let err = unreadable.expect_err("poison event should make the run projection unreadable"); - assert!( - err.to_string().contains("invalid completed stage status"), - "unexpected projection error: {err}" - ); -} - fn github_token_settings() -> ServerSettings { ServerSettingsBuilder::from_toml( r#" @@ -7767,18 +5043,6 @@ async fn github_token_strategy_reads_github_token_from_vault() { ); } -/// Build the (state, router, run_id) triple every PR-endpoint test -/// needs. Use this instead of repeating the -/// state/build_router/fixtures::RUN_1 incantation per test. -fn pr_test_app( - token: Option<&str>, - github_api_base_url: Option, -) -> (Arc, Router, RunId) { - let state = create_github_token_app_state(token, github_api_base_url); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - (state, app, fixtures::RUN_1) -} - /// Same as [`pr_test_app`] but creates a fresh minimal run via the /// HTTP create-run endpoint instead of using fixtures::RUN_1. For /// tests that exercise endpoints expecting a real on-disk run rather @@ -7793,170 +5057,6 @@ async fn pr_test_app_with_minimal_run( (state, app, run_id) } -/// Same as [`pr_test_app`] but the run is set up as a completed -/// workflow ready for `POST /runs/{id}/pull_request`. The branches -/// and diff are fixed defaults; only the origin URL varies per -/// test (None to test missing-origin rejection, gitlab.com to test -/// non-github rejection, etc.). -async fn pr_test_app_with_completed_run( - token: Option<&str>, - github_api_base_url: Option, - repo_origin_url: Option<&str>, -) -> (Arc, Router, RunId) { - let (state, app, run_id) = pr_test_app(token, github_api_base_url); - Box::pin(create_completed_run_ready_for_pull_request( - &state, - run_id, - repo_origin_url, - Some("main"), - Some("fabro/run/42"), - "diff --git a/src/lib.rs b/src/lib.rs\n+fn shipped() {}\n", - )) - .await; - (state, app, run_id) -} - -async fn create_run_with_pull_request_record( - state: &Arc, - run_id: RunId, - pr_url: &str, - pr_number: u64, - title: &str, -) { - create_durable_run_with_events(state, run_id, &[ - workflow_event::Event::PullRequestCreated { - pr_url: pr_url.to_string(), - pr_number, - owner: "acme".to_string(), - repo: "widgets".to_string(), - base_branch: "main".to_string(), - head_branch: "feature".to_string(), - head_sha: Some("final-sha".to_string()), - title: title.to_string(), - draft: false, - }, - ]) - .await; -} - -async fn create_run_with_linked_pull_request_record( - state: &Arc, - run_id: RunId, - pull_request: PullRequestLink, -) { - create_durable_run_with_events(state, run_id, &[workflow_event::Event::PullRequestLinked { - pull_request, - }]) - .await; -} - -async fn create_completed_run_ready_for_pull_request( - state: &Arc, - run_id: RunId, - repo_origin_url: Option<&str>, - base_branch: Option<&str>, - run_branch: Option<&str>, - final_patch: &str, -) { - let mut graph = Graph::new("test"); - graph.attrs.insert( - "goal".to_string(), - AttrValue::String("Ship the server-side PR".to_string()), - ); - let git = match (repo_origin_url, base_branch) { - (Some(origin), Some(branch)) => Some(fabro_types::GitContext { - origin_url: origin.to_string(), - branch: branch.to_string(), - sha: None, - dirty: fabro_types::DirtyStatus::Clean, - }), - _ => None, - }; - let run_spec = RunSpec { - run_id, - settings: fabro_types::WorkflowSettings::default(), - graph, - graph_source: None, - workflow_slug: Some("test".to_string()), - workflow_version_id: None, - target: None, - automation: None, - source_directory: Some("/tmp/project".to_string()), - git: git.clone(), - labels: HashMap::new(), - provenance: test_support::test_run_provenance(), - definition_blob: None, - spec_blob: None, - fork_source_ref: None, - admission: PetriAdmission::default(), - }; - - create_durable_run_with_events(state, run_id, &[ - workflow_event::Event::RunCreated { - run_id, - title: None, - settings: serde_json::to_value(&run_spec.settings).unwrap(), - graph: serde_json::to_value(&run_spec.graph).unwrap(), - workflow_source: None, - labels: run_spec.labels.clone().into_iter().collect(), - source_directory: run_spec.source_directory.clone(), - workflow_slug: run_spec.workflow_slug.clone(), - workflow_version_id: run_spec.workflow_version_id, - target: run_spec.target.clone(), - automation: None, - provenance: run_spec.provenance.clone(), - spec_blob: None, - git, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, - admission: PetriAdmission::default(), - }, - workflow_event::Event::WorkflowRunStarted { - name: "test".to_string(), - run_id, - base_branch: base_branch.map(str::to_string), - base_sha: None, - run_branch: run_branch.map(str::to_string), - worktree_dir: None, - goal: Some("Ship the server-side PR".to_string()), - }, - workflow_event::Event::WorkflowRunCompleted { - timing: fabro_types::RunTiming::wall_only(1), - artifact_count: 0, - status: "succeeded".to_string(), - reason: SuccessReason::Completed, - final_git_commit_sha: Some("final-sha".to_string()), - final_patch: Some(final_patch.to_string()), - diff_summary: None, - usage: None, - }, - ]) - .await; -} - -fn test_event_envelope(seq: u32, run_id: RunId, body: EventBody) -> EventEnvelope { - EventEnvelope { - seq, - event: RunEvent { - id: format!("evt-{seq}"), - ts: Utc::now(), - run_id, - node_id: None, - node_label: None, - stage_id: None, - parallel_group_id: None, - parallel_branch_id: None, - session_id: None, - parent_session_id: None, - tool_call_id: None, - actor: None, - body, - }, - } -} - #[tokio::test] async fn test_model_unknown_returns_404() { let app = test_app_with(); @@ -9151,51 +6251,6 @@ async fn get_run_status_returns_status() { assert!(body["labels"].is_object()); } -#[tokio::test] -async fn get_run_status_advances_live_active_timing_between_events() { - let state = test_app_state_with_isolated_storage(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - workflow_run_started_event(run_id), - ]) - .await; - append_scoped_stage_event( - &state, - run_id, - "work", - 1, - &stage_started_event("work", "command"), - ) - .await; - - // The SQLite summary stores timing at the StageStarted event. A later - // detail read must overlay the in-flight command's active time from the - // projection even though no newer event has arrived. - tokio::time::sleep(std::time::Duration::from_millis(20)).await; - let response = app - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - let timing = &body["timing"]; - - assert!(timing["active_time_ms"].as_u64().unwrap() > 0); - assert_eq!(timing["tool_time_ms"], timing["active_time_ms"]); - assert!(timing["wall_time_ms"].as_u64().unwrap() >= timing["active_time_ms"].as_u64().unwrap()); -} - #[tokio::test] async fn get_run_status_not_found() { let app = test_app_with(); @@ -9619,44 +6674,6 @@ async fn get_run_state_returns_projection() { assert!(body["stages"].is_object()); } -#[tokio::test] -async fn get_run_logs_returns_per_run_log_file() { - let state = test_app_state_with_isolated_storage(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - create_durable_run_with_events(&state, run_id, &[workflow_event::Event::RunSubmitted { - definition_blob: None, - }]) - .await; - let log_path = Storage::new(state.server_storage_dir()) - .run_scratch(&run_id) - .runtime_dir() - .join("server.log"); - tokio::fs::create_dir_all(log_path.parent().unwrap()) - .await - .unwrap(); - tokio::fs::write(&log_path, b"worker log line\nsecond line\n") - .await - .unwrap(); - - let req = Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/logs"))) - .body(Body::empty()) - .unwrap(); - - let response = app.oneshot(req).await.unwrap(); - let content_type = response - .headers() - .get(header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()) - .map(str::to_owned); - let body = response_bytes!(response, StatusCode::OK).await; - - assert_eq!(content_type.as_deref(), Some("text/plain; charset=utf-8")); - assert_eq!(&body[..], b"worker log line\nsecond line\n"); -} - #[tokio::test] async fn get_run_logs_returns_not_found_for_missing_run() { let state = test_app_state_with_isolated_storage(); @@ -9673,111 +6690,6 @@ async fn get_run_logs_returns_not_found_for_missing_run() { assert_status!(response, StatusCode::NOT_FOUND).await; } -#[tokio::test] -async fn get_run_logs_returns_not_found_when_log_file_is_missing() { - let state = test_app_state_with_isolated_storage(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - create_durable_run_with_events(&state, run_id, &[workflow_event::Event::RunSubmitted { - definition_blob: None, - }]) - .await; - - let req = Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/logs"))) - .body(Body::empty()) - .unwrap(); - - let response = app.oneshot(req).await.unwrap(); - assert_status!(response, StatusCode::NOT_FOUND).await; -} - -#[tokio::test] -async fn get_run_stage_command_log_returns_cas_slice() { - let state = test_app_state_with_isolated_storage(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - let run_store = state.stores.runs.create_run(&run_id).await.unwrap(); - append_default_run_created(&run_store, run_id).await; - let output_blob = run_store - .write_blob(&serde_json::to_vec("hello world").unwrap()) - .await - .unwrap(); - let output_ref = format!("blob://sha256/{output_blob}"); - for event in [ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::StageStarted { - graph_visit: None, - resumed_from_stage_id: None, - node_id: "script_node".to_string(), - name: "Script".to_string(), - index: 1, - handler_type: "command".to_string(), - attempt: 1, - max_attempts: 1, - }, - workflow_event::Event::CommandCompleted { - node_id: "script_node".to_string(), - output: output_ref.clone(), - exit_code: Some(0), - duration_ms: 5, - termination: CommandTermination::Exited, - output_bytes: 11, - live_streaming: false, - }, - ] { - workflow_event::append_event(&run_store, &run_id, &event) - .await - .unwrap(); - } - - let req = Request::builder() - .method("GET") - .uri(api(&format!( - "/runs/{run_id}/stages/script_node@1/logs/output?offset=6&limit=5" - ))) - .body(Body::empty()) - .unwrap(); - - let response = app.oneshot(req).await.unwrap(); - let body = response_json!(response, StatusCode::OK).await; - let bytes = BASE64_STANDARD - .decode(body["bytes_base64"].as_str().unwrap()) - .unwrap(); - - assert!(body.get("stream").is_none()); - assert_eq!(body["offset"], 6); - assert_eq!(body["next_offset"], 11); - assert_eq!(body["total_bytes"], 11); - assert_eq!(bytes, b"world"); - assert_eq!(body["eof"], true); - assert_eq!(body["cas_ref"], output_ref); - assert_eq!(body["live_streaming"], false); -} - -#[tokio::test] -async fn get_run_stage_command_log_returns_not_found_for_missing_stage() { - let state = test_app_state_with_isolated_storage(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - create_durable_run_with_events(&state, run_id, &[workflow_event::Event::RunSubmitted { - definition_blob: None, - }]) - .await; - - let req = Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/stages/missing@1/logs/output"))) - .body(Body::empty()) - .unwrap(); - - let response = app.oneshot(req).await.unwrap(); - assert_status!(response, StatusCode::NOT_FOUND).await; -} - #[tokio::test] async fn get_run_stage_context_window_returns_not_found_for_missing_run() { let app = crate::test_support::build_test_router(test_app_state_with_isolated_storage()); @@ -9799,288 +6711,6 @@ async fn get_run_stage_context_window_returns_not_found_for_missing_run() { assert_status!(response, StatusCode::NOT_FOUND).await; } -#[tokio::test] -async fn get_run_stage_context_window_returns_not_found_for_missing_stage() { - let state = test_app_state_with_isolated_storage(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - create_durable_run_with_events(&state, run_id, &[workflow_event::Event::RunSubmitted { - definition_blob: None, - }]) - .await; - - let response = app - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!( - "/runs/{run_id}/stages/missing@1/context-window" - ))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - - assert_status!(response, StatusCode::NOT_FOUND).await; -} - -#[tokio::test] -async fn get_run_stage_context_window_returns_unavailable_for_non_agent_stage() { - let state = test_app_state_with_isolated_storage(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - stage_started_event("script_node", "command"), - command_started_event("script_node"), - ]) - .await; - - let body = response_json!( - app.oneshot( - Request::builder() - .method("GET") - .uri(api(&format!( - "/runs/{run_id}/stages/script_node@1/context-window" - ))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(), - StatusCode::OK - ) - .await; - - assert_eq!(body["available"], false); - assert_eq!(body["unavailable_reason"], "not_agent_stage"); - assert_eq!(body["breakdown"], json!([])); - assert_eq!(body["staleness"], "unavailable"); -} - -#[tokio::test] -async fn get_run_stage_context_window_returns_not_observed_for_agent_stage_without_snapshot() { - let state = test_app_state_with_isolated_storage(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - agent_session_activated_event("agent_node", 1), - ]) - .await; - - let body = response_json!( - app.oneshot( - Request::builder() - .method("GET") - .uri(api(&format!( - "/runs/{run_id}/stages/agent_node@1/context-window" - ))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(), - StatusCode::OK - ) - .await; - - assert_eq!(body["available"], false); - assert_eq!(body["unavailable_reason"], "not_observed"); - assert_eq!(body["input_tokens"], serde_json::Value::Null); - assert!(!body["warnings"].as_array().unwrap().is_empty()); -} - -#[tokio::test] -async fn get_run_stage_context_window_returns_live_projected_snapshot() { - let state = test_app_state_with_isolated_storage(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - stage_started_event("agent_node", "agent"), - context_window_event( - "agent_node", - 1, - context_window_snapshot(123_456, Vec::new()), - ), - ]) - .await; - - let body = response_json!( - app.oneshot( - Request::builder() - .method("GET") - .uri(api(&format!( - "/runs/{run_id}/stages/agent_node@1/context-window" - ))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(), - StatusCode::OK - ) - .await; - - assert_eq!(body["stage_id"], "agent_node@1"); - assert_eq!(body["available"], true); - assert_eq!(body["provider"], "openai"); - assert_eq!(body["count_method"], "response_usage_scaled_breakdown"); - assert_eq!(body["staleness"], "live"); - assert_eq!(body["input_tokens"], 123_456); - assert_eq!(body["breakdown"][0]["category"], "conversation"); -} - -#[tokio::test] -async fn get_run_stage_context_window_marks_completed_stage_snapshot_stored() { - let state = test_app_state_with_isolated_storage(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - stage_started_event("agent_node", "agent"), - context_window_event("agent_node", 1, context_window_snapshot(100, Vec::new())), - stage_completed_event("agent_node"), - ]) - .await; - - let body = response_json!( - app.oneshot( - Request::builder() - .method("GET") - .uri(api(&format!( - "/runs/{run_id}/stages/agent_node@1/context-window" - ))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(), - StatusCode::OK - ) - .await; - - assert_eq!(body["available"], true); - assert_eq!(body["staleness"], "stored"); - assert_eq!(body["input_tokens"], 100); -} - -#[tokio::test] -async fn get_run_stage_context_window_returns_projected_warnings() { - let state = test_app_state_with_isolated_storage(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - stage_started_event("agent_node", "agent"), - context_window_event( - "agent_node", - 1, - context_window_snapshot(100, vec![ContextWindowWarning { - code: "provider_token_count_failed".to_string(), - message: "provider input token counting failed; returned local estimate" - .to_string(), - }]), - ), - ]) - .await; - - let body = response_json!( - app.oneshot( - Request::builder() - .method("GET") - .uri(api(&format!( - "/runs/{run_id}/stages/agent_node@1/context-window" - ))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(), - StatusCode::OK - ) - .await; - - assert_eq!(body["warnings"][0]["code"], "provider_token_count_failed"); -} - -#[tokio::test] -async fn get_run_pull_request_returns_live_detail_from_github() { - let github = MockServer::start(); - let github_mock = github.mock(|when, then| { - when.method("GET") - .path("/repos/acme/widgets/pulls/42") - .header("authorization", "Bearer ghu_test"); - then.status(200) - .header("content-type", "application/json") - .body( - json!({ - "number": 42, - "title": "Fix the bug", - "body": "Detailed description", - "state": "closed", - "draft": false, - "merged": true, - "merged_at": "2026-04-23T15:45:00Z", - "mergeable": false, - "additions": 10, - "deletions": 3, - "changed_files": 2, - "html_url": "https://github.com/acme/widgets/pull/42", - "user": { "login": "testuser" }, - "head": { "ref": "feature" }, - "base": { "ref": "main" }, - "created_at": "2026-04-23T15:40:00Z", - "updated_at": "2026-04-23T15:45:00Z" - }) - .to_string(), - ); - }); - let (state, app, run_id) = pr_test_app(Some("ghu_test"), Some(github.base_url())); - - create_run_with_pull_request_record( - &state, - run_id, - "https://github.com/acme/widgets/pull/42", - 42, - "Fix the bug", - ) - .await; - - let response = app - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/pull_request"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - - assert_eq!(body["data"]["link"]["number"], 42); - assert_eq!(body["data"]["link"]["owner"], "acme"); - assert_eq!(body["data"]["details"]["state"], "closed"); - assert_eq!(body["data"]["details"]["merged"], true); - assert_eq!(body["data"]["details"]["head_branch"], "feature"); - assert_eq!(body["data"]["details"]["base_branch"], "main"); - assert_eq!(body["meta"]["details_status"], "available"); - github_mock.assert(); -} - #[tokio::test] async fn get_run_pull_request_returns_not_found_when_record_missing() { let state = test_app_state(); @@ -10235,628 +6865,6 @@ async fn unlink_run_pull_request_appends_event_and_clears_projected_state() { })); } -#[tokio::test] -async fn get_run_pull_request_returns_stored_github_association_without_github_credentials() { - let (state, app, run_id) = pr_test_app(None, None); - - create_run_with_pull_request_record( - &state, - run_id, - "https://github.com/acme/widgets/pull/42", - 42, - "Fix the bug", - ) - .await; - - let response = app - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/pull_request"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - - assert_eq!(body["data"]["link"]["number"], 42); - assert_eq!( - body["data"]["link"]["html_url"], - "https://github.com/acme/widgets/pull/42" - ); - assert!(body["data"]["details"].is_null()); - assert_eq!(body["meta"]["details_status"], "unavailable"); - assert_eq!( - body["meta"]["details_unavailable_reason"], - "integration_unavailable" - ); -} - -#[tokio::test] -async fn get_run_pull_request_returns_stored_github_association_when_github_pr_is_missing() { - let github = MockServer::start(); - let github_mock = github.mock(|when, then| { - when.method("GET") - .path("/repos/acme/widgets/pulls/42") - .header("authorization", "Bearer ghu_test"); - then.status(404) - .header("content-type", "application/json") - .body(json!({ "message": "Not Found" }).to_string()); - }); - let (state, app, run_id) = pr_test_app(Some("ghu_test"), Some(github.base_url())); - - create_run_with_pull_request_record( - &state, - run_id, - "https://github.com/acme/widgets/pull/42", - 42, - "Fix the bug", - ) - .await; - - let response = app - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/pull_request"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - - assert_eq!(body["data"]["link"]["number"], 42); - assert_eq!( - body["data"]["link"]["html_url"], - "https://github.com/acme/widgets/pull/42" - ); - assert!(body["data"]["details"].is_null()); - assert_eq!(body["meta"]["details_status"], "unavailable"); - assert_eq!(body["meta"]["details_unavailable_reason"], "not_found"); - github_mock.assert(); -} - -#[tokio::test] -async fn pull_request_creation_recovers_durable_request_after_crash_gap() { - let github = MockServer::start(); - let branch_mock = github.mock(|when, then| { - when.method("GET") - .path("/repos/acme/widgets/branches/fabro/run/42") - .header("authorization", "Bearer ghu_test"); - then.status(200) - .header("content-type", "application/json") - .body(json!({ "commit": { "sha": "final-sha" } }).to_string()); - }); - let create_mock = github.mock(|when, then| { - when.method("POST") - .path("/repos/acme/widgets/pulls") - .header("authorization", "Bearer ghu_test"); - then.status(201) - .header("content-type", "application/json") - .body( - json!({ - "html_url": "https://github.com/acme/widgets/pull/42", - "number": 42, - "node_id": "PR_kwDOAA" - }) - .to_string(), - ); - }); - let find_mock = github.mock(|when, then| { - when.method("GET") - .path("/repos/acme/widgets/pulls") - .query_param("state", "open") - .query_param("base", "main") - .query_param("head", "acme:fabro/run/42") - .header("authorization", "Bearer ghu_test"); - then.status(200) - .header("content-type", "application/json") - .body("[]"); - }); - let llm = MockServer::start_async().await; - let response_mock = llm - .mock_async(|when, then| { - when.method(POST) - .path("/v1/responses") - .header("authorization", "Bearer openai-key"); - then.status(200) - .header("content-type", "application/json") - .json_body(openai_responses_payload( - &serde_json::to_string(&json!({ - "title": "Mock title", - "body": "Narrative from mock.", - })) - .unwrap(), - )); - }) - .await; - let state = create_github_token_app_state_with_env_lookup_and_llm_catalog_settings( - Some("ghu_test"), - Some(github.base_url()), - |_| None, - llm_overlay_with_provider_base_url("openai", llm.url("/v1")), - ); - state - .stores - .vault - .set("OPENAI_API_KEY", "openai-key", SecretType::Token, None) - .await - .unwrap(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = fixtures::RUN_1; - Box::pin(create_completed_run_ready_for_pull_request( - &state, - run_id, - Some("git@github.com:acme/widgets.git"), - Some("main"), - Some("fabro/run/42"), - "diff --git a/src/lib.rs b/src/lib.rs\n+fn shipped() {}\n", - )) - .await; - - let response = app - .clone() - .oneshot( - Request::builder() - .method("POST") - .uri(api(&format!("/runs/{run_id}/pull_request"))) - .header("content-type", "application/json") - .body(Body::from( - json!({ - "force": false, - "model": "gpt-5.4" - }) - .to_string(), - )) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!( - response.headers().get(header::LOCATION).unwrap(), - &format!("/api/v1/runs/{run_id}/pull_request/creation") - ); - let body = response_json!(response, StatusCode::ACCEPTED).await; - - assert_eq!(body["status"], "pending"); - assert_eq!(body["model"], "gpt-5.4"); - assert_eq!(state.pull_request_creation_queue_len(), 1); - - // Starting the supervisor after the request simulates server recovery: - // the durable pending event is enough to resume the operation. - let _ = state.drain_pull_request_creation_queue(); - let supervisor = spawn_pull_request_creation_supervisor(Arc::clone(&state)); - - let creation_body = wait_for_pull_request_creation(&app, run_id).await; - - assert_eq!(creation_body["status"], "succeeded"); - assert_eq!(creation_body["pull_request"]["number"], 42); - assert_eq!(creation_body["pull_request"]["owner"], "acme"); - assert_eq!(creation_body["pull_request"]["repo"], "widgets"); - assert_eq!( - creation_body["pull_request"]["html_url"], - "https://github.com/acme/widgets/pull/42" - ); - - let state_response = app - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/state"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let state_body = response_json!(state_response, StatusCode::OK).await; - assert_eq!(state_body["pull_request"]["number"], 42); - assert_eq!(state_body["pull_request"]["owner"], "acme"); - assert_eq!(state_body["pull_request"]["repo"], "widgets"); - - response_mock.assert_async().await; - branch_mock.assert(); - find_mock.assert(); - create_mock.assert(); - state.shutdown_token().cancel(); - supervisor.await.unwrap(); -} - -#[tokio::test] -async fn pull_request_creation_returns_the_active_durable_request() { - let github = MockServer::start(); - let (state, app, run_id) = Box::pin(pr_test_app_with_completed_run( - Some("ghu_test"), - Some(github.base_url()), - Some("https://github.com/acme/widgets.git"), - )) - .await; - - let configured_provider_ids = state - .ready_llm_provider_ids() - .await - .into_iter() - .collect::>(); - let expected_default_model = state - .catalog() - .default_offering_for(&configured_provider_ids) - .expect("a ready provider should have a default model") - .model - .id() - .to_string(); - let request_body = json!({ - "force": false, - "model": null - }) - .to_string(); - let first = app - .clone() - .oneshot( - Request::builder() - .method("POST") - .uri(api(&format!("/runs/{run_id}/pull_request"))) - .header("content-type", "application/json") - .body(Body::from(request_body.clone())) - .unwrap(), - ) - .await - .unwrap(); - let first_body = response_json!(first, StatusCode::ACCEPTED).await; - - let second = app - .oneshot( - Request::builder() - .method("POST") - .uri(api(&format!("/runs/{run_id}/pull_request"))) - .header("content-type", "application/json") - .body(Body::from(request_body)) - .unwrap(), - ) - .await - .unwrap(); - let second_body = response_json!(second, StatusCode::ACCEPTED).await; - - assert_eq!(first_body["id"], second_body["id"]); - assert_eq!(first_body["model"], expected_default_model); - assert_eq!(state.pull_request_creation_queue_len(), 1); - let run_store = state.stores.runs.open_run_reader(&run_id).await.unwrap(); - let events = run_store.list_events().await.unwrap(); - assert_eq!( - events - .iter() - .filter(|event| event.event.event_name() == "pull_request.creation_requested") - .count(), - 1 - ); -} - -#[tokio::test] -async fn pull_request_creation_queue_overflow_recovers_from_indexed_scan() { - let state = test_app_state(); - let mut creation_ids = HashMap::new(); - for _ in 0..17 { - let run_id = RunId::new(); - let creation_id = fabro_types::PullRequestCreationId::new(); - creation_ids.insert(run_id, creation_id); - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::PullRequestCreationRequested { - creation_id, - model: "test-model".to_string(), - force: false, - }, - ]) - .await; - } - - pull_request_supervisor::recover_pending_pull_request_creations( - state.as_ref(), - &HashMap::new(), - &HashMap::new(), - ) - .await - .unwrap(); - let first_batch = state.drain_pull_request_creation_queue(); - assert_eq!(first_batch.len(), 16); - - for run_id in first_batch { - let run_store = state.stores.runs.open_run(&run_id).await.unwrap(); - workflow_event::append_event( - &run_store, - &run_id, - &workflow_event::Event::PullRequestFailed { - creation_id: creation_ids.get(&run_id).copied(), - error: "test failure".to_string(), - }, - ) - .await - .unwrap(); - } - - pull_request_supervisor::recover_pending_pull_request_creations( - state.as_ref(), - &HashMap::new(), - &HashMap::new(), - ) - .await - .unwrap(); - let recovered = state.drain_pull_request_creation_queue(); - assert_eq!(recovered.len(), 1); - let recovered_id = recovered[0]; - let projection = state - .stores - .runs - .open_run_reader(&recovered_id) - .await - .unwrap() - .state() - .await - .unwrap(); - assert!( - projection - .pull_request_creation - .as_ref() - .is_some_and(fabro_types::PullRequestCreation::is_pending) - ); -} - -#[tokio::test] -async fn pull_request_creation_persists_generation_failure() { - let github = MockServer::start(); - let branch_mock = github.mock(|when, then| { - when.method("GET") - .path("/repos/acme/widgets/branches/fabro/run/42") - .header("authorization", "Bearer ghu_test"); - then.status(200) - .header("content-type", "application/json") - .body(json!({ "commit": { "sha": "final-sha" } }).to_string()); - }); - let find_mock = github.mock(|when, then| { - when.method("GET") - .path("/repos/acme/widgets/pulls") - .query_param("state", "open") - .query_param("base", "main") - .query_param("head", "acme:fabro/run/42") - .header("authorization", "Bearer ghu_test"); - then.status(200) - .header("content-type", "application/json") - .body("[]"); - }); - let (state, app, run_id) = Box::pin(pr_test_app_with_completed_run( - Some("ghu_test"), - Some(github.base_url()), - Some("https://github.com/acme/widgets.git"), - )) - .await; - - let response = app - .clone() - .oneshot( - Request::builder() - .method("POST") - .uri(api(&format!("/runs/{run_id}/pull_request"))) - .header("content-type", "application/json") - .body(Body::from( - json!({ "force": false, "model": "gpt-5.4" }).to_string(), - )) - .unwrap(), - ) - .await - .unwrap(); - response_json!(response, StatusCode::ACCEPTED).await; - let supervisor = spawn_pull_request_creation_supervisor(Arc::clone(&state)); - - let creation = wait_for_pull_request_creation(&app, run_id).await; - - assert_eq!(creation["status"], "failed"); - // The unconfigured LLM is what fails this fixture; pin the error to the - // generation step so the test cannot pass on an earlier validation error. - assert!( - creation["error"] - .as_str() - .is_some_and(|error| error.contains("LLM generation failed")), - "unexpected error: {:?}", - creation["error"] - ); - assert!(creation["pull_request"].is_null()); - branch_mock.assert(); - find_mock.assert(); - state.shutdown_token().cancel(); - supervisor.await.unwrap(); -} - -#[tokio::test] -async fn create_run_pull_request_returns_conflict_when_record_exists() { - let (state, app, run_id) = pr_test_app(None, None); - - create_run_with_pull_request_record( - &state, - run_id, - "https://github.com/acme/widgets/pull/42", - 42, - "Fix the bug", - ) - .await; - - let response = app - .oneshot( - Request::builder() - .method("POST") - .uri(api(&format!("/runs/{run_id}/pull_request"))) - .header("content-type", "application/json") - .body(Body::from( - json!({ "force": false, "model": null }).to_string(), - )) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::CONFLICT).await; - - assert_eq!(body["errors"][0]["code"], "pull_request_exists"); - assert!( - body["errors"][0]["detail"] - .as_str() - .unwrap() - .contains("https://github.com/acme/widgets/pull/42") - ); -} - -#[tokio::test] -async fn pull_request_creation_rejects_missing_repo_origin_without_enqueue() { - let (state, app, run_id) = Box::pin(pr_test_app_with_completed_run(None, None, None)).await; - - let response = app - .oneshot( - Request::builder() - .method("POST") - .uri(api(&format!("/runs/{run_id}/pull_request"))) - .header("content-type", "application/json") - .body(Body::from( - json!({ - "force": false, - "model": "claude-sonnet-4-6" - }) - .to_string(), - )) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::BAD_REQUEST).await; - - assert_eq!(body["errors"][0]["code"], "missing_repo_origin"); - assert_eq!(state.pull_request_creation_queue_len(), 0); -} - -#[tokio::test] -async fn pull_request_creation_rejects_missing_credentials_without_enqueue() { - let (state, app, run_id) = Box::pin(pr_test_app_with_completed_run( - None, - None, - Some("https://github.com/acme/widgets.git"), - )) - .await; - - let response = app - .oneshot( - Request::builder() - .method("POST") - .uri(api(&format!("/runs/{run_id}/pull_request"))) - .header("content-type", "application/json") - .body(Body::from( - json!({ - "force": false, - "model": "claude-sonnet-4-6" - }) - .to_string(), - )) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::SERVICE_UNAVAILABLE).await; - - assert_eq!(state.pull_request_creation_queue_len(), 0); - - assert_eq!(body["errors"][0]["code"], "integration_unavailable"); -} - -#[tokio::test] -async fn create_run_pull_request_rejects_non_github_origin_url() { - let (_state, app, run_id) = Box::pin(pr_test_app_with_completed_run( - Some("ghu_test"), - None, - Some("https://gitlab.com/acme/widgets.git"), - )) - .await; - - let response = app - .oneshot( - Request::builder() - .method("POST") - .uri(api(&format!("/runs/{run_id}/pull_request"))) - .header("content-type", "application/json") - .body(Body::from( - json!({ - "force": false, - "model": "claude-sonnet-4-6" - }) - .to_string(), - )) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::BAD_REQUEST).await; - - assert_eq!(body["errors"][0]["code"], "unsupported_host"); -} - -#[tokio::test] -async fn pull_request_endpoints_use_github_base_url_captured_at_startup() { - let github = MockServer::start(); - let captured_mock = github.mock(|when, then| { - when.method("GET") - .path("/repos/acme/widgets/pulls/42") - .header("authorization", "Bearer ghu_test"); - then.status(200) - .header("content-type", "application/json") - .body( - json!({ - "number": 42, - "title": "Captured", - "body": "", - "state": "open", - "draft": false, - "merged": false, - "mergeable": true, - "additions": 1, - "deletions": 0, - "changed_files": 1, - "html_url": "https://github.com/acme/widgets/pull/42", - "user": { "login": "octocat" }, - "head": { "ref": "feature" }, - "base": { "ref": "main" }, - "created_at": "2026-04-23T12:00:00Z", - "updated_at": "2026-04-23T12:00:00Z" - }) - .to_string(), - ); - }); - let state = create_github_token_app_state(Some("ghu_test"), Some(github.base_url())); - assert_eq!(state.github_api_base_url, github.base_url()); - - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = fixtures::RUN_1; - create_run_with_pull_request_record( - &state, - run_id, - "https://github.com/acme/widgets/pull/42", - 42, - "Captured", - ) - .await; - - let response = app - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/pull_request"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - response_json!(response, StatusCode::OK).await; - - // If the handler read GITHUB_BASE_URL at request time instead of using the - // value captured at AppState construction, the outbound call would miss - // this mock — no other server is running at the captured URL, and the - // process env default points elsewhere. - captured_mock.assert(); -} - #[tokio::test] async fn merge_run_pull_request_returns_not_found_when_record_missing() { let (_state, app, run_id) = pr_test_app_with_minimal_run(Some("ghu_test"), None).await; @@ -10877,102 +6885,6 @@ async fn merge_run_pull_request_returns_not_found_when_record_missing() { assert_eq!(body["errors"][0]["code"], "no_stored_record"); } -#[tokio::test] -async fn merge_run_pull_request_rejects_invalid_method() { - let (state, app, run_id) = pr_test_app(Some("ghu_test"), None); - - create_run_with_pull_request_record( - &state, - run_id, - "https://github.com/acme/widgets/pull/42", - 42, - "Fix the bug", - ) - .await; - - let response = app - .oneshot( - Request::builder() - .method("POST") - .uri(api(&format!("/runs/{run_id}/pull_request/merge"))) - .header("content-type", "application/json") - .body(Body::from(json!({ "method": "bogus" }).to_string())) - .unwrap(), - ) - .await - .unwrap(); - - assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY); -} - -#[tokio::test] -async fn merge_run_pull_request_returns_service_unavailable_without_github_credentials() { - let (state, app, run_id) = pr_test_app(None, None); - - create_run_with_pull_request_record( - &state, - run_id, - "https://github.com/acme/widgets/pull/42", - 42, - "Fix the bug", - ) - .await; - - let response = app - .oneshot( - Request::builder() - .method("POST") - .uri(api(&format!("/runs/{run_id}/pull_request/merge"))) - .header("content-type", "application/json") - .body(Body::from(json!({ "method": "squash" }).to_string())) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::SERVICE_UNAVAILABLE).await; - - assert_eq!(body["errors"][0]["code"], "integration_unavailable"); -} - -#[tokio::test] -async fn merge_run_pull_request_uses_stored_link_coordinates() { - let github = MockServer::start(); - let github_mock = github.mock(|when, then| { - when.method("PUT") - .path("/repos/acme/widgets/pulls/42/merge") - .header("authorization", "Bearer ghu_test") - .json_body(json!({ "merge_method": "squash" })); - then.status(200) - .header("content-type", "application/json") - .body(json!({}).to_string()); - }); - let (state, app, run_id) = pr_test_app(Some("ghu_test"), Some(github.base_url())); - - create_run_with_linked_pull_request_record(&state, run_id, PullRequestLink { - owner: "acme".to_string(), - repo: "widgets".to_string(), - number: 42, - }) - .await; - - let response = app - .oneshot( - Request::builder() - .method("POST") - .uri(api(&format!("/runs/{run_id}/pull_request/merge"))) - .header("content-type", "application/json") - .body(Body::from(json!({ "method": "squash" }).to_string())) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - - assert_eq!(body["number"], 42); - assert_eq!(body["html_url"], "https://github.com/acme/widgets/pull/42"); - github_mock.assert(); -} - #[tokio::test] async fn close_run_pull_request_returns_not_found_when_record_missing() { let (_state, app, run_id) = pr_test_app_with_minimal_run(Some("ghu_test"), None).await; @@ -10992,424 +6904,6 @@ async fn close_run_pull_request_returns_not_found_when_record_missing() { assert_eq!(body["errors"][0]["code"], "no_stored_record"); } -#[tokio::test] -async fn close_run_pull_request_returns_service_unavailable_without_github_credentials() { - let (state, app, run_id) = pr_test_app(None, None); - - create_run_with_pull_request_record( - &state, - run_id, - "https://github.com/acme/widgets/pull/42", - 42, - "Fix the bug", - ) - .await; - - let response = app - .oneshot( - Request::builder() - .method("POST") - .uri(api(&format!("/runs/{run_id}/pull_request/close"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::SERVICE_UNAVAILABLE).await; - - assert_eq!(body["errors"][0]["code"], "integration_unavailable"); -} - -#[tokio::test] -async fn close_run_pull_request_returns_bad_gateway_when_github_pr_is_missing() { - let github = MockServer::start(); - let github_mock = github.mock(|when, then| { - when.method("PATCH") - .path("/repos/acme/widgets/pulls/42") - .header("authorization", "Bearer ghu_test"); - then.status(404) - .header("content-type", "application/json") - .body(json!({ "message": "Not Found" }).to_string()); - }); - let (state, app, run_id) = pr_test_app(Some("ghu_test"), Some(github.base_url())); - - create_run_with_pull_request_record( - &state, - run_id, - "https://github.com/acme/widgets/pull/42", - 42, - "Fix the bug", - ) - .await; - - let response = app - .oneshot( - Request::builder() - .method("POST") - .uri(api(&format!("/runs/{run_id}/pull_request/close"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::BAD_GATEWAY).await; - - assert_eq!(body["errors"][0]["code"], "github_not_found"); - github_mock.assert(); -} - -#[tokio::test] -async fn get_run_state_exposes_pending_interviews() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = fixtures::RUN_1; - - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - ]) - .await; - append_raw_run_event( - &state, - run_id, - "pending-question", - "2026-04-19T12:00:00Z", - "interview.started", - json!({ - "question_id": "q-1", - "question": "Approve deploy?", - "stage": "gate", - "question_type": "multiple_choice", - "options": [], - "allow_freeform": false, - "context_display": null, - "timeout_seconds": null, - "review_target": { - "label": "Quarry review exercise", - "url": "https://quarry.lithos.computer/tmp/0123456789abcdef0123456789abcdef", - "kind": "document" - }, - }), - Some("gate"), - ) - .await; - - let req = Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/state"))) - .body(Body::empty()) - .unwrap(); - - let response = app.oneshot(req).await.unwrap(); - let body = response_json!(response, StatusCode::OK).await; - assert_eq!( - body["pending_interviews"]["q-1"]["question"]["text"].as_str(), - Some("Approve deploy?") - ); - assert_eq!( - body["pending_interviews"]["q-1"]["question"]["stage"].as_str(), - Some("gate") - ); -} - -/// Builds an app state over shared object, blob, and summary stores so a test -/// can drop it and open a second state that sees the same durable data. -fn test_app_state_over_shared_stores( - object_store: &Arc, - blobs: &Arc, - summaries: &Arc, -) -> Arc { - let store = Arc::new(fabro_store::test_support::test_database_with_stores( - Arc::clone(object_store), - "runs", - std::time::Duration::from_millis(1), - None, - Arc::clone(blobs), - Arc::clone(summaries), - )); - test_app_state_with_store( - default_test_server_settings(), - RunLayer::default(), - 5, - store, - ArtifactStore::new(Arc::clone(object_store), "artifacts"), - ) -} - -#[tokio::test] -async fn restarted_run_state_details_load_from_sql_and_preserve_error_statuses() { - let object_store: Arc = - Arc::new(object_store::memory::InMemory::new()); - let summaries = fabro_store::test_support::test_run_summary_store(); - let blobs = fabro_store::test_support::test_blob_store(); - let first_state = test_app_state_over_shared_stores(&object_store, &blobs, &summaries); - let healthy_id = fixtures::RUN_1; - let broken_id = fixtures::RUN_2; - create_durable_run_with_events(&first_state, healthy_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - ]) - .await; - create_succeeded_run(&first_state, broken_id).await; - first_state - .stores - .run_summaries - .test_delete_run_events(&broken_id) - .await - .unwrap(); - drop(first_state); - - let reopened_state = test_app_state_over_shared_stores(&object_store, &blobs, &summaries); - assert_eq!( - reconcile_incomplete_runs_on_startup(&reopened_state) - .await - .unwrap(), - 0, - "startup reconciliation must not replay terminal histories" - ); - let app = crate::test_support::build_test_router(reopened_state); - - let healthy = app - .clone() - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{healthy_id}/state"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let healthy_body = response_json!(healthy, StatusCode::OK).await; - assert_eq!(healthy_body["spec"]["run_id"], healthy_id.to_string()); - - let missing = app - .clone() - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{}/state", fixtures::RUN_3))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_status!(missing, StatusCode::NOT_FOUND).await; - - let broken = app - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{broken_id}/state"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_status!(broken, StatusCode::INTERNAL_SERVER_ERROR).await; -} - -#[tokio::test] -async fn run_projection_endpoints_reflect_events_appended_to_an_open_run() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = create_run(&app, MINIMAL_DOT) - .await - .parse::() - .unwrap(); - - let run_store = state.stores.runs.open_run(&run_id).await.unwrap(); - workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunRunnable { - source: fabro_types::RunRunnableSource::StartRequested, - actor: None, - }) - .await - .unwrap(); - workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunStarting) - .await - .unwrap(); - workflow_event::append_event(&run_store, &run_id, &workflow_event::Event::RunRunning) - .await - .unwrap(); - append_scoped_stage_event( - &state, - run_id, - "review", - 1, - &workflow_event::Event::StageStarted { - graph_visit: None, - resumed_from_stage_id: None, - node_id: "review".to_string(), - name: "Review".to_string(), - index: 0, - handler_type: "human".to_string(), - attempt: 1, - max_attempts: 1, - }, - ) - .await; - append_raw_run_event( - &state, - run_id, - "cache-question", - "2026-04-19T12:00:00Z", - "interview.started", - json!({ - "question_id": "q-cache", - "question": "Approve cached deploy?", - "stage": "review", - "question_type": "yes_no", - "options": [], - "allow_freeform": false, - "context_display": null, - "timeout_seconds": null, - "review_target": { - "label": "Quarry review exercise", - "url": "https://quarry.lithos.computer/tmp/0123456789abcdef0123456789abcdef", - "kind": "document" - }, - }), - Some("review"), - ) - .await; - append_raw_run_event( - &state, - run_id, - "cache-checkpoint", - "2026-04-19T12:00:01Z", - "checkpoint.completed", - json!({ - "status": "running", - "current_node": "review", - "completed_nodes": [], - "node_retries": {}, - "context_values": {}, - "node_outcomes": {}, - "next_node_id": "review", - "git_commit_sha": "cache-sha", - "loop_failure_signatures": {}, - "restart_failure_signatures": {}, - "node_visits": { "review": 1 }, - }), - Some("review"), - ) - .await; - - let status = app - .clone() - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let status = response_json!(status, StatusCode::OK).await; - assert_eq!(run_json_status(&status)["kind"].as_str(), Some("running")); - - let state_response = app - .clone() - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/state"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let state_body = response_json!(state_response, StatusCode::OK).await; - assert_eq!( - state_body["pending_interviews"]["q-cache"]["question"]["text"].as_str(), - Some("Approve cached deploy?") - ); - assert_eq!( - state_body["pending_interviews"]["q-cache"]["question"]["review_target"]["url"].as_str(), - Some("https://quarry.lithos.computer/tmp/0123456789abcdef0123456789abcdef") - ); - - let stages = app - .clone() - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/stages"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let stages = response_json!(stages, StatusCode::OK).await; - assert_eq!(stages["data"][0]["id"].as_str(), Some("review@1")); - - let questions = app - .clone() - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/questions"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let questions = response_json!(questions, StatusCode::OK).await; - assert_eq!( - questions["data"][0]["text"].as_str(), - Some("Approve cached deploy?") - ); - assert_eq!( - questions["data"][0]["review_target"]["label"].as_str(), - Some("Quarry review exercise") - ); - - let settings = app - .clone() - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/settings"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_status!(settings, StatusCode::OK).await; - - let checkpoint = app - .clone() - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/checkpoint"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let checkpoint = response_json!(checkpoint, StatusCode::OK).await; - assert_eq!(checkpoint["git_commit_sha"].as_str(), Some("cache-sha")); - - let usage = app - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/usage"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let usage = response_json!(usage, StatusCode::OK).await; - assert_eq!(usage["stages"][0]["stage"]["id"].as_str(), Some("review")); -} - #[tokio::test] async fn get_run_state_includes_provenance_from_user_agent() { let state = test_app_state(); @@ -11574,65 +7068,6 @@ async fn list_run_events_returns_paginated_json() { assert!(body["meta"]["has_more"].is_boolean()); } -#[tokio::test] -async fn list_run_events_descends_from_latest_with_exclusive_cursor() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::RunRunnable { - source: fabro_types::RunRunnableSource::StartRequested, - actor: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - ]) - .await; - - let response = app - .clone() - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/events?order=desc&limit=2"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - let seqs = body["data"] - .as_array() - .unwrap() - .iter() - .map(|event| event["seq"].as_u64().unwrap()) - .collect::>(); - assert_eq!(seqs, vec![4, 3]); - assert_eq!(body["meta"]["has_more"], true); - - let response = app - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!( - "/runs/{run_id}/events?order=desc&before_seq=3&limit=2" - ))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - let seqs = body["data"] - .as_array() - .unwrap() - .iter() - .map(|event| event["seq"].as_u64().unwrap()) - .collect::>(); - assert_eq!(seqs, vec![2, 1]); - assert_eq!(body["meta"]["has_more"], false); -} - #[tokio::test] async fn list_run_events_rejects_cursor_for_opposite_order() { let app = crate::test_support::build_test_router(test_app_state()); @@ -11983,172 +7418,6 @@ async fn stage_artifacts_keep_same_filename_per_retry() { assert_eq!(&bytes[..], b"second"); } -#[tokio::test] -async fn run_artifacts_download_streams_latest_files_as_zip() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = create_run(&app, MINIMAL_DOT) - .await - .parse::() - .unwrap(); - - let run_store = state.stores.runs.open_run(&run_id).await.unwrap(); - for event in [ - workflow_event::Event::RunRunnable { - source: fabro_types::RunRunnableSource::StartRequested, - actor: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - ] { - workflow_event::append_event(&run_store, &run_id, &event) - .await - .unwrap(); - } - append_scoped_stage_event( - &state, - run_id, - "build", - 1, - &stage_started_event("build", "command"), - ) - .await; - append_scoped_stage_event( - &state, - run_id, - "verify", - 1, - &stage_started_event("verify", "command"), - ) - .await; - - for (stage_id, retry, path, contents) in [ - ( - StageId::new("unknown", 1), - 99, - "reports/result.txt", - &b"unknown stage"[..], - ), - ( - StageId::new("build", 1), - 1, - "reports/result.txt", - &b"build result"[..], - ), - ( - StageId::new("verify", 1), - 1, - "reports/result.txt", - &b"first verify"[..], - ), - ( - StageId::new("verify", 1), - 2, - "reports/result.txt", - &b"latest verify"[..], - ), - ( - StageId::new("build", 1), - 1, - "logs/run.txt", - &b"build log"[..], - ), - ( - StageId::new("start", 1), - 1, - "control-start.txt", - &b"excluded"[..], - ), - ( - StageId::new("exit", 1), - 1, - "control-exit.txt", - &b"excluded"[..], - ), - // Neither stage reached the projection, so both rank equally on stage - // order and retry. The serialized stage ID breaks the tie the same way - // the artifacts page does: "unknown@2" sorts above "unknown@10". - ( - StageId::new("unknown", 10), - 1, - "orphan.txt", - &b"visit ten"[..], - ), - ( - StageId::new("unknown", 2), - 1, - "orphan.txt", - &b"visit two"[..], - ), - ] { - state - .artifact_store - .put(&run_id, &ArtifactKey::new(stage_id, retry, path), contents) - .await - .unwrap(); - } - - let response = app - .clone() - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/artifacts/download"))) - .header(header::ACCEPT_ENCODING, "gzip") - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!( - response - .headers() - .get(header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()), - Some("application/zip") - ); - assert_eq!( - response - .headers() - .get(header::CONTENT_DISPOSITION) - .and_then(|value| value.to_str().ok()), - Some(format!("attachment; filename=\"fabro-artifacts-{run_id}.zip\"").as_str()) - ); - assert_eq!( - response - .headers() - .get(header::CACHE_CONTROL) - .and_then(|value| value.to_str().ok()), - Some("private, no-store") - ); - assert!(response.headers().get(header::CONTENT_ENCODING).is_none()); - - let bytes = response_bytes!(response, StatusCode::OK).await; - let archive = ZipFileReader::new(bytes).await.unwrap(); - let names = archive - .file() - .entries() - .iter() - .map(|entry| entry.filename().as_str().unwrap().to_string()) - .collect::>(); - assert_eq!(names, vec![ - "logs/run.txt", - "orphan.txt", - "reports/result.txt" - ]); - - let mut contents_by_name = HashMap::new(); - for (index, name) in names.into_iter().enumerate() { - let mut entry = archive.reader_with_entry(index).await.unwrap(); - let mut contents = Vec::new(); - entry.read_to_end_checked(&mut contents).await.unwrap(); - contents_by_name.insert(name, contents); - } - assert_eq!(contents_by_name["logs/run.txt"], b"build log"); - assert_eq!(contents_by_name["reports/result.txt"], b"latest verify"); - assert_eq!(contents_by_name["orphan.txt"], b"visit two"); -} - #[tokio::test] async fn run_artifacts_download_returns_not_found_for_unknown_run() { let state = test_app_state(); @@ -12711,78 +7980,6 @@ async fn worker_token_controls_stage_artifact_route() { assert_status!(response, StatusCode::UNAUTHORIZED).await; } -#[tokio::test] -async fn worker_token_controls_command_log_route() { - let (state, app) = jwt_auth_app(); - let user_jwt = issue_test_user_jwt(); - let run_id = create_run_with_bearer(&app, &user_jwt).await; - let worker_token = issue_test_worker_token(&run_id); - let other_run_id = create_run_with_bearer(&app, &user_jwt).await; - let mismatched_worker_token = issue_test_worker_token(&other_run_id); - let run_store = state.stores.runs.open_run(&run_id).await.unwrap(); - workflow_event::append_event( - &run_store, - &run_id, - &workflow_event::Event::CommandStarted { - node_id: "code".to_string(), - script: "echo hello".to_string(), - command: "echo hello".to_string(), - language: "shell".to_string(), - timeout_ms: None, - }, - ) - .await - .unwrap(); - - let response = app - .clone() - .oneshot(bearer_request( - Method::GET, - &format!("/runs/{run_id}/stages/code@1/logs/output"), - &worker_token, - Body::empty(), - )) - .await - .unwrap(); - assert_status!(response, StatusCode::OK).await; - - let response = app - .clone() - .oneshot(bearer_request( - Method::GET, - &format!("/runs/{run_id}/stages/code@1/logs/output"), - &user_jwt, - Body::empty(), - )) - .await - .unwrap(); - assert_status!(response, StatusCode::OK).await; - - let response = app - .clone() - .oneshot(bearer_request( - Method::GET, - &format!("/runs/{run_id}/stages/code@1/logs/output"), - &mismatched_worker_token, - Body::empty(), - )) - .await - .unwrap(); - assert_status!(response, StatusCode::FORBIDDEN).await; - - let response = app - .oneshot( - Request::builder() - .method(Method::GET) - .uri(api(&format!("/runs/{run_id}/stages/code@1/logs/output"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_status!(response, StatusCode::UNAUTHORIZED).await; -} - #[tokio::test] async fn worker_token_is_rejected_on_user_only_routes() { let (_state, app) = jwt_auth_app(); @@ -13216,123 +8413,6 @@ async fn denying_pending_child_run_fails_with_approval_denied() { assert!(denied_body["lifecycle"]["approval"]["denial_reason"].is_null()); } -#[tokio::test] -async fn patch_run_title_updates_active_and_archived_runs() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = create_run(&app, MINIMAL_DOT) - .await - .parse::() - .unwrap(); - - let patch_response = app - .clone() - .oneshot( - Request::builder() - .method("PATCH") - .uri(api(&format!("/runs/{run_id}"))) - .header("content-type", "application/json") - .body(Body::from( - json!({ "title": " Active title " }).to_string(), - )) - .unwrap(), - ) - .await - .unwrap(); - let patch_body = response_json!(patch_response, StatusCode::OK).await; - assert_eq!(patch_body["title"], "Active title"); - - let run_store = state.stores.runs.open_run_reader(&run_id).await.unwrap(); - let event_count = run_store.list_events().await.unwrap().len(); - let same_title_response = app - .clone() - .oneshot( - Request::builder() - .method("PATCH") - .uri(api(&format!("/runs/{run_id}"))) - .header("content-type", "application/json") - .body(Body::from(json!({ "title": "Active title" }).to_string())) - .unwrap(), - ) - .await - .unwrap(); - let same_title_body = response_json!(same_title_response, StatusCode::OK).await; - assert_eq!(same_title_body["title"], "Active title"); - assert_eq!( - state - .stores - .runs - .open_run_reader(&run_id) - .await - .unwrap() - .list_events() - .await - .unwrap() - .len(), - event_count, - "same-title PATCH should not append an event" - ); - - let run_store = state.stores.runs.open_run(&run_id).await.unwrap(); - for event in [ - workflow_event::Event::RunRunnable { - source: fabro_types::RunRunnableSource::StartRequested, - actor: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - ] { - workflow_event::append_event(&run_store, &run_id, &event) - .await - .unwrap(); - } - workflow_event::append_event( - &run_store, - &run_id, - &workflow_event::Event::WorkflowRunCompleted { - timing: fabro_types::RunTiming::wall_only(1), - artifact_count: 0, - status: "succeeded".to_string(), - reason: SuccessReason::Completed, - final_git_commit_sha: None, - final_patch: None, - diff_summary: None, - usage: None, - }, - ) - .await - .unwrap(); - response_json!( - app.clone() - .oneshot( - Request::builder() - .method("POST") - .uri(api(&format!("/runs/{run_id}/archive"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(), - StatusCode::OK - ) - .await; - - let archived_patch_response = app - .oneshot( - Request::builder() - .method("PATCH") - .uri(api(&format!("/runs/{run_id}"))) - .header("content-type", "application/json") - .body(Body::from(json!({ "title": "Archived title" }).to_string())) - .unwrap(), - ) - .await - .unwrap(); - let archived_patch_body = response_json!(archived_patch_response, StatusCode::OK).await; - assert_eq!(archived_patch_body["title"], "Archived title"); - assert!(run_json_archived(&archived_patch_body)); -} - #[tokio::test] async fn patch_run_title_rejects_invalid_titles() { let app = test_app_with(); @@ -13389,82 +8469,6 @@ async fn start_run_conflict_when_not_submitted() { assert_status!(response, StatusCode::CONFLICT).await; } -#[tokio::test] -async fn retry_failed_run_creates_and_queues_new_run() { - let state = test_app_state_with_isolated_storage(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let source_run_id = RunId::new(); - create_durable_run_with_events(&state, source_run_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::workflow_run_failed_from_error( - &WorkflowError::engine("boom"), - fabro_types::RunTiming::wall_only(10), - FailureReason::WorkflowError, - None, - None, - None, - None, - ), - ]) - .await; - let source_events_before = state - .stores - .runs - .open_run(&source_run_id) - .await - .unwrap() - .list_events() - .await - .unwrap() - .len(); - - let response = app - .oneshot( - Request::builder() - .method("POST") - .uri(api(&format!("/runs/{source_run_id}/retry"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::CREATED).await; - let new_run_id = body["id"].as_str().unwrap().parse::().unwrap(); - - assert_ne!(new_run_id, source_run_id); - assert_eq!(body["retried_from"], source_run_id.to_string()); - assert_eq!(body["created_by"]["kind"], "user"); - assert_eq!(body["created_by"]["login"], "dev"); - assert_eq!(run_json_status(&body)["kind"], "runnable"); - - let source_store = state.stores.runs.open_run(&source_run_id).await.unwrap(); - assert_eq!( - source_store.list_events().await.unwrap().len(), - source_events_before - ); - assert_eq!( - source_store.state().await.unwrap().status, - RunStatus::Failed { - reason: FailureReason::WorkflowError, - } - ); - - let new_state = state - .stores - .runs - .open_run(&new_run_id) - .await - .unwrap() - .state() - .await - .unwrap(); - assert_eq!(new_state.retried_from, Some(source_run_id)); - assert_eq!(new_state.status, RunStatus::Runnable); - assert!(new_state.checkpoints.is_empty()); -} - #[tokio::test] async fn retry_missing_run_returns_not_found() { let state = test_app_state(); @@ -13484,103 +8488,6 @@ async fn retry_missing_run_returns_not_found() { assert_status!(response, StatusCode::NOT_FOUND).await; } -#[tokio::test] -async fn retry_succeeded_run_creates_and_queues_new_run() { - let state = test_app_state_with_isolated_storage(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let source_run_id = RunId::new(); - create_durable_run_with_events(&state, source_run_id, &[ - workflow_event::Event::WorkflowRunCompleted { - timing: fabro_types::RunTiming::wall_only(10), - artifact_count: 0, - status: "succeeded".to_string(), - reason: SuccessReason::Completed, - final_git_commit_sha: None, - final_patch: None, - diff_summary: None, - usage: None, - }, - ]) - .await; - let source_events_before = state - .stores - .runs - .open_run(&source_run_id) - .await - .unwrap() - .list_events() - .await - .unwrap() - .len(); - - let response = app - .oneshot( - Request::builder() - .method("POST") - .uri(api(&format!("/runs/{source_run_id}/retry"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::CREATED).await; - let new_run_id = body["id"].as_str().unwrap().parse::().unwrap(); - - assert_ne!(new_run_id, source_run_id); - assert_eq!(body["retried_from"], source_run_id.to_string()); - assert_eq!(run_json_status(&body)["kind"], "runnable"); - - let source_store = state.stores.runs.open_run(&source_run_id).await.unwrap(); - assert_eq!( - source_store.list_events().await.unwrap().len(), - source_events_before - ); - assert_eq!( - source_store.state().await.unwrap().status, - RunStatus::Succeeded { - reason: SuccessReason::Completed, - } - ); - - let new_state = state - .stores - .runs - .open_run(&new_run_id) - .await - .unwrap() - .state() - .await - .unwrap(); - assert_eq!(new_state.retried_from, Some(source_run_id)); - assert_eq!(new_state.status, RunStatus::Runnable); -} - -#[tokio::test] -async fn retry_active_run_returns_conflict() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let source_run_id = RunId::new(); - create_durable_run_with_events(&state, source_run_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - ]) - .await; - - let response = app - .oneshot( - Request::builder() - .method("POST") - .uri(api(&format!("/runs/{source_run_id}/retry"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - - assert_status!(response, StatusCode::CONFLICT).await; -} - #[tokio::test] async fn cancel_run_succeeds() { let state = test_app_state(); @@ -13622,39 +8529,6 @@ async fn cancel_nonexistent_run_returns_not_found() { assert_status!(response, StatusCode::NOT_FOUND).await; } -#[tokio::test] -async fn cancel_terminal_durable_run_returns_conflict() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = fixtures::RUN_1; - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::WorkflowRunCompleted { - timing: fabro_types::RunTiming::wall_only(1000), - artifact_count: 0, - status: "succeeded".to_string(), - reason: SuccessReason::Completed, - final_git_commit_sha: None, - final_patch: None, - diff_summary: None, - usage: None, - }, - ]) - .await; - - let req = Request::builder() - .method("POST") - .uri(api(&format!("/runs/{run_id}/cancel"))) - .body(Body::empty()) - .unwrap(); - - let response = app.oneshot(req).await.unwrap(); - let body = response_json!(response, StatusCode::CONFLICT).await; - assert_eq!( - body["errors"][0]["detail"], - "Run is already terminal and cannot be cancelled." - ); -} - #[tokio::test] async fn steer_nonexistent_run_returns_not_found() { let app = test_app_with(); @@ -13671,38 +8545,6 @@ async fn steer_nonexistent_run_returns_not_found() { assert_status!(response, StatusCode::NOT_FOUND).await; } -#[tokio::test] -async fn steer_terminal_durable_run_returns_run_not_steerable() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = fixtures::RUN_1; - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::WorkflowRunCompleted { - timing: fabro_types::RunTiming::wall_only(1000), - artifact_count: 0, - status: "succeeded".to_string(), - reason: SuccessReason::Completed, - final_git_commit_sha: None, - final_patch: None, - diff_summary: None, - usage: None, - }, - ]) - .await; - - let req = Request::builder() - .method("POST") - .uri(api(&format!("/runs/{run_id}/steer"))) - .header("content-type", "application/json") - .body(Body::from(r#"{"text":"try again"}"#)) - .unwrap(); - - let response = app.oneshot(req).await.unwrap(); - let body = response_json!(response, StatusCode::CONFLICT).await; - assert_eq!(body["errors"][0]["code"], "run_not_steerable"); - assert_eq!(body["errors"][0]["detail"], "Run is no longer steerable."); -} - #[tokio::test] async fn steer_empty_text_returns_bad_request() { let state = test_app_state(); @@ -13926,273 +8768,6 @@ async fn interrupt_terminal_run_returns_run_not_interruptible() { assert_eq!(body["errors"][0]["code"], "run_not_interruptible"); } -#[test] -fn injected_runnable_event_does_not_make_submitted_run_schedulable() { - let state = test_app_state(); - let run_id = fixtures::RUN_1; - let temp_dir = tempfile::tempdir().unwrap(); - { - let mut runs = state.runs.lock().expect("runs lock poisoned"); - runs.insert( - run_id, - managed_run( - String::new(), - RunStatus::Submitted, - chrono::Utc::now(), - temp_dir.path().join(run_id.to_string()), - RunExecutionMode::Start, - ), - ); - } - - let runnable = workflow_event::to_run_event(&run_id, &workflow_event::Event::RunRunnable { - source: fabro_types::RunRunnableSource::StartRequested, - actor: None, - }); - update_live_run_from_event(&state, run_id, &runnable); - - { - let runs = state.runs.lock().expect("runs lock poisoned"); - assert_eq!(runs.get(&run_id).unwrap().status, RunStatus::Submitted); - } - - let starting = workflow_event::to_run_event(&run_id, &workflow_event::Event::RunStarting); - update_live_run_from_event(&state, run_id, &starting); - - let runs = state.runs.lock().expect("runs lock poisoned"); - assert_eq!(runs.get(&run_id).unwrap().status, RunStatus::Starting); -} - -#[test] -fn active_steerable_stage_projection_ignores_stale_deactivation() { - let state = test_app_state(); - let run_id = fixtures::RUN_1; - let temp_dir = tempfile::tempdir().unwrap(); - { - let mut runs = state.runs.lock().expect("runs lock poisoned"); - runs.insert( - run_id, - managed_run( - String::new(), - RunStatus::Running, - chrono::Utc::now(), - temp_dir.path().join(run_id.to_string()), - RunExecutionMode::Start, - ), - ); - } - - let stage_id = StageId::new("agent", 1); - let activated_a = - workflow_event::to_run_event(&run_id, &workflow_event::Event::AgentSessionActivated { - node_id: "agent".to_string(), - visit: 1, - session_id: "session-a".to_string(), - thread_id: None, - provider: Some("openai".to_string()), - model: Some("gpt-5.4".to_string()), - reasoning_effort: None, - speed: None, - permission_level: None, - capabilities: vec![SessionCapability::Steer], - }); - update_live_run_from_event(&state, run_id, &activated_a); - - let deactivated_a = - workflow_event::to_run_event(&run_id, &workflow_event::Event::AgentSessionDeactivated { - node_id: "agent".to_string(), - visit: 1, - session_id: "session-a".to_string(), - }); - update_live_run_from_event(&state, run_id, &deactivated_a); - - let activated_b = - workflow_event::to_run_event(&run_id, &workflow_event::Event::AgentSessionActivated { - node_id: "agent".to_string(), - visit: 1, - session_id: "session-b".to_string(), - thread_id: None, - provider: Some("openai".to_string()), - model: Some("gpt-5.4".to_string()), - reasoning_effort: None, - speed: None, - permission_level: None, - capabilities: vec![SessionCapability::Steer], - }); - update_live_run_from_event(&state, run_id, &activated_b); - update_live_run_from_event(&state, run_id, &deactivated_a); - - let runs = state.runs.lock().expect("runs lock poisoned"); - let run = runs.get(&run_id).unwrap(); - assert_eq!( - run.active_steerable_stages - .get(&stage_id) - .map(String::as_str), - Some("session-b") - ); -} - -fn acp_event_for_stage(run_id: &RunId, event: &workflow_event::Event) -> fabro_types::RunEvent { - workflow_event::to_run_event_at( - run_id, - event, - Utc::now(), - Some(&workflow_event::StageScope { - node_id: "agent".to_string(), - visit: 1, - parallel_group_id: None, - parallel_branch_id: None, - }), - ) -} - -#[tokio::test] -async fn steer_with_active_acp_session_forwards_to_worker() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = fixtures::RUN_1; - let (transport, mut control_rx) = worker_transport_with_receiver(run_id).await; - let _temp_dir = insert_running_control_run(&state, run_id, Some(transport)); - - let started = acp_event_for_stage(&run_id, &workflow_event::Event::AgentAcpStarted { - node_id: "agent".to_string(), - visit: 1, - command: "python fake_agent.py".to_string(), - config_name: None, - }); - update_live_run_from_event(&state, run_id, &started); - let activated = - workflow_event::to_run_event(&run_id, &workflow_event::Event::AgentSessionActivated { - node_id: "agent".to_string(), - visit: 1, - session_id: "acp-session".to_string(), - thread_id: None, - provider: Some(AgentBackend::Acp.to_string()), - model: None, - reasoning_effort: None, - speed: None, - permission_level: None, - capabilities: vec![SessionCapability::Steer], - }); - update_live_run_from_event(&state, run_id, &activated); - - let req = Request::builder() - .method("POST") - .uri(api(&format!("/runs/{run_id}/steer"))) - .header("content-type", "application/json") - .body(Body::from(r#"{"text":"try again"}"#)) - .unwrap(); - - let response = app.oneshot(req).await.unwrap(); - assert_status!(response, StatusCode::ACCEPTED).await; - let envelope = recv_worker_control_envelope(&mut control_rx).await; - assert!(matches!( - envelope.message, - WorkerControlMessage::Steer { ref text, .. } if text == "try again" - )); -} - -#[tokio::test] -async fn active_acp_steerable_marker_clears_on_terminal_paths() { - let terminal_events: Vec = vec![ - workflow_event::Event::AgentAcpCompleted { - node_id: "agent".to_string(), - stdout: "done".to_string(), - stderr: String::new(), - stop_reason: "end_turn".to_string(), - duration_ms: 42, - }, - workflow_event::Event::AgentAcpCancelled { - node_id: "agent".to_string(), - stdout: "partial".to_string(), - stderr: "cancelled".to_string(), - duration_ms: 7, - }, - workflow_event::Event::AgentAcpTimedOut { - node_id: "agent".to_string(), - stdout: "partial".to_string(), - stderr: "timeout".to_string(), - duration_ms: 99, - }, - workflow_event::Event::StageCompleted { - node_id: "agent".to_string(), - name: "agent".to_string(), - index: 0, - timing: fabro_types::StageTiming::wall_only(1), - status: "success".to_string(), - preferred_label: None, - suggested_next_ids: Vec::new(), - usage_by_model: Vec::new(), - usage: None, - failure: None, - notes: None, - files_touched: Vec::new(), - context_updates: None, - jump_to_node: None, - context_values: None, - node_visits: None, - loop_failure_signatures: None, - restart_failure_signatures: None, - response: None, - attempt: 1, - max_attempts: 1, - }, - workflow_event::Event::StageFailed { - node_id: "agent".to_string(), - name: "agent".to_string(), - index: 0, - failure: FailureDetail::new("failed", FailureCategory::Deterministic), - will_retry: false, - timing: fabro_types::StageTiming::wall_only(1), - usage_by_model: Vec::new(), - usage: None, - actor: None, - }, - ]; - - for terminal_event in terminal_events { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = fixtures::RUN_1; - let (transport, _control_rx) = worker_transport_with_receiver(run_id).await; - let _temp_dir = insert_running_control_run(&state, run_id, Some(transport)); - let started = acp_event_for_stage(&run_id, &workflow_event::Event::AgentAcpStarted { - node_id: "agent".to_string(), - visit: 1, - command: "python fake_agent.py".to_string(), - config_name: None, - }); - update_live_run_from_event(&state, run_id, &started); - let activated = - workflow_event::to_run_event(&run_id, &workflow_event::Event::AgentSessionActivated { - node_id: "agent".to_string(), - visit: 1, - session_id: "acp-session".to_string(), - thread_id: None, - provider: Some(AgentBackend::Acp.to_string()), - model: None, - reasoning_effort: None, - speed: None, - permission_level: None, - capabilities: vec![SessionCapability::Steer], - }); - update_live_run_from_event(&state, run_id, &activated); - let terminal = acp_event_for_stage(&run_id, &terminal_event); - update_live_run_from_event(&state, run_id, &terminal); - - let req = Request::builder() - .method("POST") - .uri(api(&format!("/runs/{run_id}/interrupt"))) - .body(Body::empty()) - .unwrap(); - - let response = app.oneshot(req).await.unwrap(); - assert_eq!(response.status(), StatusCode::CONFLICT); - let body = body_json(response.into_body()).await; - assert_eq!(body["errors"][0]["code"], "no_active_steerable_session"); - } -} - #[tokio::test] async fn get_graph_returns_svg() { let state = test_app_state(); @@ -14512,210 +9087,6 @@ async fn list_runs_returns_started_run() { assert!(items[0]["usage"].get("cost").is_none()); } -#[tokio::test] -async fn archive_and_unarchive_updates_listing_visibility() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = fixtures::RUN_1; - - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - workflow_event::Event::WorkflowRunCompleted { - timing: fabro_types::RunTiming::wall_only(1000), - artifact_count: 0, - status: "succeeded".to_string(), - reason: SuccessReason::Completed, - final_git_commit_sha: None, - final_patch: None, - diff_summary: None, - usage: None, - }, - ]) - .await; - - let archive_response = app - .clone() - .oneshot( - Request::builder() - .method("POST") - .uri(api(&format!("/runs/{run_id}/archive"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let archive_body = response_json!(archive_response, StatusCode::OK).await; - assert!(run_json_archived(&archive_body)); - assert_eq!(run_json_status(&archive_body)["kind"], "succeeded"); - assert_eq!(run_json_status(&archive_body)["reason"], "completed"); - - let hidden_response = app - .clone() - .oneshot( - Request::builder() - .method("GET") - .uri(api("/runs")) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let hidden_body = response_json!(hidden_response, StatusCode::OK).await; - assert!( - !hidden_body["data"] - .as_array() - .unwrap() - .iter() - .any(|item| run_json_id(item) == Some(&run_id.to_string())), - "archived run should be hidden from default listing" - ); - - let visible_response = app - .clone() - .oneshot( - Request::builder() - .method("GET") - .uri(api("/runs?include_archived=true")) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let visible_body = response_json!(visible_response, StatusCode::OK).await; - let archived_item = visible_body["data"] - .as_array() - .unwrap() - .iter() - .find(|item| run_json_id(item) == Some(&run_id.to_string())) - .expect("archived run should appear when include_archived=true"); - assert!(run_json_archived(archived_item)); - assert_eq!(run_json_status(archived_item)["kind"], "succeeded"); - assert_eq!(run_json_status(archived_item)["reason"], "completed"); - - let unarchive_response = app - .clone() - .oneshot( - Request::builder() - .method("POST") - .uri(api(&format!("/runs/{run_id}/unarchive"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let unarchive_body = response_json!(unarchive_response, StatusCode::OK).await; - assert!(!run_json_archived(&unarchive_body)); - assert_eq!(run_json_status(&unarchive_body)["kind"], "succeeded"); - assert_eq!(run_json_status(&unarchive_body)["reason"], "completed"); - - let restored_response = app - .oneshot( - Request::builder() - .method("GET") - .uri(api("/runs")) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let restored_body = response_json!(restored_response, StatusCode::OK).await; - let restored_item = restored_body["data"] - .as_array() - .unwrap() - .iter() - .find(|item| run_json_id(item) == Some(&run_id.to_string())) - .expect("unarchived run should reappear in default listing"); - assert_eq!(run_json_status(restored_item)["kind"], "succeeded"); - assert_eq!(run_json_status(restored_item)["reason"], "completed"); -} - -fn run_submitted_event() -> workflow_event::Event { - workflow_event::Event::RunSubmitted { - definition_blob: None, - } -} - -fn workflow_completed_event() -> workflow_event::Event { - workflow_event::Event::WorkflowRunCompleted { - timing: fabro_types::RunTiming::wall_only(1000), - artifact_count: 0, - status: "succeeded".to_string(), - reason: SuccessReason::Completed, - final_git_commit_sha: None, - final_patch: None, - diff_summary: None, - usage: None, - } -} - -async fn create_succeeded_run(state: &Arc, run_id: RunId) { - create_durable_run_with_events(state, run_id, &[ - run_submitted_event(), - workflow_completed_event(), - ]) - .await; -} - -async fn create_running_run(state: &Arc, run_id: RunId) { - create_durable_run_with_events(state, run_id, &[ - run_submitted_event(), - workflow_event::Event::RunRunning, - ]) - .await; -} - -async fn create_preserved_local_sandbox_run(state: &Arc, run_id: RunId) { - let mut settings = fabro_types::WorkflowSettings::default(); - settings.run.environment.lifecycle.preserve = true; - let graph = Graph::new("test"); - - create_durable_run_with_events(state, run_id, &[ - workflow_event::Event::RunCreated { - run_id, - title: None, - settings: serde_json::to_value(settings).unwrap(), - graph: serde_json::to_value(graph).unwrap(), - workflow_source: None, - labels: std::collections::BTreeMap::default(), - source_directory: Some("/tmp/fabro-run".to_string()), - workflow_slug: Some("test".to_string()), - workflow_version_id: None, - target: None, - automation: None, - provenance: test_support::test_run_provenance(), - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, - admission: PetriAdmission::default(), - }, - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::SandboxInitialized { - provider: SandboxProviderKind::LOCAL, - id: "sandbox-preserve-1".to_string(), - working_directory: "/tmp/fabro-preserved-sandbox".to_string(), - image: None, - snapshot: None, - repo_cloned: None, - clone_origin_url: None, - clone_branch: None, - workspace_root: None, - repos_root: None, - primary_repo_path: None, - primary_repo_link: None, - }, - ]) - .await; -} - fn batch_lifecycle_body(run_ids: &[RunId]) -> serde_json::Value { json!({ "run_ids": run_ids.iter().map(ToString::to_string).collect::>(), @@ -14729,275 +9100,6 @@ fn batch_delete_body(run_ids: &[RunId], force: bool) -> serde_json::Value { }) } -fn assert_batch_result(result: &serde_json::Value, run_id: RunId, ok: bool, outcome: &str) { - assert_eq!(result["run_id"], run_id.to_string()); - assert_eq!(result["ok"], ok); - assert_eq!(result["outcome"], outcome); - if ok { - assert!( - result["run"].is_object(), - "successful result should include run: {result}" - ); - assert!( - result["error"].is_null(), - "successful result should omit error: {result}" - ); - } else { - assert!( - result["error"].is_object(), - "failed result should include error: {result}" - ); - assert!( - result["run"].is_null(), - "failed result should omit run: {result}" - ); - } -} - -fn assert_batch_delete_result(result: &serde_json::Value, run_id: RunId, ok: bool, outcome: &str) { - assert_eq!(result["run_id"], run_id.to_string()); - assert_eq!(result["ok"], ok); - assert_eq!(result["outcome"], outcome); - if ok { - assert!( - result["error"].is_null(), - "successful delete result should omit error: {result}" - ); - } else { - assert!( - result["error"].is_object(), - "failed delete result should include error: {result}" - ); - } -} - -#[tokio::test] -async fn batch_archive_and_unarchive_updates_listing_visibility() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let first_id = RunId::new(); - let second_id = RunId::new(); - create_succeeded_run(&state, first_id).await; - create_succeeded_run(&state, second_id).await; - - let archive_response = app - .clone() - .oneshot(json_request( - Method::POST, - "/runs/archive", - &batch_lifecycle_body(&[first_id, second_id]), - )) - .await - .unwrap(); - let archive_body = response_json!(archive_response, StatusCode::OK).await; - assert_eq!(archive_body["summary"]["requested"], 2); - assert_eq!(archive_body["summary"]["succeeded"], 2); - assert_eq!(archive_body["summary"]["failed"], 0); - let archive_results = archive_body["results"].as_array().unwrap(); - assert_batch_result(&archive_results[0], first_id, true, "archived"); - assert!(run_json_archived(&archive_results[0]["run"])); - assert_batch_result(&archive_results[1], second_id, true, "archived"); - assert!(run_json_archived(&archive_results[1]["run"])); - - let hidden_response = app - .clone() - .oneshot( - Request::builder() - .method("GET") - .uri(api("/runs")) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let hidden_body = response_json!(hidden_response, StatusCode::OK).await; - assert!( - hidden_body["data"].as_array().unwrap().iter().all(|item| { - let item_id = run_json_id(item); - item_id != Some(&first_id.to_string()) && item_id != Some(&second_id.to_string()) - }), - "archived runs should be hidden from default listing" - ); - - let unarchive_response = app - .clone() - .oneshot(json_request( - Method::POST, - "/runs/unarchive", - &batch_lifecycle_body(&[first_id, second_id]), - )) - .await - .unwrap(); - let unarchive_body = response_json!(unarchive_response, StatusCode::OK).await; - assert_eq!(unarchive_body["summary"]["requested"], 2); - assert_eq!(unarchive_body["summary"]["succeeded"], 2); - assert_eq!(unarchive_body["summary"]["failed"], 0); - let unarchive_results = unarchive_body["results"].as_array().unwrap(); - assert_batch_result(&unarchive_results[0], first_id, true, "unarchived"); - assert!(!run_json_archived(&unarchive_results[0]["run"])); - assert_batch_result(&unarchive_results[1], second_id, true, "unarchived"); - assert!(!run_json_archived(&unarchive_results[1]["run"])); - - let restored_response = app - .oneshot( - Request::builder() - .method("GET") - .uri(api("/runs")) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let restored_body = response_json!(restored_response, StatusCode::OK).await; - for run_id in [first_id, second_id] { - let restored_item = restored_body["data"] - .as_array() - .unwrap() - .iter() - .find(|item| run_json_id(item) == Some(&run_id.to_string())) - .expect("unarchived run should reappear in default listing"); - assert_eq!(run_json_status(restored_item)["kind"], "succeeded"); - } -} - -#[tokio::test] -async fn batch_archive_reports_ordered_mixed_results_without_rollback() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let already_archived_id = RunId::new(); - let terminal_id = RunId::new(); - let running_id = RunId::new(); - let missing_id = RunId::new(); - create_succeeded_run(&state, already_archived_id).await; - create_succeeded_run(&state, terminal_id).await; - create_running_run(&state, running_id).await; - - let already_archived_response = app - .clone() - .oneshot( - Request::builder() - .method("POST") - .uri(api(&format!("/runs/{already_archived_id}/archive"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_status!(already_archived_response, StatusCode::OK).await; - - let response = app - .clone() - .oneshot(json_request( - Method::POST, - "/runs/archive", - &batch_lifecycle_body(&[already_archived_id, terminal_id, running_id, missing_id]), - )) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - assert_eq!(body["summary"]["requested"], 4); - assert_eq!(body["summary"]["succeeded"], 2); - assert_eq!(body["summary"]["failed"], 2); - let results = body["results"].as_array().unwrap(); - assert_batch_result(&results[0], already_archived_id, true, "already_archived"); - assert_batch_result(&results[1], terminal_id, true, "archived"); - assert_batch_result(&results[2], running_id, false, "conflict"); - assert_eq!(results[2]["error"]["status"], "409"); - assert_batch_result(&results[3], missing_id, false, "not_found"); - assert_eq!(results[3]["error"]["status"], "404"); - - let terminal_response = app - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{terminal_id}"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let terminal_body = response_json!(terminal_response, StatusCode::OK).await; - assert!(run_json_archived(&terminal_body)); -} - -#[tokio::test] -async fn batch_unarchive_treats_terminal_not_archived_as_success() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let archived_id = RunId::new(); - let not_archived_id = RunId::new(); - create_succeeded_run(&state, archived_id).await; - create_succeeded_run(&state, not_archived_id).await; - - let archive_response = app - .clone() - .oneshot( - Request::builder() - .method("POST") - .uri(api(&format!("/runs/{archived_id}/archive"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_status!(archive_response, StatusCode::OK).await; - - let response = app - .oneshot(json_request( - Method::POST, - "/runs/unarchive", - &batch_lifecycle_body(&[archived_id, not_archived_id]), - )) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - assert_eq!(body["summary"]["requested"], 2); - assert_eq!(body["summary"]["succeeded"], 2); - assert_eq!(body["summary"]["failed"], 0); - let results = body["results"].as_array().unwrap(); - assert_batch_result(&results[0], archived_id, true, "unarchived"); - assert_batch_result(&results[1], not_archived_id, true, "not_archived"); -} - -#[tokio::test] -async fn batch_lifecycle_rejects_invalid_requests_before_mutating_runs() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - create_succeeded_run(&state, run_id).await; - let too_many_ids = (0..251) - .map(|_| RunId::new().to_string()) - .collect::>(); - let invalid_requests = [ - json!({ "run_ids": [] }), - json!({ "run_ids": [run_id.to_string(), run_id.to_string()] }), - json!({ "run_ids": ["not-a-run-id"] }), - json!({ "run_ids": too_many_ids }), - ]; - - for body in invalid_requests { - let response = app - .clone() - .oneshot(json_request(Method::POST, "/runs/archive", &body)) - .await - .unwrap(); - assert_status!(response, StatusCode::BAD_REQUEST).await; - } - - let response = app - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - assert!(!run_json_archived(&body)); -} - #[tokio::test] async fn batch_lifecycle_requires_user_authentication() { let (_state, app) = jwt_auth_app(); @@ -15035,215 +9137,6 @@ async fn batch_lifecycle_requires_user_authentication() { } } -#[tokio::test] -async fn batch_delete_removes_runs_and_reports_ordered_results() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let first_id = RunId::new(); - let second_id = RunId::new(); - create_succeeded_run(&state, first_id).await; - create_succeeded_run(&state, second_id).await; - - let response = app - .clone() - .oneshot(json_request( - Method::POST, - "/runs/delete", - &batch_delete_body(&[first_id, second_id], false), - )) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - assert_eq!(body["summary"]["requested"], 2); - assert_eq!(body["summary"]["succeeded"], 2); - assert_eq!(body["summary"]["failed"], 0); - let results = body["results"].as_array().unwrap(); - assert_batch_delete_result(&results[0], first_id, true, "deleted"); - assert_batch_delete_result(&results[1], second_id, true, "deleted"); - - for run_id in [first_id, second_id] { - let response = app - .clone() - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_status!(response, StatusCode::NOT_FOUND).await; - } -} - -#[tokio::test] -async fn batch_delete_reports_mixed_results_without_rollback() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let terminal_id = RunId::new(); - let running_id = RunId::new(); - let missing_id = RunId::new(); - create_succeeded_run(&state, terminal_id).await; - create_running_run(&state, running_id).await; - - let response = app - .clone() - .oneshot(json_request( - Method::POST, - "/runs/delete", - &batch_delete_body(&[terminal_id, running_id, missing_id], false), - )) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - assert_eq!(body["summary"]["requested"], 3); - assert_eq!(body["summary"]["succeeded"], 2); - assert_eq!(body["summary"]["failed"], 1); - let results = body["results"].as_array().unwrap(); - assert_batch_delete_result(&results[0], terminal_id, true, "deleted"); - assert_batch_delete_result(&results[1], running_id, false, "conflict"); - assert_eq!(results[1]["error"]["status"], "409"); - assert_batch_delete_result(&results[2], missing_id, true, "already_absent"); - - let deleted_response = app - .clone() - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{terminal_id}"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_status!(deleted_response, StatusCode::NOT_FOUND).await; - - let running_response = app - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{running_id}"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_status!(running_response, StatusCode::OK).await; -} - -#[tokio::test] -async fn batch_delete_force_removes_active_runs() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - create_running_run(&state, run_id).await; - - let response = app - .clone() - .oneshot(json_request( - Method::POST, - "/runs/delete", - &batch_delete_body(&[run_id], true), - )) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - assert_eq!(body["summary"]["requested"], 1); - assert_eq!(body["summary"]["succeeded"], 1); - assert_eq!(body["summary"]["failed"], 0); - let results = body["results"].as_array().unwrap(); - assert_batch_delete_result(&results[0], run_id, true, "deleted"); - - let response = app - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_status!(response, StatusCode::NOT_FOUND).await; -} - -#[tokio::test] -async fn batch_delete_with_preserved_sandbox_returns_handoff() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - create_preserved_local_sandbox_run(&state, run_id).await; - - let response = app - .clone() - .oneshot(json_request( - Method::POST, - "/runs/delete", - &batch_delete_body(&[run_id], true), - )) - .await - .unwrap(); - let body = response_json!(response, StatusCode::OK).await; - assert_eq!(body["summary"]["requested"], 1); - assert_eq!(body["summary"]["succeeded"], 1); - assert_eq!(body["summary"]["failed"], 0); - let results = body["results"].as_array().unwrap(); - assert_batch_delete_result(&results[0], run_id, true, "sandbox_preserved"); - assert_eq!(results[0]["sandbox"]["provider"], "local"); - assert_eq!(results[0]["sandbox"]["id"], "sandbox-preserve-1"); - - let response = app - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_status!(response, StatusCode::NOT_FOUND).await; -} - -#[tokio::test] -async fn batch_delete_rejects_invalid_requests_before_mutating_runs() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - create_succeeded_run(&state, run_id).await; - let too_many_ids = (0..251) - .map(|_| RunId::new().to_string()) - .collect::>(); - let invalid_requests = [ - json!({ "run_ids": [], "force": false }), - json!({ "run_ids": [run_id.to_string(), run_id.to_string()], "force": false }), - json!({ "run_ids": ["not-a-run-id"], "force": false }), - json!({ "run_ids": too_many_ids, "force": false }), - ]; - - for body in invalid_requests { - let response = app - .clone() - .oneshot(json_request(Method::POST, "/runs/delete", &body)) - .await - .unwrap(); - assert_status!(response, StatusCode::BAD_REQUEST).await; - } - - let response = app - .oneshot( - Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}"))) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_status!(response, StatusCode::OK).await; -} - #[tokio::test] async fn batch_delete_requires_user_authentication() { let (_state, app) = jwt_auth_app(); @@ -15330,180 +9223,6 @@ async fn delete_run_removes_durable_run() { assert_status!(response, StatusCode::NOT_FOUND).await; } -#[tokio::test] -async fn delete_run_force_removes_unreadable_durable_run() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - create_unreadable_durable_run(&state, run_id).await; - - let req = Request::builder() - .method("GET") - .uri(api("/system/repair/runs")) - .body(Body::empty()) - .unwrap(); - let body = response_json!(app.clone().oneshot(req).await.unwrap(), StatusCode::OK).await; - assert_eq!(body["total_count"], 1); - assert_eq!(body["runs"][0]["run_id"], run_id.to_string()); - - let req = Request::builder() - .method("DELETE") - .uri(api(&format!("/runs/{run_id}?force=true"))) - .body(Body::empty()) - .unwrap(); - let response = app.clone().oneshot(req).await.unwrap(); - assert_status!(response, StatusCode::NO_CONTENT).await; - - let req = Request::builder() - .method("GET") - .uri(api("/system/repair/runs")) - .body(Body::empty()) - .unwrap(); - let body = response_json!(app.oneshot(req).await.unwrap(), StatusCode::OK).await; - assert_eq!(body["total_count"], 0); - assert!(body["runs"].as_array().unwrap().is_empty()); -} - -#[tokio::test] -async fn delete_run_without_force_keeps_active_durable_run() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - create_unreadable_durable_run(&state, run_id).await; - - let req = Request::builder() - .method("DELETE") - .uri(api(&format!("/runs/{run_id}"))) - .body(Body::empty()) - .unwrap(); - let response = app.clone().oneshot(req).await.unwrap(); - response_json!(response, StatusCode::CONFLICT).await; - - let req = Request::builder() - .method("GET") - .uri(api("/system/repair/runs")) - .body(Body::empty()) - .unwrap(); - let body = response_json!(app.oneshot(req).await.unwrap(), StatusCode::OK).await; - assert_eq!(body["total_count"], 1); - assert_eq!(body["runs"][0]["run_id"], run_id.to_string()); -} - -#[tokio::test] -async fn delete_run_with_preserved_sandbox_returns_handoff() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - create_preserved_local_sandbox_run(&state, run_id).await; - - let req = Request::builder() - .method("DELETE") - .uri(api(&format!("/runs/{run_id}?force=true"))) - .body(Body::empty()) - .unwrap(); - let response = app.clone().oneshot(req).await.unwrap(); - let body = response_json!(response, StatusCode::OK).await; - assert_eq!(body["deleted"].as_bool(), Some(true)); - assert_eq!(body["sandbox_preserved"].as_bool(), Some(true)); - assert_eq!(body["sandbox"]["provider"].as_str(), Some("local")); - assert_eq!(body["sandbox"]["id"].as_str(), Some("sandbox-preserve-1")); - assert!(body["sandbox"].get("identifier").is_none()); - - let req = Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}"))) - .body(Body::empty()) - .unwrap(); - let response = app.oneshot(req).await.unwrap(); - assert_status!(response, StatusCode::NOT_FOUND).await; -} - -#[tokio::test] -async fn delete_run_retry_after_missing_provider_resource_removes_metadata() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = RunId::new(); - let graph = Graph::new("test"); - - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::RunCreated { - run_id, - title: None, - settings: serde_json::to_value(fabro_types::WorkflowSettings::default()).unwrap(), - graph: serde_json::to_value(graph).unwrap(), - workflow_source: None, - labels: std::collections::BTreeMap::default(), - source_directory: Some("/tmp/fabro-run".to_string()), - workflow_slug: Some("test".to_string()), - workflow_version_id: None, - target: None, - automation: None, - provenance: test_support::test_run_provenance(), - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, - admission: PetriAdmission::default(), - }, - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - workflow_event::Event::SandboxInitialized { - provider: SandboxProviderKind::DOCKER, - id: "missing-sandbox".to_string(), - working_directory: "/tmp/fabro-missing-sandbox".to_string(), - image: None, - snapshot: None, - repo_cloned: Some(false), - clone_origin_url: None, - clone_branch: None, - workspace_root: None, - repos_root: None, - primary_repo_path: None, - primary_repo_link: None, - }, - workflow_event::Event::WorkflowRunCompleted { - timing: fabro_types::RunTiming::wall_only(1), - artifact_count: 0, - status: "succeeded".to_string(), - reason: SuccessReason::Completed, - final_git_commit_sha: None, - final_patch: None, - diff_summary: None, - usage: None, - }, - ]) - .await; - - let req = Request::builder() - .method("DELETE") - .uri(api(&format!("/runs/{run_id}"))) - .body(Body::empty()) - .unwrap(); - let response = app.clone().oneshot(req).await.unwrap(); - response_json!(response, StatusCode::CONFLICT).await; - - let req = Request::builder() - .method("DELETE") - .uri(api(&format!("/runs/{run_id}"))) - .body(Body::empty()) - .unwrap(); - let response = app.clone().oneshot(req).await.unwrap(); - assert_status!(response, StatusCode::NO_CONTENT).await; - - let req = Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}"))) - .body(Body::empty()) - .unwrap(); - let response = app.oneshot(req).await.unwrap(); - assert_status!(response, StatusCode::NOT_FOUND).await; -} - #[tokio::test] async fn delete_active_run_requires_force() { let state = test_app_state(); @@ -16538,374 +10257,6 @@ async fn unpause_run_returns_blocked_when_human_gate_is_still_unresolved() { assert_eq!(summary.lifecycle.pending_control, None); } -#[tokio::test] -async fn reconcile_incomplete_runs_relaunches_runnable_runs_and_fails_cancelled_ones() { - let object_store: Arc = - Arc::new(object_store::memory::InMemory::new()); - let summaries = fabro_store::test_support::test_run_summary_store(); - let blobs = fabro_store::test_support::test_blob_store(); - let first_state = test_app_state_over_shared_stores(&object_store, &blobs, &summaries); - let mut histories = Vec::new(); - - for (run_id, cancel_pending) in [(fixtures::RUN_1, false), (fixtures::RUN_2, true)] { - let mut events = vec![ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunRunnable { - source: fabro_types::RunRunnableSource::StartRequested, - actor: None, - }, - ]; - if cancel_pending { - events.push(workflow_event::Event::RunCancelRequested { actor: None }); - } - create_durable_run_with_events(&first_state, run_id, &events).await; - - let reader = first_state - .stores - .runs - .open_run_reader(&run_id) - .await - .unwrap(); - let run = reader.state().await.unwrap(); - assert_eq!(run.status, RunStatus::Runnable); - assert_eq!( - run.pending_control, - cancel_pending.then_some(RunControlAction::Cancel) - ); - let history = reader.list_events().await.unwrap(); - assert_eq!(history.len(), events.len() + 1); - histories.push((run_id, cancel_pending, history)); - } - assert!(first_state.runs.lock().unwrap().is_empty()); - drop(first_state); - - // The run with no cancel pending goes back to a worker: it is asked to - // start again, stays runnable for the scheduler, and is managed in - // start mode since no worker ever created its Petri record. The run - // whose cancel was pending ends cancelled. - let reopened_state = test_app_state_over_shared_stores(&object_store, &blobs, &summaries); - assert!(reopened_state.runs.lock().unwrap().is_empty()); - assert_eq!( - reconcile_incomplete_runs_on_startup(&reopened_state) - .await - .unwrap(), - 2 - ); - - for (run_id, cancel_pending, before) in histories { - let reader = reopened_state - .stores - .runs - .open_run_reader(&run_id) - .await - .unwrap(); - let run = reader.state().await.unwrap(); - let after = reader.list_events().await.unwrap(); - assert_eq!(&after[..before.len()], before.as_slice()); - let appended = after[before.len()..] - .iter() - .map(|envelope| envelope.event.event_name()) - .collect::>(); - let summary = summaries.get(&run_id, Utc::now()).await.unwrap().unwrap(); - assert_eq!(summary.lifecycle.status, run.status); - assert_eq!(run.pending_control, None); - assert_eq!(summary.lifecycle.pending_control, None); - let managed = reopened_state.runs.lock().unwrap(); - if cancel_pending { - assert_eq!(run.status, RunStatus::Failed { - reason: FailureReason::Cancelled, - }); - assert_eq!(appended, vec!["run.failed"]); - assert!(!managed.contains_key(&run_id)); - } else { - assert_eq!(run.status, RunStatus::Runnable); - assert_eq!(appended, vec!["run.start_requested", "run.runnable"]); - let managed_run = managed.get(&run_id).expect("the run is managed again"); - assert_eq!(managed_run.status, RunStatus::Runnable); - assert!(matches!( - managed_run.execution_mode, - RunExecutionMode::Start - )); - assert!(managed_run.worker_ref.is_none()); - } - } -} - -#[tokio::test] -async fn reconcile_incomplete_runs_relaunches_inflight_runs() { - let state = test_app_state(); - - create_durable_run_with_events(&state, fixtures::RUN_1, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - ]) - .await; - create_durable_run_with_events(&state, fixtures::RUN_2, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - ]) - .await; - create_durable_run_with_events(&state, fixtures::RUN_3, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - workflow_event::Event::RunPaused, - workflow_event::Event::RunCancelRequested { actor: None }, - ]) - .await; - - create_durable_run_with_events(&state, fixtures::RUN_4, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunPending { - reason: fabro_types::PendingReason::ApprovalRequired, - actor: None, - }, - ]) - .await; - let mut untouched_histories = Vec::new(); - for (run_id, expected_status) in [ - (fixtures::RUN_1, RunStatus::Submitted), - (fixtures::RUN_4, RunStatus::Pending { - reason: fabro_types::PendingReason::ApprovalRequired, - }), - ] { - let reader = state.stores.runs.open_run_reader(&run_id).await.unwrap(); - assert_eq!(reader.state().await.unwrap().status, expected_status); - untouched_histories.push((run_id, expected_status, reader.list_events().await.unwrap())); - } - - let reconciled = reconcile_incomplete_runs_on_startup(&state).await.unwrap(); - assert_eq!(reconciled, 2); - - for (run_id, expected_status, expected_history) in untouched_histories { - let reader = state.stores.runs.open_run_reader(&run_id).await.unwrap(); - assert_eq!(reader.state().await.unwrap().status, expected_status); - assert_eq!(reader.list_events().await.unwrap(), expected_history); - let summary = state - .stores - .run_summaries - .get(&run_id, Utc::now()) - .await - .unwrap() - .unwrap(); - assert_eq!(summary.lifecycle.status, expected_status); - } - - // The running run continues in a new worker: runnable again, managed - // for the scheduler. - let run_2 = state - .stores - .runs - .open_run_reader(&fixtures::RUN_2) - .await - .unwrap() - .state() - .await - .unwrap(); - assert_eq!(run_2.status, RunStatus::Runnable); - assert!(state.runs.lock().unwrap().contains_key(&fixtures::RUN_2)); - - // The paused run whose cancel was pending ends cancelled. - let run_3 = state - .stores - .runs - .open_run_reader(&fixtures::RUN_3) - .await - .unwrap() - .state() - .await - .unwrap(); - assert_eq!(run_3.status, RunStatus::Failed { - reason: FailureReason::Cancelled, - }); - assert_eq!(run_3.pending_control, None); - assert!(!state.runs.lock().unwrap().contains_key(&fixtures::RUN_3)); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn shutdown_active_workers_uses_worker_runtime_for_live_refs() { - let runtime = StdArc::new(RecordingWorkerRuntime::default()); - runtime.set_alive(true); - let state = TestAppStateBuilder::new() - .worker_runtime(runtime.clone()) - .build(); - let worker_refs = [test_worker_ref(u32::MAX - 1), test_worker_ref(u32::MAX)]; - let run_ids = [RunId::new(), RunId::new()]; - let temp_dir = tempfile::tempdir().unwrap(); - - for (run_id, worker_ref) in run_ids.iter().zip(worker_refs.iter()) { - create_durable_run_with_events(&state, *run_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - ]) - .await; - - let mut run = managed_run( - String::new(), - RunStatus::Running, - chrono::Utc::now(), - temp_dir.path().join(run_id.to_string()), - RunExecutionMode::Start, - ); - run.worker_ref = Some(worker_ref.clone()); - state - .runs - .lock() - .expect("runs lock poisoned") - .insert(*run_id, run); - } - - let terminated = shutdown_active_workers_with_grace( - &state, - Duration::from_millis(0), - Duration::from_millis(1), - ) - .await - .unwrap(); - - assert_eq!(terminated, 2); - assert_eq!(runtime.requested_refs().len(), 2); - assert_eq!(runtime.forced_refs().len(), 2); -} - -#[cfg(unix)] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn shutdown_active_workers_terminates_process_groups() { - let state = test_app_state(); - let run_id = fixtures::RUN_4; - - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - ]) - .await; - - let temp_dir = tempfile::tempdir().unwrap(); - let mut child = tokio::process::Command::new("sh"); - child - .arg("-c") - .arg("trap '' TERM; while :; do sleep 1; done") - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - fabro_proc::pre_exec_setpgid(child.as_std_mut()); - let mut child = child.spawn().unwrap(); - let worker_process_id = child.id().expect("worker pid should be available"); - - { - let mut runs = state.runs.lock().expect("runs lock poisoned"); - let mut run = managed_run( - String::new(), - RunStatus::Running, - chrono::Utc::now(), - temp_dir.path().join(run_id.to_string()), - RunExecutionMode::Start, - ); - run.worker_ref = Some(test_worker_ref(worker_process_id)); - runs.insert(run_id, run); - } - - let terminated = shutdown_active_workers_with_grace( - &state, - Duration::from_millis(50), - Duration::from_millis(10), - ) - .await - .unwrap(); - assert_eq!(terminated, 1); - - let exit_status = tokio::time::timeout(Duration::from_secs(2), child.wait()) - .await - .expect("worker should exit after shutdown") - .expect("wait should succeed"); - assert!(!exit_status.success()); - assert!(!fabro_proc::process_group_alive(worker_process_id)); - - let run_state = state - .stores - .runs - .open_run_reader(&run_id) - .await - .unwrap() - .state() - .await - .unwrap(); - let run_status = run_state.status; - assert_eq!(run_status, RunStatus::Failed { - reason: FailureReason::Terminated, - }); -} - -/// Reasoning has to survive the whole durable path, not just the local -/// struct conversion: emitted event → run store → attach SSE JSON. -#[tokio::test] -async fn attach_stream_replays_agent_message_reasoning() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = fixtures::RUN_1; - - create_durable_run_with_events(&state, run_id, &[ - stage_started_event("code", "agent"), - agent_message_event( - "code", - 1, - "session-1", - "", - None, - Some(ReasoningOutput::new( - "inspect the sink first", - "read events.rs, then attach", - )), - ), - workflow_event::Event::WorkflowRunCompleted { - timing: fabro_types::RunTiming::wall_only(1000), - artifact_count: 0, - status: "succeeded".to_string(), - reason: SuccessReason::Completed, - final_git_commit_sha: None, - final_patch: None, - diff_summary: None, - usage: None, - }, - ]) - .await; - - let req = Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/attach?since_seq=1"))) - .body(Body::empty()) - .unwrap(); - let response = app.oneshot(req).await.unwrap(); - let body = response_bytes!(response, StatusCode::OK).await; - let text = String::from_utf8(body.clone()).unwrap(); - - let message = text - .lines() - .filter_map(|line| line.strip_prefix("data: ")) - .filter_map(|data| serde_json::from_str::(data).ok()) - .find(|value| value["event"] == "agent.message") - .expect("attach stream should replay the agent message"); - let reasoning = &message["properties"]["event"]["AssistantMessage"]["reasoning"]; - assert_eq!(reasoning["summary"], "inspect the sink first"); - assert_eq!(reasoning["trace"], "read events.rs, then attach"); -} - #[tokio::test] async fn queue_position_reported_for_runnable_runs() { let state = test_app_state(); @@ -17472,768 +10823,6 @@ async fn list_runs_returns_run_list_items() { assert!(item["usage"].get("cost").is_none()); } -#[tokio::test] -async fn list_runs_excludes_removing_status_by_default() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = fixtures::RUN_1; - - // A run in Removing status should not appear by default - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - workflow_event::Event::RunRemoving, - ]) - .await; - - let req = Request::builder() - .method("GET") - .uri(api("/runs")) - .body(Body::empty()) - .unwrap(); - let response = app.clone().oneshot(req).await.unwrap(); - let body = response_json!(response, StatusCode::OK).await; - let data = body["data"].as_array().expect("data should be array"); - assert!( - !data - .iter() - .any(|i| run_json_id(i) == Some(&run_id.to_string())), - "removing run should not appear by default" - ); - - // ?status=removing opts the bucket in. - let req = Request::builder() - .method("GET") - .uri(api("/runs?status=removing")) - .body(Body::empty()) - .unwrap(); - let response = app.oneshot(req).await.unwrap(); - let body = response_json!(response, StatusCode::OK).await; - let data = body["data"].as_array().expect("data should be array"); - assert!( - data.iter() - .any(|i| run_json_id(i) == Some(&run_id.to_string())), - "?status=removing should opt removing runs in" - ); -} - -#[tokio::test] -async fn list_runs_excludes_archived_by_default() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = fixtures::RUN_1; - - create_durable_run_with_events(&state, run_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - workflow_event::Event::WorkflowRunCompleted { - timing: fabro_types::RunTiming::wall_only(1000), - artifact_count: 0, - status: "succeeded".to_string(), - reason: SuccessReason::Completed, - final_git_commit_sha: None, - final_patch: None, - diff_summary: None, - usage: None, - }, - workflow_event::Event::RunArchived { actor: None }, - ]) - .await; - - let req = Request::builder() - .method("GET") - .uri(api("/runs")) - .body(Body::empty()) - .unwrap(); - let response = app.oneshot(req).await.unwrap(); - let body = response_json!(response, StatusCode::OK).await; - let data = body["data"].as_array().expect("data should be array"); - assert!( - !data - .iter() - .any(|i| run_json_id(i) == Some(&run_id.to_string())), - "archived run should be hidden when include_archived is unset", - ); -} - -#[tokio::test] -async fn list_runs_includes_archived_when_flag_set() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let archived_id = fixtures::RUN_1; - let succeeded_id = fixtures::RUN_2; - - create_durable_run_with_events(&state, archived_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - workflow_event::Event::WorkflowRunCompleted { - timing: fabro_types::RunTiming::wall_only(1000), - artifact_count: 0, - status: "succeeded".to_string(), - reason: SuccessReason::Completed, - final_git_commit_sha: None, - final_patch: None, - diff_summary: None, - usage: None, - }, - workflow_event::Event::RunArchived { actor: None }, - ]) - .await; - create_durable_run_with_events(&state, succeeded_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - workflow_event::Event::WorkflowRunCompleted { - timing: fabro_types::RunTiming::wall_only(1000), - artifact_count: 0, - status: "succeeded".to_string(), - reason: SuccessReason::Completed, - final_git_commit_sha: None, - final_patch: None, - diff_summary: None, - usage: None, - }, - ]) - .await; - - let req = Request::builder() - .method("GET") - .uri(api("/runs?include_archived=true")) - .body(Body::empty()) - .unwrap(); - let response = app.oneshot(req).await.unwrap(); - let body = response_json!(response, StatusCode::OK).await; - let data = body["data"].as_array().expect("data should be array"); - - let archived_item = data - .iter() - .find(|i| run_json_id(i) == Some(&archived_id.to_string())) - .expect("archived run should appear when include_archived=true"); - assert!(run_json_archived(archived_item)); - assert_eq!( - run_json_status(archived_item)["kind"].as_str().unwrap(), - "succeeded" - ); - - let succeeded_item = data - .iter() - .find(|i| run_json_id(i) == Some(&succeeded_id.to_string())) - .expect("non-archived run should still appear"); - assert_eq!( - run_json_status(succeeded_item)["kind"].as_str().unwrap(), - "succeeded" - ); -} - -#[tokio::test] -async fn get_run_exposes_canonical_operator_statuses() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - - let succeeded_id = fixtures::RUN_1; - let removing_id = fixtures::RUN_2; - let blocked_id = fixtures::RUN_3; - - create_durable_run_with_events(&state, succeeded_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - workflow_event::Event::WorkflowRunCompleted { - timing: fabro_types::RunTiming::wall_only(1000), - artifact_count: 0, - status: "succeeded".to_string(), - reason: SuccessReason::Completed, - final_git_commit_sha: None, - final_patch: None, - diff_summary: None, - usage: None, - }, - ]) - .await; - - create_durable_run_with_events(&state, removing_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - workflow_event::Event::RunRemoving, - ]) - .await; - create_durable_run_with_events(&state, blocked_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - ]) - .await; - append_raw_run_event( - &state, - blocked_id, - "status-blocked", - "2026-04-19T12:00:00Z", - "run.blocked", - json!({ "blocked_reason": "human_input_required" }), - None, - ) - .await; - - for (run_id, expected_status) in [ - (succeeded_id, "succeeded"), - (removing_id, "removing"), - (blocked_id, "blocked"), - ] { - let req = Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}"))) - .body(Body::empty()) - .unwrap(); - let response = app.clone().oneshot(req).await.unwrap(); - let body = response_json!(response, StatusCode::OK).await; - assert_eq!( - run_json_status(&body)["kind"].as_str(), - Some(expected_status) - ); - } -} - -#[tokio::test] -async fn list_runs_preserves_underlying_run_status_payloads() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - - let paused_id = fixtures::RUN_1; - let succeeded_id = fixtures::RUN_2; - let blocked_id = fixtures::RUN_3; - - create_durable_run_with_events(&state, paused_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - workflow_event::Event::RunPaused, - ]) - .await; - create_durable_run_with_events(&state, succeeded_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - workflow_event::Event::WorkflowRunCompleted { - timing: fabro_types::RunTiming::wall_only(1000), - artifact_count: 0, - status: "succeeded".to_string(), - reason: SuccessReason::Completed, - final_git_commit_sha: None, - final_patch: None, - diff_summary: None, - usage: None, - }, - ]) - .await; - create_durable_run_with_events(&state, blocked_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - ]) - .await; - append_raw_run_event( - &state, - blocked_id, - "blocked-question-1", - "2026-04-19T12:00:00Z", - "interview.started", - json!({ - "question_id": "q-older", - "question": "Older unresolved question?", - "stage": "gate", - "question_type": "multiple_choice", - "options": [], - "allow_freeform": false, - "context_display": null, - "timeout_seconds": null, - }), - Some("gate"), - ) - .await; - append_raw_run_event( - &state, - blocked_id, - "blocked-question-2", - "2026-04-19T12:00:01Z", - "interview.started", - json!({ - "question_id": "q-newer", - "question": "Newer unresolved question?", - "stage": "gate", - "question_type": "multiple_choice", - "options": [], - "allow_freeform": false, - "context_display": null, - "timeout_seconds": null, - }), - Some("gate"), - ) - .await; - append_raw_run_event( - &state, - blocked_id, - "blocked-status", - "2026-04-19T12:00:02Z", - "run.blocked", - json!({ "blocked_reason": "human_input_required" }), - None, - ) - .await; - - let req = Request::builder() - .method("GET") - .uri(api("/runs")) - .body(Body::empty()) - .unwrap(); - let response = app.oneshot(req).await.unwrap(); - let body = body_json(response.into_body()).await; - let data = body["data"].as_array().expect("data should be array"); - - let paused_item = data - .iter() - .find(|i| run_json_id(i) == Some(&paused_id.to_string())) - .expect("paused run should be on board"); - assert_eq!( - run_json_status(paused_item)["kind"].as_str().unwrap(), - "paused" - ); - assert!(run_json_status(paused_item)["prior_block"].is_null()); - - let succeeded_item = data - .iter() - .find(|i| run_json_id(i) == Some(&succeeded_id.to_string())) - .expect("succeeded run should be on board"); - assert_eq!( - run_json_status(succeeded_item)["kind"].as_str().unwrap(), - "succeeded" - ); - assert_eq!( - run_json_status(succeeded_item)["reason"].as_str().unwrap(), - "completed" - ); - - let blocked_item = data - .iter() - .find(|i| run_json_id(i) == Some(&blocked_id.to_string())) - .expect("blocked run should be on board"); - assert_eq!( - run_json_status(blocked_item)["kind"].as_str().unwrap(), - "blocked" - ); - assert_eq!( - run_json_status(blocked_item)["blocked_reason"] - .as_str() - .unwrap(), - "human_input_required" - ); - assert!( - blocked_item["current_question"].is_object(), - "blocked item should include the current question" - ); -} - -#[tokio::test] -async fn list_runs_includes_live_metadata_from_run_state() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - let run_id = create_and_start_run(&app, MINIMAL_DOT) - .await - .parse::() - .unwrap(); - let run_store = state.stores.runs.open_run(&run_id).await.unwrap(); - for event in [ - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - workflow_event::Event::SandboxInitialized { - provider: SandboxProviderKind::LOCAL, - id: "sb-test".to_string(), - working_directory: "/sandbox/workdir".to_string(), - image: None, - snapshot: None, - repo_cloned: None, - clone_origin_url: None, - clone_branch: None, - workspace_root: None, - repos_root: None, - primary_repo_path: None, - primary_repo_link: None, - }, - workflow_event::Event::PullRequestCreated { - pr_url: "https://github.com/acme/repo/pull/42".to_string(), - pr_number: 42, - owner: "acme".to_string(), - repo: "repo".to_string(), - base_branch: "main".to_string(), - head_branch: "fabro/run".to_string(), - head_sha: Some("final-sha".to_string()), - title: "Fix board metadata".to_string(), - draft: false, - }, - workflow_event::Event::InterviewStarted { - question_id: "q-1".to_string(), - question: "Ship it?".to_string(), - stage: "review".to_string(), - question_type: "yes_no".to_string(), - options: vec![], - allow_freeform: false, - timeout_seconds: None, - context_display: None, - review_target: None, - }, - ] { - workflow_event::append_event(&run_store, &run_id, &event) - .await - .unwrap(); - } - - let req = Request::builder() - .method("GET") - .uri(api("/runs")) - .body(Body::empty()) - .unwrap(); - let response = app.oneshot(req).await.unwrap(); - let body = response_json!(response, StatusCode::OK).await; - let data = body["data"].as_array().expect("data should be array"); - let item = data - .iter() - .find(|i| run_json_id(i) == Some(&run_id.to_string())) - .expect("run should be in board"); - - assert_eq!(item["pull_request"]["number"].as_u64(), Some(42)); - assert_eq!(item["sandbox"]["kind"].as_str(), Some("ready")); - assert_eq!( - item["sandbox"]["instance"]["runtime"]["id"].as_str(), - Some("sb-test") - ); - assert_eq!( - item["sandbox"]["instance"]["runtime"]["working_directory"].as_str(), - Some("/sandbox/workdir") - ); - assert!(item["current_question"].is_object()); -} - -#[tokio::test] -async fn list_runs_page_limit_preserves_metadata_for_paged_items() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - - let first_run_id = create_and_start_run(&app, MINIMAL_DOT) - .await - .parse::() - .unwrap(); - let second_run_id = create_and_start_run(&app, MINIMAL_DOT) - .await - .parse::() - .unwrap(); - - for (run_id, sandbox_id) in [(first_run_id, "sb-first"), (second_run_id, "sb-second")] { - let run_store = state.stores.runs.open_run(&run_id).await.unwrap(); - for event in [ - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - workflow_event::Event::SandboxInitialized { - provider: SandboxProviderKind::LOCAL, - id: sandbox_id.to_string(), - working_directory: "/sandbox/workdir".to_string(), - image: None, - snapshot: None, - repo_cloned: None, - clone_origin_url: None, - clone_branch: None, - workspace_root: None, - repos_root: None, - primary_repo_path: None, - primary_repo_link: None, - }, - ] { - workflow_event::append_event(&run_store, &run_id, &event) - .await - .unwrap(); - } - } - - let req = Request::builder() - .method("GET") - .uri(api("/runs?page[limit]=1")) - .body(Body::empty()) - .unwrap(); - let response = app.oneshot(req).await.unwrap(); - let body = response_json!(response, StatusCode::OK).await; - assert_eq!(body["meta"]["has_more"].as_bool(), Some(true)); - - let data = body["data"].as_array().expect("data should be array"); - assert_eq!(data.len(), 1); - - let item = &data[0]; - let sandbox_id = item["sandbox"]["instance"]["runtime"]["id"] - .as_str() - .expect("paged item should still include sandbox metadata"); - assert!(matches!(sandbox_id, "sb-first" | "sb-second")); -} - -#[tokio::test] -async fn list_runs_status_filter_accepts_repeated_values() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - - // Running run (will map to BoardColumn::Running) - let running_id = fixtures::RUN_1; - create_durable_run_with_events(&state, running_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - ]) - .await; - - // Succeeded run (BoardColumn::Succeeded) - let succeeded_id = fixtures::RUN_2; - create_durable_run_with_events(&state, succeeded_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - workflow_event::Event::WorkflowRunCompleted { - timing: fabro_types::RunTiming::wall_only(1000), - artifact_count: 0, - status: "succeeded".to_string(), - reason: SuccessReason::Completed, - final_git_commit_sha: None, - final_patch: None, - diff_summary: None, - usage: None, - }, - ]) - .await; - - // Pending run (BoardColumn::Pending via Submitted) - let pending_id = fixtures::RUN_3; - create_durable_run_with_events(&state, pending_id, &[workflow_event::Event::RunSubmitted { - definition_blob: None, - }]) - .await; - - // Single value: only running. - let req = Request::builder() - .method("GET") - .uri(api("/runs?status=running")) - .body(Body::empty()) - .unwrap(); - let response = app.clone().oneshot(req).await.unwrap(); - let body = response_json!(response, StatusCode::OK).await; - let ids: Vec<&str> = body["data"] - .as_array() - .unwrap() - .iter() - .filter_map(run_json_id) - .collect(); - assert!(ids.contains(&running_id.to_string().as_str())); - assert!(!ids.contains(&succeeded_id.to_string().as_str())); - assert!(!ids.contains(&pending_id.to_string().as_str())); - - // Repeated values: running + succeeded. - let req = Request::builder() - .method("GET") - .uri(api("/runs?status=running&status=succeeded")) - .body(Body::empty()) - .unwrap(); - let response = app.oneshot(req).await.unwrap(); - let body = response_json!(response, StatusCode::OK).await; - let ids: Vec<&str> = body["data"] - .as_array() - .unwrap() - .iter() - .filter_map(run_json_id) - .collect(); - assert!(ids.contains(&running_id.to_string().as_str())); - assert!(ids.contains(&succeeded_id.to_string().as_str())); - assert!(!ids.contains(&pending_id.to_string().as_str())); -} - -#[tokio::test] -async fn list_runs_sort_direction_reverses_order_with_stable_tiebreak() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - - // All fixtures share timestamp=0; the id-desc tiebreak controls order. - let ids = [fixtures::RUN_1, fixtures::RUN_2, fixtures::RUN_3]; - for id in &ids { - create_durable_run_with_events(&state, *id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - ]) - .await; - } - - // Default (sort=created_at desc): tiebreak puts higher ids first. - let req = Request::builder() - .method("GET") - .uri(api("/runs")) - .body(Body::empty()) - .unwrap(); - let response = app.clone().oneshot(req).await.unwrap(); - let body = response_json!(response, StatusCode::OK).await; - let observed: Vec = body["data"] - .as_array() - .unwrap() - .iter() - .filter_map(run_json_id) - .map(str::to_string) - .collect(); - let mut expected: Vec = ids.iter().map(std::string::ToString::to_string).collect(); - expected.sort_by(|a, b| b.cmp(a)); // desc by id - assert_eq!(observed, expected, "default desc order with id tiebreak"); - - // Ascending: timestamps still tie, then id desc tiebreak still applies. - let req = Request::builder() - .method("GET") - .uri(api("/runs?sort=created_at&direction=asc")) - .body(Body::empty()) - .unwrap(); - let response = app.oneshot(req).await.unwrap(); - let body = response_json!(response, StatusCode::OK).await; - let observed: Vec = body["data"] - .as_array() - .unwrap() - .iter() - .filter_map(run_json_id) - .map(str::to_string) - .collect(); - assert_eq!( - observed, expected, - "asc still uses id-desc tiebreak for tied keys" - ); -} - -#[tokio::test] -async fn list_runs_sort_by_status_groups_by_bucket() { - let state = test_app_state(); - let app = crate::test_support::build_test_router(Arc::clone(&state)); - - // BoardColumn enum order: pending < runnable < initializing < running < - // blocked < succeeded < failed < archived < removing. Use three distinct - // buckets. - let pending_id = fixtures::RUN_1; - create_durable_run_with_events(&state, pending_id, &[workflow_event::Event::RunSubmitted { - definition_blob: None, - }]) - .await; - - let succeeded_id = fixtures::RUN_2; - create_durable_run_with_events(&state, succeeded_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - workflow_event::Event::WorkflowRunCompleted { - timing: fabro_types::RunTiming::wall_only(1000), - artifact_count: 0, - status: "succeeded".to_string(), - reason: SuccessReason::Completed, - final_git_commit_sha: None, - final_patch: None, - diff_summary: None, - usage: None, - }, - ]) - .await; - - let running_id = fixtures::RUN_3; - create_durable_run_with_events(&state, running_id, &[ - workflow_event::Event::RunSubmitted { - definition_blob: None, - }, - workflow_event::Event::RunStarting, - workflow_event::Event::RunRunning, - ]) - .await; - - // sort=status asc: pending < running < succeeded. - let req = Request::builder() - .method("GET") - .uri(api("/runs?sort=status&direction=asc")) - .body(Body::empty()) - .unwrap(); - let response = app.oneshot(req).await.unwrap(); - let body = response_json!(response, StatusCode::OK).await; - let observed: Vec = body["data"] - .as_array() - .unwrap() - .iter() - .filter_map(run_json_id) - .map(str::to_string) - .collect(); - assert_eq!(observed, vec![ - pending_id.to_string(), - running_id.to_string(), - succeeded_id.to_string(), - ]); -} - -#[tokio::test] -async fn filtered_global_events_streams_only_matching_run_ids() { - let run_one = fixtures::RUN_1; - let run_two = fixtures::RUN_2; - let (event_tx, _) = broadcast::channel(8); - - let stream = filtered_global_events(event_tx.subscribe(), Some(HashSet::from([run_one]))); - - event_tx - .send(test_event_envelope( - 1, - run_two, - EventBody::RunRunnable(fabro_types::run_event::RunRunnableProps { - source: fabro_types::RunRunnableSource::StartRequested, - }), - )) - .unwrap(); - event_tx - .send(test_event_envelope( - 2, - run_one, - EventBody::RunRunnable(fabro_types::run_event::RunRunnableProps { - source: fabro_types::RunRunnableSource::StartRequested, - }), - )) - .unwrap(); - drop(event_tx); - - let events = stream.collect::>().await; - assert_eq!(events.len(), 1); - assert_eq!(events[0].seq, 2); - assert_eq!(events[0].event.run_id, run_one); -} - #[test] fn validate_github_slug_accepts_real_names() { assert!(super::validate_github_slug("owner", "anthropic", 39).is_ok()); diff --git a/lib/apps/fabro-server/tests/it/api/events.rs b/lib/apps/fabro-server/tests/it/api/events.rs deleted file mode 100644 index 9419e4768..000000000 --- a/lib/apps/fabro-server/tests/it/api/events.rs +++ /dev/null @@ -1,187 +0,0 @@ -use std::sync::Arc; -use std::time::Duration; - -use axum::body::{Body, to_bytes}; -use axum::http::{Request, StatusCode}; -use chrono::{SecondsFormat, Utc}; -use fabro_static::EnvVars; -use object_store::ObjectStore; -use object_store::memory::InMemory; -use tokio::sync::Barrier; -use tower::ServiceExt; - -use crate::helpers::{MINIMAL_DOT, api, minimal_intent_json, response_json, test_settings}; - -fn app_with_store( - object_store: Arc, - blobs: Arc, - run_summaries: Arc, -) -> axum::Router { - let settings = test_settings(); - let store = Arc::new(fabro_store::test_support::test_database_with_stores( - Arc::clone(&object_store), - "event-race", - Duration::from_millis(1), - None, - blobs, - run_summaries, - )); - let artifact_store = fabro_store::ArtifactStore::new(object_store, "artifacts"); - let state = fabro_server::test_support::TestAppStateBuilder::new() - .runtime_settings(settings.server_settings, settings.manifest_run_defaults) - .env_lookup(|_| None) - .vault_entries([(EnvVars::OPENAI_API_KEY, "test-openai-api-key")]) - .store_bundle(store, artifact_store) - .build(); - fabro_server::test_support::build_test_router(state) -} - -async fn create_run(app: &axum::Router) -> (String, tempfile::TempDir) { - let workspace = tempfile::tempdir().expect("run target workspace should be created"); - let request = Request::builder() - .method("POST") - .uri(api("/runs")) - .header("content-type", "application/json") - .body(Body::from( - serde_json::to_string(&minimal_intent_json(app, MINIMAL_DOT, workspace.path()).await) - .expect("intent should serialize"), - )) - .expect("create-run request should build"); - let body = response_json( - app.clone().oneshot(request).await.unwrap(), - StatusCode::CREATED, - "POST /api/v1/runs", - ) - .await; - let run_id = body["id"] - .as_str() - .expect("create-run response should include an id") - .to_string(); - (run_id, workspace) -} - -fn append_stage_started_request(run_id: &str, index: usize) -> Request { - Request::builder() - .method("POST") - .uri(api(&format!("/runs/{run_id}/events"))) - .header("content-type", "application/json") - .body(Body::from( - serde_json::to_string(&serde_json::json!({ - "id": ulid::Ulid::new().to_string(), - "ts": Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), - "run_id": run_id, - "event": "stage.started", - "node_id": format!("race-{index}"), - "node_label": format!("Race {index}"), - "stage_id": format!("race-{index}@1"), - "actor": { - "kind": "worker", - "run_id": run_id, - }, - "properties": { - "index": index, - "handler_type": "noop", - "attempt": 1, - "max_attempts": 1, - }, - })) - .expect("event should serialize"), - )) - .expect("append-event request should build") -} - -async fn append_status_and_body( - app: axum::Router, - run_id: String, - index: usize, - barrier: Arc, -) -> (usize, StatusCode, String) { - barrier.wait().await; - let response = app - .oneshot(append_stage_started_request(&run_id, index)) - .await - .expect("append-event response should execute"); - let status = response.status(); - let bytes = to_bytes(response.into_body(), usize::MAX) - .await - .expect("append-event body should buffer"); - (index, status, String::from_utf8_lossy(&bytes).into_owned()) -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 8)] -async fn concurrent_event_appends_after_restart_keep_projection_cache_contiguous() { - let object_store: Arc = Arc::new(InMemory::new()); - let blobs = fabro_store::test_support::test_blob_store(); - let run_summaries = fabro_store::test_support::test_run_summary_store(); - let first_app = app_with_store( - Arc::clone(&object_store), - Arc::clone(&blobs), - Arc::clone(&run_summaries), - ); - let (run_id, _run_id_workspace) = create_run(&first_app).await; - - tokio::time::sleep(Duration::from_millis(25)).await; - - // Simulate a server restart: a fresh AppState opens the existing run with - // an empty active-run cache, so concurrent appends all race through the - // public event endpoint instead of sharing an already-open RunDatabase. - let restarted_app = app_with_store(object_store, blobs, run_summaries); - let appends = 64; - let barrier = Arc::new(Barrier::new(appends)); - let mut tasks = Vec::with_capacity(appends); - for index in 0..appends { - tasks.push(tokio::spawn(append_status_and_body( - restarted_app.clone(), - run_id.clone(), - index, - Arc::clone(&barrier), - ))); - } - - let mut results = Vec::with_capacity(appends); - for task in tasks { - results.push(task.await.expect("append task should not panic")); - } - let failures = results - .iter() - .filter(|(_, status, _)| *status != StatusCode::OK) - .collect::>(); - assert!( - failures.is_empty(), - "all concurrent event appends should succeed, got failures: {failures:#?}" - ); - - let request = Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/events"))) - .body(Body::empty()) - .expect("list-events request should build"); - let body = response_json( - restarted_app.clone().oneshot(request).await.unwrap(), - StatusCode::OK, - format!("GET /api/v1/runs/{run_id}/events"), - ) - .await; - let events = body["data"] - .as_array() - .expect("events response should include data"); - let seqs = events - .iter() - .map(|event| { - event["seq"] - .as_u64() - .expect("event should include numeric seq") - }) - .collect::>(); - - assert_eq!( - events.len(), - appends + 2, - "every append should be durable; observed seqs: {seqs:?}" - ); - assert_eq!( - seqs, - (1..=u64::try_from(appends + 2).unwrap()).collect::>(), - "event seqs should be contiguous" - ); -} diff --git a/lib/apps/fabro-server/tests/it/api/mcp_servers.rs b/lib/apps/fabro-server/tests/it/api/mcp_servers.rs index 5c71d671e..cc6d56c59 100644 --- a/lib/apps/fabro-server/tests/it/api/mcp_servers.rs +++ b/lib/apps/fabro-server/tests/it/api/mcp_servers.rs @@ -565,8 +565,13 @@ async fn invalid_mcp_server_id_is_bad_request() { .await; } +/// A `run.agent.mcps.` entry that names a server catalog entry by +/// `id`: Petri's Fabro frontend reads `workflow.toml` itself and has no +/// server catalog to resolve the reference against, so the check refuses +/// it (`unsupported.workflow_toml.run.agent.mcps.reference`) until the +/// frontend takes the catalog. The run create path shares the gap. #[tokio::test] -async fn created_mcp_server_can_be_referenced_by_manifest_validation() { +async fn manifest_validation_reports_a_catalog_mcp_reference_as_unsupported() { let (app, _temp_dir, _mcp_dir) = mcp_server_app(); create_mcp_server(&app, "sentry", "Sentry").await; @@ -587,7 +592,18 @@ id = "sentry" .expect("manifest validation should respond"); let body = response_json(response, StatusCode::OK, "POST /api/v1/validate").await; - assert_eq!(body["ok"], true); + assert_eq!(body["ok"], false, "{body}"); + let rules: Vec<&str> = body["workflow"]["diagnostics"] + .as_array() + .expect("diagnostics") + .iter() + .filter_map(|diagnostic| diagnostic["rule"].as_str()) + .collect(); + assert_eq!( + rules, + vec!["unsupported.workflow_toml.run.agent.mcps.reference"], + "{body}" + ); } #[tokio::test] diff --git a/lib/apps/fabro-server/tests/it/api/mod.rs b/lib/apps/fabro-server/tests/it/api/mod.rs index 8833cfa21..63fc5f346 100644 --- a/lib/apps/fabro-server/tests/it/api/mod.rs +++ b/lib/apps/fabro-server/tests/it/api/mod.rs @@ -4,7 +4,6 @@ mod cli_auth_token; mod compression; mod docs; mod environments; -mod events; mod install; mod install_openai_compatible; mod mcp_servers; diff --git a/lib/apps/fabro-server/tests/it/scenario/petri_stream.rs b/lib/apps/fabro-server/tests/it/scenario/petri_stream.rs index f084a5891..b079b4e59 100644 --- a/lib/apps/fabro-server/tests/it/scenario/petri_stream.rs +++ b/lib/apps/fabro-server/tests/it/scenario/petri_stream.rs @@ -358,20 +358,6 @@ async fn a_reconnecting_client_receives_every_stream_item_once_in_order() { .count(); assert_eq!(finished, 1, "the stream ends with the run's finish"); - // The legacy cursors are refused for a Petri run; the stream cursor is - // refused for nothing else. - let req = Request::builder() - .method("GET") - .uri(api(&format!("/runs/{run_id}/events?since_seq=1"))) - .body(Body::empty()) - .expect("events request should build"); - let response = app - .clone() - .oneshot(req) - .await - .expect("events request routes"); - assert_eq!(response.status(), StatusCode::BAD_REQUEST); - capture_fixture(&app, &run_id, "parallel", &projection).await; } diff --git a/lib/components/fabro-petri/src/interview.rs b/lib/components/fabro-petri/src/interview.rs index ee0496add..19727f29c 100644 --- a/lib/components/fabro-petri/src/interview.rs +++ b/lib/components/fabro-petri/src/interview.rs @@ -98,12 +98,10 @@ use fabro_interview::{ Answer as LegacyAnswer, AnswerSubmission, AnswerValue, AutoApproveInterviewer, ControlInterviewer, Interviewer as LegacyInterviewer, Question as LegacyQuestion, }; -use fabro_store::RunDatabase; use fabro_types::{ - InterviewOption, Principal, QuestionType, ReviewTarget, ReviewTargetKind, RunId, StageId, + InterviewOption, Principal, QuestionType, ReviewTarget, ReviewTargetKind, StageId, SystemActorKind, }; -use fabro_workflow::event::{self as workflow_event, Event, RunEventSink}; use petri_execution::events::{Parsed, Projection, ViewEvent}; use petri_execution::{ CoordinatorRecord, ExecutionId, ExecutionObserver, InterviewError, InterviewReply, @@ -199,63 +197,6 @@ pub enum QuestionNotice { } impl QuestionNotice { - /// The run event the legacy `human` stage emits for the same fact. - #[must_use] - pub fn into_event(self) -> Event { - match self { - Self::Asked(asked) => Event::InterviewStarted { - question_id: asked.question_id, - question: asked.text, - stage: asked.stage, - question_type: asked.question_type.to_string(), - options: asked.options, - allow_freeform: asked.allow_freeform, - timeout_seconds: asked.timeout_seconds, - context_display: None, - review_target: asked.review_target, - }, - Self::Answered { - question_id, - text, - answer, - actor, - duration_ms, - } => Event::InterviewCompleted { - actor: Some(actor), - question_id, - question: text, - answer, - duration_ms, - }, - Self::Expired { - question_id, - text, - stage, - duration_ms, - } => Event::InterviewTimeout { - actor: None, - question_id, - question: text, - stage, - duration_ms, - }, - Self::Interrupted { - question_id, - text, - stage, - reason, - duration_ms, - } => Event::InterviewInterrupted { - actor: None, - question_id, - question: text, - stage, - reason, - duration_ms, - }, - } - } - fn question_id(&self) -> &str { match self { Self::Asked(asked) => &asked.question_id, @@ -266,52 +207,21 @@ impl QuestionNotice { } } -/// Where the adapter posts what happens to a question: the run's event -/// stream, whichever way the process reaches it. +/// Where the adapter reports what happens to a question, when something +/// observes it: a test's board. A run's own record of a question is +/// Petri's, and who answered it is the server's platform record. #[async_trait::async_trait] pub trait QuestionSink: Send + Sync { async fn post(&self, notice: QuestionNotice) -> anyhow::Result<()>; } -/// The worker's sink: the run event sink its lifecycle events go through. -pub struct EventSinkQuestions { - sink: RunEventSink, - run_id: RunId, -} - -impl EventSinkQuestions { - #[must_use] - pub fn new(sink: RunEventSink, run_id: RunId) -> Self { - Self { sink, run_id } - } -} +/// A sink that drops every notice: the adapter with nothing observing it. +struct Unobserved; #[async_trait::async_trait] -impl QuestionSink for EventSinkQuestions { - async fn post(&self, notice: QuestionNotice) -> anyhow::Result<()> { - workflow_event::append_event_to_sink(&self.sink, &self.run_id, ¬ice.into_event()) - .await - .map_err(anyhow::Error::new) - } -} - -/// The server's sink for a run in its own process: the run's database. -pub struct DatabaseQuestions { - store: RunDatabase, - run_id: RunId, -} - -impl DatabaseQuestions { - #[must_use] - pub fn new(store: RunDatabase, run_id: RunId) -> Self { - Self { store, run_id } - } -} - -#[async_trait::async_trait] -impl QuestionSink for DatabaseQuestions { - async fn post(&self, notice: QuestionNotice) -> anyhow::Result<()> { - workflow_event::append_event(&self.store, &self.run_id, ¬ice.into_event()).await +impl QuestionSink for Unobserved { + async fn post(&self, _notice: QuestionNotice) -> anyhow::Result<()> { + Ok(()) } } @@ -415,22 +325,24 @@ pub struct FabroInterviewer { } impl FabroInterviewer { - /// Over the control interviewer the run's answers are delivered to, - /// and the sink its questions are posted through. + /// Over the control interviewer the run's answers are delivered to. #[must_use] - pub fn new( - answers: Arc, - sink: Arc, - approval: Approval, - ) -> Self { + pub fn new(answers: Arc, approval: Approval) -> Self { Self { answers, - sink, + sink: Arc::new(Unobserved), approval, observed: Arc::new(Observed::default()), } } + /// Report what happens to each question to `sink` as well. + #[must_use] + pub fn with_sink(mut self, sink: Arc) -> Self { + self.sink = sink; + self + } + /// The observer that labels each firing as the projection does and /// sees a gate report a question's expiry. A run registers it ahead of /// the interview dispatcher, so a question's stage is known when it is diff --git a/lib/components/fabro-petri/src/projection.rs b/lib/components/fabro-petri/src/projection.rs index 46d2bfea9..360dc4d2d 100644 --- a/lib/components/fabro-petri/src/projection.rs +++ b/lib/components/fabro-petri/src/projection.rs @@ -36,11 +36,11 @@ use fabro_types::{ BlockedReason, CheckpointRecord as ViewCheckpoint, CodingAgentEvent, CodingEvent, Conclusion, FailureCategory, FailureDetail, FailureReason, InterviewOption, InterviewQuestionRecord, ModelRef, ModelUsage, ParallelBranchId, ParallelBranchResult, PendingInterviewRecord, - PullRequestLink, RunApproval, RunApprovalState, RunControlAction, RunDiff, RunFailure, RunId, - RunProjection, RunSandbox, RunSandboxPlan, RunStatus, RunTiming, SandboxProviderKind, - StageCompletion, StageHandler, StageId, StageInferenceProjection, StageModelUsage, - StageOutcome, StageProjection, StageState, StageTiming, StartRecord, SuccessReason, - first_event_seq, timing, usage_rollup, + PullRequestCreation, PullRequestCreationStatus, PullRequestLink, RunApproval, RunApprovalState, + RunControlAction, RunDiff, RunFailure, RunId, RunProjection, RunSandbox, RunSandboxPlan, + RunStatus, RunTiming, SandboxProviderKind, StageCompletion, StageHandler, StageId, + StageInferenceProjection, StageModelUsage, StageOutcome, StageProjection, StageState, + StageTiming, StartRecord, SuccessReason, first_event_seq, timing, usage_rollup, }; use lithos_llm::catalog::{ModelId, ProviderId}; use lithos_llm::types::Usage; @@ -258,12 +258,62 @@ impl RunView { }, }); } + PlatformRecord::PullRequestRequested(record) => { + projection.pull_request_creation = Some(PullRequestCreation { + id: record.creation_id, + status: PullRequestCreationStatus::Pending, + model: record.model.clone(), + force: record.force, + requested_at: at, + updated_at: at, + pull_request: None, + error: None, + }); + } PlatformRecord::PullRequestCreated(record) => { - projection.pull_request = Some(PullRequestLink { + let link = PullRequestLink { owner: record.owner.clone(), repo: record.repo.clone(), number: record.number, - }); + }; + projection.pull_request = Some(link.clone()); + if let Some(creation) = projection + .pull_request_creation + .as_mut() + .filter(|creation| creation.is_pending()) + { + creation.succeed(link, at); + } + } + PlatformRecord::PullRequestFailed(record) => { + if let Some(creation) = + projection + .pull_request_creation + .as_mut() + .filter(|creation| { + creation.is_pending() + && record + .creation_id + .is_none_or(|creation_id| creation_id == creation.id) + }) + { + creation.fail(record.error.clone(), at); + } + } + PlatformRecord::PullRequestLinked(record) => { + let link = record.link(); + projection.pull_request = Some(link.clone()); + if let Some(creation) = projection + .pull_request_creation + .as_mut() + .filter(|creation| creation.is_pending()) + { + creation.succeed(link, at); + } + } + PlatformRecord::PullRequestUnlinked(_) => { + projection.pull_request = None; + projection.pull_request_creation = None; } } } @@ -1101,8 +1151,24 @@ fn fold_lifecycle(projection: &mut RunProjection, record: &RunLifecycleRecord, a at, ); } - Kind::Runnable - | Kind::Starting + Kind::Runnable => { + // A run left in flight by a restart goes back to the queue: the + // resume's `runnable` steps back from wherever the run stood. + let in_flight = matches!( + projection.status, + RunStatus::Starting + | RunStatus::Running + | RunStatus::Blocked { .. } + | RunStatus::Paused { .. } + ); + if in_flight && record.status == Some(RunStatus::Runnable) { + projection.status = RunStatus::Runnable; + projection.status_updated_at = at; + } else if let Some(status) = record.status { + apply_status(projection, status, at); + } + } + Kind::Starting | Kind::Running | Kind::Blocked | Kind::Unblocked diff --git a/lib/components/fabro-petri/tests/interview.rs b/lib/components/fabro-petri/tests/interview.rs index 9a5f5e29d..78b86f429 100644 --- a/lib/components/fabro-petri/tests/interview.rs +++ b/lib/components/fabro-petri/tests/interview.rs @@ -156,7 +156,7 @@ impl Gate { } fn interviewer(&self, approval: Approval) -> FabroInterviewer { - FabroInterviewer::new(Arc::clone(&self.control), self.board.clone(), approval) + FabroInterviewer::new(Arc::clone(&self.control), approval).with_sink(self.board.clone()) } fn marker(&self, name: &str) -> bool { diff --git a/lib/components/fabro-petri/tests/projection.rs b/lib/components/fabro-petri/tests/projection.rs index 9c4698137..73cac347a 100644 --- a/lib/components/fabro-petri/tests/projection.rs +++ b/lib/components/fabro-petri/tests/projection.rs @@ -960,11 +960,7 @@ impl GateRun { Launch::default(), &runtime, ); - let interviewer = FabroInterviewer::new( - Arc::new(ControlInterviewer::new()), - Arc::new(support::Silent), - approval, - ); + let interviewer = FabroInterviewer::new(Arc::new(ControlInterviewer::new()), approval); let store = self .projector .observe_store(Arc::new(SqliteRunStore::new(self.scenario.pool.clone()))); diff --git a/lib/components/fabro-petri/tests/support/mod.rs b/lib/components/fabro-petri/tests/support/mod.rs index 3c8d00cef..7cc8fd0c3 100644 --- a/lib/components/fabro-petri/tests/support/mod.rs +++ b/lib/components/fabro-petri/tests/support/mod.rs @@ -129,9 +129,9 @@ pub(crate) fn run_request( pub(crate) fn no_questions(sink: Arc) -> FabroInterviewer { FabroInterviewer::new( Arc::new(fabro_interview::ControlInterviewer::new()), - sink, Approval::Prompt, ) + .with_sink(sink) } /// A sink that drops every notice. diff --git a/lib/components/fabro-store/src/keys.rs b/lib/components/fabro-store/src/keys.rs index 6e0feb7c1..28adb69ba 100644 --- a/lib/components/fabro-store/src/keys.rs +++ b/lib/components/fabro-store/src/keys.rs @@ -40,6 +40,7 @@ impl SlateKey { &self.0 } + #[cfg(test)] pub(crate) fn segments(raw: &str) -> impl Iterator { raw.split(Self::SEP) } @@ -53,38 +54,6 @@ impl AsRef<[u8]> for SlateKey { // --- Construction --- -pub(crate) fn run_events_prefix(run_id: &RunId) -> SlateKey { - SlateKey::new("runs") - .with(run_id) - .with("events") - .into_prefix() -} - -/// Prefix of the retired `runs/_index/by-start/` catalog markers that -/// the legacy layout kept beside each run's events. -pub(crate) fn run_catalog_prefix() -> SlateKey { - run_catalog_root().into_prefix() -} - -#[cfg(test)] -pub(crate) fn run_catalog_key(run_id: &RunId) -> SlateKey { - run_catalog_root().with(run_id) -} - -/// Extracts the run id from a full catalog marker key, or `None` when the key -/// is not exactly `runs/_index/by-start/`. -pub(crate) fn parse_run_catalog_key(raw: &str) -> Option { - let segments = SlateKey::segments(raw).collect::>(); - let ["runs", "_index", "by-start", run_id] = segments.as_slice() else { - return None; - }; - run_id.parse().ok() -} - -fn run_catalog_root() -> SlateKey { - SlateKey::new("runs").with("_index").with("by-start") -} - // Sequence keys zero-pad `seq` to six digits so lexicographic key order // matches numeric seq order through `MAX_EVENT_SEQ`. Seek-based event listing // (`run_events_range`) depends on this invariant, so event allocation rejects @@ -116,10 +85,6 @@ pub(crate) fn run_events_range(run_id: &RunId, start_seq: u32) -> Range SlateKey { - SlateKey::new("sessions").with("by-id").into_prefix() -} - #[cfg(test)] pub(crate) fn session_by_id_key(session_id: &fabro_types::SessionId) -> SlateKey { SlateKey::new("sessions").with("by-id").with(session_id) diff --git a/lib/components/fabro-store/src/legacy_blob_import.rs b/lib/components/fabro-store/src/legacy_blob_import.rs deleted file mode 100644 index 2894f12de..000000000 --- a/lib/components/fabro-store/src/legacy_blob_import.rs +++ /dev/null @@ -1,2007 +0,0 @@ -//! Temporary compatibility importer for the legacy SlateDB blob keyspace. -//! -//! Remove this module with the Slate blob backend after the approved legacy -//! support window ends. - -use std::error::Error as StdError; -use std::fmt; - -use bytes::Bytes; -use fabro_types::BlobHash; -use fabro_util::error; -use futures::TryStreamExt as _; -use sqlx::pool::PoolConnection; -use sqlx::{Acquire as _, Sqlite, SqlitePool}; -#[cfg(test)] -use tokio::sync::Barrier; -use tracing::{debug, error}; - -use crate::Database; -use crate::keys::SlateKey; - -const MAX_BATCH_ROWS: usize = 100; -const MAX_BATCH_BYTES: u64 = 1024 * 1024; -const PASSIVE_CHECKPOINT_BYTES: u64 = 8 * 1024 * 1024; - -/// Aggregate size and row count of the exact legacy blob keyspace. -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] -pub struct LegacyBlobInventory { - pub rows: u64, - pub bytes: u64, - /// Legacy rows whose hash is not yet present in the SQLite blobs table. - pub pending_rows: u64, - /// Bytes belonging to [`Self::pending_rows`]. - pub pending_bytes: u64, -} - -/// A failed strict inventory and the aggregate progress observed before it. -pub struct LegacyBlobInventoryError { - report: LegacyBlobInventory, - failure: LegacyBlobInventoryFailure, -} - -impl LegacyBlobInventoryError { - #[must_use] - pub fn report(&self) -> &LegacyBlobInventory { - &self.report - } -} - -impl fmt::Debug for LegacyBlobInventoryError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("LegacyBlobInventoryError") - .field("report", &self.report) - .field("failure", &self.failure.kind()) - .finish() - } -} - -impl fmt::Display for LegacyBlobInventoryError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - formatter, - "legacy blob inventory failed after scanning {} rows: {}", - self.report.rows, self.failure - ) - } -} - -impl StdError for LegacyBlobInventoryError { - fn source(&self) -> Option<&(dyn StdError + 'static)> { - Some(&self.failure) - } -} - -#[derive(strum::IntoStaticStr, thiserror::Error)] -#[strum(serialize_all = "snake_case")] -enum LegacyBlobInventoryFailure { - #[error("opening the legacy blob source")] - OpenSource(#[source] crate::Error), - #[error("opening the legacy blob scan")] - OpenSourceScan(#[source] slatedb::Error), - #[error("reading the legacy blob scan")] - ReadSourceScan(#[source] slatedb::Error), - #[error("a legacy blob key is not canonical")] - InvalidSourceKey, - #[error("reading a SQLite blob row for the legacy inventory")] - ReadDestination(#[source] sqlx::Error), - #[error("a legacy blob inventory counter overflowed")] - CounterOverflow, -} - -impl LegacyBlobInventoryFailure { - fn kind(&self) -> &'static str { - self.into() - } -} - -impl fmt::Debug for LegacyBlobInventoryFailure { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("LegacyBlobInventoryFailure") - .field("kind", &self.kind()) - .finish() - } -} - -/// Aggregate proof produced by a complete legacy-source and SQLite-target -/// verification pass. -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] -pub struct LegacyBlobVerificationReport { - pub source_rows: u64, - pub source_bytes: u64, - pub matched_rows: u64, - pub matched_bytes: u64, - pub target_rows: u64, - pub target_bytes: u64, - pub missing_rows: u64, - pub invalid_source_rows: u64, - pub invalid_target_rows: u64, - pub conflicting_rows: u64, -} - -/// A failed complete verification and its aggregate partial report. -pub struct LegacyBlobVerificationError { - report: LegacyBlobVerificationReport, - failure: LegacyBlobVerificationFailure, -} - -impl LegacyBlobVerificationError { - #[must_use] - pub fn report(&self) -> &LegacyBlobVerificationReport { - &self.report - } -} - -impl fmt::Debug for LegacyBlobVerificationError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("LegacyBlobVerificationError") - .field("report", &self.report) - .field("failure", &self.failure.kind()) - .finish() - } -} - -impl fmt::Display for LegacyBlobVerificationError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - formatter, - "legacy blob verification failed after checking {} source rows and {} target rows: {}", - self.report.source_rows, self.report.target_rows, self.failure - ) - } -} - -impl StdError for LegacyBlobVerificationError { - fn source(&self) -> Option<&(dyn StdError + 'static)> { - Some(&self.failure) - } -} - -#[derive(strum::IntoStaticStr, thiserror::Error)] -#[strum(serialize_all = "snake_case")] -enum LegacyBlobVerificationFailure { - #[error("opening the legacy blob source")] - OpenSource(#[source] crate::Error), - #[error("opening the legacy blob scan")] - OpenSourceScan(#[source] slatedb::Error), - #[error("reading the legacy blob scan")] - ReadSourceScan(#[source] slatedb::Error), - #[error("a legacy blob key is not canonical")] - InvalidSourceKey, - #[error("legacy blob bytes do not match their key digest")] - SourceDigestMismatch, - #[error("reading a SQLite blob row for legacy verification")] - ReadDestination(#[source] sqlx::Error), - #[error("SQLite is missing a legacy blob row")] - MissingDestination, - #[error("SQLite contains different bytes for a legacy blob hash")] - DestinationConflict, - #[error("scanning SQLite blob rows")] - ScanTarget(#[source] sqlx::Error), - #[error("a SQLite blob hash is not canonical")] - InvalidTargetHash, - #[error("SQLite blob bytes do not match their hash")] - TargetDigestMismatch, - #[error("a legacy blob verification counter overflowed")] - CounterOverflow, -} - -impl LegacyBlobVerificationFailure { - fn kind(&self) -> &'static str { - self.into() - } -} - -impl fmt::Debug for LegacyBlobVerificationFailure { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("LegacyBlobVerificationFailure") - .field("kind", &self.kind()) - .finish() - } -} - -/// Aggregate progress from one legacy blob import attempt. -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] -pub struct LegacyBlobImportReport { - /// Source rows observed under the exact legacy blob prefix. - pub scanned_rows: u64, - /// Raw source value bytes observed under the exact legacy blob prefix. - pub scanned_bytes: u64, - /// Destination rows newly inserted by committed transactions. - pub imported_rows: u64, - /// Destination bytes newly inserted by committed transactions. - pub imported_bytes: u64, - /// Byte-equal destination rows accepted by committed transactions. - pub existing_rows: u64, - /// Bytes belonging to byte-equal destination rows. - pub existing_bytes: u64, - /// Source rows rejected for malformed keys or digest mismatches. - pub invalid_rows: u64, - /// Destination rows rejected because their bytes differed. - pub conflicting_rows: u64, - /// Successfully committed import transactions. - pub committed_batches: u64, - /// Successful passive WAL checkpoints during the import. - pub passive_checkpoints: u64, -} - -/// A failed legacy blob import and the durable progress completed before it. -pub struct LegacyBlobImportError { - report: LegacyBlobImportReport, - failure: LegacyBlobImportFailure, -} - -impl LegacyBlobImportError { - /// Returns the aggregate durable progress completed before the failure. - #[must_use] - pub fn report(&self) -> &LegacyBlobImportReport { - &self.report - } - - /// Returns secondary errors encountered while cleaning up the failed - /// import. - /// - /// The standard error source chain preserves the failure that interrupted - /// the import. Because that chain is linear, rollback, connection-setting - /// restoration, and connection-retirement errors are exposed separately. - pub fn cleanup_errors(&self) -> impl Iterator { - let mut errors = Vec::new(); - self.failure.collect_cleanup_errors(&mut errors); - errors.into_iter() - } -} - -impl fmt::Debug for LegacyBlobImportError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("LegacyBlobImportError") - .field("report", &self.report) - .field("failure", &self.failure.kind()) - .finish() - } -} - -impl fmt::Display for LegacyBlobImportError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - formatter, - "legacy blob import failed after scanning {} rows and committing {} batches: {}", - self.report.scanned_rows, self.report.committed_batches, self.failure - ) - } -} - -impl StdError for LegacyBlobImportError { - fn source(&self) -> Option<&(dyn StdError + 'static)> { - Some(self.failure.primary_failure()) - } -} - -#[derive(strum::IntoStaticStr, thiserror::Error)] -#[strum(serialize_all = "snake_case")] -enum LegacyBlobImportFailure { - #[error("opening the legacy blob source")] - OpenSource(#[source] crate::Error), - - #[error("opening the legacy blob scan")] - OpenSourceScan(#[source] slatedb::Error), - - #[error("reading the legacy blob scan")] - ReadSourceScan(#[source] slatedb::Error), - - #[error("a legacy blob key is not canonical")] - InvalidSourceKey, - - #[error("legacy blob bytes do not match their key digest")] - SourceDigestMismatch, - - #[error("a legacy blob import counter overflowed")] - CounterOverflow, - - #[error("acquiring the SQLite import connection")] - AcquireConnection(#[source] sqlx::Error), - - #[error("reading the SQLite automatic checkpoint setting")] - ReadAutomaticCheckpoint(#[source] sqlx::Error), - - #[error("disabling SQLite automatic checkpointing")] - DisableAutomaticCheckpoint(#[source] sqlx::Error), - - #[error("starting a SQLite blob import transaction")] - BeginTransaction(#[source] sqlx::Error), - - #[error("inserting a SQLite blob row")] - InsertDestination(#[source] sqlx::Error), - - #[error("reading an existing SQLite blob row")] - ReadDestination(#[source] sqlx::Error), - - #[error("SQLite contains different bytes for a legacy blob hash")] - DestinationConflict, - - #[error("committing a SQLite blob import transaction")] - CommitTransaction(#[source] sqlx::Error), - - #[error("rolling back a failed SQLite blob import transaction")] - RollbackTransaction { - #[source] - source: sqlx::Error, - prior: Box, - }, - - #[error("running a passive SQLite WAL checkpoint")] - PassiveCheckpoint(#[source] sqlx::Error), - - #[error("the passive SQLite WAL checkpoint could not complete")] - PassiveCheckpointBusy, - - #[error("restoring the SQLite automatic checkpoint setting")] - RestoreAutomaticCheckpoint { - #[source] - source: sqlx::Error, - prior: Option>, - retirement_error: Option, - }, -} - -impl LegacyBlobImportFailure { - fn kind(&self) -> &'static str { - self.into() - } - - fn primary_failure(&self) -> &Self { - match self { - Self::RollbackTransaction { prior, .. } - | Self::RestoreAutomaticCheckpoint { - prior: Some(prior), .. - } => prior.primary_failure(), - _ => self, - } - } - - fn collect_cleanup_errors<'a>(&'a self, errors: &mut Vec<&'a (dyn StdError + 'static)>) { - match self { - Self::RollbackTransaction { source, prior } => { - errors.push(source); - prior.collect_cleanup_errors(errors); - } - Self::RestoreAutomaticCheckpoint { - source, - prior, - retirement_error, - } => { - errors.push(source); - if let Some(retirement_error) = retirement_error { - errors.push(retirement_error); - } - if let Some(prior) = prior { - prior.collect_cleanup_errors(errors); - } - } - _ => {} - } - } -} - -impl fmt::Debug for LegacyBlobImportFailure { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - let mut debug = formatter.debug_struct("LegacyBlobImportFailure"); - debug.field("kind", &self.kind()); - match self { - Self::RollbackTransaction { prior, .. } => { - debug.field("prior_failure", &prior.kind()); - } - Self::RestoreAutomaticCheckpoint { - prior, - retirement_error, - .. - } => { - debug - .field("prior_failure", &prior.as_deref().map(Self::kind)) - .field("retirement_failed", &retirement_error.is_some()); - } - _ => {} - } - debug.finish() - } -} - -#[derive(Default)] -struct ImportControls { - #[cfg(test)] - after_automatic_checkpoint_disabled: Option>, - #[cfg(test)] - source_after_rows: Option, - #[cfg(test)] - passive_checkpoint: bool, - #[cfg(test)] - restore_automatic_checkpoint: bool, -} - -impl ImportControls { - #[cfg(test)] - async fn after_automatic_checkpoint_disabled(&self) { - if let Some(barrier) = &self.after_automatic_checkpoint_disabled { - barrier.wait().await; - barrier.wait().await; - } - } - - fn source_scan_error(&self, scanned_rows: u64) -> Option { - #[cfg(test)] - if self.source_after_rows == Some(scanned_rows) { - return Some(slatedb::Error::unavailable( - "injected legacy source transport failure".to_owned(), - )); - } - let _ = (self, scanned_rows); - None - } - - fn passive_checkpoint_error(&self) -> Option { - #[cfg(test)] - if self.passive_checkpoint { - return Some(sqlx::Error::Protocol( - "injected passive checkpoint failure".to_owned(), - )); - } - let _ = self; - None - } - - fn restore_automatic_checkpoint_error(&self) -> Option { - #[cfg(test)] - if self.restore_automatic_checkpoint { - return Some(sqlx::Error::Protocol( - "injected automatic checkpoint restoration failure".to_owned(), - )); - } - let _ = self; - None - } -} - -struct ImportConnection { - connection: Option>, - retire_on_drop: bool, -} - -impl ImportConnection { - fn new(connection: PoolConnection) -> Self { - Self { - connection: Some(connection), - retire_on_drop: false, - } - } - - fn get_mut(&mut self) -> &mut PoolConnection { - self.connection - .as_mut() - .expect("import connection exists until explicit retirement") - } - - fn retire_if_dropped(&mut self) { - self.retire_on_drop = true; - } - - fn checkpoint_setting_restored(&mut self) { - self.retire_on_drop = false; - } - - async fn retire(mut self) -> Result<(), sqlx::Error> { - let Some(mut connection) = self.connection.take() else { - return Ok(()); - }; - connection.close_on_drop(); - connection.close().await - } -} - -impl Drop for ImportConnection { - fn drop(&mut self) { - if self.retire_on_drop { - if let Some(connection) = &mut self.connection { - connection.close_on_drop(); - } - } - } -} - -struct PendingBlob { - hash: BlobHash, - bytes: Bytes, -} - -#[derive(Clone, Copy, Default)] -struct BatchReport { - imported_rows: u64, - imported_bytes: u64, - existing_rows: u64, - existing_bytes: u64, -} - -impl Database { - /// Inventories the exact legacy SlateDB blob keyspace against the SQLite - /// blobs table in `pool`. - /// - /// Keys must be canonical, but value digests are not rehashed here: the - /// inventory only sizes the keyspace, and the import pass validates every - /// digest before any row is persisted. Rows whose hash the blobs table - /// does not contain yet are reported as pending so callers can size the - /// remaining import work. - pub async fn legacy_blob_inventory( - &self, - pool: &SqlitePool, - ) -> std::result::Result { - let mut report = LegacyBlobInventory::default(); - let result = self.run_legacy_blob_inventory(pool, &mut report).await; - match result { - Ok(()) => Ok(report), - Err(failure) => Err(LegacyBlobInventoryError { report, failure }), - } - } - - async fn run_legacy_blob_inventory( - &self, - pool: &SqlitePool, - report: &mut LegacyBlobInventory, - ) -> Result<(), LegacyBlobInventoryFailure> { - let source = self - .open_db() - .await - .map_err(LegacyBlobInventoryFailure::OpenSource)?; - let prefix = legacy_blob_prefix(); - let mut entries = source - .scan_prefix(&prefix) - .await - .map_err(LegacyBlobInventoryFailure::OpenSourceScan)?; - while let Some(entry) = entries - .next() - .await - .map_err(LegacyBlobInventoryFailure::ReadSourceScan)? - { - inventory_checked_add(&mut report.rows, 1)?; - let value_bytes = inventory_usize_to_u64(entry.value.len())?; - inventory_checked_add(&mut report.bytes, value_bytes)?; - let hash = parse_source_key(&entry.key, &prefix) - .ok_or(LegacyBlobInventoryFailure::InvalidSourceKey)?; - let imported: bool = - sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM blobs WHERE hash = ?)") - .bind(hash.to_string()) - .fetch_one(pool) - .await - .map_err(LegacyBlobInventoryFailure::ReadDestination)?; - if !imported { - inventory_checked_add(&mut report.pending_rows, 1)?; - inventory_checked_add(&mut report.pending_bytes, value_bytes)?; - } - } - Ok(()) - } - - /// Verifies every legacy blob against SQLite and validates every SQLite - /// blob row independently. - pub async fn verify_legacy_blobs_in( - &self, - pool: &SqlitePool, - ) -> std::result::Result { - let mut report = LegacyBlobVerificationReport::default(); - let result = self.run_legacy_blob_verification(pool, &mut report).await; - match result { - Ok(()) => Ok(report), - Err(failure) => Err(LegacyBlobVerificationError { report, failure }), - } - } - - async fn run_legacy_blob_verification( - &self, - pool: &SqlitePool, - report: &mut LegacyBlobVerificationReport, - ) -> Result<(), LegacyBlobVerificationFailure> { - let source = self - .open_db() - .await - .map_err(LegacyBlobVerificationFailure::OpenSource)?; - let prefix = legacy_blob_prefix(); - let mut entries = source - .scan_prefix(&prefix) - .await - .map_err(LegacyBlobVerificationFailure::OpenSourceScan)?; - while let Some(entry) = entries - .next() - .await - .map_err(LegacyBlobVerificationFailure::ReadSourceScan)? - { - verification_checked_add(&mut report.source_rows, 1)?; - let value_bytes = verification_usize_to_u64(entry.value.len())?; - verification_checked_add(&mut report.source_bytes, value_bytes)?; - let hash = match validate_source_entry_common(&entry.key, &entry.value, &prefix) { - Ok(hash) => hash, - Err(SourceEntryFailure::InvalidKey) => { - verification_checked_add(&mut report.invalid_source_rows, 1)?; - return Err(LegacyBlobVerificationFailure::InvalidSourceKey); - } - Err(SourceEntryFailure::DigestMismatch) => { - verification_checked_add(&mut report.invalid_source_rows, 1)?; - return Err(LegacyBlobVerificationFailure::SourceDigestMismatch); - } - }; - let equal: Option = - sqlx::query_scalar("SELECT data = ? FROM blobs WHERE hash = ?") - .bind(entry.value.as_ref()) - .bind(hash.to_string()) - .fetch_optional(pool) - .await - .map_err(LegacyBlobVerificationFailure::ReadDestination)?; - match equal { - Some(true) => { - verification_checked_add(&mut report.matched_rows, 1)?; - verification_checked_add(&mut report.matched_bytes, value_bytes)?; - } - Some(false) => { - verification_checked_add(&mut report.conflicting_rows, 1)?; - return Err(LegacyBlobVerificationFailure::DestinationConflict); - } - None => { - verification_checked_add(&mut report.missing_rows, 1)?; - return Err(LegacyBlobVerificationFailure::MissingDestination); - } - } - } - - let mut rows = - sqlx::query_as::<_, (String, Vec)>("SELECT hash, data FROM blobs").fetch(pool); - while let Some((hash_text, bytes)) = rows - .try_next() - .await - .map_err(LegacyBlobVerificationFailure::ScanTarget)? - { - verification_checked_add(&mut report.target_rows, 1)?; - verification_checked_add( - &mut report.target_bytes, - verification_usize_to_u64(bytes.len())?, - )?; - let Some(hash) = parse_canonical_hash(&hash_text) else { - verification_checked_add(&mut report.invalid_target_rows, 1)?; - return Err(LegacyBlobVerificationFailure::InvalidTargetHash); - }; - if BlobHash::new(&bytes) != hash { - verification_checked_add(&mut report.invalid_target_rows, 1)?; - return Err(LegacyBlobVerificationFailure::TargetDigestMismatch); - } - } - Ok(()) - } - - /// Strictly imports the legacy SlateDB blob keyspace into a SQLite blob - /// store. - /// - /// # Errors - /// - /// Returns a typed error with partial durable progress if source - /// validation, destination persistence, checkpointing, or connection - /// cleanup fails. - pub async fn import_legacy_blobs_into( - &self, - pool: &SqlitePool, - ) -> std::result::Result { - self.import_legacy_blobs_with_controls(pool, &ImportControls::default()) - .await - } - - async fn import_legacy_blobs_with_controls( - &self, - pool: &SqlitePool, - controls: &ImportControls, - ) -> std::result::Result { - let mut report = LegacyBlobImportReport::default(); - let result = self - .run_legacy_blob_import(pool, controls, &mut report) - .await; - - match result { - Ok(()) => { - debug_import_outcome("complete", &report, None); - Ok(report) - } - Err(failure) => { - debug_import_outcome("failed", &report, Some(failure.kind())); - let import_error = LegacyBlobImportError { report, failure }; - for cleanup_error in import_error.cleanup_errors() { - let rendered = error::collect_chain(cleanup_error).join(": "); - error!( - error = %rendered, - "Legacy blob import cleanup failed" - ); - } - Err(import_error) - } - } - } - - async fn run_legacy_blob_import( - &self, - pool: &SqlitePool, - controls: &ImportControls, - report: &mut LegacyBlobImportReport, - ) -> Result<(), LegacyBlobImportFailure> { - let mut connection = pool - .acquire() - .await - .map_err(LegacyBlobImportFailure::AcquireConnection)?; - let previous_automatic_checkpoint = sqlx::query_scalar("PRAGMA wal_autocheckpoint") - .fetch_one(&mut *connection) - .await - .map_err(LegacyBlobImportFailure::ReadAutomaticCheckpoint)?; - let mut connection = ImportConnection::new(connection); - // A cancelled PRAGMA future may already have changed connection-local - // state. Arm retirement before the first mutating await so an altered - // connection can never return to the pool without restoration. - connection.retire_if_dropped(); - - let import_result = match set_automatic_checkpoint(connection.get_mut(), 0).await { - Ok(()) => { - #[cfg(test)] - controls.after_automatic_checkpoint_disabled().await; - self.copy_legacy_blobs(connection.get_mut(), controls, report) - .await - } - Err(source) => Err(LegacyBlobImportFailure::DisableAutomaticCheckpoint(source)), - }; - - let restore_result = if let Some(error) = controls.restore_automatic_checkpoint_error() { - Err(error) - } else { - set_automatic_checkpoint(connection.get_mut(), previous_automatic_checkpoint).await - }; - - if let Err(source) = restore_result { - let retirement_error = connection.retire().await.err(); - return Err(LegacyBlobImportFailure::RestoreAutomaticCheckpoint { - source, - prior: import_result.err().map(Box::new), - retirement_error, - }); - } - - connection.checkpoint_setting_restored(); - import_result - } - - async fn copy_legacy_blobs( - &self, - connection: &mut PoolConnection, - controls: &ImportControls, - report: &mut LegacyBlobImportReport, - ) -> Result<(), LegacyBlobImportFailure> { - let source = self - .open_db() - .await - .map_err(LegacyBlobImportFailure::OpenSource)?; - let prefix = SlateKey::new("blobs").with("sha256").into_prefix(); - let prefix_bytes = prefix.as_ref().to_vec(); - let mut entries = source - .scan_prefix(&prefix_bytes) - .await - .map_err(LegacyBlobImportFailure::OpenSourceScan)?; - let mut pending = Vec::with_capacity(MAX_BATCH_ROWS); - let mut pending_bytes = 0_u64; - let mut bytes_since_checkpoint = 0_u64; - - loop { - if let Some(source) = controls.source_scan_error(report.scanned_rows) { - return Err(LegacyBlobImportFailure::ReadSourceScan(source)); - } - let Some(entry) = entries - .next() - .await - .map_err(LegacyBlobImportFailure::ReadSourceScan)? - else { - break; - }; - checked_add(&mut report.scanned_rows, 1)?; - let value_bytes = usize_to_u64(entry.value.len())?; - checked_add(&mut report.scanned_bytes, value_bytes)?; - - let hash = validate_source_entry(&entry.key, &entry.value, &prefix_bytes, report)?; - - let would_exceed_rows = pending.len() == MAX_BATCH_ROWS; - let would_exceed_bytes = pending_bytes - .checked_add(value_bytes) - .ok_or(LegacyBlobImportFailure::CounterOverflow)? - > MAX_BATCH_BYTES; - if !pending.is_empty() && (would_exceed_rows || would_exceed_bytes) { - commit_pending_batch( - connection, - &mut pending, - &mut pending_bytes, - &mut bytes_since_checkpoint, - controls, - report, - ) - .await?; - } - - pending.push(PendingBlob { - hash, - bytes: entry.value, - }); - checked_add(&mut pending_bytes, value_bytes)?; - - if value_bytes > MAX_BATCH_BYTES { - commit_pending_batch( - connection, - &mut pending, - &mut pending_bytes, - &mut bytes_since_checkpoint, - controls, - report, - ) - .await?; - } - } - - commit_pending_batch( - connection, - &mut pending, - &mut pending_bytes, - &mut bytes_since_checkpoint, - controls, - report, - ) - .await?; - Ok(()) - } -} - -#[derive(Clone, Copy)] -enum SourceEntryFailure { - InvalidKey, - DigestMismatch, -} - -fn legacy_blob_prefix() -> Vec { - SlateKey::new("blobs") - .with("sha256") - .into_prefix() - .as_ref() - .to_vec() -} - -fn parse_canonical_hash(value: &str) -> Option { - let canonical = value.len() == 64 - && value - .bytes() - .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)); - canonical.then(|| value.parse().ok()).flatten() -} - -fn parse_source_key(key: &[u8], prefix: &[u8]) -> Option { - let suffix = key.strip_prefix(prefix)?; - let hash_text = std::str::from_utf8(suffix).ok()?; - parse_canonical_hash(hash_text) -} - -fn validate_source_entry_common( - key: &[u8], - value: &[u8], - prefix: &[u8], -) -> Result { - let hash = parse_source_key(key, prefix).ok_or(SourceEntryFailure::InvalidKey)?; - if BlobHash::new(value) != hash { - return Err(SourceEntryFailure::DigestMismatch); - } - Ok(hash) -} - -fn validate_source_entry( - key: &[u8], - value: &[u8], - prefix: &[u8], - report: &mut LegacyBlobImportReport, -) -> Result { - match validate_source_entry_common(key, value, prefix) { - Ok(hash) => Ok(hash), - Err(SourceEntryFailure::InvalidKey) => { - invalid_source_row(report, LegacyBlobImportFailure::InvalidSourceKey) - } - Err(SourceEntryFailure::DigestMismatch) => { - invalid_source_row(report, LegacyBlobImportFailure::SourceDigestMismatch) - } - } -} - -fn invalid_source_row( - report: &mut LegacyBlobImportReport, - failure: LegacyBlobImportFailure, -) -> Result { - checked_add(&mut report.invalid_rows, 1)?; - Err(failure) -} - -async fn commit_pending_batch( - connection: &mut PoolConnection, - pending: &mut Vec, - pending_bytes: &mut u64, - bytes_since_checkpoint: &mut u64, - controls: &ImportControls, - report: &mut LegacyBlobImportReport, -) -> Result<(), LegacyBlobImportFailure> { - if pending.is_empty() { - return Ok(()); - } - - let batch = std::mem::take(pending); - *pending_bytes = 0; - let batch_report = commit_batch(connection, batch, report).await?; - - let mut updated = *report; - checked_add(&mut updated.imported_rows, batch_report.imported_rows)?; - checked_add(&mut updated.imported_bytes, batch_report.imported_bytes)?; - checked_add(&mut updated.existing_rows, batch_report.existing_rows)?; - checked_add(&mut updated.existing_bytes, batch_report.existing_bytes)?; - checked_add(&mut updated.committed_batches, 1)?; - *report = updated; - - checked_add(bytes_since_checkpoint, batch_report.imported_bytes)?; - if *bytes_since_checkpoint >= PASSIVE_CHECKPOINT_BYTES { - run_checkpoint(connection, controls).await?; - checked_add(&mut report.passive_checkpoints, 1)?; - *bytes_since_checkpoint = 0; - } - - Ok(()) -} - -async fn commit_batch( - connection: &mut PoolConnection, - batch: Vec, - report: &mut LegacyBlobImportReport, -) -> Result { - let mut transaction = connection - .begin() - .await - .map_err(LegacyBlobImportFailure::BeginTransaction)?; - let batch_result = async { - let mut batch_report = BatchReport::default(); - for blob in batch { - let value_bytes = usize_to_u64(blob.bytes.len())?; - let result = sqlx::query( - "INSERT INTO blobs (hash, data) VALUES (?, ?) ON CONFLICT(hash) DO NOTHING", - ) - .bind(blob.hash.to_string()) - .bind(blob.bytes.as_ref()) - .execute(&mut *transaction) - .await - .map_err(LegacyBlobImportFailure::InsertDestination)?; - - if result.rows_affected() == 1 { - checked_add(&mut batch_report.imported_rows, 1)?; - checked_add(&mut batch_report.imported_bytes, value_bytes)?; - continue; - } - - let stored: Vec = sqlx::query_scalar("SELECT data FROM blobs WHERE hash = ?") - .bind(blob.hash.to_string()) - .fetch_one(&mut *transaction) - .await - .map_err(LegacyBlobImportFailure::ReadDestination)?; - if stored != blob.bytes { - checked_add(&mut report.conflicting_rows, 1)?; - return Err(LegacyBlobImportFailure::DestinationConflict); - } - checked_add(&mut batch_report.existing_rows, 1)?; - checked_add(&mut batch_report.existing_bytes, value_bytes)?; - } - Ok(batch_report) - } - .await; - - match batch_result { - Ok(batch_report) => { - transaction - .commit() - .await - .map_err(LegacyBlobImportFailure::CommitTransaction)?; - Ok(batch_report) - } - Err(prior) => match transaction.rollback().await { - Ok(()) => Err(prior), - Err(source) => Err(LegacyBlobImportFailure::RollbackTransaction { - source, - prior: Box::new(prior), - }), - }, - } -} - -async fn run_checkpoint( - connection: &mut PoolConnection, - controls: &ImportControls, -) -> Result<(), LegacyBlobImportFailure> { - if let Some(source) = controls.passive_checkpoint_error() { - return Err(LegacyBlobImportFailure::PassiveCheckpoint(source)); - } - - let result = sqlx::query_as::<_, (i64, i64, i64)>("PRAGMA wal_checkpoint(PASSIVE)") - .fetch_one(&mut **connection) - .await; - let (busy, _, _) = result.map_err(LegacyBlobImportFailure::PassiveCheckpoint)?; - if busy != 0 { - return Err(LegacyBlobImportFailure::PassiveCheckpointBusy); - } - Ok(()) -} - -fn inventory_usize_to_u64(value: usize) -> Result { - u64::try_from(value).map_err(|_| LegacyBlobInventoryFailure::CounterOverflow) -} - -fn inventory_checked_add(value: &mut u64, amount: u64) -> Result<(), LegacyBlobInventoryFailure> { - *value = value - .checked_add(amount) - .ok_or(LegacyBlobInventoryFailure::CounterOverflow)?; - Ok(()) -} - -fn verification_usize_to_u64(value: usize) -> Result { - u64::try_from(value).map_err(|_| LegacyBlobVerificationFailure::CounterOverflow) -} - -fn verification_checked_add( - value: &mut u64, - amount: u64, -) -> Result<(), LegacyBlobVerificationFailure> { - *value = value - .checked_add(amount) - .ok_or(LegacyBlobVerificationFailure::CounterOverflow)?; - Ok(()) -} - -async fn set_automatic_checkpoint( - connection: &mut PoolConnection, - pages: i64, -) -> Result<(), sqlx::Error> { - // `pages` comes directly from SQLite as an integer, so the dynamic PRAGMA - // contains no caller-controlled text and cannot change the SQL shape. - let statement = sqlx::AssertSqlSafe(format!("PRAGMA wal_autocheckpoint = {pages}")); - sqlx::query(statement).execute(&mut **connection).await?; - Ok(()) -} - -fn usize_to_u64(value: usize) -> Result { - u64::try_from(value).map_err(|_| LegacyBlobImportFailure::CounterOverflow) -} - -fn checked_add(value: &mut u64, amount: u64) -> Result<(), LegacyBlobImportFailure> { - *value = value - .checked_add(amount) - .ok_or(LegacyBlobImportFailure::CounterOverflow)?; - Ok(()) -} - -fn debug_import_outcome( - outcome: &'static str, - report: &LegacyBlobImportReport, - failure_kind: Option<&'static str>, -) { - debug!( - outcome, - failure_kind, - scanned_rows = report.scanned_rows, - scanned_bytes = report.scanned_bytes, - imported_rows = report.imported_rows, - imported_bytes = report.imported_bytes, - existing_rows = report.existing_rows, - existing_bytes = report.existing_bytes, - invalid_rows = report.invalid_rows, - conflicting_rows = report.conflicting_rows, - committed_batches = report.committed_batches, - passive_checkpoints = report.passive_checkpoints, - "Legacy blob import finished" - ); -} - -#[cfg(test)] -mod tests { - use std::fmt::{self, Write as _}; - use std::sync::atomic::{AtomicU64, Ordering}; - use std::sync::{Arc, Mutex}; - use std::time::Duration; - - use bytes::Bytes; - use fabro_types::BlobHash; - use object_store::memory::InMemory; - use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions}; - use tokio::sync::Barrier; - use tracing::field::{Field, Visit}; - use tracing::instrument::WithSubscriber as _; - use tracing::span::{Attributes, Id, Record}; - use tracing::{Event, Metadata, Subscriber}; - - use super::{ - ImportControls, LegacyBlobImportFailure, LegacyBlobImportReport, MAX_BATCH_BYTES, - PASSIVE_CHECKPOINT_BYTES, set_automatic_checkpoint, - }; - use crate::keys::SlateKey; - use crate::{BlobStore, Database, test_support as store_test_support}; - - type TestResult = std::result::Result>; - - struct TestContext { - _dir: tempfile::TempDir, - source: Database, - source_db: slatedb::Db, - sqlite: sqlx::SqlitePool, - target: Arc, - } - - impl TestContext { - async fn new() -> TestResult { - let dir = tempfile::tempdir()?; - let sqlite = fabro_db::Database::connect(dir.path().join("fabro.sqlite3")).await?; - sqlite.migrate().await?; - let sqlite = sqlite.clone_pool(); - let target = Arc::new(BlobStore::new(sqlite.clone())); - let source = Database::new( - Arc::new(InMemory::new()), - "legacy-blob-import-tests", - Duration::from_millis(1), - None, - Arc::clone(&target), - store_test_support::test_run_summary_store(), - ); - let source_db = source.open_db().await?; - Ok(Self { - _dir: dir, - source, - source_db, - sqlite, - target, - }) - } - - async fn new_with_single_sqlite_connection() -> TestResult { - let mut context = Self::new().await?; - let options = context.sqlite.connect_options().as_ref().clone(); - context.sqlite.close().await; - let sqlite = SqlitePoolOptions::new() - .max_connections(1) - .connect_with(options) - .await?; - context.target = Arc::new(BlobStore::new(sqlite.clone())); - context.sqlite = sqlite; - Ok(context) - } - - async fn put_blob(&self, bytes: &[u8]) -> TestResult { - let hash = BlobHash::new(bytes); - let key = SlateKey::new("blobs").with("sha256").with(hash); - self.source_db.put(key, bytes).await?; - Ok(hash) - } - - async fn put_raw(&self, key: Vec, bytes: &[u8]) -> TestResult<()> { - self.source_db.put(key, bytes).await?; - Ok(()) - } - - async fn source_entries(&self) -> TestResult, Vec)>> { - let mut entries = self.source_db.scan_prefix(Vec::::new()).await?; - let mut snapshot = Vec::new(); - while let Some(entry) = entries.next().await? { - snapshot.push((entry.key.to_vec(), entry.value.to_vec())); - } - Ok(snapshot) - } - - async fn destination_rows(&self) -> TestResult { - Ok(sqlx::query_scalar("SELECT COUNT(*) FROM blobs") - .fetch_one(&self.sqlite) - .await?) - } - - async fn insert_destination(&self, hash: BlobHash, bytes: &[u8]) -> TestResult<()> { - sqlx::query("INSERT INTO blobs (hash, data) VALUES (?, ?)") - .bind(hash.to_string()) - .bind(bytes) - .execute(&self.sqlite) - .await?; - Ok(()) - } - - async fn delete_destination(&self, hash: BlobHash) -> TestResult<()> { - sqlx::query("DELETE FROM blobs WHERE hash = ?") - .bind(hash.to_string()) - .execute(&self.sqlite) - .await?; - Ok(()) - } - - async fn set_automatic_checkpoint(&self, pages: i64) -> TestResult<()> { - let mut connection = self.sqlite.acquire().await?; - let statement = sqlx::AssertSqlSafe(format!("PRAGMA wal_autocheckpoint = {pages}")); - sqlx::query(statement).execute(&mut *connection).await?; - Ok(()) - } - - async fn automatic_checkpoint(&self) -> TestResult { - let mut connection = self.sqlite.acquire().await?; - Ok(sqlx::query_scalar("PRAGMA wal_autocheckpoint") - .fetch_one(&mut *connection) - .await?) - } - - async fn import(&self) -> TestResult { - Ok(self.source.import_legacy_blobs_into(&self.sqlite).await?) - } - } - - fn legacy_prefix() -> Vec { - SlateKey::new("blobs") - .with("sha256") - .into_prefix() - .as_ref() - .to_vec() - } - - async fn seed_blobs( - context: &TestContext, - count: usize, - ) -> TestResult)>> { - let mut blobs = Vec::with_capacity(count); - for index in 0..count { - let bytes = format!("legacy-blob-{index:04}").into_bytes(); - let hash = context.put_blob(&bytes).await?; - blobs.push((hash, bytes)); - } - blobs.sort_by_key(|(hash, _)| *hash); - Ok(blobs) - } - - #[tokio::test] - async fn imports_valid_raw_blobs_and_reports_aggregate_progress() -> TestResult<()> { - let context = TestContext::new().await?; - let binary = [0_u8, 0xff, 0x80, b'a']; - let binary_hash = context.put_blob(&binary).await?; - let empty_hash = context.put_blob(b"").await?; - let source_before = context.source_entries().await?; - - let report = context.import().await?; - - assert_eq!(report, LegacyBlobImportReport { - scanned_rows: 2, - scanned_bytes: 4, - imported_rows: 2, - imported_bytes: 4, - committed_batches: 1, - ..LegacyBlobImportReport::default() - }); - assert_eq!( - context.target.read(&binary_hash).await?, - Some(Bytes::copy_from_slice(&binary)) - ); - assert_eq!(context.target.read(&empty_hash).await?, Some(Bytes::new())); - assert_eq!(context.source_entries().await?, source_before); - Ok(()) - } - - #[tokio::test] - async fn empty_source_succeeds_without_committing_a_batch() -> TestResult<()> { - let context = TestContext::new().await?; - - let report = context.import().await?; - - assert_eq!(report, LegacyBlobImportReport::default()); - assert_eq!(context.destination_rows().await?, 0); - Ok(()) - } - - #[tokio::test] - async fn scan_is_limited_to_the_exact_legacy_prefix() -> TestResult<()> { - let context = TestContext::new().await?; - context.put_blob(b"included").await?; - context - .source_db - .put( - SlateKey::new("blobs").with("other").with("ignored"), - b"nearby", - ) - .await?; - - let report = context.import().await?; - - assert_eq!(report.scanned_rows, 1); - assert_eq!(report.imported_rows, 1); - assert_eq!(context.destination_rows().await?, 1); - Ok(()) - } - - #[tokio::test] - async fn inventory_streams_the_exact_prefix_and_reports_logical_bytes() -> TestResult<()> { - let context = TestContext::new().await?; - context.put_blob(b"").await?; - context.put_blob(&[0, 0xff, 0x80, b'a']).await?; - context - .source_db - .put( - SlateKey::new("blobs").with("other").with("ignored"), - b"nearby", - ) - .await?; - - let inventory = context - .source - .legacy_blob_inventory(&context.sqlite) - .await?; - - assert_eq!(inventory.rows, 2); - assert_eq!(inventory.bytes, 4); - assert_eq!(inventory.pending_rows, 2); - assert_eq!(inventory.pending_bytes, 4); - Ok(()) - } - - #[tokio::test] - async fn inventory_reports_already_imported_rows_as_not_pending() -> TestResult<()> { - let context = TestContext::new().await?; - context.put_blob(b"imported-before-inventory").await?; - context.import().await?; - context.put_blob(b"still-pending").await?; - - let inventory = context - .source - .legacy_blob_inventory(&context.sqlite) - .await?; - - assert_eq!(inventory.rows, 2); - assert_eq!(inventory.pending_rows, 1); - assert_eq!( - inventory.pending_bytes, - u64::try_from(b"still-pending".len())? - ); - Ok(()) - } - - #[tokio::test] - async fn verification_checks_every_source_and_allows_valid_sqlite_only_rows() -> TestResult<()> - { - let context = TestContext::new().await?; - context.put_blob(b"legacy").await?; - context.import().await?; - let sqlite_only = b"written-after-activation"; - context - .insert_destination(BlobHash::new(sqlite_only), sqlite_only) - .await?; - - let report = context - .source - .verify_legacy_blobs_in(&context.sqlite) - .await?; - - assert_eq!(report.source_rows, 1); - assert_eq!(report.matched_rows, 1); - assert_eq!(report.target_rows, 2); - assert_eq!(report.missing_rows, 0); - assert_eq!(report.invalid_source_rows, 0); - assert_eq!(report.invalid_target_rows, 0); - assert_eq!(report.conflicting_rows, 0); - Ok(()) - } - - #[tokio::test] - async fn verification_reports_a_missing_destination_without_exposing_its_hash() -> TestResult<()> - { - let context = TestContext::new().await?; - let bytes = b"missing-sensitive-content"; - let hash = context.put_blob(bytes).await?; - - let error = context - .source - .verify_legacy_blobs_in(&context.sqlite) - .await - .expect_err("a missing destination row must fail verification"); - - assert_eq!(error.report().source_rows, 1); - assert_eq!(error.report().missing_rows, 1); - let rendered = format!("{error} {error:?}"); - assert!(!rendered.contains(&hash.to_string())); - assert!(!rendered.contains(std::str::from_utf8(bytes)?)); - Ok(()) - } - - #[tokio::test] - async fn verification_rejects_invalid_sqlite_hashes_and_bytes() -> TestResult<()> { - let malformed_hash = TestContext::new().await?; - let mut connection = malformed_hash.sqlite.acquire().await?; - sqlx::query("PRAGMA ignore_check_constraints = ON") - .execute(&mut *connection) - .await?; - sqlx::query("INSERT INTO blobs (hash, data) VALUES (?, ?)") - .bind("not-a-canonical-hash") - .bind(b"bytes".as_slice()) - .execute(&mut *connection) - .await?; - sqlx::query("PRAGMA ignore_check_constraints = OFF") - .execute(&mut *connection) - .await?; - drop(connection); - let error = malformed_hash - .source - .verify_legacy_blobs_in(&malformed_hash.sqlite) - .await - .expect_err("malformed SQLite hashes must fail verification"); - assert_eq!(error.report().invalid_target_rows, 1); - - let mismatched_bytes = TestContext::new().await?; - mismatched_bytes - .insert_destination(BlobHash::new(b"expected"), b"different") - .await?; - let error = mismatched_bytes - .source - .verify_legacy_blobs_in(&mismatched_bytes.sqlite) - .await - .expect_err("SQLite bytes must match their hash"); - assert_eq!(error.report().invalid_target_rows, 1); - Ok(()) - } - - #[tokio::test] - async fn rejects_every_noncanonical_legacy_key_shape() -> TestResult<()> { - let valid_hash = BlobHash::new(b"valid-shape").to_string(); - let cases = [ - ("empty", Vec::new()), - ("short", vec![b'0'; 63]), - ("long", vec![b'0'; 65]), - ("uppercase", vec![b'A'; 64]), - ("non_hex", vec![b'g'; 64]), - ("non_utf8", vec![0xff; 64]), - ( - "extra_segment", - [valid_hash.as_bytes(), b"\0extra"].concat(), - ), - ]; - - for (case, suffix) in cases { - let context = TestContext::new().await?; - let mut key = legacy_prefix(); - key.extend_from_slice(&suffix); - context.put_raw(key, b"source-bytes").await?; - let source_before = context.source_entries().await?; - - let error = context - .source - .import_legacy_blobs_into(&context.sqlite) - .await - .expect_err("noncanonical key should fail import"); - - assert_eq!(error.report().scanned_rows, 1, "case {case}"); - assert_eq!(error.report().invalid_rows, 1, "case {case}"); - assert_eq!(error.report().committed_batches, 0, "case {case}"); - assert!( - matches!(&error.failure, LegacyBlobImportFailure::InvalidSourceKey), - "case {case}: {error:?}" - ); - assert_eq!(context.destination_rows().await?, 0, "case {case}"); - assert_eq!( - context.source_entries().await?, - source_before, - "case {case}" - ); - } - Ok(()) - } - - #[tokio::test] - async fn rejects_source_bytes_that_do_not_match_the_key_digest() -> TestResult<()> { - let context = TestContext::new().await?; - let mut key = legacy_prefix(); - key.extend_from_slice(BlobHash::new(b"expected").to_string().as_bytes()); - context.put_raw(key, b"different").await?; - - let error = context - .source - .import_legacy_blobs_into(&context.sqlite) - .await - .expect_err("digest mismatch should fail import"); - - assert_eq!(error.report().scanned_rows, 1); - assert_eq!(error.report().scanned_bytes, 9); - assert_eq!(error.report().invalid_rows, 1); - assert!(matches!( - &error.failure, - LegacyBlobImportFailure::SourceDigestMismatch - )); - assert_eq!(context.destination_rows().await?, 0); - Ok(()) - } - - #[tokio::test] - async fn invalid_source_row_discards_the_uncommitted_pending_batch() -> TestResult<()> { - let context = TestContext::new().await?; - context.put_blob(b"valid-but-not-yet-committed").await?; - let mut invalid_key = legacy_prefix(); - invalid_key.extend_from_slice(&[b'z'; 64]); - context.put_raw(invalid_key, b"invalid").await?; - - let error = context - .source - .import_legacy_blobs_into(&context.sqlite) - .await - .expect_err("invalid row should stop the import"); - - assert_eq!(error.report().scanned_rows, 2); - assert_eq!(error.report().invalid_rows, 1); - assert_eq!(error.report().imported_rows, 0); - assert_eq!(error.report().committed_batches, 0); - assert_eq!(context.destination_rows().await?, 0); - Ok(()) - } - - #[tokio::test] - async fn equal_destination_rows_are_retry_progress() -> TestResult<()> { - let context = TestContext::new().await?; - context.put_blob(b"first").await?; - context.put_blob(b"second").await?; - - let first = context.import().await?; - let second = context.import().await?; - - assert_eq!(first.imported_rows, 2); - assert_eq!(second.scanned_rows, 2); - assert_eq!(second.imported_rows, 0); - assert_eq!(second.existing_rows, 2); - assert_eq!(second.existing_bytes, 11); - assert_eq!(second.committed_batches, 1); - assert_eq!(context.destination_rows().await?, 2); - Ok(()) - } - - #[tokio::test] - async fn destination_conflict_rolls_back_the_current_batch() -> TestResult<()> { - let context = TestContext::new().await?; - let blobs = seed_blobs(&context, 2).await?; - context - .insert_destination(blobs[1].0, b"conflicting-destination-bytes") - .await?; - - let error = context - .source - .import_legacy_blobs_into(&context.sqlite) - .await - .expect_err("differing destination row should fail import"); - - assert_eq!(error.report().scanned_rows, 2); - assert_eq!(error.report().conflicting_rows, 1); - assert_eq!(error.report().imported_rows, 0); - assert_eq!(error.report().committed_batches, 0); - assert!(matches!( - &error.failure, - LegacyBlobImportFailure::DestinationConflict - )); - assert_eq!(context.target.read(&blobs[0].0).await?, None); - assert_eq!(context.destination_rows().await?, 1); - Ok(()) - } - - #[tokio::test] - async fn interrupted_import_preserves_commits_and_retry_converges() -> TestResult<()> { - let context = TestContext::new().await?; - let blobs = seed_blobs(&context, 101).await?; - let conflict = blobs[100].0; - context - .insert_destination(conflict, b"conflicting-destination-bytes") - .await?; - - let error = context - .source - .import_legacy_blobs_into(&context.sqlite) - .await - .expect_err("last-row conflict should interrupt import"); - - assert_eq!(error.report().scanned_rows, 101); - assert_eq!(error.report().imported_rows, 100); - assert_eq!(error.report().conflicting_rows, 1); - assert_eq!(error.report().committed_batches, 1); - assert_eq!(context.destination_rows().await?, 101); - - context.delete_destination(conflict).await?; - let retry = context.import().await?; - assert_eq!(retry.scanned_rows, 101); - assert_eq!(retry.existing_rows, 100); - assert_eq!(retry.imported_rows, 1); - assert_eq!(retry.committed_batches, 2); - assert_eq!(context.destination_rows().await?, 101); - Ok(()) - } - - #[tokio::test] - async fn row_limit_splits_one_hundred_and_one_values() -> TestResult<()> { - let context = TestContext::new().await?; - seed_blobs(&context, 101).await?; - - let report = context.import().await?; - - assert_eq!(report.imported_rows, 101); - assert_eq!(report.committed_batches, 2); - Ok(()) - } - - #[tokio::test] - async fn byte_limit_splits_batches_and_allows_exact_limit() -> TestResult<()> { - let split_context = TestContext::new().await?; - split_context.put_blob(&vec![b'a'; 600 * 1024]).await?; - split_context.put_blob(&vec![b'b'; 600 * 1024]).await?; - let split = split_context.import().await?; - assert_eq!(split.committed_batches, 2); - - let exact_context = TestContext::new().await?; - exact_context - .put_blob(&vec![b'x'; usize::try_from(MAX_BATCH_BYTES)?]) - .await?; - exact_context.put_blob(b"").await?; - let exact = exact_context.import().await?; - assert_eq!(exact.committed_batches, 1); - Ok(()) - } - - #[tokio::test] - async fn oversized_value_is_committed_alone_between_surrounding_batches() -> TestResult<()> { - let context = TestContext::new().await?; - let oversized = vec![b'o'; usize::try_from(MAX_BATCH_BYTES + 1)?]; - let oversized_hash = BlobHash::new(&oversized); - let mut lower = None; - let mut upper = None; - for index in 0_u32..10_000 { - let bytes = format!("surrounding-{index}").into_bytes(); - let hash = BlobHash::new(&bytes); - if hash < oversized_hash && lower.is_none() { - lower = Some(bytes); - } else if hash > oversized_hash && upper.is_none() { - upper = Some(bytes); - } - if lower.is_some() && upper.is_some() { - break; - } - } - let lower = lower.expect("search should find a hash below the oversized value"); - let upper = upper.expect("search should find a hash above the oversized value"); - context.put_blob(&lower).await?; - context.put_blob(&oversized).await?; - context.put_blob(&upper).await?; - - let report = context.import().await?; - - assert_eq!(report.imported_rows, 3); - assert_eq!(report.committed_batches, 3); - Ok(()) - } - - #[tokio::test] - async fn passive_checkpoint_runs_once_after_crossing_each_batch_threshold() -> TestResult<()> { - let crossing_context = TestContext::new().await?; - crossing_context - .put_blob(&vec![b'c'; usize::try_from(PASSIVE_CHECKPOINT_BYTES + 1)?]) - .await?; - let crossing = crossing_context.import().await?; - assert_eq!(crossing.passive_checkpoints, 1); - - let multi_context = TestContext::new().await?; - multi_context - .put_blob(&vec![ - b'm'; - usize::try_from(PASSIVE_CHECKPOINT_BYTES * 2 + 1)? - ]) - .await?; - let multiple = multi_context.import().await?; - assert_eq!(multiple.passive_checkpoints, 1); - Ok(()) - } - - #[tokio::test] - async fn passive_checkpoint_failure_returns_durable_partial_report() -> TestResult<()> { - let context = TestContext::new().await?; - context - .put_blob(&vec![b'p'; usize::try_from(PASSIVE_CHECKPOINT_BYTES + 1)?]) - .await?; - let controls = ImportControls { - passive_checkpoint: true, - ..ImportControls::default() - }; - - let error = context - .source - .import_legacy_blobs_with_controls(&context.sqlite, &controls) - .await - .expect_err("injected passive checkpoint should fail import"); - - assert_eq!(error.report().imported_rows, 1); - assert_eq!(error.report().committed_batches, 1); - assert_eq!(error.report().passive_checkpoints, 0); - assert!(matches!( - &error.failure, - LegacyBlobImportFailure::PassiveCheckpoint(sqlx::Error::Protocol(_)) - )); - let failure_source = std::error::Error::source(&error) - .and_then(std::error::Error::source) - .expect("checkpoint failure should preserve its SQL source"); - assert!(failure_source.downcast_ref::().is_some()); - assert_eq!(context.destination_rows().await?, 1); - Ok(()) - } - - #[tokio::test] - async fn source_transport_failure_preserves_commits_and_typed_source() -> TestResult<()> { - let context = TestContext::new().await?; - seed_blobs(&context, 101).await?; - let controls = ImportControls { - source_after_rows: Some(101), - ..ImportControls::default() - }; - - let error = context - .source - .import_legacy_blobs_with_controls(&context.sqlite, &controls) - .await - .expect_err("injected source transport failure should stop import"); - - assert_eq!(error.report().scanned_rows, 101); - assert_eq!(error.report().imported_rows, 100); - assert_eq!(error.report().committed_batches, 1); - assert!(matches!( - &error.failure, - LegacyBlobImportFailure::ReadSourceScan(_) - )); - let failure_source = std::error::Error::source(&error) - .and_then(std::error::Error::source) - .expect("transport failure should preserve its SlateDB source"); - assert!(failure_source.downcast_ref::().is_some()); - - let retry = context.import().await?; - assert_eq!(retry.existing_rows, 100); - assert_eq!(retry.imported_rows, 1); - assert_eq!(context.destination_rows().await?, 101); - Ok(()) - } - - #[tokio::test] - async fn restoration_failure_preserves_the_prior_source_chain() -> TestResult<()> { - let context = TestContext::new().await?; - let controls = ImportControls { - source_after_rows: Some(0), - restore_automatic_checkpoint: true, - ..ImportControls::default() - }; - let capture = CapturedEvents::default(); - - let error = context - .source - .import_legacy_blobs_with_controls(&context.sqlite, &controls) - .with_subscriber(capture.clone()) - .await - .expect_err("both injected failures should fail import"); - - let mut current: Option<&(dyn std::error::Error + 'static)> = Some(&error); - let mut saw_slate_source = false; - while let Some(source) = current { - saw_slate_source |= source.downcast_ref::().is_some(); - current = source.source(); - } - assert!(saw_slate_source, "prior source was absent from the chain"); - assert!( - error - .cleanup_errors() - .any(|source| source.downcast_ref::().is_some()), - "restoration source was absent from cleanup errors" - ); - let events = capture.events().join("\n"); - assert!( - events.contains("Legacy blob import cleanup failed"), - "captured: {events}" - ); - assert!( - events.contains("injected automatic checkpoint restoration failure"), - "captured: {events}" - ); - Ok(()) - } - - #[tokio::test] - async fn automatic_checkpoint_setting_is_restored_after_success_and_failure() -> TestResult<()> - { - let success = TestContext::new_with_single_sqlite_connection().await?; - success.set_automatic_checkpoint(37).await?; - success.put_blob(b"success").await?; - success.import().await?; - assert_eq!(success.automatic_checkpoint().await?, 37); - - let failure = TestContext::new_with_single_sqlite_connection().await?; - failure.set_automatic_checkpoint(41).await?; - let mut invalid_key = legacy_prefix(); - invalid_key.extend_from_slice(&[b'z'; 64]); - failure.put_raw(invalid_key, b"invalid").await?; - failure - .source - .import_legacy_blobs_into(&failure.sqlite) - .await - .expect_err("invalid source should fail import"); - assert_eq!(failure.automatic_checkpoint().await?, 41); - Ok(()) - } - - #[tokio::test] - async fn cancellation_retires_a_connection_with_disabled_checkpointing() -> TestResult<()> { - let dir = tempfile::tempdir()?; - let options = SqliteConnectOptions::new() - .filename(dir.path().join("fabro.sqlite3")) - .create_if_missing(true) - .journal_mode(SqliteJournalMode::Wal); - let pool = SqlitePoolOptions::new() - .max_connections(1) - .connect_with(options) - .await?; - sqlx::query("CREATE TABLE blobs (hash TEXT PRIMARY KEY NOT NULL, data BLOB NOT NULL)") - .execute(&pool) - .await?; - let target = Arc::new(BlobStore::new(pool.clone())); - let source = Database::new( - Arc::new(InMemory::new()), - "legacy-blob-import-cancellation-test", - Duration::from_millis(1), - None, - Arc::clone(&target), - store_test_support::test_run_summary_store(), - ); - - let mut connection = pool.acquire().await?; - set_automatic_checkpoint(&mut connection, 73).await?; - drop(connection); - - let barrier = Arc::new(Barrier::new(2)); - let controls = ImportControls { - after_automatic_checkpoint_disabled: Some(Arc::clone(&barrier)), - ..ImportControls::default() - }; - let task = tokio::spawn({ - let source = source.clone(); - let pool = pool.clone(); - async move { - let mut report = LegacyBlobImportReport::default(); - source - .run_legacy_blob_import(&pool, &controls, &mut report) - .await - } - }); - - barrier.wait().await; - task.abort(); - let join_error = task.await.expect_err("aborted import should be cancelled"); - assert!(join_error.is_cancelled()); - - let mut connection = pool.acquire().await?; - let observed: i64 = sqlx::query_scalar("PRAGMA wal_autocheckpoint") - .fetch_one(&mut *connection) - .await?; - assert_ne!(observed, 0); - Ok(()) - } - - #[tokio::test] - async fn failed_connection_restoration_retires_the_connection() -> TestResult<()> { - let context = TestContext::new().await?; - context.set_automatic_checkpoint(43).await?; - context.put_blob(b"committed-before-restore").await?; - let controls = ImportControls { - restore_automatic_checkpoint: true, - ..ImportControls::default() - }; - - let error = context - .source - .import_legacy_blobs_with_controls(&context.sqlite, &controls) - .await - .expect_err("injected restoration failure should fail import"); - - assert_eq!(error.report().imported_rows, 1); - assert!(matches!( - &error.failure, - LegacyBlobImportFailure::RestoreAutomaticCheckpoint { - source: sqlx::Error::Protocol(_), - .. - } - )); - assert_ne!(context.automatic_checkpoint().await?, 0); - Ok(()) - } - - #[tokio::test] - async fn logs_and_error_rendering_expose_counts_but_not_row_data() -> TestResult<()> { - let context = TestContext::new().await?; - let sensitive_key_fragment = "sensitive-invalid-legacy-key"; - let sensitive_content = b"sensitive-blob-content"; - let mut key = legacy_prefix(); - key.extend_from_slice(sensitive_key_fragment.as_bytes()); - context.put_raw(key, sensitive_content).await?; - let capture = CapturedEvents::default(); - - let error = context - .source - .import_legacy_blobs_into(&context.sqlite) - .with_subscriber(capture.clone()) - .await - .expect_err("invalid key should fail import"); - - let rendered = format!("{error} {error:?}"); - let events = capture.events().join("\n"); - for output in [&rendered, &events] { - assert!(!output.contains(sensitive_key_fragment)); - assert!(!output.contains(std::str::from_utf8(sensitive_content)?)); - } - assert!(events.contains("scanned_rows=1"), "captured: {events}"); - assert!(events.contains("invalid_rows=1"), "captured: {events}"); - assert!( - events.contains("failure_kind=\"invalid_source_key\""), - "captured: {events}" - ); - assert_eq!(capture.events().len(), 1); - Ok(()) - } - - #[derive(Clone, Default)] - struct CapturedEvents { - events: Arc>>, - next_span_id: Arc, - } - - impl CapturedEvents { - fn events(&self) -> Vec { - self.events - .lock() - .expect("captured tracing events mutex should not be poisoned") - .clone() - } - } - - impl Subscriber for CapturedEvents { - fn enabled(&self, _metadata: &Metadata<'_>) -> bool { - true - } - - fn new_span(&self, _span: &Attributes<'_>) -> Id { - Id::from_u64(self.next_span_id.fetch_add(1, Ordering::Relaxed) + 1) - } - - fn record(&self, _span: &Id, _values: &Record<'_>) {} - - fn record_follows_from(&self, _span: &Id, _follows: &Id) {} - - fn event(&self, event: &Event<'_>) { - let mut visitor = CapturedFields::default(); - event.record(&mut visitor); - self.events - .lock() - .expect("captured tracing events mutex should not be poisoned") - .push(visitor.output); - } - - fn enter(&self, _span: &Id) {} - - fn exit(&self, _span: &Id) {} - } - - #[derive(Default)] - struct CapturedFields { - output: String, - } - - impl Visit for CapturedFields { - fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) { - write!(&mut self.output, "{}={value:?};", field.name()) - .expect("writing tracing fields to String cannot fail"); - } - } -} diff --git a/lib/components/fabro-store/src/legacy_run_history_import.rs b/lib/components/fabro-store/src/legacy_run_history_import.rs deleted file mode 100644 index b253248e5..000000000 --- a/lib/components/fabro-store/src/legacy_run_history_import.rs +++ /dev/null @@ -1,2234 +0,0 @@ -//! Temporary compatibility importer for legacy SlateDB run history. -//! -//! Keep this source reader, its reports, and its verification path through the -//! 30-day run-history compatibility window. Remove them only after the -//! production evidence gate for that window has been accepted. - -use std::collections::HashSet; -use std::error::Error as StdError; -use std::fmt; - -use fabro_types::{EventEnvelope, RunEvent, RunId}; -use sha2::{Digest as _, Sha256}; -use sqlx::SqlitePool; -#[cfg(test)] -use tokio::sync::Barrier; -use tracing::debug; - -use crate::keys::SlateKey; -use crate::run_state::ProjectedRun; -use crate::{Database, EventPayload, RunSummaryStore, keys}; - -/// Count-only observations about the legacy catalog and session indexes. -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] -pub struct LegacyRunHistoryDiagnostics { - pub catalog_markers: u64, - pub empty_catalog_markers: u64, - pub session_reverse_rows: u64, -} - -/// Durable progress from one legacy run-history import attempt. -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] -pub struct LegacyRunHistoryImportReport { - pub scanned_source_runs: u64, - pub scanned_source_events: u64, - pub imported_runs: u64, - pub imported_events: u64, - pub verified_existing_runs: u64, - pub verified_existing_events: u64, - pub discarded_projection_only_rows: u64, - pub committed_run_transactions: u64, - pub tombstoned_source_runs: u64, - pub tombstoned_source_events: u64, - pub diagnostics: LegacyRunHistoryDiagnostics, -} - -/// Aggregate proof from legacy-prefix and full-SQL-destination verification. -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] -pub struct LegacyRunHistoryVerificationReport { - pub source_runs: u64, - pub source_events: u64, - pub matched_prefix_runs: u64, - pub matched_prefix_events: u64, - pub target_runs: u64, - pub target_events: u64, - pub sql_only_runs: u64, - pub sql_only_events: u64, - pub tombstoned_source_runs: u64, - pub tombstoned_source_events: u64, - pub diagnostics: LegacyRunHistoryDiagnostics, -} - -/// Stable, aggregate-only identity of the exact legacy run-event source. -#[derive(Clone, Copy, Eq, PartialEq)] -pub struct LegacyRunHistorySourceIdentity { - fingerprint: [u8; 32], - pub runs: u64, - pub events: u64, -} - -impl LegacyRunHistorySourceIdentity { - #[must_use] - pub fn fingerprint(&self) -> &[u8; 32] { - &self.fingerprint - } -} - -impl fmt::Debug for LegacyRunHistorySourceIdentity { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("LegacyRunHistorySourceIdentity") - .field("runs", &self.runs) - .field("events", &self.events) - .finish_non_exhaustive() - } -} - -/// Failure while strictly identifying the legacy run-event source. -pub struct LegacyRunHistorySourceIdentityError { - failure: LegacyRunHistorySourceFailure, -} - -impl From for LegacyRunHistorySourceIdentityError { - fn from(failure: LegacyRunHistorySourceFailure) -> Self { - Self { failure } - } -} - -impl fmt::Debug for LegacyRunHistorySourceIdentityError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("LegacyRunHistorySourceIdentityError") - .field("failure", &self.failure.kind()) - .finish() - } -} - -impl fmt::Display for LegacyRunHistorySourceIdentityError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - formatter, - "identifying the legacy run-history source: {}", - self.failure - ) - } -} - -impl StdError for LegacyRunHistorySourceIdentityError { - fn source(&self) -> Option<&(dyn StdError + 'static)> { - Some(&self.failure) - } -} - -/// An import failure plus the durable progress completed before it. -pub struct LegacyRunHistoryImportError { - report: LegacyRunHistoryImportReport, - failure: LegacyRunHistoryImportFailure, -} - -impl LegacyRunHistoryImportError { - #[must_use] - pub fn report(&self) -> &LegacyRunHistoryImportReport { - &self.report - } - - /// Returns secondary errors encountered while rolling back a failed run - /// import transaction. - /// - /// The standard error source chain preserves the failure that interrupted - /// the import. Because that chain is linear, rollback errors are exposed - /// separately. - pub fn cleanup_errors(&self) -> impl Iterator { - let mut errors = Vec::new(); - self.failure.collect_cleanup_errors(&mut errors); - errors.into_iter() - } -} - -impl fmt::Debug for LegacyRunHistoryImportError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("LegacyRunHistoryImportError") - .field("report", &self.report) - .field("failure", &self.failure.kind()) - .finish() - } -} - -impl fmt::Display for LegacyRunHistoryImportError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - formatter, - "legacy run-history import failed after scanning {} runs and committing {} run transactions: {}", - self.report.scanned_source_runs, self.report.committed_run_transactions, self.failure - ) - } -} - -impl StdError for LegacyRunHistoryImportError { - fn source(&self) -> Option<&(dyn StdError + 'static)> { - Some(self.failure.primary_failure()) - } -} - -/// A verification failure plus the aggregate proof completed before it. -pub struct LegacyRunHistoryVerificationError { - report: LegacyRunHistoryVerificationReport, - failure: LegacyRunHistoryVerificationFailure, -} - -impl LegacyRunHistoryVerificationError { - #[must_use] - pub fn report(&self) -> &LegacyRunHistoryVerificationReport { - &self.report - } -} - -impl fmt::Debug for LegacyRunHistoryVerificationError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("LegacyRunHistoryVerificationError") - .field("report", &self.report) - .field("failure", &self.failure.kind()) - .finish() - } -} - -impl fmt::Display for LegacyRunHistoryVerificationError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - formatter, - "legacy run-history verification failed after checking {} source runs and {} target runs: {}", - self.report.source_runs, self.report.target_runs, self.failure - ) - } -} - -impl StdError for LegacyRunHistoryVerificationError { - fn source(&self) -> Option<&(dyn StdError + 'static)> { - Some(&self.failure) - } -} - -#[derive(strum::IntoStaticStr, thiserror::Error)] -#[strum(serialize_all = "snake_case")] -enum LegacyRunHistoryImportFailure { - #[error("reading and validating the legacy run-history source")] - Source(#[source] LegacyRunHistorySourceFailure), - #[error("reading legacy run-history activation state")] - ActivationState(#[source] sqlx::Error), - #[error("reading legacy run-history deletion tombstones")] - DeletionState(#[source] sqlx::Error), - #[error("a legacy run-history deletion tombstone still has canonical SQLite data")] - TombstonedDestinationPresent, - #[error("starting the projection-only row cleanup transaction")] - BeginCleanup(#[source] sqlx::Error), - #[error("deleting projection-only run rows")] - DeleteProjectionOnlyRows(#[source] sqlx::Error), - #[error("committing the projection-only row cleanup")] - CommitCleanup(#[source] sqlx::Error), - #[error("starting a per-run import transaction")] - BeginRunTransaction(#[source] sqlx::Error), - #[error("reading an existing destination run history")] - ReadDestination(#[source] crate::Error), - #[error("the destination history is partial or conflicts with the legacy prefix")] - DestinationConflict, - #[error("SQLite is missing an activated legacy run-history prefix")] - MissingDestinationAfterActivation, - #[error("replaying an existing destination run history")] - ReplayDestination(#[source] crate::Error), - #[error("verifying an existing destination run row")] - VerifyDestination(#[source] crate::Error), - #[error("inserting the imported final run row")] - InsertRun(#[source] crate::Error), - #[error("inserting an imported run event")] - InsertEvent(#[source] crate::Error), - #[error("committing a per-run import transaction")] - CommitRunTransaction(#[source] sqlx::Error), - #[error("rolling back a failed per-run import transaction")] - RollbackRunTransaction { - #[source] - source: sqlx::Error, - prior: Box, - }, - #[error("collecting count-only legacy index diagnostics")] - Diagnostics(#[source] LegacyRunHistoryDiagnosticsFailure), - #[error("a legacy run-history import counter overflowed")] - CounterOverflow, -} - -impl LegacyRunHistoryImportFailure { - fn kind(&self) -> &'static str { - self.into() - } - - fn primary_failure(&self) -> &Self { - match self { - Self::RollbackRunTransaction { prior, .. } => prior.primary_failure(), - _ => self, - } - } - - fn collect_cleanup_errors<'a>(&'a self, errors: &mut Vec<&'a (dyn StdError + 'static)>) { - if let Self::RollbackRunTransaction { source, prior } = self { - errors.push(source); - prior.collect_cleanup_errors(errors); - } - } -} - -impl fmt::Debug for LegacyRunHistoryImportFailure { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - let mut debug = formatter.debug_struct("LegacyRunHistoryImportFailure"); - debug.field("kind", &self.kind()); - if let Self::RollbackRunTransaction { prior, .. } = self { - debug.field("prior_failure", &prior.kind()); - } - debug.finish() - } -} - -#[derive(strum::IntoStaticStr, thiserror::Error)] -#[strum(serialize_all = "snake_case")] -enum LegacyRunHistoryVerificationFailure { - #[error("reading and validating the legacy run-history source")] - Source(#[source] LegacyRunHistorySourceFailure), - #[error("reading legacy run-history deletion tombstones")] - DeletionState(#[source] sqlx::Error), - #[error("a legacy run-history deletion tombstone still has canonical SQLite data")] - TombstonedDestinationPresent, - #[error("acquiring a SQLite verification connection")] - AcquireConnection(#[source] sqlx::Error), - #[error("reading a destination run history")] - ReadDestination(#[source] crate::Error), - #[error("SQLite is missing all or part of a legacy run-history prefix")] - MissingDestinationPrefix, - #[error("a destination run-history prefix conflicts with legacy JSON or sequence identity")] - DestinationPrefixConflict, - #[error("enumerating destination run rows")] - ListDestinationRuns(#[source] sqlx::Error), - #[error("a destination run row has an invalid identity")] - InvalidDestinationRunId, - #[error("a destination run has no event history")] - EmptyDestinationHistory, - #[error("replaying a destination run history")] - ReplayDestination(#[source] crate::Error), - #[error("verifying a destination run row")] - VerifyDestination(#[source] crate::Error), - #[error("collecting count-only legacy index diagnostics")] - Diagnostics(#[source] LegacyRunHistoryDiagnosticsFailure), - #[error("a legacy run-history verification counter overflowed")] - CounterOverflow, -} - -impl LegacyRunHistoryVerificationFailure { - fn kind(&self) -> &'static str { - self.into() - } -} - -impl fmt::Debug for LegacyRunHistoryVerificationFailure { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("LegacyRunHistoryVerificationFailure") - .field("kind", &self.kind()) - .finish() - } -} - -#[derive(strum::IntoStaticStr, thiserror::Error)] -#[strum(serialize_all = "snake_case")] -enum LegacyRunHistorySourceFailure { - #[error("opening the legacy run-history source")] - OpenSource(#[source] crate::Error), - #[error("opening the legacy run-history scan")] - OpenScan(#[source] slatedb::Error), - #[error("reading the legacy run-history scan")] - ReadScan(#[source] slatedb::Error), - #[error("a legacy run-event key is not UTF-8")] - KeyUtf8(#[source] std::str::Utf8Error), - #[error("a legacy run-event key is not canonical")] - InvalidKey, - #[error("a legacy run-event value is not UTF-8")] - ValueUtf8(#[source] std::str::Utf8Error), - #[error("a legacy run-event value is not valid JSON")] - DecodeEvent(#[source] serde_json::Error), - #[error("a legacy run-event value does not match its key or event contract")] - ValidateEvent(#[source] crate::Error), - #[error("a legacy run history has invalid sequence ordering")] - InvalidSequence, - #[error("a legacy run history does not begin with sequence 1 run.created")] - InvalidFirstEvent, - #[error("replaying a legacy run history")] - Replay(#[source] crate::Error), - #[error("a legacy run-history source counter overflowed")] - CounterOverflow, -} - -impl LegacyRunHistorySourceFailure { - fn kind(&self) -> &'static str { - self.into() - } -} - -impl fmt::Debug for LegacyRunHistorySourceFailure { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("LegacyRunHistorySourceFailure") - .field("kind", &self.kind()) - .finish() - } -} - -#[derive(strum::IntoStaticStr, thiserror::Error)] -#[strum(serialize_all = "snake_case")] -enum LegacyRunHistoryDiagnosticsFailure { - #[error("reading the legacy run catalog")] - ReadCatalog(#[source] slatedb::Error), - #[error("a legacy run-catalog key is not UTF-8")] - CatalogKeyUtf8(#[source] std::str::Utf8Error), - #[error("a legacy run-catalog key is not canonical")] - InvalidCatalogKey, - #[error("opening the legacy run source for diagnostics")] - OpenSource(#[source] crate::Error), - #[error("opening a legacy run-event probe")] - OpenEventProbe(#[source] slatedb::Error), - #[error("reading a legacy run-event probe")] - ReadEventProbe(#[source] slatedb::Error), - #[error("opening the legacy session reverse-row scan")] - OpenSessionScan(#[source] slatedb::Error), - #[error("reading the legacy session reverse-row scan")] - ReadSessionScan(#[source] slatedb::Error), - #[error("a legacy run-history diagnostics counter overflowed")] - CounterOverflow, -} - -impl LegacyRunHistoryDiagnosticsFailure { - fn kind(&self) -> &'static str { - self.into() - } -} - -impl fmt::Debug for LegacyRunHistoryDiagnosticsFailure { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("LegacyRunHistoryDiagnosticsFailure") - .field("kind", &self.kind()) - .finish() - } -} - -#[derive(Default)] -struct ImportControls { - #[cfg(test)] - source_after_events: Option, - #[cfg(test)] - after_run_inserted: Option>, -} - -impl ImportControls { - fn source_scan_error(&self, observed_events: u64) -> Option { - #[cfg(test)] - if self.source_after_events == Some(observed_events) { - return Some(slatedb::Error::unavailable( - "injected legacy run-history source failure".to_owned(), - )); - } - let _ = (self, observed_events); - None - } - - #[cfg(test)] - async fn after_run_inserted(&self) { - if let Some(barrier) = &self.after_run_inserted { - barrier.wait().await; - barrier.wait().await; - } - } -} - -struct ValidatedLegacyRunEvent { - run_id: RunId, - payload: EventPayload, - envelope: EventEnvelope, - event_json: String, - raw_key: Vec, -} - -struct ValidatedLegacyRunHistory { - run_id: RunId, - events: Vec, - current: ProjectedRun, -} - -struct LegacyRunHistorySource { - entries: slatedb::DbIterator, - buffered: Option, - observed_events: u64, -} - -impl LegacyRunHistorySource { - async fn open(database: &Database) -> Result { - let source = database - .open_db() - .await - .map_err(LegacyRunHistorySourceFailure::OpenSource)?; - let prefix = SlateKey::new("runs").into_prefix(); - let entries = source - .scan_prefix(prefix) - .await - .map_err(LegacyRunHistorySourceFailure::OpenScan)?; - Ok(Self { - entries, - buffered: None, - observed_events: 0, - }) - } - - async fn next_run( - &mut self, - controls: Option<&ImportControls>, - ) -> Result, LegacyRunHistorySourceFailure> { - let first = if let Some(entry) = self.buffered.take() { - parse_source_event(&entry.key, &entry.value)? - .expect("the buffered source entry belongs to the event namespace") - } else { - let Some(event) = self.next_event(controls).await? else { - return Ok(None); - }; - event - }; - let run_id = first.run_id; - let run_id_text = run_id.to_string(); - let mut events = vec![first]; - loop { - let Some(entry) = self.next_event_entry(controls).await? else { - break; - }; - if event_run_segment(&entry.key) != Some(run_id_text.as_bytes()) { - self.buffered = Some(entry); - break; - } - let event = parse_source_event(&entry.key, &entry.value)? - .expect("an event-namespace entry parses as an event or fails"); - events.push(event); - } - - if events[0].envelope.seq != 1 || events[0].envelope.event.event_name() != "run.created" { - return Err(LegacyRunHistorySourceFailure::InvalidFirstEvent); - } - if events - .windows(2) - .any(|pair| pair[0].envelope.seq >= pair[1].envelope.seq) - { - return Err(LegacyRunHistorySourceFailure::InvalidSequence); - } - let envelopes = events - .iter() - .map(|event| event.envelope.clone()) - .collect::>(); - let current = ProjectedRun::replay(run_id, &envelopes) - .map_err(LegacyRunHistorySourceFailure::Replay)?; - Ok(Some(ValidatedLegacyRunHistory { - run_id, - events, - current, - })) - } - - async fn next_event( - &mut self, - controls: Option<&ImportControls>, - ) -> Result, LegacyRunHistorySourceFailure> { - let Some(entry) = self.next_event_entry(controls).await? else { - return Ok(None); - }; - parse_source_event(&entry.key, &entry.value) - } - - async fn next_event_entry( - &mut self, - controls: Option<&ImportControls>, - ) -> Result, LegacyRunHistorySourceFailure> { - loop { - if let Some(source) = - controls.and_then(|controls| controls.source_scan_error(self.observed_events)) - { - return Err(LegacyRunHistorySourceFailure::ReadScan(source)); - } - let Some(entry) = self - .entries - .next() - .await - .map_err(LegacyRunHistorySourceFailure::ReadScan)? - else { - return Ok(None); - }; - if event_run_segment(&entry.key).is_none() { - continue; - } - self.observed_events = self - .observed_events - .checked_add(1) - .ok_or(LegacyRunHistorySourceFailure::CounterOverflow)?; - return Ok(Some(entry)); - } - } -} - -impl Database { - /// Strictly identifies the exact legacy run-event key/value stream. - pub async fn legacy_run_history_source_identity( - &self, - ) -> Result { - const DOMAIN_SEPARATOR: &[u8] = b"fabro.legacy-run-history-source.v1\0"; - - let mut source = LegacyRunHistorySource::open(self).await?; - let mut hasher = Sha256::new(); - hasher.update(DOMAIN_SEPARATOR); - let mut runs = 0_u64; - let mut events = 0_u64; - while let Some(history) = source.next_run(None).await? { - runs = runs - .checked_add(1) - .ok_or(LegacyRunHistorySourceFailure::CounterOverflow)?; - for event in &history.events { - hash_source_part(&mut hasher, &event.raw_key)?; - hash_source_part(&mut hasher, event.event_json.as_bytes())?; - events = events - .checked_add(1) - .ok_or(LegacyRunHistorySourceFailure::CounterOverflow)?; - } - } - Ok(LegacyRunHistorySourceIdentity { - fingerprint: hasher.finalize().into(), - runs, - events, - }) - } - - /// Strictly imports legacy SlateDB run history into the inactive SQLite - /// run store, committing one complete run at a time. - /// - /// The caller must prevent writes to both stores for the duration of the - /// import. This operation does not establish a cross-store snapshot. - pub async fn import_legacy_run_history_into( - &self, - pool: &SqlitePool, - ) -> Result { - self.import_legacy_run_history_with_controls(pool, &ImportControls::default()) - .await - } - - async fn import_legacy_run_history_with_controls( - &self, - pool: &SqlitePool, - controls: &ImportControls, - ) -> Result { - let mut report = LegacyRunHistoryImportReport::default(); - let result = self - .run_legacy_run_history_import(pool, controls, &mut report) - .await; - debug_import_outcome(result.as_ref().map_or("failed", |()| "complete"), &report); - match result { - Ok(()) => Ok(report), - Err(failure) => Err(LegacyRunHistoryImportError { report, failure }), - } - } - - async fn run_legacy_run_history_import( - &self, - pool: &SqlitePool, - controls: &ImportControls, - report: &mut LegacyRunHistoryImportReport, - ) -> Result<(), LegacyRunHistoryImportFailure> { - let activated = legacy_run_history_is_activated(pool) - .await - .map_err(LegacyRunHistoryImportFailure::ActivationState)?; - let tombstones = legacy_run_history_tombstones(pool) - .await - .map_err(LegacyRunHistoryImportFailure::DeletionState)?; - if tombstoned_destination_present(pool) - .await - .map_err(LegacyRunHistoryImportFailure::DeletionState)? - { - return Err(LegacyRunHistoryImportFailure::TombstonedDestinationPresent); - } - if !activated { - discard_projection_only_rows(pool, report).await?; - } - let mut source = LegacyRunHistorySource::open(self) - .await - .map_err(LegacyRunHistoryImportFailure::Source)?; - - loop { - let history = match source.next_run(Some(controls)).await { - Ok(history) => history, - Err(error) => { - report.scanned_source_events = source.observed_events; - return Err(LegacyRunHistoryImportFailure::Source(error)); - } - }; - report.scanned_source_events = source.observed_events; - let Some(history) = history else { - break; - }; - import_checked_add(&mut report.scanned_source_runs, 1)?; - if tombstones.contains(&history.run_id.to_string()) { - import_checked_add(&mut report.tombstoned_source_runs, 1)?; - import_checked_add( - &mut report.tombstoned_source_events, - usize_to_import_count(history.events.len())?, - )?; - continue; - } - import_one_run(pool, controls, history, activated, report).await?; - } - - report.diagnostics = self - .legacy_run_history_diagnostics() - .await - .map_err(LegacyRunHistoryImportFailure::Diagnostics)?; - Ok(()) - } - - /// Verifies every legacy history as an exact SQLite prefix and then - /// independently replays and verifies every SQLite run. - /// - /// The caller must prevent writes to both stores for the duration of - /// verification. This operation does not establish a cross-store snapshot. - pub async fn verify_legacy_run_history_in( - &self, - pool: &SqlitePool, - ) -> Result { - let mut report = LegacyRunHistoryVerificationReport::default(); - let result = self - .run_legacy_run_history_verification(pool, &mut report) - .await; - debug_verification_outcome(result.as_ref().map_or("failed", |()| "complete"), &report); - match result { - Ok(()) => Ok(report), - Err(failure) => Err(LegacyRunHistoryVerificationError { report, failure }), - } - } - - async fn run_legacy_run_history_verification( - &self, - pool: &SqlitePool, - report: &mut LegacyRunHistoryVerificationReport, - ) -> Result<(), LegacyRunHistoryVerificationFailure> { - let tombstones = legacy_run_history_tombstones(pool) - .await - .map_err(LegacyRunHistoryVerificationFailure::DeletionState)?; - if tombstoned_destination_present(pool) - .await - .map_err(LegacyRunHistoryVerificationFailure::DeletionState)? - { - return Err(LegacyRunHistoryVerificationFailure::TombstonedDestinationPresent); - } - let mut source_ids = HashSet::new(); - let mut source = LegacyRunHistorySource::open(self) - .await - .map_err(LegacyRunHistoryVerificationFailure::Source)?; - loop { - let history = match source.next_run(None).await { - Ok(history) => history, - Err(error) => { - report.source_events = source.observed_events; - return Err(LegacyRunHistoryVerificationFailure::Source(error)); - } - }; - report.source_events = source.observed_events; - let Some(history) = history else { - break; - }; - verification_checked_add(&mut report.source_runs, 1)?; - source_ids.insert(history.run_id); - if tombstones.contains(&history.run_id.to_string()) { - verification_checked_add(&mut report.tombstoned_source_runs, 1)?; - verification_checked_add( - &mut report.tombstoned_source_events, - usize_to_verification_count(history.events.len())?, - )?; - continue; - } - verify_source_prefix(pool, &history).await?; - verification_checked_add(&mut report.matched_prefix_runs, 1)?; - verification_checked_add( - &mut report.matched_prefix_events, - usize_to_verification_count(history.events.len())?, - )?; - } - - let stored_ids: Vec = sqlx::query_scalar("SELECT id FROM runs ORDER BY id ASC") - .fetch_all(pool) - .await - .map_err(LegacyRunHistoryVerificationFailure::ListDestinationRuns)?; - for stored_id in stored_ids { - let run_id = stored_id - .parse::() - .map_err(|_| LegacyRunHistoryVerificationFailure::InvalidDestinationRunId)?; - let mut connection = pool - .acquire() - .await - .map_err(LegacyRunHistoryVerificationFailure::AcquireConnection)?; - let events = - RunSummaryStore::list_events_with_json_on_connection(&mut connection, &run_id) - .await - .map_err(LegacyRunHistoryVerificationFailure::ReadDestination)?; - if events.is_empty() { - return Err(LegacyRunHistoryVerificationFailure::EmptyDestinationHistory); - } - let current = replay_destination(&run_id, &events) - .map_err(LegacyRunHistoryVerificationFailure::ReplayDestination)?; - RunSummaryStore::verify_current_run_on_connection(&mut connection, ¤t) - .await - .map_err(LegacyRunHistoryVerificationFailure::VerifyDestination)?; - - let event_count = usize_to_verification_count(events.len())?; - verification_checked_add(&mut report.target_runs, 1)?; - verification_checked_add(&mut report.target_events, event_count)?; - if !source_ids.contains(&run_id) { - verification_checked_add(&mut report.sql_only_runs, 1)?; - verification_checked_add(&mut report.sql_only_events, event_count)?; - } - } - - report.diagnostics = self - .legacy_run_history_diagnostics() - .await - .map_err(LegacyRunHistoryVerificationFailure::Diagnostics)?; - Ok(()) - } - - async fn legacy_run_history_diagnostics( - &self, - ) -> Result { - let source = self - .open_db() - .await - .map_err(LegacyRunHistoryDiagnosticsFailure::OpenSource)?; - let mut catalog_ids = Vec::new(); - let mut catalog = source - .scan_prefix(keys::run_catalog_prefix()) - .await - .map_err(LegacyRunHistoryDiagnosticsFailure::ReadCatalog)?; - while let Some(entry) = catalog - .next() - .await - .map_err(LegacyRunHistoryDiagnosticsFailure::ReadCatalog)? - { - let key = std::str::from_utf8(&entry.key) - .map_err(LegacyRunHistoryDiagnosticsFailure::CatalogKeyUtf8)?; - catalog_ids.push( - keys::parse_run_catalog_key(key) - .ok_or(LegacyRunHistoryDiagnosticsFailure::InvalidCatalogKey)?, - ); - } - let mut diagnostics = LegacyRunHistoryDiagnostics { - catalog_markers: u64::try_from(catalog_ids.len()) - .map_err(|_| LegacyRunHistoryDiagnosticsFailure::CounterOverflow)?, - ..LegacyRunHistoryDiagnostics::default() - }; - for run_id in catalog_ids { - let mut events = source - .scan_prefix(keys::run_events_prefix(&run_id)) - .await - .map_err(LegacyRunHistoryDiagnosticsFailure::OpenEventProbe)?; - if events - .next() - .await - .map_err(LegacyRunHistoryDiagnosticsFailure::ReadEventProbe)? - .is_none() - { - diagnostics.empty_catalog_markers = diagnostics - .empty_catalog_markers - .checked_add(1) - .ok_or(LegacyRunHistoryDiagnosticsFailure::CounterOverflow)?; - } - } - let mut sessions = source - .scan_prefix(keys::sessions_by_id_prefix()) - .await - .map_err(LegacyRunHistoryDiagnosticsFailure::OpenSessionScan)?; - while sessions - .next() - .await - .map_err(LegacyRunHistoryDiagnosticsFailure::ReadSessionScan)? - .is_some() - { - diagnostics.session_reverse_rows = diagnostics - .session_reverse_rows - .checked_add(1) - .ok_or(LegacyRunHistoryDiagnosticsFailure::CounterOverflow)?; - } - Ok(diagnostics) - } -} - -fn parse_source_event( - key: &[u8], - value: &[u8], -) -> Result, LegacyRunHistorySourceFailure> { - let key_text = std::str::from_utf8(key).map_err(LegacyRunHistorySourceFailure::KeyUtf8)?; - let segments = SlateKey::segments(key_text).collect::>(); - if segments.get(2).copied() != Some("events") { - return Ok(None); - } - let ["runs", run_id_text, "events", leaf] = segments.as_slice() else { - return Err(LegacyRunHistorySourceFailure::InvalidKey); - }; - let run_id = run_id_text - .parse::() - .map_err(|_| LegacyRunHistorySourceFailure::InvalidKey)?; - let Some((sequence_text, epoch_ms_text)) = leaf.split_once('-') else { - return Err(LegacyRunHistorySourceFailure::InvalidKey); - }; - if sequence_text.len() != 6 || !sequence_text.bytes().all(|byte| byte.is_ascii_digit()) { - return Err(LegacyRunHistorySourceFailure::InvalidKey); - } - let sequence = sequence_text - .parse::() - .ok() - .filter(|sequence| (1..=keys::MAX_EVENT_SEQ).contains(sequence)) - .ok_or(LegacyRunHistorySourceFailure::InvalidKey)?; - let epoch_ms = epoch_ms_text - .parse::() - .map_err(|_| LegacyRunHistorySourceFailure::InvalidKey)?; - if keys::run_event_key(&run_id, sequence, epoch_ms).as_ref() != key { - return Err(LegacyRunHistorySourceFailure::InvalidKey); - } - - let event_json = std::str::from_utf8(value) - .map_err(LegacyRunHistorySourceFailure::ValueUtf8)? - .to_owned(); - let payload: EventPayload = - serde_json::from_str(&event_json).map_err(LegacyRunHistorySourceFailure::DecodeEvent)?; - payload - .validate(&run_id) - .map_err(LegacyRunHistorySourceFailure::ValidateEvent)?; - let event = - RunEvent::try_from(&payload).map_err(LegacyRunHistorySourceFailure::ValidateEvent)?; - if event.run_id != run_id { - return Err(LegacyRunHistorySourceFailure::ValidateEvent( - crate::Error::RunEventMismatch { - run_id: run_id.to_string(), - seq: sequence, - field: "run_id", - }, - )); - } - Ok(Some(ValidatedLegacyRunEvent { - run_id, - payload, - envelope: EventEnvelope { - seq: sequence, - event, - }, - event_json, - raw_key: key.to_vec(), - })) -} - -fn hash_source_part( - hasher: &mut Sha256, - bytes: &[u8], -) -> Result<(), LegacyRunHistorySourceFailure> { - let length = - u64::try_from(bytes.len()).map_err(|_| LegacyRunHistorySourceFailure::CounterOverflow)?; - hasher.update(length.to_be_bytes()); - hasher.update(bytes); - Ok(()) -} - -fn event_run_segment(key: &[u8]) -> Option<&[u8]> { - let mut segments = key.split(|byte| *byte == 0); - (segments.next()? == b"runs").then_some(())?; - let run_id = segments.next()?; - (segments.next()? == b"events").then_some(run_id) -} - -async fn legacy_run_history_is_activated(pool: &SqlitePool) -> Result { - sqlx::query_scalar( - "SELECT EXISTS(SELECT 1 FROM legacy_run_history_activation WHERE singleton = 1)", - ) - .fetch_one(pool) - .await -} - -async fn legacy_run_history_tombstones(pool: &SqlitePool) -> Result, sqlx::Error> { - Ok( - sqlx::query_scalar("SELECT run_id FROM legacy_run_history_deletions") - .fetch_all(pool) - .await? - .into_iter() - .collect(), - ) -} - -/// Returns whether any tombstoned run still has canonical SQLite rows. -async fn tombstoned_destination_present(pool: &SqlitePool) -> Result { - sqlx::query_scalar( - r" -SELECT EXISTS( - SELECT 1 FROM legacy_run_history_deletions AS deletion - WHERE EXISTS(SELECT 1 FROM runs WHERE id = deletion.run_id) - OR EXISTS(SELECT 1 FROM run_events WHERE run_id = deletion.run_id) -) -", - ) - .fetch_one(pool) - .await -} - -async fn discard_projection_only_rows( - pool: &SqlitePool, - report: &mut LegacyRunHistoryImportReport, -) -> Result<(), LegacyRunHistoryImportFailure> { - let mut transaction = pool - .begin() - .await - .map_err(LegacyRunHistoryImportFailure::BeginCleanup)?; - let result = sqlx::query( - r" -DELETE FROM runs -WHERE NOT EXISTS ( - SELECT 1 FROM run_events WHERE run_events.run_id = runs.id -) -", - ) - .execute(&mut *transaction) - .await - .map_err(LegacyRunHistoryImportFailure::DeleteProjectionOnlyRows)?; - let discarded = result.rows_affected(); - transaction - .commit() - .await - .map_err(LegacyRunHistoryImportFailure::CommitCleanup)?; - report.discarded_projection_only_rows = discarded; - Ok(()) -} - -async fn import_one_run( - pool: &SqlitePool, - controls: &ImportControls, - history: ValidatedLegacyRunHistory, - activated: bool, - report: &mut LegacyRunHistoryImportReport, -) -> Result<(), LegacyRunHistoryImportFailure> { - #[cfg(not(test))] - let _ = controls; - let mut transaction = pool - .begin() - .await - .map_err(LegacyRunHistoryImportFailure::BeginRunTransaction)?; - let result = async { - let has_destination: bool = - sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM run_events WHERE run_id = ?)") - .bind(history.run_id.to_string()) - .fetch_one(&mut *transaction) - .await - .map_err(|source| { - LegacyRunHistoryImportFailure::ReadDestination(crate::Error::Sqlite(source)) - })?; - let mut updated = *report; - if has_destination { - let destination = RunSummaryStore::list_events_with_json_in_transaction( - &mut transaction, - &history.run_id, - ) - .await - .map_err(LegacyRunHistoryImportFailure::ReadDestination)?; - require_exact_prefix(&history, &destination) - .map_err(|()| LegacyRunHistoryImportFailure::DestinationConflict)?; - let current = replay_destination(&history.run_id, &destination) - .map_err(LegacyRunHistoryImportFailure::ReplayDestination)?; - RunSummaryStore::verify_current_run_on_connection(&mut transaction, ¤t) - .await - .map_err(LegacyRunHistoryImportFailure::VerifyDestination)?; - import_checked_add(&mut updated.verified_existing_runs, 1)?; - import_checked_add( - &mut updated.verified_existing_events, - usize_to_import_count(destination.len())?, - )?; - } else if activated { - return Err(LegacyRunHistoryImportFailure::MissingDestinationAfterActivation); - } else { - RunSummaryStore::insert_imported_run_on_connection(&mut transaction, &history.current) - .await - .map_err(LegacyRunHistoryImportFailure::InsertRun)?; - #[cfg(test)] - controls.after_run_inserted().await; - for event in &history.events { - RunSummaryStore::insert_imported_event_on_connection( - &mut transaction, - &history.run_id, - &event.payload, - &event.envelope, - &event.event_json, - ) - .await - .map_err(LegacyRunHistoryImportFailure::InsertEvent)?; - } - import_checked_add(&mut updated.imported_runs, 1)?; - import_checked_add( - &mut updated.imported_events, - usize_to_import_count(history.events.len())?, - )?; - } - import_checked_add(&mut updated.committed_run_transactions, 1)?; - Ok(updated) - } - .await; - - match result { - Ok(updated) => { - transaction - .commit() - .await - .map_err(LegacyRunHistoryImportFailure::CommitRunTransaction)?; - *report = updated; - Ok(()) - } - Err(prior) => match transaction.rollback().await { - Ok(()) => Err(prior), - Err(source) => Err(LegacyRunHistoryImportFailure::RollbackRunTransaction { - source, - prior: Box::new(prior), - }), - }, - } -} - -async fn verify_source_prefix( - pool: &SqlitePool, - history: &ValidatedLegacyRunHistory, -) -> Result<(), LegacyRunHistoryVerificationFailure> { - let mut connection = pool - .acquire() - .await - .map_err(LegacyRunHistoryVerificationFailure::AcquireConnection)?; - let has_destination: bool = - sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM run_events WHERE run_id = ?)") - .bind(history.run_id.to_string()) - .fetch_one(&mut *connection) - .await - .map_err(|source| { - LegacyRunHistoryVerificationFailure::ReadDestination(crate::Error::Sqlite(source)) - })?; - if !has_destination { - return Err(LegacyRunHistoryVerificationFailure::MissingDestinationPrefix); - } - let destination = - RunSummaryStore::list_events_with_json_on_connection(&mut connection, &history.run_id) - .await - .map_err(LegacyRunHistoryVerificationFailure::ReadDestination)?; - if destination.len() < history.events.len() { - return Err(LegacyRunHistoryVerificationFailure::MissingDestinationPrefix); - } - require_exact_prefix(history, &destination) - .map_err(|()| LegacyRunHistoryVerificationFailure::DestinationPrefixConflict) -} - -fn require_exact_prefix( - history: &ValidatedLegacyRunHistory, - destination: &[(EventEnvelope, String)], -) -> Result<(), ()> { - if destination.len() < history.events.len() { - return Err(()); - } - if history - .events - .iter() - .zip(destination) - .any(|(source, (target, target_json))| { - source.envelope.seq != target.seq || source.event_json != *target_json - }) - { - return Err(()); - } - Ok(()) -} - -fn replay_destination( - run_id: &RunId, - events: &[(EventEnvelope, String)], -) -> crate::Result { - if let Some((first, _event_json)) = events.first() { - if first.seq != 1 { - return Err(crate::Error::RunEventMismatch { - run_id: run_id.to_string(), - seq: first.seq, - field: "seq", - }); - } - } - let envelopes = events - .iter() - .map(|(envelope, _event_json)| envelope.clone()) - .collect::>(); - ProjectedRun::replay(*run_id, &envelopes) -} - -fn usize_to_import_count(value: usize) -> Result { - u64::try_from(value).map_err(|_| LegacyRunHistoryImportFailure::CounterOverflow) -} - -fn import_checked_add(value: &mut u64, amount: u64) -> Result<(), LegacyRunHistoryImportFailure> { - *value = value - .checked_add(amount) - .ok_or(LegacyRunHistoryImportFailure::CounterOverflow)?; - Ok(()) -} - -fn usize_to_verification_count(value: usize) -> Result { - u64::try_from(value).map_err(|_| LegacyRunHistoryVerificationFailure::CounterOverflow) -} - -fn verification_checked_add( - value: &mut u64, - amount: u64, -) -> Result<(), LegacyRunHistoryVerificationFailure> { - *value = value - .checked_add(amount) - .ok_or(LegacyRunHistoryVerificationFailure::CounterOverflow)?; - Ok(()) -} - -fn debug_import_outcome(outcome: &'static str, report: &LegacyRunHistoryImportReport) { - debug!( - outcome, - scanned_source_runs = report.scanned_source_runs, - scanned_source_events = report.scanned_source_events, - imported_runs = report.imported_runs, - imported_events = report.imported_events, - verified_existing_runs = report.verified_existing_runs, - verified_existing_events = report.verified_existing_events, - discarded_projection_only_rows = report.discarded_projection_only_rows, - committed_run_transactions = report.committed_run_transactions, - tombstoned_source_runs = report.tombstoned_source_runs, - tombstoned_source_events = report.tombstoned_source_events, - catalog_markers = report.diagnostics.catalog_markers, - empty_catalog_markers = report.diagnostics.empty_catalog_markers, - session_reverse_rows = report.diagnostics.session_reverse_rows, - "Legacy run-history import finished" - ); -} - -fn debug_verification_outcome(outcome: &'static str, report: &LegacyRunHistoryVerificationReport) { - debug!( - outcome, - source_runs = report.source_runs, - source_events = report.source_events, - matched_prefix_runs = report.matched_prefix_runs, - matched_prefix_events = report.matched_prefix_events, - target_runs = report.target_runs, - target_events = report.target_events, - sql_only_runs = report.sql_only_runs, - sql_only_events = report.sql_only_events, - tombstoned_source_runs = report.tombstoned_source_runs, - tombstoned_source_events = report.tombstoned_source_events, - catalog_markers = report.diagnostics.catalog_markers, - empty_catalog_markers = report.diagnostics.empty_catalog_markers, - session_reverse_rows = report.diagnostics.session_reverse_rows, - "Legacy run-history verification finished" - ); -} - -#[cfg(test)] -mod tests { - use std::error::Error as _; - use std::sync::Arc; - use std::time::Duration; - - use chrono::{TimeZone as _, Utc}; - use fabro_types::{Graph, RunEvent, RunId, SessionId, WorkflowSettings, test_support}; - use fabro_util::error; - use object_store::memory::InMemory; - use tokio::sync::Barrier; - use ulid::Ulid; - - use super::{ - ImportControls, LegacyRunHistoryDiagnostics, LegacyRunHistoryImportError, - LegacyRunHistoryImportFailure, LegacyRunHistoryImportReport, - LegacyRunHistoryVerificationFailure, LegacyRunHistoryVerificationReport, - parse_source_event, - }; - use crate::keys::SlateKey; - use crate::run_state::ProjectedRun; - use crate::{ - Database, EventEnvelope, EventPayload, RunSummaryStore, keys, - test_support as store_test_support, - }; - - type TestResult = std::result::Result>; - - struct TestContext { - _directory: tempfile::TempDir, - source: Database, - source_db: slatedb::Db, - sqlite: sqlx::SqlitePool, - } - - impl TestContext { - async fn new() -> TestResult { - let directory = tempfile::tempdir()?; - let sqlite = - fabro_db::Database::connect(directory.path().join("fabro.sqlite3")).await?; - sqlite.migrate().await?; - let sqlite = sqlite.clone_pool(); - let source = Database::new( - Arc::new(InMemory::new()), - "legacy-run-history-import-tests", - Duration::from_millis(1), - None, - store_test_support::test_blob_store(), - store_test_support::test_run_summary_store(), - ); - let source_db = source.open_db().await?; - Ok(Self { - _directory: directory, - source, - source_db, - sqlite, - }) - } - - async fn put_event( - &self, - run_id: &RunId, - seq: u32, - epoch_ms: i64, - event_json: &str, - ) -> TestResult<()> { - self.source_db - .put( - keys::run_event_key(run_id, seq, epoch_ms), - event_json.as_bytes(), - ) - .await?; - Ok(()) - } - - async fn put_raw(&self, key: impl AsRef<[u8]>, value: &[u8]) -> TestResult<()> { - self.source_db.put(key, value).await?; - Ok(()) - } - - async fn source_entries(&self) -> TestResult, Vec)>> { - let mut entries = self.source_db.scan_prefix(Vec::::new()).await?; - let mut snapshot = Vec::new(); - while let Some(entry) = entries.next().await? { - snapshot.push((entry.key.to_vec(), entry.value.to_vec())); - } - Ok(snapshot) - } - - async fn import(&self) -> TestResult { - Ok(self - .source - .import_legacy_run_history_into(&self.sqlite) - .await?) - } - } - - fn run_id(index: u128) -> RunId { - RunId::from(Ulid::from_parts( - 1_788_000_000_000 + u64::try_from(index).unwrap(), - index, - )) - } - - fn event_value( - run_id: &RunId, - seq: u32, - event: &str, - properties: &serde_json::Value, - ) -> serde_json::Value { - serde_json::json!({ - "id": format!("evt-{seq}-{event}"), - "ts": Utc - .timestamp_millis_opt(1_788_000_000_000 + i64::from(seq)) - .single() - .unwrap() - .to_rfc3339(), - "run_id": run_id.to_string(), - "event": event, - "properties": properties, - }) - } - - fn created_value(run_id: &RunId, title: &str) -> serde_json::Value { - event_value( - run_id, - 1, - "run.created", - &serde_json::json!({ - "title": title, - "settings": WorkflowSettings::default(), - "graph": Graph::new("test"), - "workflow_slug": "test-workflow", - "labels": {}, - "provenance": test_support::test_run_provenance(), - }), - ) - } - - fn submitted_value(run_id: &RunId, seq: u32) -> serde_json::Value { - event_value(run_id, seq, "run.submitted", &serde_json::json!({})) - } - - fn session_created_value( - run_id: &RunId, - seq: u32, - session_id: &SessionId, - ) -> serde_json::Value { - let mut value = event_value( - run_id, - seq, - "run.session.created", - &serde_json::json!({ "title": "Imported session" }), - ); - value - .as_object_mut() - .unwrap() - .insert("session_id".to_string(), session_id.to_string().into()); - value - } - - #[tokio::test] - async fn legacy_source_identity_covers_exact_keys_and_json_bytes() -> TestResult<()> { - let context = TestContext::new().await?; - let run_id = run_id(0); - let value = created_value(&run_id, "identity"); - let compact = serde_json::to_string(&value)?; - context.put_event(&run_id, 1, 1, &compact).await?; - let compact_identity = context.source.legacy_run_history_source_identity().await?; - - let pretty = serde_json::to_string_pretty(&value)?; - context.put_event(&run_id, 1, 1, &pretty).await?; - let pretty_identity = context.source.legacy_run_history_source_identity().await?; - assert_eq!( - (compact_identity.runs, compact_identity.events), - (pretty_identity.runs, pretty_identity.events) - ); - assert_ne!( - compact_identity.fingerprint(), - pretty_identity.fingerprint(), - "semantically equivalent JSON bytes must still change the source identity" - ); - - context - .source_db - .delete(keys::run_event_key(&run_id, 1, 1)) - .await?; - context.put_event(&run_id, 1, 2, &pretty).await?; - let moved_key_identity = context.source.legacy_run_history_source_identity().await?; - assert_eq!( - (pretty_identity.runs, pretty_identity.events), - (moved_key_identity.runs, moved_key_identity.events) - ); - assert_ne!( - pretty_identity.fingerprint(), - moved_key_identity.fingerprint(), - "changing only the raw legacy key must change the source identity" - ); - Ok(()) - } - - fn decode_event( - run_id: &RunId, - seq: u32, - event_json: &str, - ) -> TestResult<(EventPayload, EventEnvelope)> { - let payload: EventPayload = serde_json::from_str(event_json)?; - payload.validate(run_id)?; - let event = RunEvent::try_from(&payload)?; - Ok((payload, EventEnvelope { seq, event })) - } - - async fn seed_destination_history( - pool: &sqlx::SqlitePool, - run_id: &RunId, - events: &[(u32, String)], - ) -> TestResult<()> { - let decoded = events - .iter() - .map(|(seq, event_json)| decode_event(run_id, *seq, event_json)) - .collect::>>()?; - let envelopes = decoded - .iter() - .map(|(_payload, envelope)| envelope.clone()) - .collect::>(); - let current = ProjectedRun::replay(*run_id, &envelopes)?; - let mut transaction = pool.begin().await?; - RunSummaryStore::insert_imported_run_on_connection(&mut transaction, ¤t).await?; - for ((_, event_json), (payload, envelope)) in events.iter().zip(&decoded) { - RunSummaryStore::insert_imported_event_on_connection( - &mut transaction, - run_id, - payload, - envelope, - event_json, - ) - .await?; - } - transaction.commit().await?; - Ok(()) - } - - async fn append_destination_event( - pool: &sqlx::SqlitePool, - run_id: &RunId, - prior: &[(u32, String)], - next: (u32, String), - ) -> TestResult<()> { - let mut all = prior.to_vec(); - all.push(next.clone()); - let decoded = all - .iter() - .map(|(seq, event_json)| decode_event(run_id, *seq, event_json)) - .collect::>>()?; - let envelopes = decoded - .iter() - .map(|(_payload, envelope)| envelope.clone()) - .collect::>(); - let current = ProjectedRun::replay(*run_id, &envelopes)?; - let mut transaction = pool.begin().await?; - RunSummaryStore::append_event_on_connection( - &mut transaction, - prior.last().unwrap().0, - ¤t, - &decoded.last().unwrap().0, - ) - .await?; - transaction.commit().await?; - Ok(()) - } - - #[test] - fn legacy_run_history_import_exposes_rollback_cleanup_errors() { - let error = LegacyRunHistoryImportError { - report: LegacyRunHistoryImportReport::default(), - failure: LegacyRunHistoryImportFailure::RollbackRunTransaction { - source: sqlx::Error::Protocol("injected rollback failure".to_owned()), - prior: Box::new(LegacyRunHistoryImportFailure::DestinationConflict), - }, - }; - - assert_eq!( - error.source().map(ToString::to_string), - Some( - "the destination history is partial or conflicts with the legacy prefix".to_owned() - ) - ); - assert!( - error - .cleanup_errors() - .any(|source| source.downcast_ref::().is_some()), - "rollback source was absent from cleanup errors" - ); - } - - #[tokio::test] - async fn session_owner_imports_typed_event_and_counts_opaque_legacy_reverse_rows() - -> TestResult<()> { - let context = TestContext::new().await?; - let first = run_id(1); - let second = run_id(2); - let empty_marker = run_id(3); - let stale = run_id(4); - let session_id = SessionId::new(); - let first_created = serde_json::to_string_pretty(&created_value(&first, "first"))?; - let first_session = serde_json::to_string(&session_created_value(&first, 3, &session_id))?; - let first_submitted = serde_json::to_string(&submitted_value(&first, 4))?; - let second_created = serde_json::to_string(&created_value(&second, "second"))?; - context.put_event(&first, 1, 10, &first_created).await?; - context.put_event(&first, 3, 30, &first_session).await?; - context.put_event(&first, 4, 40, &first_submitted).await?; - context.put_event(&second, 1, 20, &second_created).await?; - context.put_raw(keys::run_catalog_key(&first), b"").await?; - context - .put_raw(keys::run_catalog_key(&empty_marker), b"") - .await?; - context - .put_raw( - keys::session_by_id_key(&session_id), - b"opaque legacy reverse row", - ) - .await?; - - let stale_created = serde_json::to_string(&created_value(&stale, "stale"))?; - seed_destination_history(&context.sqlite, &stale, &[(1, stale_created)]).await?; - sqlx::query("DELETE FROM run_events WHERE run_id = ?") - .bind(stale.to_string()) - .execute(&context.sqlite) - .await?; - sqlx::query("UPDATE runs SET summary_json = '{}' WHERE id = ?") - .bind(stale.to_string()) - .execute(&context.sqlite) - .await?; - let source_before = context.source_entries().await?; - - let report = context.import().await?; - - assert_eq!(report, LegacyRunHistoryImportReport { - scanned_source_runs: 2, - scanned_source_events: 4, - imported_runs: 2, - imported_events: 4, - discarded_projection_only_rows: 1, - committed_run_transactions: 2, - diagnostics: LegacyRunHistoryDiagnostics { - catalog_markers: 2, - empty_catalog_markers: 1, - session_reverse_rows: 1, - }, - ..LegacyRunHistoryImportReport::default() - }); - let stored: Vec<(i64, String)> = - sqlx::query_as("SELECT seq, event_json FROM run_events WHERE run_id = ? ORDER BY seq") - .bind(first.to_string()) - .fetch_all(&context.sqlite) - .await?; - assert_eq!(stored, vec![ - (1, first_created), - (3, first_session), - (4, first_submitted) - ]); - let summaries = RunSummaryStore::new(context.sqlite.clone()); - assert_eq!( - summaries.find_session_owner(&session_id).await?, - Some(first) - ); - assert_eq!( - sqlx::query_scalar::<_, i64>("SELECT source_last_seq FROM runs WHERE id = ?") - .bind(first.to_string()) - .fetch_one(&context.sqlite) - .await?, - 4 - ); - assert_eq!(context.source_entries().await?, source_before); - let verification = context - .source - .verify_legacy_run_history_in(&context.sqlite) - .await?; - assert_eq!(verification.target_runs, 2); - assert_eq!(verification.target_events, 4); - Ok(()) - } - - #[tokio::test] - async fn legacy_run_history_retry_uses_complete_destination_histories_as_progress() - -> TestResult<()> { - let context = TestContext::new().await?; - let run_id = run_id(10); - context - .put_event( - &run_id, - 1, - 10, - &serde_json::to_string(&created_value(&run_id, "retry"))?, - ) - .await?; - let first = context.import().await?; - let second = context.import().await?; - - assert_eq!(first.imported_runs, 1); - assert_eq!(second, LegacyRunHistoryImportReport { - scanned_source_runs: 1, - scanned_source_events: 1, - verified_existing_runs: 1, - verified_existing_events: 1, - committed_run_transactions: 1, - ..LegacyRunHistoryImportReport::default() - }); - Ok(()) - } - - #[tokio::test] - async fn legacy_run_history_retry_and_verification_compare_summary_json_semantically() - -> TestResult<()> { - let context = TestContext::new().await?; - let run_id = run_id(11); - let mut created = created_value(&run_id, "labeled"); - created["properties"]["labels"] = serde_json::json!({ - "alpha": "one", - "beta": "two", - "gamma": "three", - }); - context - .put_event(&run_id, 1, 10, &serde_json::to_string(&created)?) - .await?; - context.import().await?; - - let compact: String = sqlx::query_scalar("SELECT summary_json FROM runs WHERE id = ?") - .bind(run_id.to_string()) - .fetch_one(&context.sqlite) - .await?; - let reformatted = - serde_json::to_string_pretty(&serde_json::from_str::(&compact)?)?; - assert_ne!(compact, reformatted); - sqlx::query("UPDATE runs SET summary_json = ? WHERE id = ?") - .bind(reformatted) - .bind(run_id.to_string()) - .execute(&context.sqlite) - .await?; - - let retry = context.import().await?; - assert_eq!(retry.verified_existing_runs, 1); - assert_eq!(retry.verified_existing_events, 1); - let verification = context - .source - .verify_legacy_run_history_in(&context.sqlite) - .await?; - assert_eq!(verification.target_runs, 1); - assert_eq!(verification.target_events, 1); - Ok(()) - } - - #[test] - fn legacy_run_history_rejects_noncanonical_event_key_shapes() -> TestResult<()> { - let run_id = run_id(20); - let event_json = serde_json::to_string(&created_value(&run_id, "key"))?; - let invalid = [ - format!("runs\0{run_id}\0events\0000001-1\0extra").into_bytes(), - b"runs\0not-a-run-id\0events\x00000001-1".to_vec(), - format!("runs\0{run_id}\0events\000001-1").into_bytes(), - format!("runs\0{run_id}\0events\0000000-1").into_bytes(), - format!("runs\0{run_id}\0events\01000000-1").into_bytes(), - format!("runs\0{run_id}\0events\0000001-nope").into_bytes(), - format!("runs\0{run_id}\0events\0000001-01").into_bytes(), - ]; - for key in invalid { - assert!(parse_source_event(&key, event_json.as_bytes()).is_err()); - } - let mut invalid_utf8 = b"runs\0".to_vec(); - invalid_utf8.extend_from_slice(&[0xff, 0xfe]); - invalid_utf8.extend_from_slice(b"\0events\x00000001-1"); - assert!(parse_source_event(&invalid_utf8, event_json.as_bytes()).is_err()); - Ok(()) - } - - #[tokio::test] - async fn legacy_run_history_rejects_invalid_values_sequences_and_replay() -> TestResult<()> { - let invalid_utf8 = TestContext::new().await?; - let first = run_id(30); - invalid_utf8 - .put_raw(keys::run_event_key(&first, 1, 1), &[0xff]) - .await?; - assert!( - invalid_utf8 - .source - .import_legacy_run_history_into(&invalid_utf8.sqlite) - .await - .is_err() - ); - - let invalid_json = TestContext::new().await?; - invalid_json.put_event(&first, 1, 1, "{not-json").await?; - assert!( - invalid_json - .source - .import_legacy_run_history_into(&invalid_json.sqlite) - .await - .is_err() - ); - - let missing_first = TestContext::new().await?; - missing_first - .put_event( - &first, - 2, - 2, - &serde_json::to_string(&submitted_value(&first, 2))?, - ) - .await?; - assert!( - missing_first - .source - .import_legacy_run_history_into(&missing_first.sqlite) - .await - .is_err() - ); - - let wrong_first_event = TestContext::new().await?; - wrong_first_event - .put_event( - &first, - 1, - 1, - &serde_json::to_string(&submitted_value(&first, 1))?, - ) - .await?; - assert!( - wrong_first_event - .source - .import_legacy_run_history_into(&wrong_first_event.sqlite) - .await - .is_err() - ); - - let mismatched_payload = TestContext::new().await?; - let other_run = run_id(31); - mismatched_payload - .put_event( - &first, - 1, - 1, - &serde_json::to_string(&created_value(&other_run, "mismatch"))?, - ) - .await?; - assert!( - mismatched_payload - .source - .import_legacy_run_history_into(&mismatched_payload.sqlite) - .await - .is_err() - ); - - let duplicate = TestContext::new().await?; - let created = serde_json::to_string(&created_value(&first, "duplicate"))?; - duplicate.put_event(&first, 1, 1, &created).await?; - duplicate.put_event(&first, 1, 2, &created).await?; - assert!( - duplicate - .source - .import_legacy_run_history_into(&duplicate.sqlite) - .await - .is_err() - ); - - let unreplayable = TestContext::new().await?; - unreplayable.put_event(&first, 1, 1, &created).await?; - unreplayable.put_event(&first, 2, 2, &created).await?; - assert!( - unreplayable - .source - .import_legacy_run_history_into(&unreplayable.sqlite) - .await - .is_err() - ); - Ok(()) - } - - #[tokio::test] - async fn legacy_run_history_rolls_back_a_run_when_event_insertion_fails() -> TestResult<()> { - let context = TestContext::new().await?; - let run_id = run_id(40); - context - .put_event( - &run_id, - 1, - 1, - &serde_json::to_string(&created_value(&run_id, "rollback"))?, - ) - .await?; - sqlx::query( - "CREATE TRIGGER reject_imported_event BEFORE INSERT ON run_events BEGIN SELECT RAISE(FAIL, 'injected'); END", - ) - .execute(&context.sqlite) - .await?; - - let error = context - .source - .import_legacy_run_history_into(&context.sqlite) - .await - .unwrap_err(); - - assert_eq!(error.report().imported_runs, 0); - assert_eq!( - sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM runs") - .fetch_one(&context.sqlite) - .await?, - 0 - ); - assert_eq!( - sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM run_events") - .fetch_one(&context.sqlite) - .await?, - 0 - ); - Ok(()) - } - - #[tokio::test] - async fn legacy_run_history_interruption_preserves_complete_runs_and_retry_converges() - -> TestResult<()> { - let context = TestContext::new().await?; - let first = run_id(50); - let second = run_id(51); - for run_id in [first, second] { - context - .put_event( - &run_id, - 1, - 1, - &serde_json::to_string(&created_value(&run_id, "interrupted"))?, - ) - .await?; - } - let controls = ImportControls { - source_after_events: Some(2), - ..ImportControls::default() - }; - - let error = context - .source - .import_legacy_run_history_with_controls(&context.sqlite, &controls) - .await - .unwrap_err(); - assert_eq!(error.report().imported_runs, 1); - assert_eq!(error.report().committed_run_transactions, 1); - assert_eq!( - sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM runs") - .fetch_one(&context.sqlite) - .await?, - 1 - ); - - let retry = context.import().await?; - assert_eq!(retry.imported_runs, 1); - assert_eq!(retry.verified_existing_runs, 1); - assert_eq!( - sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM runs") - .fetch_one(&context.sqlite) - .await?, - 2 - ); - Ok(()) - } - - #[tokio::test] - async fn legacy_run_history_cancellation_leaves_no_half_imported_run() -> TestResult<()> { - let context = TestContext::new().await?; - let run_id = run_id(60); - context - .put_event( - &run_id, - 1, - 1, - &serde_json::to_string(&created_value(&run_id, "cancel"))?, - ) - .await?; - let barrier = Arc::new(Barrier::new(2)); - let source = context.source.clone(); - let sqlite = context.sqlite.clone(); - let task_barrier = Arc::clone(&barrier); - let task = tokio::spawn(async move { - let controls = ImportControls { - after_run_inserted: Some(task_barrier), - ..ImportControls::default() - }; - source - .import_legacy_run_history_with_controls(&sqlite, &controls) - .await - }); - barrier.wait().await; - task.abort(); - assert!(task.await.unwrap_err().is_cancelled()); - - assert_eq!( - sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM runs") - .fetch_one(&context.sqlite) - .await?, - 0 - ); - assert_eq!(context.import().await?.imported_runs, 1); - Ok(()) - } - - #[tokio::test] - async fn legacy_run_history_rejects_partial_conflicting_and_corrupt_destinations() - -> TestResult<()> { - let run_id = run_id(70); - let source_created = serde_json::to_string(&created_value(&run_id, "source"))?; - let source_submitted = serde_json::to_string(&submitted_value(&run_id, 2))?; - - let partial = TestContext::new().await?; - partial.put_event(&run_id, 1, 1, &source_created).await?; - partial.put_event(&run_id, 2, 2, &source_submitted).await?; - seed_destination_history(&partial.sqlite, &run_id, &[(1, source_created.clone())]).await?; - assert!( - partial - .source - .import_legacy_run_history_into(&partial.sqlite) - .await - .is_err() - ); - assert_eq!( - sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM run_events") - .fetch_one(&partial.sqlite) - .await?, - 1 - ); - - let conflicting = TestContext::new().await?; - conflicting - .put_event(&run_id, 1, 1, &source_created) - .await?; - let target_created = serde_json::to_string(&created_value(&run_id, "target"))?; - seed_destination_history(&conflicting.sqlite, &run_id, &[(1, target_created)]).await?; - assert!( - conflicting - .source - .import_legacy_run_history_into(&conflicting.sqlite) - .await - .is_err() - ); - - let corrupt_event = TestContext::new().await?; - corrupt_event - .put_event(&run_id, 1, 1, &source_created) - .await?; - seed_destination_history(&corrupt_event.sqlite, &run_id, &[( - 1, - source_created.clone(), - )]) - .await?; - sqlx::query("UPDATE run_events SET event_name = 'run.failed' WHERE run_id = ?") - .bind(run_id.to_string()) - .execute(&corrupt_event.sqlite) - .await?; - assert!( - corrupt_event - .source - .import_legacy_run_history_into(&corrupt_event.sqlite) - .await - .is_err() - ); - - let corrupt_summary = TestContext::new().await?; - corrupt_summary - .put_event(&run_id, 1, 1, &source_created) - .await?; - seed_destination_history(&corrupt_summary.sqlite, &run_id, &[( - 1, - source_created.clone(), - )]) - .await?; - sqlx::query("UPDATE runs SET title = 'corrupt' WHERE id = ?") - .bind(run_id.to_string()) - .execute(&corrupt_summary.sqlite) - .await?; - assert!( - corrupt_summary - .source - .import_legacy_run_history_into(&corrupt_summary.sqlite) - .await - .is_err() - ); - - let corrupt_head = TestContext::new().await?; - corrupt_head - .put_event(&run_id, 1, 1, &source_created) - .await?; - seed_destination_history(&corrupt_head.sqlite, &run_id, &[(1, source_created)]).await?; - sqlx::query("UPDATE runs SET source_last_seq = 2 WHERE id = ?") - .bind(run_id.to_string()) - .execute(&corrupt_head.sqlite) - .await?; - assert!( - corrupt_head - .source - .import_legacy_run_history_into(&corrupt_head.sqlite) - .await - .is_err() - ); - Ok(()) - } - - #[tokio::test] - async fn legacy_run_history_verification_accepts_sql_suffixes_and_sql_only_runs() - -> TestResult<()> { - let context = TestContext::new().await?; - let source_run = run_id(80); - let sql_only_run = run_id(81); - let source_created = serde_json::to_string(&created_value(&source_run, "source"))?; - context - .put_event(&source_run, 1, 1, &source_created) - .await?; - context.import().await?; - let suffix = serde_json::to_string(&submitted_value(&source_run, 2))?; - append_destination_event( - &context.sqlite, - &source_run, - &[(1, source_created)], - (2, suffix), - ) - .await?; - let sql_only_created = serde_json::to_string(&created_value(&sql_only_run, "sql-only"))?; - seed_destination_history(&context.sqlite, &sql_only_run, &[(1, sql_only_created)]).await?; - - let report = context - .source - .verify_legacy_run_history_in(&context.sqlite) - .await?; - - assert_eq!(report, LegacyRunHistoryVerificationReport { - source_runs: 1, - source_events: 1, - matched_prefix_runs: 1, - matched_prefix_events: 1, - target_runs: 2, - target_events: 3, - sql_only_runs: 1, - sql_only_events: 1, - ..LegacyRunHistoryVerificationReport::default() - }); - Ok(()) - } - - #[tokio::test] - async fn legacy_run_history_verification_rejects_invalid_sql_only_histories() -> TestResult<()> - { - let missing_first = TestContext::new().await?; - let missing_first_id = run_id(82); - let created = serde_json::to_string(&created_value(&missing_first_id, "missing-first"))?; - seed_destination_history(&missing_first.sqlite, &missing_first_id, &[(2, created)]).await?; - let error = missing_first - .source - .verify_legacy_run_history_in(&missing_first.sqlite) - .await - .unwrap_err(); - assert!(matches!( - error.failure, - LegacyRunHistoryVerificationFailure::ReplayDestination( - crate::Error::RunEventMismatch { field: "seq", .. } - ) - )); - - let noncanonical = TestContext::new().await?; - let noncanonical_id = run_id(83); - let canonical_json = - serde_json::to_string(&created_value(&noncanonical_id, "noncanonical"))?; - seed_destination_history(&noncanonical.sqlite, &noncanonical_id, &[( - 1, - canonical_json.clone(), - )]) - .await?; - let canonical_id = noncanonical_id.to_string(); - let lowercase_id = canonical_id.to_lowercase(); - assert_ne!(canonical_id, lowercase_id); - sqlx::query("UPDATE run_events SET event_json = ? WHERE run_id = ?") - .bind(canonical_json.replace(&canonical_id, &lowercase_id)) - .bind(canonical_id) - .execute(&noncanonical.sqlite) - .await?; - let error = noncanonical - .source - .verify_legacy_run_history_in(&noncanonical.sqlite) - .await - .unwrap_err(); - assert!(matches!( - error.failure, - LegacyRunHistoryVerificationFailure::ReadDestination(crate::Error::RunEventMismatch { - field: "run_id", - .. - }) - )); - Ok(()) - } - - #[tokio::test] - async fn legacy_run_history_verification_fails_on_prefix_and_current_row_corruption() - -> TestResult<()> { - let run_id = run_id(90); - let created = serde_json::to_string(&created_value(&run_id, "verify"))?; - - let changed_prefix = TestContext::new().await?; - changed_prefix.put_event(&run_id, 1, 1, &created).await?; - let changed = serde_json::to_string(&created_value(&run_id, "changed"))?; - seed_destination_history(&changed_prefix.sqlite, &run_id, &[(1, changed)]).await?; - assert!( - changed_prefix - .source - .verify_legacy_run_history_in(&changed_prefix.sqlite) - .await - .is_err() - ); - - let corrupt_row = TestContext::new().await?; - corrupt_row.put_event(&run_id, 1, 1, &created).await?; - seed_destination_history(&corrupt_row.sqlite, &run_id, &[(1, created)]).await?; - sqlx::query("UPDATE runs SET workflow_slug = 'corrupt' WHERE id = ?") - .bind(run_id.to_string()) - .execute(&corrupt_row.sqlite) - .await?; - assert!( - corrupt_row - .source - .verify_legacy_run_history_in(&corrupt_row.sqlite) - .await - .is_err() - ); - - let corrupt_json = TestContext::new().await?; - let created = serde_json::to_string(&created_value(&run_id, "verify"))?; - corrupt_json.put_event(&run_id, 1, 1, &created).await?; - seed_destination_history(&corrupt_json.sqlite, &run_id, &[(1, created)]).await?; - sqlx::query("UPDATE runs SET summary_json = '{}' WHERE id = ?") - .bind(run_id.to_string()) - .execute(&corrupt_json.sqlite) - .await?; - assert!( - corrupt_json - .source - .verify_legacy_run_history_in(&corrupt_json.sqlite) - .await - .is_err() - ); - Ok(()) - } - - #[tokio::test] - async fn legacy_run_history_errors_expose_safe_counts_without_event_contents() -> TestResult<()> - { - let context = TestContext::new().await?; - let run_id = run_id(100); - let secret = "DO-NOT-RENDER-THIS-TITLE"; - let mut value = created_value(&run_id, secret); - value - .get_mut("properties") - .and_then(serde_json::Value::as_object_mut) - .unwrap() - .remove("settings"); - context - .put_event(&run_id, 1, 1, &serde_json::to_string(&value)?) - .await?; - - let error = context - .source - .import_legacy_run_history_into(&context.sqlite) - .await - .unwrap_err(); - let rendered = format!("{error:?} {error}"); - let chain = error::collect_chain(&error).join(": "); - assert!(!rendered.contains(secret)); - assert!(!chain.contains(secret)); - assert!(error.source().is_some()); - Ok(()) - } - - #[tokio::test] - async fn legacy_run_history_scan_ignores_non_event_run_namespaces() -> TestResult<()> { - let context = TestContext::new().await?; - let run_id = run_id(110); - context - .put_event( - &run_id, - 1, - 1, - &serde_json::to_string(&created_value(&run_id, "event"))?, - ) - .await?; - context - .put_raw( - SlateKey::new("runs").with(run_id).with("state"), - b"not event JSON", - ) - .await?; - assert_eq!(context.import().await?.imported_events, 1); - Ok(()) - } -} diff --git a/lib/components/fabro-store/src/lib.rs b/lib/components/fabro-store/src/lib.rs index 46eb46351..b0eca994b 100644 --- a/lib/components/fabro-store/src/lib.rs +++ b/lib/components/fabro-store/src/lib.rs @@ -5,8 +5,6 @@ mod blob_store; mod error; mod keyed_mutex; mod keys; -mod legacy_blob_import; -mod legacy_run_history_import; pub mod platform_records; #[cfg(test)] mod record; @@ -35,15 +33,6 @@ pub use fabro_types::{ BlobHash, EventEnvelope, PendingInterviewRecord, Run, RunProjection, StageId, StageProjection, }; pub use keyed_mutex::{KeyedMutex, KeyedMutexGuard}; -pub use legacy_blob_import::{ - LegacyBlobImportError, LegacyBlobImportReport, LegacyBlobInventory, LegacyBlobInventoryError, - LegacyBlobVerificationError, LegacyBlobVerificationReport, -}; -pub use legacy_run_history_import::{ - LegacyRunHistoryDiagnostics, LegacyRunHistoryImportError, LegacyRunHistoryImportReport, - LegacyRunHistorySourceIdentity, LegacyRunHistorySourceIdentityError, - LegacyRunHistoryVerificationError, LegacyRunHistoryVerificationReport, -}; pub use platform_records::{ PlatformRecord, PlatformRecordHook, PlatformRecordKind, PlatformRecordStore, StagePosition, StoredPlatformRecord, diff --git a/lib/components/fabro-store/src/platform_records.rs b/lib/components/fabro-store/src/platform_records.rs index 7d82c5761..c41faf134 100644 --- a/lib/components/fabro-store/src/platform_records.rs +++ b/lib/components/fabro-store/src/platform_records.rs @@ -32,8 +32,8 @@ use fabro_types::run_event::{ RunNoticeLevel, RunPairStartedProps, RunRunnableSource, RunStartedProps, RunSupersededByProps, }; use fabro_types::{ - BlobHash, DiffSummary, EventBody, GitIdentity, PairId, PairTarget, Principal, RunControlAction, - RunEvent, RunId, RunSpec, RunStatus, + BlobHash, DiffSummary, EventBody, GitIdentity, PairId, PairTarget, Principal, + PullRequestCreationId, PullRequestLink, RunControlAction, RunEvent, RunId, RunSpec, RunStatus, }; use serde::{Deserialize, Serialize}; use sqlx::sqlite::{SqliteConnection, SqliteRow}; @@ -125,9 +125,21 @@ pub enum PlatformRecordKind { #[serde(rename = "checkpoint")] #[strum(serialize = "checkpoint")] Checkpoint, + #[serde(rename = "pull_request.requested")] + #[strum(serialize = "pull_request.requested")] + PullRequestRequested, #[serde(rename = "pull_request.created")] #[strum(serialize = "pull_request.created")] PullRequestCreated, + #[serde(rename = "pull_request.failed")] + #[strum(serialize = "pull_request.failed")] + PullRequestFailed, + #[serde(rename = "pull_request.linked")] + #[strum(serialize = "pull_request.linked")] + PullRequestLinked, + #[serde(rename = "pull_request.unlinked")] + #[strum(serialize = "pull_request.unlinked")] + PullRequestUnlinked, #[serde(rename = "notification.sent")] #[strum(serialize = "notification.sent")] NotificationSent, @@ -176,8 +188,19 @@ pub enum PlatformRecord { /// record, written after the commit succeeds. #[serde(rename = "checkpoint")] Checkpoint(CheckpointRecord), + /// A pull request was asked for: the supervisor creates it. + #[serde(rename = "pull_request.requested")] + PullRequestRequested(PullRequestRequestedRecord), #[serde(rename = "pull_request.created")] PullRequestCreated(PullRequestCreatedRecord), + /// The requested pull request could not be created. + #[serde(rename = "pull_request.failed")] + PullRequestFailed(PullRequestFailedRecord), + /// An existing pull request was linked to the run by hand. + #[serde(rename = "pull_request.linked")] + PullRequestLinked(PullRequestLinkedRecord), + #[serde(rename = "pull_request.unlinked")] + PullRequestUnlinked(PullRequestLinkedRecord), #[serde(rename = "notification.sent")] NotificationSent(NotificationSentRecord), #[serde(rename = "run.paired")] @@ -200,7 +223,11 @@ impl PlatformRecord { Self::RunBranch(_) => PlatformRecordKind::RunBranch, Self::GitIdentity(_) => PlatformRecordKind::GitIdentity, Self::Checkpoint(_) => PlatformRecordKind::Checkpoint, + Self::PullRequestRequested(_) => PlatformRecordKind::PullRequestRequested, Self::PullRequestCreated(_) => PlatformRecordKind::PullRequestCreated, + Self::PullRequestFailed(_) => PlatformRecordKind::PullRequestFailed, + Self::PullRequestLinked(_) => PlatformRecordKind::PullRequestLinked, + Self::PullRequestUnlinked(_) => PlatformRecordKind::PullRequestUnlinked, Self::NotificationSent(_) => PlatformRecordKind::NotificationSent, Self::RunPaired(_) => PlatformRecordKind::RunPaired, } @@ -225,6 +252,10 @@ impl PlatformRecord { | Self::InterviewAnswered(_) | Self::RunBranch(_) | Self::GitIdentity(_) + | Self::PullRequestRequested(_) + | Self::PullRequestFailed(_) + | Self::PullRequestLinked(_) + | Self::PullRequestUnlinked(_) | Self::RunPaired(_) => None, } } @@ -356,6 +387,12 @@ pub struct InterviewAnsweredRecord { pub principal: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub channel: Option, + /// The question's text, for a reader that shows the answer beside it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub text: Option, + /// The answer as the person gave it, rendered as text. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub answer: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -407,6 +444,48 @@ pub struct PullRequestCreatedRecord { pub operation: Option, } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PullRequestRequestedRecord { + pub creation_id: PullRequestCreationId, + pub model: String, + #[serde(default)] + pub force: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PullRequestFailedRecord { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub creation_id: Option, + pub error: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PullRequestLinkedRecord { + pub owner: String, + pub repo: String, + pub number: u64, +} + +impl PullRequestLinkedRecord { + #[must_use] + pub fn link(&self) -> PullRequestLink { + PullRequestLink { + owner: self.owner.clone(), + repo: self.repo.clone(), + number: self.number, + } + } + + #[must_use] + pub fn from_link(link: &PullRequestLink) -> Self { + Self { + owner: link.owner.clone(), + repo: link.repo.clone(), + number: link.number, + } + } +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct NotificationSentRecord { pub route: String, @@ -786,6 +865,8 @@ fn interview_answered_record( question: props.question_id.clone(), principal, channel: None, + text: Some(props.question.clone()), + answer: Some(props.answer.clone()), } } @@ -848,6 +929,8 @@ mod tests { question: "q-1".to_string(), principal: None, channel: Some("web".to_string()), + text: None, + answer: None, }) } PlatformRecordKind::RunBranch => PlatformRecord::RunBranch(RunBranchRecord { @@ -893,6 +976,33 @@ mod tests { operation: None, }) } + PlatformRecordKind::PullRequestRequested => { + PlatformRecord::PullRequestRequested(PullRequestRequestedRecord { + creation_id: PullRequestCreationId::new(), + model: "gpt-5.4".to_string(), + force: false, + }) + } + PlatformRecordKind::PullRequestFailed => { + PlatformRecord::PullRequestFailed(PullRequestFailedRecord { + creation_id: None, + error: "no remote".to_string(), + }) + } + PlatformRecordKind::PullRequestLinked => { + PlatformRecord::PullRequestLinked(PullRequestLinkedRecord { + owner: "fabro-sh".to_string(), + repo: "fabro".to_string(), + number: 7, + }) + } + PlatformRecordKind::PullRequestUnlinked => { + PlatformRecord::PullRequestUnlinked(PullRequestLinkedRecord { + owner: "fabro-sh".to_string(), + repo: "fabro".to_string(), + number: 7, + }) + } PlatformRecordKind::NotificationSent => { PlatformRecord::NotificationSent(NotificationSentRecord { route: "slack".to_string(), diff --git a/lib/components/fabro-store/src/run_summary_store.rs b/lib/components/fabro-store/src/run_summary_store.rs index 24b38756e..fe6ec7c04 100644 --- a/lib/components/fabro-store/src/run_summary_store.rs +++ b/lib/components/fabro-store/src/run_summary_store.rs @@ -355,31 +355,6 @@ impl RunSummaryStore { Ok(()) } - #[cfg(test)] - pub(crate) async fn test_mark_run_history_activated(&self) -> Result<()> { - sqlx::query( - r" -INSERT INTO legacy_run_history_activation ( - singleton, source_fingerprint, source_runs, source_events, activated_at_ms -) VALUES (1, zeroblob(32), 0, 0, 1) -ON CONFLICT(singleton) DO NOTHING -", - ) - .execute(&self.pool) - .await?; - Ok(()) - } - - #[cfg(test)] - pub(crate) async fn test_is_run_history_tombstoned(&self, run_id: &RunId) -> Result { - Ok(sqlx::query_scalar( - "SELECT EXISTS(SELECT 1 FROM legacy_run_history_deletions WHERE run_id = ?)", - ) - .bind(run_id.to_string()) - .fetch_one(&self.pool) - .await?) - } - pub(crate) async fn acquire(&self) -> Result> { Ok(self.pool.acquire().await?) } @@ -571,31 +546,11 @@ ON CONFLICT(singleton) DO NOTHING Ok(Some(run_id)) } - pub(crate) async fn delete_canonical(&self, run_id: &RunId, deleted_at_ms: i64) -> Result<()> { - // This transaction reads the activation marker before it writes the - // tombstone and run deletion. A deferred SQLite transaction can fail - // immediately when that read transaction is upgraded while another - // writer is active, bypassing the configured busy timeout. Reserve - // the write lock up front so concurrent deletes wait normally. + pub(crate) async fn delete_canonical(&self, run_id: &RunId) -> Result<()> { + // Reserve the write lock up front: a deferred transaction upgraded + // while another writer is active can fail at once, bypassing the + // configured busy timeout, so concurrent deletes wait normally. let mut transaction = self.pool.begin_with("BEGIN IMMEDIATE").await?; - let activated: bool = sqlx::query_scalar( - "SELECT EXISTS(SELECT 1 FROM legacy_run_history_activation WHERE singleton = 1)", - ) - .fetch_one(&mut *transaction) - .await?; - if activated { - sqlx::query( - r" -INSERT INTO legacy_run_history_deletions (run_id, deleted_at_ms) -VALUES (?, ?) -ON CONFLICT(run_id) DO UPDATE SET deleted_at_ms = excluded.deleted_at_ms -", - ) - .bind(run_id.to_string()) - .bind(deleted_at_ms) - .execute(&mut *transaction) - .await?; - } sqlx::query("DELETE FROM runs WHERE id = ?") .bind(run_id.to_string()) .execute(&mut *transaction) @@ -851,71 +806,6 @@ impl RunSummaryStore { decode_event_rows_with_json(&rows, run_id) } - pub(crate) async fn insert_imported_run_on_connection( - connection: &mut SqliteConnection, - entry: &ProjectedRun, - ) -> Result<()> { - let record = PreparedRunSummary::from_entry(entry); - ensure_entry_identity(entry, &record, entry.last_seq)?; - if !(1..=keys::MAX_EVENT_SEQ).contains(&entry.last_seq) { - return Err(Error::RunHeadMismatch { - run_id: entry.run_id.to_string(), - expected_last_seq: entry.last_seq, - actual_last_seq: None, - }); - } - insert_run_on_connection(connection, &record).await - } - - pub(crate) async fn insert_imported_event_on_connection( - connection: &mut SqliteConnection, - run_id: &RunId, - payload: &EventPayload, - envelope: &EventEnvelope, - event_json: &str, - ) -> Result<()> { - payload.validate(run_id)?; - let decoded = RunEvent::try_from(payload)?; - if envelope.event != decoded || envelope.event.run_id != *run_id { - return Err(run_event_mismatch(run_id, envelope.seq, "event_json")); - } - if !(1..=keys::MAX_EVENT_SEQ).contains(&envelope.seq) { - return Err(run_event_mismatch(run_id, envelope.seq, "seq")); - } - insert_event_json_on_connection(connection, run_id, envelope, event_json).await - } - - pub(crate) async fn verify_current_run_on_connection( - connection: &mut SqliteConnection, - entry: &ProjectedRun, - ) -> Result<()> { - let record = PreparedRunSummary::from_entry(entry); - ensure_entry_identity(entry, &record, entry.last_seq)?; - let row = sqlx::query( - r" -SELECT id, source_last_seq, created_at_ms, started_at_ms, last_event_at_ms, completed_at_ms, - status, archived_at_ms, parent_id, title, workflow_slug, workflow_name, - repository_name, automation_id, diff_files_changed, diff_additions, diff_deletions, - input_tokens, output_tokens, reasoning_tokens, cache_read_tokens, cache_write_tokens, - total_usd_micros, summary_json -FROM runs -WHERE id = ? -", - ) - .bind(entry.run_id.to_string()) - .fetch_optional(connection) - .await? - .ok_or_else(|| Error::RunNotFound(entry.run_id.to_string()))?; - - let run = &record.run; - verify_run_field(&row, run, "id", &run.id.to_string())?; - verify_run_field(&row, run, "source_last_seq", &i64::from(record.last_seq))?; - // The run's row is written by its projector from Petri's records - // and the platform records; the legacy fold knows the lifecycle - // alone, so only the identity and the legacy guard are checked here. - Ok(()) - } - pub(crate) async fn list_events_from_with_limit_on_connection( connection: &mut SqliteConnection, run_id: &RunId, @@ -1228,20 +1118,6 @@ fn decode_event_rows_with_json( .collect() } -fn verify_run_field(row: &SqliteRow, run: &Run, field: &'static str, expected: &T) -> Result<()> -where - T: for<'row> sqlx::Decode<'row, Sqlite> + sqlx::Type + PartialEq, -{ - let stored: T = row.try_get(field)?; - if &stored != expected { - return Err(Error::RunSummaryMismatch { - run_id: run.id.to_string(), - field, - }); - } - Ok(()) -} - fn decode_event_row( row: &SqliteRow, expected_run_id: &RunId, @@ -2184,11 +2060,9 @@ mod tests { .await .unwrap(); transaction.commit().await.unwrap(); - store.test_mark_run_history_activated().await.unwrap(); - let blocker = store.pool.begin_with("BEGIN IMMEDIATE").await.unwrap(); let contender = store.clone(); - let delete = tokio::spawn(async move { contender.delete_canonical(&id, 2).await }); + let delete = tokio::spawn(async move { contender.delete_canonical(&id).await }); time::sleep(Duration::from_millis(25)).await; assert!( !delete.is_finished(), @@ -2198,7 +2072,6 @@ mod tests { blocker.commit().await.unwrap(); delete.await.unwrap().unwrap(); assert!(!store.contains(&id).await.unwrap()); - assert!(store.test_is_run_history_tombstoned(&id).await.unwrap()); } #[tokio::test] diff --git a/lib/components/fabro-store/src/slate/mod.rs b/lib/components/fabro-store/src/slate/mod.rs index 8fa4d3a05..7206c7c6d 100644 --- a/lib/components/fabro-store/src/slate/mod.rs +++ b/lib/components/fabro-store/src/slate/mod.rs @@ -238,24 +238,11 @@ impl Database { Ok(()) } - /// The run's projection: for a legacy run the reducer's fold of its - /// events; for a Petri run the projection its projector last committed - /// over Petri's records and the platform records, falling back to the - /// legacy fold (the lifecycle alone) until the first view pass commits. + /// The run's projection: the one its projector last committed over + /// Petri's records and the platform records, or `None` before the + /// first view pass commits (or for no such run). pub async fn load_run_projection(&self, run_id: &RunId) -> Result>> { - let legacy = if let Some(active) = self.get_active_run(run_id).await { - active.projection_snapshot().await? - } else { - match self.run_summary_store.load_projection(run_id).await { - Ok(projected) => projected.projection, - Err(Error::RunNotFound(_)) => return Ok(None), - Err(error) => return Err(error), - } - }; - if let Some(petri) = self.run_summary_store.load_petri_projection(run_id).await? { - return Ok(Some(petri)); - } - Ok(Some(legacy)) + self.run_summary_store.load_petri_projection(run_id).await } /// Install the wake-up called after a platform record of a Petri run is @@ -277,9 +264,7 @@ impl Database { Some(active) => Some(active.state_lock.lock().await), None => None, }; - self.run_summary_store - .delete_canonical(run_id, Utc::now().timestamp_millis()) - .await?; + self.run_summary_store.delete_canonical(run_id).await?; active_runs.remove(run_id); Ok(()) } @@ -865,67 +850,6 @@ mod tests { assert_eq!(read.as_deref(), Some(shared_blob.as_slice())); } - #[tokio::test] - async fn activated_delete_tombstone_prevents_legacy_history_resurrection() { - let (directory, summaries) = make_run_summary_store().await; - let (_object_store, store) = make_store_with_run_summaries(Arc::clone(&summaries)); - let run_id = test_run_id("run-1"); - let created = event_payload( - "run-1", - "2026-03-27T12:00:00Z", - "run.created", - &serde_json::json!({ - "settings": sample_run_spec("run-1").settings, - "graph": sample_run_spec("run-1").graph, - "provenance": test_support::test_run_provenance(), - }), - ); - store - .put_unvalidated_legacy_run_event(&run_id, 1, created.as_value()) - .await - .unwrap(); - let run = store.create_run(&run_id).await.unwrap(); - run.append_event(&created).await.unwrap(); - summaries.test_mark_run_history_activated().await.unwrap(); - - store.delete_run(&run_id).await.unwrap(); - - assert!( - summaries - .test_is_run_history_tombstoned(&run_id) - .await - .unwrap() - ); - assert!(store.open_run(&run_id).await.is_err()); - let database = fabro_db::Database::connect(directory.path().join("fabro.sqlite3")) - .await - .unwrap(); - let report = store - .import_legacy_run_history_into(database.pool()) - .await - .unwrap(); - assert_eq!(report.tombstoned_source_runs, 1); - assert_eq!(report.tombstoned_source_events, 1); - assert!(store.open_run(&run_id).await.is_err()); - - let verification = store - .verify_legacy_run_history_in(database.pool()) - .await - .unwrap(); - assert_eq!(verification.tombstoned_source_runs, 1); - assert_eq!(verification.tombstoned_source_events, 1); - - let recreated = store.create_run(&run_id).await.unwrap(); - recreated.append_event(&created).await.unwrap(); - assert!( - store - .verify_legacy_run_history_in(database.pool()) - .await - .is_err(), - "a tombstone and live canonical data must fail verification" - ); - } - #[tokio::test] async fn missing_run_open_paths_return_run_not_found() { let (_object_store, store) = make_store(); diff --git a/lib/components/fabro-workflow/src/operations/archive.rs b/lib/components/fabro-workflow/src/operations/archive.rs deleted file mode 100644 index 3f19adac9..000000000 --- a/lib/components/fabro-workflow/src/operations/archive.rs +++ /dev/null @@ -1,470 +0,0 @@ -use fabro_store::Database; -use fabro_types::{Principal, RunId, RunStatus, TerminalStatus}; - -use super::run_store::map_open_run_error; -use crate::error::Error; -use crate::event::{self, Event}; - -/// The canonical "run is archived — mutation rejected" error message. Shared -/// by the operations layer, the CLI rewind precheck, and the server HTTP -/// guards so the user sees the same actionable guidance everywhere. -#[must_use] -pub fn archived_rejection_message(run_id: &RunId) -> String { - format!("run {run_id} is archived; run `fabro unarchive {run_id}` to restore it and try again") -} - -/// Returns `Err(Error::Precondition)` when the given status represents an -/// archived run. Use this at any mutation entry point that would otherwise -/// transition or emit events against the run (rewind, resume, etc.). -pub fn ensure_not_archived(archived: bool, run_id: &RunId) -> Result<(), Error> { - if archived { - Err(Error::Precondition(archived_rejection_message(run_id))) - } else { - Ok(()) - } -} - -/// Outcome of an `archive` call. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum ArchiveOutcome { - /// Event was appended; projection marks the run archived. - Archived { prior_status: TerminalStatus }, - /// Run was already archived; no event emitted. - AlreadyArchived, -} - -/// Outcome of an `unarchive` call. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum UnarchiveOutcome { - /// Event was appended; projection clears archive metadata. - Unarchived { restored_status: TerminalStatus }, - /// Run was terminal but not archived; no event emitted. Symmetric with - /// `ArchiveOutcome::AlreadyArchived`. - NotArchived { status: RunStatus }, -} - -/// Archive a terminal run. Idempotent if already archived. -pub async fn archive( - store: &Database, - run_id: &RunId, - actor: Option, -) -> Result { - let run_store = store - .open_run(run_id) - .await - .map_err(|err| map_open_run_error(run_id, err))?; - let projection = run_store - .state() - .await - .map_err(|err| Error::engine(err.to_string()))?; - let current = projection.status; - - if projection.archived_at.is_some() { - return Ok(ArchiveOutcome::AlreadyArchived); - } - - if !matches!( - current, - RunStatus::Succeeded { .. } | RunStatus::Failed { .. } | RunStatus::Dead - ) { - return Err(Error::Precondition(format!( - "run {run_id} must be terminal (succeeded, failed, or dead) to archive; \ - current status is {current}" - ))); - } - - event::append_event(&run_store, run_id, &Event::RunArchived { actor }) - .await - .map_err(|err| Error::engine(err.to_string()))?; - - let prior_status = current.terminal_status().ok_or_else(|| { - Error::engine(format!( - "run {run_id} passed archive precondition but had non-terminal status {current}" - )) - })?; - - Ok(ArchiveOutcome::Archived { prior_status }) -} - -/// Unarchive a previously archived run, restoring its prior terminal status. -/// Idempotent on terminal-but-not-archived runs (returns `NotArchived` without -/// emitting an event). -pub async fn unarchive( - store: &Database, - run_id: &RunId, - actor: Option, -) -> Result { - let run_store = store - .open_run(run_id) - .await - .map_err(|err| map_open_run_error(run_id, err))?; - let projection = run_store - .state() - .await - .map_err(|err| Error::engine(err.to_string()))?; - let current = projection.status; - - if projection.archived_at.is_some() { - event::append_event(&run_store, run_id, &Event::RunUnarchived { actor }) - .await - .map_err(|err| Error::engine(err.to_string()))?; - let prior = current.terminal_status().ok_or_else(|| { - Error::engine(format!( - "run {run_id} is archived but has non-terminal status {current}" - )) - })?; - return Ok(UnarchiveOutcome::Unarchived { - restored_status: prior, - }); - } - - if matches!( - current, - RunStatus::Succeeded { .. } | RunStatus::Failed { .. } | RunStatus::Dead - ) { - return Ok(UnarchiveOutcome::NotArchived { status: current }); - } - - Err(Error::Precondition(format!( - "run {run_id} is not archived (status: {current}); nothing to unarchive" - ))) -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - use std::time::Duration; - - use fabro_store::Database; - use fabro_types::{ - FailureReason, PetriAdmission, RunId, SuccessReason, TerminalStatus, fixtures, test_support, - }; - use object_store::memory::InMemory; - - use super::*; - - fn memory_store() -> Arc { - Arc::new(fabro_store::test_support::test_database( - Arc::new(InMemory::new()), - "", - Duration::from_millis(1), - None, - )) - } - - async fn seed_succeeded(store: &Database, run_id: &RunId) { - let run_store = store.create_run(run_id).await.unwrap(); - seed_created(&run_store, run_id).await; - seed_runnable(&run_store, run_id).await; - event::append_event(&run_store, run_id, &Event::RunStarting) - .await - .unwrap(); - event::append_event(&run_store, run_id, &Event::RunRunning) - .await - .unwrap(); - event::append_event(&run_store, run_id, &Event::WorkflowRunCompleted { - timing: fabro_types::RunTiming::wall_only(10), - artifact_count: 0, - status: "succeeded".to_string(), - reason: SuccessReason::Completed, - final_git_commit_sha: None, - final_patch: None, - diff_summary: None, - usage: None, - }) - .await - .unwrap(); - } - - async fn seed_failed(store: &Database, run_id: &RunId) { - let run_store = store.create_run(run_id).await.unwrap(); - seed_created(&run_store, run_id).await; - seed_runnable(&run_store, run_id).await; - event::append_event(&run_store, run_id, &Event::RunStarting) - .await - .unwrap(); - event::append_event(&run_store, run_id, &Event::RunRunning) - .await - .unwrap(); - let failure_event = Event::workflow_run_failed_from_error( - &crate::error::Error::engine("boom"), - fabro_types::RunTiming::wall_only(10), - FailureReason::WorkflowError, - None, - None, - None, - None, - ); - event::append_event(&run_store, run_id, &failure_event) - .await - .unwrap(); - } - - async fn seed_running(store: &Database, run_id: &RunId) { - let run_store = store.create_run(run_id).await.unwrap(); - seed_created(&run_store, run_id).await; - seed_runnable(&run_store, run_id).await; - event::append_event(&run_store, run_id, &Event::RunStarting) - .await - .unwrap(); - event::append_event(&run_store, run_id, &Event::RunRunning) - .await - .unwrap(); - } - - async fn seed_created(run_store: &fabro_store::RunDatabase, run_id: &RunId) { - event::append_event(run_store, run_id, &Event::RunCreated { - run_id: *run_id, - title: None, - settings: serde_json::to_value(fabro_types::WorkflowSettings::default()) - .unwrap(), - graph: serde_json::to_value(fabro_types::Graph::new("test")).unwrap(), - workflow_source: None, - labels: std::collections::BTreeMap::default(), - source_directory: None, - workflow_slug: None, - workflow_version_id: None, - target: None, - automation: None, - provenance: test_support::test_run_provenance(), - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, - admission: PetriAdmission::default(), - }) - .await - .unwrap(); - } - - async fn seed_runnable(run_store: &fabro_store::RunDatabase, run_id: &RunId) { - event::append_event(run_store, run_id, &Event::RunRunnable { - source: fabro_types::RunRunnableSource::StartRequested, - actor: None, - }) - .await - .unwrap(); - } - - async fn current_status(store: &Database, run_id: &RunId) -> RunStatus { - let run_store = store.open_run_reader(run_id).await.unwrap(); - run_store.state().await.unwrap().status - } - - async fn is_archived(store: &Database, run_id: &RunId) -> bool { - let run_store = store.open_run_reader(run_id).await.unwrap(); - run_store.state().await.unwrap().archived_at.is_some() - } - - async fn event_count(store: &Database, run_id: &RunId) -> usize { - let run_store = store.open_run_reader(run_id).await.unwrap(); - run_store.list_events().await.unwrap().len() - } - - #[tokio::test] - async fn archive_on_succeeded_emits_event_and_transitions_to_archived() { - let store = memory_store(); - let run_id = fixtures::RUN_1; - seed_succeeded(&store, &run_id).await; - - let outcome = archive(&store, &run_id, None).await.unwrap(); - assert_eq!(outcome, ArchiveOutcome::Archived { - prior_status: TerminalStatus::Succeeded { - reason: SuccessReason::Completed, - }, - }); - assert_eq!( - current_status(&store, &run_id).await, - RunStatus::Succeeded { - reason: SuccessReason::Completed, - } - ); - assert!(is_archived(&store, &run_id).await); - - let projection = store - .open_run_reader(&run_id) - .await - .unwrap() - .state() - .await - .unwrap(); - assert_eq!(projection.status, RunStatus::Succeeded { - reason: SuccessReason::Completed, - }); - assert!(projection.archived_at.is_some()); - } - - #[tokio::test] - async fn archive_on_failed_captures_failed_as_prior_status() { - let store = memory_store(); - let run_id = fixtures::RUN_2; - seed_failed(&store, &run_id).await; - - let outcome = archive(&store, &run_id, None).await.unwrap(); - assert_eq!(outcome, ArchiveOutcome::Archived { - prior_status: TerminalStatus::Failed { - reason: FailureReason::WorkflowError, - }, - }); - assert_eq!(current_status(&store, &run_id).await, RunStatus::Failed { - reason: FailureReason::WorkflowError, - }); - assert!(is_archived(&store, &run_id).await); - } - - #[tokio::test] - async fn archive_on_already_archived_is_idempotent_and_emits_no_event() { - let store = memory_store(); - let run_id = fixtures::RUN_1; - seed_succeeded(&store, &run_id).await; - archive(&store, &run_id, None).await.unwrap(); - - let events_before = event_count(&store, &run_id).await; - let outcome = archive(&store, &run_id, None).await.unwrap(); - let events_after = event_count(&store, &run_id).await; - - assert_eq!(outcome, ArchiveOutcome::AlreadyArchived); - assert_eq!(events_before, events_after); - } - - #[tokio::test] - async fn archive_on_running_rejects_with_precondition_error() { - let store = memory_store(); - let run_id = fixtures::RUN_1; - seed_running(&store, &run_id).await; - - let err = archive(&store, &run_id, None).await.unwrap_err(); - let Error::Precondition(message) = err else { - panic!("expected Precondition, got {err:?}"); - }; - assert!( - message.contains("must be terminal"), - "message should explain terminal requirement, got: {message}" - ); - } - - #[tokio::test] - async fn unarchive_restores_succeeded_and_clears_prior_status() { - let store = memory_store(); - let run_id = fixtures::RUN_1; - seed_succeeded(&store, &run_id).await; - archive(&store, &run_id, None).await.unwrap(); - - let outcome = unarchive(&store, &run_id, None).await.unwrap(); - assert_eq!(outcome, UnarchiveOutcome::Unarchived { - restored_status: TerminalStatus::Succeeded { - reason: SuccessReason::Completed, - }, - }); - let projection = store - .open_run_reader(&run_id) - .await - .unwrap() - .state() - .await - .unwrap(); - assert_eq!(projection.status, RunStatus::Succeeded { - reason: SuccessReason::Completed, - }); - } - - #[tokio::test] - async fn unarchive_restores_failed_when_prior_was_failed() { - let store = memory_store(); - let run_id = fixtures::RUN_2; - seed_failed(&store, &run_id).await; - archive(&store, &run_id, None).await.unwrap(); - - let outcome = unarchive(&store, &run_id, None).await.unwrap(); - assert_eq!(outcome, UnarchiveOutcome::Unarchived { - restored_status: TerminalStatus::Failed { - reason: FailureReason::WorkflowError, - }, - }); - assert_eq!(current_status(&store, &run_id).await, RunStatus::Failed { - reason: FailureReason::WorkflowError, - }); - } - - #[tokio::test] - async fn unarchive_on_terminal_non_archived_run_is_idempotent_no_op() { - let store = memory_store(); - let run_id = fixtures::RUN_1; - seed_succeeded(&store, &run_id).await; - - let events_before = event_count(&store, &run_id).await; - let outcome = unarchive(&store, &run_id, None).await.unwrap(); - let events_after = event_count(&store, &run_id).await; - - assert_eq!(outcome, UnarchiveOutcome::NotArchived { - status: RunStatus::Succeeded { - reason: SuccessReason::Completed, - }, - }); - assert_eq!(events_before, events_after); - } - - #[tokio::test] - async fn archive_on_unknown_run_returns_run_not_found() { - let store = memory_store(); - let run_id = fixtures::RUN_3; - - let err = archive(&store, &run_id, None).await.unwrap_err(); - assert!( - matches!(err, Error::RunNotFound(_)), - "expected RunNotFound, got {err:?}" - ); - } - - #[tokio::test] - async fn unarchive_on_unknown_run_returns_run_not_found() { - let store = memory_store(); - let run_id = fixtures::RUN_3; - - let err = unarchive(&store, &run_id, None).await.unwrap_err(); - assert!( - matches!(err, Error::RunNotFound(_)), - "expected RunNotFound, got {err:?}" - ); - } - - #[tokio::test] - async fn unarchive_on_running_rejects_with_precondition_error() { - let store = memory_store(); - let run_id = fixtures::RUN_1; - seed_running(&store, &run_id).await; - - let err = unarchive(&store, &run_id, None).await.unwrap_err(); - let Error::Precondition(message) = err else { - panic!("expected Precondition, got {err:?}"); - }; - assert!( - message.contains("not archived"), - "message should explain run is not archived, got: {message}" - ); - } - - #[tokio::test] - async fn archive_unarchive_archive_cycle_produces_three_events() { - let store = memory_store(); - let run_id = fixtures::RUN_1; - seed_succeeded(&store, &run_id).await; - - let events_before = event_count(&store, &run_id).await; - archive(&store, &run_id, None).await.unwrap(); - unarchive(&store, &run_id, None).await.unwrap(); - archive(&store, &run_id, None).await.unwrap(); - let events_after = event_count(&store, &run_id).await; - - assert_eq!(events_after - events_before, 3); - assert_eq!( - current_status(&store, &run_id).await, - RunStatus::Succeeded { - reason: SuccessReason::Completed, - } - ); - assert!(is_archived(&store, &run_id).await); - } -} diff --git a/lib/components/fabro-workflow/src/operations/create.rs b/lib/components/fabro-workflow/src/operations/create.rs index 554226cd6..5e5e2b24b 100644 --- a/lib/components/fabro-workflow/src/operations/create.rs +++ b/lib/components/fabro-workflow/src/operations/create.rs @@ -6,23 +6,24 @@ ) )] -use std::collections::{BTreeMap, HashMap}; +use std::collections::HashMap; use std::path::{Path, PathBuf}; use fabro_config::Storage; use fabro_graphviz::graph::{AttrValue, Graph}; +use fabro_store::platform_records::{ + PlatformRecord, RunCreatedRecord, RunLifecycleKind, RunLifecycleRecord, +}; use fabro_store::{BlobStore, Database}; use fabro_template::TemplateContext; use fabro_types::{ AutomationRef, BlobHash, ForkSourceRef, GitContext, ManifestPath, PetriAdmission, RunId, - RunProvenance, RunTarget, WorkflowSettings, WorkflowVersionId, + RunProvenance, RunStatus, RunTarget, WorkflowSettings, WorkflowVersionId, }; -use fabro_util::json::normalize_json_value; use tokio::task::spawn_blocking; use super::source::{ResolveWorkflowInput, WorkflowInput, resolve_workflow}; use crate::error::Error; -use crate::event::{self, Event, append_event}; use crate::pipeline::types::PersistOptions; use crate::pipeline::{self, Persisted, TransformOptions, Validated}; use crate::records::RunSpec; @@ -378,6 +379,8 @@ pub async fn persist_create_run( }) } +/// The run's first records: `run.created` with the spec Fabro built, and +/// the `submitted` lifecycle transition. Both wake the run's projector. async fn persist_created_run( store: &Database, persisted: &Persisted, @@ -399,50 +402,32 @@ async fn persist_created_run( write_optional_blob(&blob_store, definition_bytes.as_deref()), async { blob_store.write(&spec_bytes).await.map_err(store_error) }, )?; + let _ = workflow_source; let title = explicit_title.unwrap_or_else(|| fabro_types::infer_run_title(record.graph.goal())); - let first_event = Event::RunCreated { - run_id: record.run_id, + let mut spec = record.clone(); + spec.definition_blob = definition_blob; + spec.spec_blob = Some(spec_blob); + let created = PlatformRecord::RunCreated(RunCreatedRecord { + spec, title: Some(title), - settings: normalize_json_value( - serde_json::to_value(&record.settings).map_err(|err| Error::engine(err.to_string()))?, - ), - graph: normalize_json_value( - serde_json::to_value(&record.graph).map_err(|err| Error::engine(err.to_string()))?, - ), - workflow_source: (!workflow_source.is_empty()).then(|| workflow_source.to_string()), - labels: record - .labels - .clone() - .into_iter() - .collect::>(), - source_directory: record.source_directory.clone(), - workflow_slug: record.workflow_slug.clone(), - workflow_version_id: record.workflow_version_id, - target: record.target.clone(), - automation: record.automation.clone(), - provenance: record.provenance.clone(), - spec_blob: Some(spec_blob), - git: record.git.clone(), - fork_source_ref: record.fork_source_ref.clone(), - retried_from: None, parent_id, + retried_from: None, web_url, - admission: record.admission.clone(), - }; - let run_store = event::create_run( - store, - &record.run_id, - &first_event, - record.run_id.created_at(), - ) - .await - .map_err(|err| Error::engine_with_source("failed to create run store", err))?; - append_event(&run_store, &record.run_id, &Event::RunSubmitted { - definition_blob, - }) - .await - .map_err(store_error) + }); + let submitted = PlatformRecord::RunLifecycle( + RunLifecycleRecord::new(RunLifecycleKind::Submitted).with_status(RunStatus::Submitted), + ); + let summaries = store.run_summary_store(); + let platform_records = summaries.platform_records(); + for platform_record in [created, submitted] { + platform_records + .append(&record.run_id, &platform_record, None) + .await + .map_err(store_error)?; + } + summaries.notify_platform_record(record.run_id); + Ok(()) } async fn write_optional_blob( diff --git a/lib/components/fabro-workflow/src/operations/mod.rs b/lib/components/fabro-workflow/src/operations/mod.rs index a3d06f304..232ef5e00 100644 --- a/lib/components/fabro-workflow/src/operations/mod.rs +++ b/lib/components/fabro-workflow/src/operations/mod.rs @@ -1,19 +1,34 @@ -mod archive; mod create; -mod run_store; mod source; mod validate; -pub use archive::{ - ArchiveOutcome, UnarchiveOutcome, archive, archived_rejection_message, ensure_not_archived, - unarchive, -}; pub use create::{ CompiledRun, CreateRunCompileInput, CreateRunPersistenceInput, CreateRunPersistenceMetadata, CreatedRun, MaterializedRun, assemble_create_run_persistence_input, compile_admitted_run, make_run_dir, materialize_admitted_run, persist_create_run, }; +use fabro_types::RunId; pub use source::WorkflowInput; pub use validate::{ValidateInput, validate}; +pub use crate::error::Error; pub use crate::transforms::RenderMode; + +/// The canonical "run is archived — mutation rejected" error message. Shared +/// by the server's HTTP guards and the CLI so the user sees the same +/// actionable guidance everywhere. +#[must_use] +pub fn archived_rejection_message(run_id: &RunId) -> String { + format!("run {run_id} is archived; run `fabro unarchive {run_id}` to restore it and try again") +} + +/// Returns `Err(Error::Precondition)` when the given status represents an +/// archived run. Use this at any mutation entry point that would otherwise +/// transition the run. +pub fn ensure_not_archived(archived: bool, run_id: &RunId) -> Result<(), Error> { + if archived { + Err(Error::Precondition(archived_rejection_message(run_id))) + } else { + Ok(()) + } +} diff --git a/lib/components/fabro-workflow/src/operations/run_store.rs b/lib/components/fabro-workflow/src/operations/run_store.rs deleted file mode 100644 index d60e56ce2..000000000 --- a/lib/components/fabro-workflow/src/operations/run_store.rs +++ /dev/null @@ -1,11 +0,0 @@ -use fabro_store::Error as StoreError; -use fabro_types::RunId; - -use crate::error::Error; - -pub(super) fn map_open_run_error(run_id: &RunId, err: StoreError) -> Error { - match err { - StoreError::RunNotFound(id) => Error::RunNotFound(id), - other => Error::engine(format!("failed to open run {run_id}: {other}")), - } -} diff --git a/lib/components/fabro-workflow/src/pull_request.rs b/lib/components/fabro-workflow/src/pull_request.rs index 63b990837..5232637d4 100644 --- a/lib/components/fabro-workflow/src/pull_request.rs +++ b/lib/components/fabro-workflow/src/pull_request.rs @@ -18,7 +18,6 @@ use tracing::{debug, info, warn}; use crate::outcome::format_cost as outcome_format_cost; use crate::records::{Conclusion, RunSpec}; -use crate::runtime_store::RunStoreHandle; /// Maximum length of a PR title (Unicode scalar values). const PR_TITLE_MAX_CHARS: usize = 72; @@ -333,7 +332,6 @@ pub async fn build_pr_content( diff: &str, goal: &str, model: &str, - run_store: &RunStoreHandle, llm_source: Arc, catalog: Arc, conclusion: Option<&Conclusion>, @@ -352,7 +350,6 @@ pub async fn build_pr_content( diff, goal, model, - run_store, catalog.as_ref(), conclusion, run_state, @@ -365,7 +362,6 @@ async fn build_pr_content_with_client( diff: &str, goal: &str, model: &str, - run_store: &RunStoreHandle, catalog: &Catalog, conclusion: Option<&Conclusion>, run_state: Option<&RunProjection>, @@ -373,18 +369,6 @@ async fn build_pr_content_with_client( ) -> Result { info!("Building PR content"); - let loaded_run_state = if run_state.is_none() { - run_store - .state() - .await - .inspect_err(|err| { - tracing::warn!(error = %err, "Failed to load run state from store for PR body"); - }) - .ok() - } else { - None - }; - let run_state = run_state.or(loaded_run_state.as_ref()); let conclusion = conclusion.or_else(|| run_state.and_then(|state| state.conclusion.as_ref())); let plan_text = run_state.and_then(read_plan_text); let run_spec = run_state.map(|state| state.spec.clone()); @@ -462,7 +446,6 @@ pub struct OpenPullRequestRequest<'a> { pub model: &'a str, pub draft: bool, pub auto_merge: Option, - pub run_store: &'a RunStoreHandle, pub llm_source: Arc, pub catalog: Arc, pub conclusion: Option<&'a Conclusion>, @@ -616,7 +599,6 @@ pub async fn open_pull_request( req.diff, req.goal, req.model, - req.run_store, Arc::clone(&req.llm_source), Arc::clone(&req.catalog), req.conclusion, @@ -1073,12 +1055,11 @@ capabilities = { text = true, tools = true, response_format = { json_object = tr #[tokio::test] async fn build_pr_content_uses_in_memory_conclusion() { let store = test_store(); - let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); + let _run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); let PrContent { title, body } = build_pr_content_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "mock-model", - &run_store.clone().into(), &mock_catalog(), Some(&make_test_conclusion()), None, @@ -1152,7 +1133,6 @@ capabilities = { text = true, tools = true, response_format = { json_object = tr "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "mock-model", - &run_store.clone().into(), &mock_catalog(), Some(&make_test_conclusion()), None, @@ -1251,7 +1231,6 @@ capabilities = { text = true, tools = true, response_format = { json_object = tr "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "mock-model", - &run_store.clone().into(), &mock_catalog(), Some(&make_test_conclusion()), None, @@ -1271,12 +1250,11 @@ capabilities = { text = true, tools = true, response_format = { json_object = tr #[tokio::test] async fn build_pr_content_uses_explicit_llm_client() { let store = test_store(); - let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); + let _run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); let body = build_pr_content_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "gpt-5.4", - &run_store.clone().into(), &mock_catalog(), Some(&make_test_conclusion()), None, @@ -1327,14 +1305,12 @@ capabilities = { text = true, tools = true, response_format = { json_object = tr let catalog = test_catalog_with_provider_base_url("openai", &server.url("/v1")); let store = test_store(); - let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); - let run_store_handle: RunStoreHandle = run_store.into(); + let _run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); let PrContent { title, body } = build_pr_content( "diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n", "Implement feature", "gpt-5.4", - &run_store_handle, llm_source, catalog, Some(&make_test_conclusion()), @@ -1522,7 +1498,6 @@ capabilities = { text = true, tools = true, response_format = { json_object = tr model: "claude-sonnet-4-20250514", draft: false, auto_merge: None, - run_store: &harness.run_store, llm_source: Arc::clone(&harness.llm_source), catalog: harness.catalog.clone(), conclusion: None, @@ -1554,14 +1529,13 @@ capabilities = { text = true, tools = true, response_format = { json_object = tr #[tokio::test] async fn build_pr_content_truncates_long_title() { let store = test_store(); - let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); + let _run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); let long_title = "x".repeat(200); let payload = pr_content_json(&long_title, "Body content."); let title = build_pr_content_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n", "Implement feature", "mock-model", - &run_store.clone().into(), &mock_catalog(), Some(&make_test_conclusion()), None, @@ -1578,13 +1552,12 @@ capabilities = { text = true, tools = true, response_format = { json_object = tr #[tokio::test] async fn build_pr_content_uses_default_title_when_generated_and_goal_titles_empty() { let store = test_store(); - let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); + let _run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); let payload = pr_content_json("", "Body content."); let title = build_pr_content_with_client( "diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n", "## Plan:", "mock-model", - &run_store.clone().into(), &mock_catalog(), Some(&make_test_conclusion()), None, @@ -1675,7 +1648,6 @@ capabilities = { text = true, tools = true, response_format = { json_object = tr "diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n", "Implement feature", "mock-model", - &run_store.clone().into(), &mock_catalog(), Some(&make_test_conclusion()), None, @@ -1712,7 +1684,6 @@ capabilities = { text = true, tools = true, response_format = { json_object = tr llm_source: Arc, catalog: Arc, creds: fabro_github::GitHubCredentials, - run_store: RunStoreHandle, } impl FallbackHarness { @@ -1912,7 +1883,6 @@ capabilities = { text = true, tools = true, response_format = { json_object = tr llm_source, catalog, creds, - run_store: run_store.into(), } } @@ -1950,7 +1920,6 @@ capabilities = { text = true, tools = true, response_format = { json_object = tr model: "gpt-5.4", draft: false, auto_merge: None, - run_store: &harness.run_store, llm_source: Arc::clone(&harness.llm_source), catalog: harness.catalog.clone(), conclusion: None, @@ -1998,7 +1967,6 @@ capabilities = { text = true, tools = true, response_format = { json_object = tr model: "gpt-5.4", draft: false, auto_merge: None, - run_store: &harness.run_store, llm_source: Arc::clone(&harness.llm_source), catalog: harness.catalog.clone(), conclusion: None, @@ -2035,7 +2003,6 @@ capabilities = { text = true, tools = true, response_format = { json_object = tr model: "gpt-5.4", draft: false, auto_merge: None, - run_store: &harness.run_store, llm_source: Arc::clone(&harness.llm_source), catalog: harness.catalog.clone(), conclusion: None,