From 90b911c92775652fff70cc5151a12c53c190836a Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Mon, 20 Apr 2026 18:26:46 -0400 Subject: [PATCH 01/12] docs(plan): clarify unknown-event handling in client extraction Document that unknown Fabro run events already fall back through EventBody::Unknown, and that the refactor only needs to preserve that behavior during the EventEnvelope move. --- ...20-002-refactor-extract-fabro-client-crate-plan.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/plans/2026-04-20-002-refactor-extract-fabro-client-crate-plan.md b/docs/plans/2026-04-20-002-refactor-extract-fabro-client-crate-plan.md index 931b9d497..9a431d4f5 100644 --- a/docs/plans/2026-04-20-002-refactor-extract-fabro-client-crate-plan.md +++ b/docs/plans/2026-04-20-002-refactor-extract-fabro-client-crate-plan.md @@ -57,7 +57,7 @@ These were all reasonable while the client had exactly one caller. They prevent - Writing a second consumer of `fabro-client` (SDK example, IDE integration, etc.) — the crate exists to enable that, not to deliver it. - Changing wire semantics (OpenAPI contract stays identical). - Migrating existing `~/.fabro/auth.json` entries. The canonical key format for HTTP targets remains identical (same `normalized_http_base_url` logic); Unix-socket canonicalization changes from `fs::canonicalize` (symlink-resolving) to lexical-only (`.`/`..` cleanup, no symlink chasing). Existing sessions against socket paths written as absolute non-symlinked paths continue to match. Users with symlinked socket paths may need to re-login — acceptable per `CLAUDE.md` ("We don't care about migration"). -- (Previously noted as out of scope; now in scope per review feedback.) `RunEvent` unknown-variant handling: add a fallback `Unknown` variant so older CLIs against newer servers survive deserialization. Covered under Unit 2. +- `RunEvent` unknown-variant handling. Already present today via `EventBody::Unknown` at `lib/crates/fabro-types/src/run_event/mod.rs:695` (covered by `:974`). Unit 2 verifies the `EventEnvelope` restructure preserves this existing behavior — no new variant, no new code. - Splitting the ~40 client methods into separate `client/runs.rs`, `client/secrets.rs`, etc. modules. A single `client.rs` is acceptable at ~900 lines; cosmetic domain splitting can happen later if the file grows. ## Context & Research @@ -107,7 +107,7 @@ None. Internal refactor; the patterns are all established locally. ``` On refresh failure modes that previously triggered fallback, `OAuthSession` calls `fallback.resolve()`. If `Some(Credential)`, the session rebuilds the client bundle with that credential; if `None`, it surfaces "session expired." The CLI provides a concrete impl in `fabro-cli` that wraps the existing dev-token-loading logic. Named trait (vs bare `Fn` closure) makes the role obvious at the `OAuthSession::builder` call site. -- **`fabro_types::RunEvent` gains unknown-variant tolerance.** Add `#[serde(other)] Unknown` (or `#[serde(untagged)] Unknown(serde_json::Value)` depending on the enum shape) so a newer server's unknown event type doesn't break an older CLI at deserialize. Preserves today's accidental forward-compat that came from storing events as raw JSON. Small addition to the event enum; benefit extends beyond this plan (any event-consuming path becomes version-mismatch-tolerant). +- **`fabro_types::RunEvent` already tolerates unknown event names.** `EventBody::Unknown { name, properties }` at `lib/crates/fabro-types/src/run_event/mod.rs:695` is the existing fallback — a newer server's unknown event type deserializes into this variant without error. Unit 2's responsibility is to *preserve* this behavior across the `EventEnvelope { seq, #[serde(flatten)] event: RunEvent }` restructure, not to add a new variant. - **Phased delivery.** DTO lift must land before the client extraction — otherwise `fabro-client`'s deps can't be correct. We split the work into three phases (DTO lift → `fabro-client` extraction → `fabro-cli` rewire) and land each phase as a coherent chunk. Each phase compiles and tests pass at its boundary. ## Open Questions @@ -117,7 +117,7 @@ None. Internal refactor; the patterns are all established locally. - **Where do `apply_events`/`apply_event` live after `RunProjection` moves?** In `fabro-store` as an extension trait `RunProjectionReducer` implemented on `fabro_types::RunProjection`. Preserves OOP-style method-call syntax at call sites via `use fabro_store::RunProjectionReducer;`. Chosen over free functions because the workspace leans toward methods over free helpers. - **HTTP canonicalization scope in `ServerTarget::from_url`.** Preserve today's full normalization: lowercase scheme + lowercase host + strip default ports + strip trailing `/` + strip `/api/v1` suffix + rebuild authority. Matches today's `canonical_http_target` in `auth_store.rs:350-383` and the existing `https_normalization_collapses_equivalent_urls` test at line 507. Simpler "strip slash + api-v1 only" form was rejected because it would silently invalidate AuthStore entries for users whose URLs differed only in case or default-port. - **OAuthSession refresh fallback mechanism.** Add a `CredentialFallback` trait; `fabro-cli` implements it against its dev-token sources; `OAuthSession` holds `Option>` and calls `fallback.resolve()` on the failure modes that previously fell back to dev-token. Named trait (not bare `Fn`) chosen for role clarity. -- **`RunEvent` unknown-variant forward-compat.** Add an `Unknown` fallback variant in `fabro_types::RunEvent` so the `EventEnvelope` restructure doesn't regress today's accidental pass-through tolerance for unknown event types. Addressed in Unit 2. +- **`RunEvent` unknown-variant forward-compat.** Already handled today via `EventBody::Unknown` at `lib/crates/fabro-types/src/run_event/mod.rs:695` (tested at `:974`). Unit 2 verifies the `EventEnvelope` restructure preserves this existing behavior — no new variant, no new code. - **Does `EventEnvelope` need `EventPayload` to travel with it?** No. OpenAPI already defines the wire shape as `seq + RunEvent flattened`. `EventPayload` is a storage-internal validation helper and stays behind. - **Does `AuthStore` need a trait-based abstraction?** No. Concrete type with a configurable file path is sufficient; we extract a trait the day a second implementation exists. - **Does `ArtifactUpload` live in `fabro-types` or `fabro-client`?** `fabro-types`. Source is `fabro-workflow` (capture) → sink is `fabro-client` (upload). Placing the DTO in `fabro-types` prevents `fabro-workflow` from having to depend on `fabro-client`. @@ -301,7 +301,6 @@ Client::builder() Construction sites (switch to `fabro_types::EventEnvelope { seq, event }`): - Create: `lib/crates/fabro-types/src/event_envelope.rs` - Modify: `lib/crates/fabro-types/src/lib.rs` (module + re-export) -- Modify: `lib/crates/fabro-types/src/run_event/` (add `Unknown` fallback variant — exact form depends on current enum tagging) - Modify: `lib/crates/fabro-store/src/types.rs` (delete old `EventEnvelope`; keep `EventPayload` internal; update tests at lines 127 and 171) - Modify: `lib/crates/fabro-store/src/slate/run_store.rs` (production constructors at lines 194 and 342) - Modify: `lib/crates/fabro-store/src/run_state.rs` (apply_event body at line 79; test fixtures at lines 728, 741, 1086, 1112; test assertions at lines 1133, 1137) @@ -343,7 +342,7 @@ Write-path `EventPayload::new` sites (UNAFFECTED — stay in fabro-store as inte pub event: RunEvent, } ``` -- Add unknown-variant tolerance to `fabro_types::RunEvent` so the read-path typed deserialization doesn't regress today's forward-compat pass-through behavior. Shape depends on `RunEvent`'s current enum tagging — likely a catch-all `Unknown(serde_json::Value)` variant with `#[serde(other)]` or an `#[serde(untagged)]` fallback. Inspect the existing `fabro_types::RunEvent` definition during implementation and pick the minimal form that preserves lossless round-trip of unknown payloads. +- Confirm today's `EventBody::Unknown { name, properties }` fallback (at `lib/crates/fabro-types/src/run_event/mod.rs:695`) continues to round-trip through the new `EventEnvelope { seq, #[serde(flatten)] event: RunEvent }` shape — the `#[serde(flatten)]` interacts cleanly with the existing catch-all. No new variant is added; we are only confirming the existing forward-compat behavior survives. - `fabro-store` keeps `EventPayload` strictly internal — every existing write path continues to call `EventPayload::new(value, run_id)?` for `expected_run_id` validation before persisting (unchanged behavior). No write path bypasses that check. Only the external-facing read-path return type changes. - At the store's read boundary (`slate/run_store.rs:194, 342` — where the old `EventEnvelope { seq, payload }` was constructed by pairing a separately-tracked seq with decoded payload bytes): deserialize the raw bytes as `RunEvent` (the on-disk format is raw RunEvent JSON without `seq`), then wrap in `fabro_types::EventEnvelope { seq, event }`. The write path at `slate/run_store.rs:201-203` continues to persist `serde_json::to_vec(payload)?` of the `EventPayload`; the `seq` comes from a separate counter, same as today. - CLI callers that did `event.payload.as_value()` + `RunEvent::from_ref(...)` → change to `&event.event`. This is a win at every call site (fewer conversions). @@ -357,7 +356,7 @@ Write-path `EventPayload::new` sites (UNAFFECTED — stay in fabro-store as inte - Happy path: `EventEnvelope` round-trips through `serde_json` preserving both `seq` and all `RunEvent` fields, and deserializes the existing wire format identically. - Integration: existing `fabro-store` test `wire_event_envelope_round_trips` still passes after porting (adapt it to construct the new struct shape). - Integration: CLI integration tests that consume events over SSE or HTTP (e.g., `fabro-cli/tests/it/workflow/mod.rs`, `cmd/attach.rs`) continue to pass — proves the wire shape is unchanged. -- Forward-compat: an `EventEnvelope` JSON payload whose `type` (or equivalent discriminator) is unknown to this build of `fabro-types` deserializes into a `RunEvent::Unknown` fallback variant without error, and re-serializes losslessly. +- Forward-compat regression guard: an `EventEnvelope` JSON payload whose event name is unknown to this build of `fabro-types` deserializes into the existing `EventBody::Unknown { name, properties }` without error and re-serializes losslessly. (Covers: the `EventEnvelope` restructure did not break today's fallback at `run_event/mod.rs:695`.) **Verification:** - `cargo build --workspace` succeeds. From f0f04abf44aa80c69f969673a83ff1cd474612bc Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Mon, 20 Apr 2026 20:42:20 -0400 Subject: [PATCH 02/12] refactor(client): extract fabro-client crate Lift shared client DTOs into fabro-types, move auth/target/error/session logic into fabro-client, and reduce fabro-cli to orchestration around the builder-based client path. This also lands the remaining plan cleanup for ApiError, ServerTarget canonicalization, and the RunEventStream rename at the CLI boundary. --- Cargo.lock | 27 + ...efactor-extract-fabro-client-crate-plan.md | 25 +- lib/crates/fabro-cli/Cargo.toml | 1 + .../fabro-cli/src/commands/auth/login.rs | 24 +- .../fabro-cli/src/commands/auth/logout.rs | 74 +- .../fabro-cli/src/commands/auth/status.rs | 15 +- lib/crates/fabro-cli/src/commands/doctor.rs | 2 +- lib/crates/fabro-cli/src/commands/exec.rs | 4 +- .../fabro-cli/src/commands/run/attach.rs | 14 +- .../fabro-cli/src/commands/run/create.rs | 9 +- lib/crates/fabro-cli/src/commands/run/logs.rs | 8 +- .../fabro-cli/src/commands/run/runner.rs | 15 +- .../fabro-cli/src/commands/store/rebuild.rs | 2 +- .../fabro-cli/src/commands/system/events.rs | 2 +- lib/crates/fabro-cli/src/commands/version.rs | 10 +- lib/crates/fabro-cli/src/main.rs | 3 - lib/crates/fabro-cli/src/server_client.rs | 1750 ++--------------- lib/crates/fabro-cli/src/server_runs.rs | 3 +- lib/crates/fabro-cli/src/user_config.rs | 73 +- lib/crates/fabro-cli/tests/it/cmd/attach.rs | 1 + lib/crates/fabro-cli/tests/it/cmd/rewind.rs | 17 +- lib/crates/fabro-cli/tests/it/cmd/runner.rs | 2 +- lib/crates/fabro-cli/tests/it/cmd/support.rs | 9 +- lib/crates/fabro-cli/tests/it/workflow/mod.rs | 11 +- lib/crates/fabro-client/Cargo.toml | 37 + .../src/auth_store.rs | 229 +-- lib/crates/fabro-client/src/client.rs | 1392 +++++++++++++ lib/crates/fabro-client/src/credential.rs | 40 + lib/crates/fabro-client/src/error.rs | 184 ++ lib/crates/fabro-client/src/lib.rs | 26 + .../src/loopback.rs} | 64 +- lib/crates/fabro-client/src/session.rs | 48 + .../{fabro-cli => fabro-client}/src/sse.rs | 2 +- lib/crates/fabro-client/src/target.rs | 220 +++ lib/crates/fabro-retro/src/retro_agent.rs | 2 +- lib/crates/fabro-server/src/server.rs | 48 +- lib/crates/fabro-store/src/lib.rs | 8 +- lib/crates/fabro-store/src/run_state.rs | 387 ++-- lib/crates/fabro-store/src/slate/mod.rs | 9 +- lib/crates/fabro-store/src/slate/run_store.rs | 12 +- lib/crates/fabro-store/src/types.rs | 130 +- lib/crates/fabro-types/src/artifact.rs | 30 + lib/crates/fabro-types/src/event_envelope.rs | 135 ++ lib/crates/fabro-types/src/lib.rs | 8 + lib/crates/fabro-types/src/run_projection.rs | 109 + lib/crates/fabro-types/src/run_summary.rs | 56 + .../fabro-workflow/src/artifact_snapshot.rs | 22 +- .../fabro-workflow/src/artifact_upload.rs | 6 +- lib/crates/fabro-workflow/src/event.rs | 2 +- lib/crates/fabro-workflow/src/lib.rs | 18 +- .../fabro-workflow/src/lifecycle/artifact.rs | 10 +- .../fabro-workflow/src/operations/create.rs | 5 +- lib/crates/fabro-workflow/src/run_lookup.rs | 4 +- 53 files changed, 2890 insertions(+), 2454 deletions(-) create mode 100644 lib/crates/fabro-client/Cargo.toml rename lib/crates/{fabro-cli => fabro-client}/src/auth_store.rs (68%) create mode 100644 lib/crates/fabro-client/src/client.rs create mode 100644 lib/crates/fabro-client/src/credential.rs create mode 100644 lib/crates/fabro-client/src/error.rs create mode 100644 lib/crates/fabro-client/src/lib.rs rename lib/crates/{fabro-cli/src/loopback_target.rs => fabro-client/src/loopback.rs} (71%) create mode 100644 lib/crates/fabro-client/src/session.rs rename lib/crates/{fabro-cli => fabro-client}/src/sse.rs (94%) create mode 100644 lib/crates/fabro-client/src/target.rs create mode 100644 lib/crates/fabro-types/src/artifact.rs create mode 100644 lib/crates/fabro-types/src/event_envelope.rs create mode 100644 lib/crates/fabro-types/src/run_projection.rs create mode 100644 lib/crates/fabro-types/src/run_summary.rs diff --git a/Cargo.lock b/Cargo.lock index c779b34bf..7af5fd082 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1616,6 +1616,7 @@ dependencies = [ "fabro-api", "fabro-auth", "fabro-checkpoint", + "fabro-client", "fabro-config", "fabro-devcontainer", "fabro-github", @@ -1682,6 +1683,32 @@ dependencies = [ "x509-parser", ] +[[package]] +name = "fabro-client" +version = "0.208.0-nightly.1" +dependencies = [ + "anyhow", + "bytes", + "chrono", + "fabro-api", + "fabro-http", + "fabro-model", + "fabro-types", + "fabro-util", + "fs2", + "futures", + "libc", + "progenitor-client", + "rand 0.9.4", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "fabro-config" version = "0.208.0-nightly.1" diff --git a/docs/plans/2026-04-20-002-refactor-extract-fabro-client-crate-plan.md b/docs/plans/2026-04-20-002-refactor-extract-fabro-client-crate-plan.md index 9a431d4f5..04faa0652 100644 --- a/docs/plans/2026-04-20-002-refactor-extract-fabro-client-crate-plan.md +++ b/docs/plans/2026-04-20-002-refactor-extract-fabro-client-crate-plan.md @@ -41,6 +41,19 @@ These were all reasonable while the client had exactly one caller. They prevent - R8. `fabro-cli` keeps CLI-owned orchestration: subprocess autostart, server-record lookup, dev-token-from-disk loading, `[cli.target]` TOML resolution, and the wrapper that stitches these together into a ready-to-use `fabro_client::Client`. - R9. The full workspace build passes (`cargo build --workspace`), clippy is clean (`cargo +nightly-2026-04-14 clippy --workspace --all-targets -- -D warnings`), rustfmt check passes, and all workspace tests continue to pass (`cargo nextest run --workspace`). +## Review Adjustments + +- Remaining gaps against this plan in the current landed code: + - `HttpResponseFailure` → `ApiError` rename is still open. + - `ServerTarget` canonical-by-construction / lexical-only Unix-path canonicalization is still open; canonicalization still partly lives in `AuthStore`. + - `RunAttachEventStream` → `RunEventStream` rename is still open at the CLI boundary. + - Because the two rename cleanups above are still open, Unit 11's "no leaked old names" grep is currently expected to fail until that follow-up cleanup lands. +- Clarifications after implementation review: + - `RunProjection` ownership moved to `fabro-types`; external callers may continue to import it through `fabro_store`'s re-export. Verification should check type ownership and dependency boundaries, not the literal import spelling. + - `Client::from_http_client(...)` is the required public constructor. `Client::new_no_proxy(...)` may remain as a convenience wrapper, primarily for tests. + - `TransportConnector` is an acceptable internal helper inside `fabro-client` when needed to preserve caller-specific transport configuration across refresh/rebuild flows. + - `fabro-cli` may retain multiple thin `connect_*` convenience wrappers so long as they preserve CLI-only orchestration and funnel into `Client::builder()` under the hood. + ## Scope Boundaries **In scope:** @@ -96,7 +109,8 @@ None. Internal refactor; the patterns are all established locally. - **`ServerTarget` canonical-by-construction, lexical only.** `ServerTarget::from_url` applies today's full HTTP canonicalization inline: lowercase scheme, lowercase host, strip default ports (`:443` on https, `:80` on http), strip trailing `/`, strip `/api/v1` suffix, rebuild authority as `{scheme}://{host}[:{port}]`. `ServerTarget::from_unix_path` applies lexical `.`/`..` resolution (no FS access, no symlink chasing). `PartialEq`/`Eq`/`Hash` operate on the canonical form directly. `ServerTargetKey` is deleted; `ServerTarget` itself serves as the `AuthStore` map key. In OOP style: construction *is* the canonicalizer — no separate `canonicalize()` method. Related helpers attach to `ServerTarget` as inherent methods (`target.loopback_classification()`, `target.build_public_http_client()`) rather than free functions. - **`AuthStore` moves to `fabro-client`.** Default path remains `~/.fabro/auth.json` via `fabro_util::Home`. The public API (`get`/`put`/`remove`/`list`) is narrow enough that we keep it concrete — no `TokenRefresher` trait abstraction. Callers wanting alternative storage can pass an explicit path to `AuthStore::new`. If external-consumer flexibility becomes a real need later, we extract a trait then, not now. - **Renames applied inline with moves.** We don't do a separate rename pass — the DTOs and internal types are renamed as they move. This keeps the compiler-driven find-all-callers loop honest: every broken import is both a move and a rename in one commit. -- **Connection API collapses to one builder.** Today's four `connect_*` functions in `server_client.rs` are all CLI-opinionated. `fabro-client` exposes a single `Client::builder().target(t).credential(c).oauth_session(s).connect().await?`. The CLI's `connect_server_with_settings` becomes a thin orchestrator: resolve target → autostart if needed → build `Credential` from dev-token/env/AuthStore → call `Client::builder()`. +- **Connection API centers on one builder.** Today's `connect_*` functions in `server_client.rs` are CLI-opinionated convenience wrappers. `fabro-client` exposes `Client::builder().target(t).credential(c).oauth_session(s).connect().await?` as the underlying connection API. `fabro-cli` may keep several thin `connect_*` wrappers, but they must remain orchestration-only and funnel into the builder instead of duplicating transport/session assembly logic. +- **Refresh rebuilds may use a transport connector hook.** If the CLI needs request-transport customization across OAuth refresh rebuilds (for example, preserving a CLI-specific user-agent), `fabro-client` may carry a `TransportConnector`-style helper as an internal implementation detail. This does not count as a second public connection API. - **`OAuthSession` refresh fallback via `CredentialFallback` trait.** Today's refresh flow falls back to a dev-token-from-disk when the OAuth entry is missing, expired, or revoked. That fallback lookup reads CLI-owned sources (`FABRO_DEV_TOKEN` env, `~/.fabro/dev-token`, storage-dir dev-token file, fabro-server pidfile record) that don't belong in `fabro-client`. `OAuthSession` takes an optional `Box` at build time: ```text @@ -121,7 +135,7 @@ None. Internal refactor; the patterns are all established locally. - **Does `EventEnvelope` need `EventPayload` to travel with it?** No. OpenAPI already defines the wire shape as `seq + RunEvent flattened`. `EventPayload` is a storage-internal validation helper and stays behind. - **Does `AuthStore` need a trait-based abstraction?** No. Concrete type with a configurable file path is sufficient; we extract a trait the day a second implementation exists. - **Does `ArtifactUpload` live in `fabro-types` or `fabro-client`?** `fabro-types`. Source is `fabro-workflow` (capture) → sink is `fabro-client` (upload). Placing the DTO in `fabro-types` prevents `fabro-workflow` from having to depend on `fabro-client`. -- **How do we handle the `Client::new_no_proxy(base_url)` constructor used by CLI tests today?** Expose `Client::from_http_client(base_url, http_client)` as a public `pub fn`; the CLI's `new_no_proxy` wrapper stays in `fabro-cli` test code. +- **How do we handle the `Client::new_no_proxy(base_url)` constructor used by CLI tests today?** Expose `Client::from_http_client(base_url, http_client)` as the stable public `pub fn`. `Client::new_no_proxy(base_url)` may remain as a small convenience wrapper (in `fabro-client` or CLI-local test code) if it continues to earn its keep. - **`convert_type` serde round-trip helper — does it stay in the CLI or move with the client?** Moves with the client. It's how the client bridges `fabro_api::types::RunSummary` (wire) → `fabro_types::RunSummary` (domain) at response boundaries. ### Deferred to Implementation @@ -421,7 +435,7 @@ Write-path `EventPayload::new` sites (UNAFFECTED — stay in fabro-store as inte **Verification:** - `cargo build --workspace` succeeds. - `cargo nextest run --workspace` passes. -- `grep -rn "fabro_store::RunProjection" lib/` returns only re-export lines and internal fabro-store uses; external callers use `fabro_types::RunProjection`. +- `grep -rn "struct RunProjection" lib/crates/fabro-store lib/crates/fabro-types` shows the concrete struct definition only in `fabro-types`; external callers may import either `fabro_types::RunProjection` or the `fabro-store` re-export. --- @@ -628,11 +642,12 @@ Write-path `EventPayload::new` sites (UNAFFECTED — stay in fabro-store as inte } ``` The old `ClientBundle` name disappears; `ClientState` is private to the module. -- `Client::builder()`: new public API. Replaces today's four `connect_*` functions that blend CLI opinions with transport. The CLI's own `connect_server_with_settings` becomes a thin orchestrator over `Client::builder()` (handled in Unit 9). +- `Client::builder()`: new public API. It becomes the underlying connection API. The CLI may keep thin `connect_*` orchestration wrappers around it (handled in Unit 9), but transport/session assembly should live in the builder path rather than being duplicated across wrappers. - `RunEventStream` rename: the struct, its `next_event`/`buffer_sse_events` methods, and the `VecDeque` field. `EventEnvelope` is now `fabro_types::EventEnvelope`. - Method bodies: the 40 wrappers move verbatim. They call `send_api(|client| ...)` — `client` is the `fabro_api::ApiClient` from `ClientState`. `convert_type::<_, fabro_types::RunSummary>(...)` continues to bridge wire → domain. -- `Client::from_http_client(base_url, http_client)` — public `pub fn` constructor for test use (replaces today's `new_no_proxy`). The CLI's test code can still build one via this with a `no_proxy()` builder. +- `Client::from_http_client(base_url, http_client)` — public `pub fn` constructor for test use and non-builder callers. `Client::new_no_proxy(base_url)` may remain as a small convenience wrapper built on top of it. - Preserve `send_api`'s 401 → refresh → retry auto-logic. `OAuthSession` owns the refresh state it needs (`target`, `auth_store`, optional `fallback`); the actual refresh HTTP call uses a bespoke HTTP client built via `target.build_public_http_client()` (method on `ServerTarget`, not a free function — OOP style). +- If preserving caller-specific transport behavior across refresh rebuilds requires it, `Client` may carry an internal `TransportConnector` helper that can rebuild the transport with the same customization after credentials change. - `CredentialFallback` trait lives in `fabro-client::credential`: ```text // Directional — not implementation diff --git a/lib/crates/fabro-cli/Cargo.toml b/lib/crates/fabro-cli/Cargo.toml index 6aadb6032..4687f7df9 100644 --- a/lib/crates/fabro-cli/Cargo.toml +++ b/lib/crates/fabro-cli/Cargo.toml @@ -39,6 +39,7 @@ graphviz-sys.workspace = true fabro-validate = { path = "../fabro-validate" } fabro-workflow = { path = "../fabro-workflow" } fabro-server = { path = "../fabro-server" } +fabro-client = { path = "../fabro-client" } fabro-api = { path = "../fabro-api" } fabro-telemetry = { path = "../fabro-telemetry" } fabro-store = { path = "../fabro-store" } diff --git a/lib/crates/fabro-cli/src/commands/auth/login.rs b/lib/crates/fabro-cli/src/commands/auth/login.rs index b71437d1f..227ff8936 100644 --- a/lib/crates/fabro-cli/src/commands/auth/login.rs +++ b/lib/crates/fabro-cli/src/commands/auth/login.rs @@ -3,6 +3,7 @@ use std::time::Duration; use anyhow::{Context as _, Result, anyhow, bail}; use chrono::{DateTime, Utc}; use fabro_api::types; +use fabro_client::{AuthEntry, AuthStore, StoredSubject}; use fabro_http::header::CONTENT_TYPE; use fabro_types::settings::CliSettings; use fabro_types::settings::cli::CliLayer; @@ -11,9 +12,7 @@ use serde::Deserialize; use tokio::time::timeout; use crate::args::{AuthLoginArgs, require_no_json_override}; -use crate::auth_store::{AuthEntry, AuthStore, ServerTargetKey, StoredSubject}; use crate::command_context::CommandContext; -use crate::loopback_target::{LoopbackClassification, is_loopback_or_unix_socket}; use crate::user_config; use crate::user_config::ServerTarget; @@ -56,7 +55,6 @@ pub(super) async fn login_command( { let ctx = CommandContext::base(printer, cli.clone(), cli_layer)?; let target = user_config::resolve_server_target(&args.server, ctx.machine_settings())?; - let server_key = ServerTargetKey::new(&target)?; let config = fetch_cli_auth_config(&target).await?; if !config.enabled { bail!("{}", cli_auth_unavailable_message(config.reason.as_deref())); @@ -98,11 +96,11 @@ pub(super) async fn login_command( } }; - match is_loopback_or_unix_socket(&target)? { - LoopbackClassification::Https - | LoopbackClassification::LoopbackHttp - | LoopbackClassification::UnixSocket => {} - LoopbackClassification::Rejected => { + match target.loopback_classification()? { + fabro_client::LoopbackClassification::Https + | fabro_client::LoopbackClassification::LoopbackHttp + | fabro_client::LoopbackClassification::UnixSocket => {} + fabro_client::LoopbackClassification::Rejected => { bail!("{}", token_transport_error(&target)); } } @@ -123,8 +121,8 @@ pub(super) async fn login_command( logged_in_at: Utc::now(), }; let summary = identity_summary(&entry.subject); - AuthStore::default().put(&server_key, entry)?; - fabro_util::printerr!(printer, "Logged in to {} as {}", server_key, summary); + AuthStore::default().put(&target, entry)?; + fabro_util::printerr!(printer, "Logged in to {} as {}", target, summary); Ok(()) } } @@ -270,11 +268,11 @@ struct OAuthErrorBody { mod tests { use base64::Engine as _; use base64::engine::general_purpose::URL_SAFE_NO_PAD; + use fabro_client::LoopbackClassification; use insta::assert_snapshot; use sha2::{Digest, Sha256}; use super::{build_browser_url, cli_auth_unavailable_message, login_failure_message}; - use crate::loopback_target::{LoopbackClassification, is_loopback_or_unix_socket}; use crate::user_config::ServerTarget; #[test] @@ -343,9 +341,9 @@ mod tests { #[test] fn token_transport_accepts_only_https_loopback_or_unix() { - let target = ServerTarget::HttpUrl("https://fabro.example.com".to_string()); + let target = ServerTarget::http_url("https://fabro.example.com").unwrap(); assert_eq!( - is_loopback_or_unix_socket(&target).unwrap(), + target.loopback_classification().unwrap(), LoopbackClassification::Https ); } diff --git a/lib/crates/fabro-cli/src/commands/auth/logout.rs b/lib/crates/fabro-cli/src/commands/auth/logout.rs index db18379fe..4743eb42b 100644 --- a/lib/crates/fabro-cli/src/commands/auth/logout.rs +++ b/lib/crates/fabro-cli/src/commands/auth/logout.rs @@ -1,11 +1,11 @@ use anyhow::{Result, bail}; +use fabro_client::{AuthEntry, AuthStore}; use fabro_http::header::AUTHORIZATION; use fabro_types::settings::CliSettings; use fabro_types::settings::cli::CliLayer; use fabro_util::printer::Printer; use crate::args::{AuthLogoutArgs, require_no_json_override}; -use crate::auth_store::{AuthEntry, AuthStore, ServerTargetKey}; use crate::command_context::CommandContext; use crate::user_config; use crate::user_config::ServerTarget; @@ -29,13 +29,11 @@ pub(super) async fn logout_command( } let mut warnings = Vec::new(); - for (key, entry) in entries { - if let Ok(target) = server_target_from_key(&key) { - if let Err(error) = revoke_remote_session(&target, &entry).await { - warnings.push(format_warning(&key, &error.to_string())); - } + for (target, entry) in entries { + if let Err(error) = revoke_remote_session(&target, &entry).await { + warnings.push(format_warning(&target, &error.to_string())); } - store.remove(&key)?; + store.remove(&target)?; } for warning in warnings { @@ -46,17 +44,16 @@ pub(super) async fn logout_command( } let target = user_config::resolve_server_target(&args.server, ctx.machine_settings())?; - let key = ServerTargetKey::new(&target)?; - let Some(entry) = store.get(&key)? else { - fabro_util::printerr!(printer, "Not logged in to {}.", key); + let Some(entry) = store.get(&target)? else { + fabro_util::printerr!(printer, "Not logged in to {}.", target); return Ok(()); }; if let Err(error) = revoke_remote_session(&target, &entry).await { - fabro_util::printerr!(printer, "{}", format_warning(&key, &error.to_string())); + fabro_util::printerr!(printer, "{}", format_warning(&target, &error.to_string())); } - store.remove(&key)?; - fabro_util::printerr!(printer, "Logged out from {}.", key); + store.remove(&target)?; + fabro_util::printerr!(printer, "Logged out from {}.", target); Ok(()) } @@ -79,62 +76,21 @@ async fn revoke_remote_session(target: &ServerTarget, entry: &AuthEntry) -> Resu bail!("request failed with status {status}: {body}") } -fn server_target_from_key(key: &ServerTargetKey) -> Result { - let value = key.to_string(); - if let Some(path) = value.strip_prefix("unix://") { - return Ok(ServerTarget::UnixSocket(path.into())); - } - if value.starts_with("http://") || value.starts_with("https://") { - return Ok(ServerTarget::HttpUrl(value)); - } - bail!("invalid auth store server key `{value}`") -} - -fn format_warning(key: &ServerTargetKey, error: &str) -> String { +fn format_warning(target: &ServerTarget, error: &str) -> String { format!( - "Warning: removed local session for {key}, but remote revocation failed: {error}. The refresh token may remain valid until it expires." + "Warning: removed local session for {target}, but remote revocation failed: {error}. The refresh token may remain valid until it expires." ) } #[cfg(test)] mod tests { - use std::path::PathBuf; - - use super::{format_warning, server_target_from_key}; - use crate::auth_store::ServerTargetKey; + use super::format_warning; use crate::user_config::ServerTarget; - #[test] - fn rebuilds_server_target_from_http_key() { - let key = ServerTargetKey::new(&ServerTarget::HttpUrl( - "https://fabro.example.com/api/v1".to_string(), - )) - .unwrap(); - - assert_eq!( - server_target_from_key(&key).unwrap(), - ServerTarget::HttpUrl("https://fabro.example.com".to_string()) - ); - } - - #[test] - fn rebuilds_server_target_from_unix_key() { - let key = ServerTargetKey::new(&ServerTarget::UnixSocket(PathBuf::from("/tmp/fabro.sock"))) - .unwrap(); - - assert_eq!( - server_target_from_key(&key).unwrap(), - ServerTarget::UnixSocket(PathBuf::from("/tmp/fabro.sock")) - ); - } - #[test] fn warning_mentions_local_removal_and_remote_failure() { - let key = ServerTargetKey::new(&ServerTarget::HttpUrl( - "https://fabro.example.com".to_string(), - )) - .unwrap(); - let warning = format_warning(&key, "request failed with status 500"); + let target = ServerTarget::http_url("https://fabro.example.com").unwrap(); + let warning = format_warning(&target, "request failed with status 500"); assert!(warning.contains("removed local session")); assert!(warning.contains("remote revocation failed")); } diff --git a/lib/crates/fabro-cli/src/commands/auth/status.rs b/lib/crates/fabro-cli/src/commands/auth/status.rs index ccdcc2023..de81e60c2 100644 --- a/lib/crates/fabro-cli/src/commands/auth/status.rs +++ b/lib/crates/fabro-cli/src/commands/auth/status.rs @@ -1,5 +1,6 @@ use anyhow::Result; use chrono::{DateTime, Utc}; +use fabro_client::{AuthEntry, AuthStore}; use fabro_types::settings::CliSettings; use fabro_types::settings::cli::CliLayer; use fabro_util::dev_token::{read_dev_token_file, validate_dev_token_format}; @@ -7,7 +8,6 @@ use fabro_util::printer::Printer; use serde::Serialize; use crate::args::AuthStatusArgs; -use crate::auth_store::{AuthEntry, AuthStore, ServerTargetKey}; use crate::command_context::CommandContext; use crate::shared::print_json_pretty; use crate::user_config; @@ -126,7 +126,7 @@ fn all_rows(store: &AuthStore, now: DateTime) -> Result> { Ok(store .list()? .into_iter() - .map(|(key, entry)| status_row(&key, entry, now)) + .map(|(target, entry)| status_row(&target, entry, now)) .collect()) } @@ -135,17 +135,16 @@ fn filter_rows( target: &ServerTarget, now: DateTime, ) -> Result> { - let key = ServerTargetKey::new(target)?; Ok(store - .get(&key)? + .get(target)? .into_iter() - .map(|entry| status_row(&key, entry, now)) + .map(|entry| status_row(target, entry, now)) .collect()) } -fn status_row(key: &ServerTargetKey, entry: AuthEntry, now: DateTime) -> StatusRow { +fn status_row(target: &ServerTarget, entry: AuthEntry, now: DateTime) -> StatusRow { StatusRow { - server: key.to_string(), + server: target.to_string(), oauth_state: oauth_state(&entry, now), access_token_expires_at: entry.access_token_expires_at, refresh_token_expires_at: entry.refresh_token_expires_at, @@ -187,9 +186,9 @@ fn load_dev_token_if_available() -> bool { #[cfg(test)] mod tests { use chrono::Duration; + use fabro_client::{AuthEntry, StoredSubject}; use super::{OAuthState, human_state, oauth_state}; - use crate::auth_store::{AuthEntry, StoredSubject}; fn entry(access_offset_secs: i64, refresh_offset_secs: i64) -> AuthEntry { let now = chrono::Utc::now(); diff --git a/lib/crates/fabro-cli/src/commands/doctor.rs b/lib/crates/fabro-cli/src/commands/doctor.rs index f7543e41d..2a5fc1ed5 100644 --- a/lib/crates/fabro-cli/src/commands/doctor.rs +++ b/lib/crates/fabro-cli/src/commands/doctor.rs @@ -343,7 +343,7 @@ pub(crate) async fn run_doctor( checks: vec![CheckResult { name: "Location".to_string(), status: CheckStatus::Pass, - summary: server.base_url().to_string(), + summary: server.base_url().clone(), details: vec![], remediation: None, }], diff --git a/lib/crates/fabro-cli/src/commands/exec.rs b/lib/crates/fabro-cli/src/commands/exec.rs index 7d3026557..2a63e9337 100644 --- a/lib/crates/fabro-cli/src/commands/exec.rs +++ b/lib/crates/fabro-cli/src/commands/exec.rs @@ -115,7 +115,7 @@ struct AuthenticatedFabroServerAdapter { impl AuthenticatedFabroServerAdapter { fn new(client: server_client::Client, provider_name: impl Into) -> Self { - let base_url = client.base_url().to_string(); + let base_url = client.base_url().clone(); Self { client, base_url, @@ -212,7 +212,7 @@ fn transport_error(provider: &str, err: &anyhow::Error) -> LlmError { } } -fn map_response_failure(provider: &str, failure: &server_client::HttpResponseFailure) -> LlmError { +fn map_response_failure(provider: &str, failure: &fabro_client::ApiError) -> LlmError { let retry_after = parse_retry_after(&failure.headers); let (message, code, raw) = parse_server_error_body(&failure.body); error_from_status_code( diff --git a/lib/crates/fabro-cli/src/commands/run/attach.rs b/lib/crates/fabro-cli/src/commands/run/attach.rs index c39fc605c..5b3eca612 100644 --- a/lib/crates/fabro-cli/src/commands/run/attach.rs +++ b/lib/crates/fabro-cli/src/commands/run/attach.rs @@ -21,7 +21,7 @@ use fabro_interview::{AnswerValue, ConsoleInterviewer, Question, QuestionOption, use fabro_store::EventEnvelope; use fabro_types::settings::cli::OutputVerbosity; use fabro_types::settings::run::ApprovalMode; -use fabro_types::{EventBody, RunEvent, RunId}; +use fabro_types::{EventBody, RunId}; use fabro_util::json::normalize_json_value; use fabro_util::printer::Printer; use fabro_util::terminal::Styles; @@ -155,7 +155,7 @@ async fn attach_live_run_with_client( client: &server_client::Client, run_id: &RunId, existing_events: Vec, - mut stream: server_client::RunAttachEventStream, + mut stream: server_client::RunEventStream, styles: &'static Styles, opts: AttachOptions, printer: Printer, @@ -392,7 +392,7 @@ fn show_progress(progress_ui: &mut run_progress::ProgressUI, json_output: bool) } fn event_payload_line(event: &EventEnvelope) -> Result { - let mut value = normalize_json_value(event.payload.as_value().clone()); + let mut value = normalize_json_value(event.event.to_value()?); restore_empty_run_properties(&mut value); serde_json::to_string(&value).map_err(Into::into) } @@ -462,8 +462,7 @@ fn state_exit_code(state: &server_client::RunProjection) -> Option { } fn event_exit_code(event: &EventEnvelope) -> Option { - let run_event = RunEvent::try_from(&event.payload).ok()?; - match run_event.body { + match &event.event.body { EventBody::RunCompleted(props) => Some( if props.status == "success" || props.status == "partial_success" { ExitCode::from(0) @@ -477,10 +476,7 @@ fn event_exit_code(event: &EventEnvelope) -> Option { } fn event_starts_interview(event: &EventEnvelope) -> bool { - let Ok(run_event) = RunEvent::try_from(&event.payload) else { - return false; - }; - matches!(run_event.body, EventBody::InterviewStarted(_)) + matches!(event.event.body, EventBody::InterviewStarted(_)) } #[cfg(test)] diff --git a/lib/crates/fabro-cli/src/commands/run/create.rs b/lib/crates/fabro-cli/src/commands/run/create.rs index 48685ee1d..6ee251680 100644 --- a/lib/crates/fabro-cli/src/commands/run/create.rs +++ b/lib/crates/fabro-cli/src/commands/run/create.rs @@ -73,14 +73,15 @@ pub(crate) async fn create_run( } let created_run_id = client.create_run_from_manifest(built.manifest).await?; - let local_run_dir = match &target { - ServerTarget::UnixSocket(_) => Some( + let local_run_dir = if target.is_unix_socket() { + Some( Storage::new(user_config::storage_dir(ctx.machine_settings())?) .run_scratch(&created_run_id) .root() .to_path_buf(), - ), - ServerTarget::HttpUrl(_) => None, + ) + } else { + None }; Ok(CreatedRun { diff --git a/lib/crates/fabro-cli/src/commands/run/logs.rs b/lib/crates/fabro-cli/src/commands/run/logs.rs index dae03d121..a732bc023 100644 --- a/lib/crates/fabro-cli/src/commands/run/logs.rs +++ b/lib/crates/fabro-cli/src/commands/run/logs.rs @@ -87,8 +87,8 @@ pub(crate) async fn run( Ok(()) } -fn event_name(event: &fabro_store::EventEnvelope) -> Option<&str> { - event.payload.as_value().get("event")?.as_str() +fn event_name(event: &fabro_store::EventEnvelope) -> &str { + event.event.event_name() } fn apply_filters( @@ -175,7 +175,7 @@ async fn follow_store_logs( let had_events = !events.is_empty(); let saw_terminal = events .iter() - .any(|event| matches!(event_name(event), Some("run.completed" | "run.failed"))); + .any(|event| matches!(event_name(event), "run.completed" | "run.failed")); for event in events { let line = event_payload_line(&event)?; if pretty { @@ -268,7 +268,7 @@ async fn flush_remaining_store_events( } fn event_payload_line(event: &fabro_store::EventEnvelope) -> Result { - let mut value = normalize_json_value(event.payload.as_value().clone()); + let mut value = normalize_json_value(event.event.to_value()?); restore_empty_run_properties(&mut value); let line = serde_json::to_string(&value)?; Ok(redact_jsonl_line(&line)) diff --git a/lib/crates/fabro-cli/src/commands/run/runner.rs b/lib/crates/fabro-cli/src/commands/run/runner.rs index bfc4e60be..7b6033dce 100644 --- a/lib/crates/fabro-cli/src/commands/run/runner.rs +++ b/lib/crates/fabro-cli/src/commands/run/runner.rs @@ -14,12 +14,11 @@ use anyhow::{Context, Result, anyhow}; use async_trait::async_trait; use fabro_config::Storage; use fabro_interview::{ControlInterviewer, WorkerControlEnvelope, WorkerControlMessage}; -use fabro_store::{EventEnvelope, EventPayload, RunProjection}; +use fabro_store::{EventEnvelope, RunProjection, RunProjectionReducer}; use fabro_types::settings::run::RunMode; use fabro_types::settings::{InterpString, SettingsLayer}; -use fabro_types::{EventBody, RunBlobId, RunEvent, RunId, StatusReason}; +use fabro_types::{ArtifactUpload, EventBody, RunBlobId, RunEvent, RunId, StatusReason}; use fabro_vault::Vault; -use fabro_workflow::artifact_snapshot::CapturedArtifactInfo; use fabro_workflow::artifact_upload::{ArtifactSink, StageArtifactUploader}; use fabro_workflow::event::{Emitter, RunEventSink}; use fabro_workflow::operations::{self, StartServices}; @@ -264,7 +263,7 @@ impl StageArtifactUploader for HttpArtifactUploader { &self, stage_id: &fabro_types::StageId, artifact_capture_dir: &Path, - artifacts: &[CapturedArtifactInfo], + artifacts: &[ArtifactUpload], ) -> Result<()> { if artifacts.is_empty() { return Ok(()); @@ -306,7 +305,7 @@ impl StageArtifactUploader for MissingArtifactUploadTokenUploader { &self, _stage_id: &fabro_types::StageId, _artifact_capture_dir: &Path, - _artifacts: &[CapturedArtifactInfo], + _artifacts: &[ArtifactUpload], ) -> Result<()> { Err(anyhow!( "run {} could not upload artifacts because the worker did not receive an artifact upload token", @@ -369,8 +368,10 @@ impl HttpRunStore { } async fn apply_acknowledged_event(&self, seq: u32, event: &RunEvent) -> Result<()> { - let payload = EventPayload::new(event.to_value()?, &self.run_id)?; - let envelope = EventEnvelope { seq, payload }; + let envelope = EventEnvelope { + seq, + event: event.clone(), + }; { let mut state = self.state.lock().await; diff --git a/lib/crates/fabro-cli/src/commands/store/rebuild.rs b/lib/crates/fabro-cli/src/commands/store/rebuild.rs index 91003770b..526fc8ff2 100644 --- a/lib/crates/fabro-cli/src/commands/store/rebuild.rs +++ b/lib/crates/fabro-cli/src/commands/store/rebuild.rs @@ -17,7 +17,7 @@ pub(crate) async fn rebuild_run_store( )); let run_store = store.create_run(run_id).await?; for event in events { - let payload = EventPayload::new(event.payload.as_value().clone(), run_id)?; + let payload = EventPayload::new(event.event.to_value()?, run_id)?; run_store.append_event(&payload).await?; } Ok(run_store) diff --git a/lib/crates/fabro-cli/src/commands/system/events.rs b/lib/crates/fabro-cli/src/commands/system/events.rs index bba7bebc3..da5bdad57 100644 --- a/lib/crates/fabro-cli/src/commands/system/events.rs +++ b/lib/crates/fabro-cli/src/commands/system/events.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use fabro_client::sse; use fabro_types::settings::CliSettings; use fabro_types::settings::cli::{CliLayer, OutputFormat}; use fabro_util::printer::Printer; @@ -6,7 +7,6 @@ use futures::StreamExt; use crate::args::SystemEventsArgs; use crate::command_context::CommandContext; -use crate::sse; pub(super) async fn events_command( args: &SystemEventsArgs, diff --git a/lib/crates/fabro-cli/src/commands/version.rs b/lib/crates/fabro-cli/src/commands/version.rs index a4eeb3dcd..a58e0541f 100644 --- a/lib/crates/fabro-cli/src/commands/version.rs +++ b/lib/crates/fabro-cli/src/commands/version.rs @@ -132,9 +132,13 @@ fn is_non_release_profile(profile: &str) -> bool { } fn format_server_target(target: &ServerTarget) -> String { - match target { - ServerTarget::HttpUrl(api_url) => api_url.clone(), - ServerTarget::UnixSocket(path) => path.display().to_string(), + if let Some(api_url) = target.as_http_url() { + api_url.to_string() + } else { + target + .as_unix_socket_path() + .map(|path| path.display().to_string()) + .unwrap_or_default() } } diff --git a/lib/crates/fabro-cli/src/main.rs b/lib/crates/fabro-cli/src/main.rs index 4d7924742..fc64910b4 100644 --- a/lib/crates/fabro-cli/src/main.rs +++ b/lib/crates/fabro-cli/src/main.rs @@ -4,20 +4,17 @@ )] mod args; -mod auth_store; mod command_context; mod commands; mod gh; mod landing; mod logging; -mod loopback_target; mod manifest_builder; mod server_client; mod server_runs; mod shared; #[cfg(feature = "sleep_inhibitor")] mod sleep_inhibitor; -mod sse; mod user_config; #[cfg(test)] diff --git a/lib/crates/fabro-cli/src/server_client.rs b/lib/crates/fabro-cli/src/server_client.rs index 730dc1db8..4f12f2c0c 100644 --- a/lib/crates/fabro-cli/src/server_client.rs +++ b/lib/crates/fabro-cli/src/server_client.rs @@ -1,199 +1,63 @@ -use std::collections::VecDeque; -use std::num::NonZeroU64; -use std::path::Path; -use std::sync::{Arc, RwLock}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; use std::time::Duration; use anyhow::{Context as _, Result, anyhow, bail}; -use bytes::Bytes; -use fabro_api::types; +use fabro_client::{ + AuthStore, Credential, CredentialFallback, OAuthSession, ServerTarget, TransportConnector, +}; +pub(crate) use fabro_client::{Client, RunEventStream}; use fabro_config::Storage; -use fabro_http::header::{AUTHORIZATION, CONTENT_LENGTH, CONTENT_TYPE}; -use fabro_http::multipart::{Form, Part}; -use fabro_model::Model; +use fabro_http::header::AUTHORIZATION; use fabro_server::bind::Bind; -use fabro_store::{EventEnvelope, RunSummary, StageId}; +pub(crate) use fabro_types::RunProjection; use fabro_types::settings::SettingsLayer; -use fabro_types::{RunBlobId, RunEvent, RunId}; use fabro_util::dev_token::validate_dev_token_format; use fabro_util::{Home, dev_token}; -use fabro_workflow::artifact_snapshot::CapturedArtifactInfo; -use futures::StreamExt; -use serde::de::DeserializeOwned; -use serde::{Deserialize, Serialize}; -use tokio::fs::File; -use tokio::sync::Mutex; use tokio::time::sleep; -use tokio_util::io::ReaderStream; use crate::args::ServerTargetArgs; -use crate::auth_store::{AuthEntry, AuthStore, ServerTargetKey, StoredSubject}; use crate::commands::server::{record, start}; -use crate::loopback_target::{LoopbackClassification, is_loopback_or_unix_socket}; -use crate::user_config::cli_http_client_builder; -use crate::{sse, user_config}; +use crate::user_config::{self, cli_http_client_builder}; -#[derive(Clone)] -pub(crate) struct Client { - state: Arc>, - base_url: String, - refreshable_oauth: Option, - refresh_lock: Arc>, +#[derive(Debug)] +struct CliDevTokenFallback { + storage_dir: Option, } -#[derive(Clone)] -struct ClientBundle { - client: fabro_api::ApiClient, - http_client: fabro_http::HttpClient, - bearer_token: Option, -} - -#[derive(Debug, Clone)] -enum ResolvedBearer { - DevToken(String), - OAuth(AuthEntry), -} - -impl ResolvedBearer { - fn bearer_token(&self) -> &str { - match self { - Self::DevToken(token) => token, - Self::OAuth(entry) => &entry.access_token, - } - } -} - -#[derive(Debug, Clone)] -struct RefreshableOAuth { - target: user_config::ServerTarget, - key: ServerTargetKey, - auth_store: AuthStore, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -struct ApiFailure { - status: fabro_http::StatusCode, - code: Option, -} - -struct StructuredApiError { - error: anyhow::Error, - failure: Option, -} - -#[derive(Debug, Deserialize)] -struct CliTokenResponse { - access_token: String, - access_token_expires_at: chrono::DateTime, - refresh_token: String, - refresh_token_expires_at: chrono::DateTime, - subject: CliTokenSubject, -} - -#[derive(Debug, Deserialize)] -struct CliTokenSubject { - idp_issuer: String, - idp_subject: String, - login: String, - name: String, - email: String, -} - -#[derive(Debug, Deserialize)] -struct OAuthErrorBody { - error: String, - #[serde(default)] - error_description: Option, -} - -pub(crate) struct RunAttachEventStream { - stream: progenitor_client::ByteStream, - pending_bytes: Vec, - buffered_events: VecDeque, -} - -impl RunAttachEventStream { - fn new(stream: progenitor_client::ByteStream) -> Self { - Self { - stream, - pending_bytes: Vec::new(), - buffered_events: VecDeque::new(), - } - } - - pub(crate) async fn next_event(&mut self) -> Result> { - loop { - if let Some(event) = self.buffered_events.pop_front() { - return Ok(Some(event)); - } - - if let Some(chunk) = self.stream.next().await { - let chunk = chunk.map_err(|err| anyhow!("{err}"))?; - self.pending_bytes.extend_from_slice(&chunk); - self.buffer_sse_events(false)?; - } else { - self.buffer_sse_events(true)?; - return Ok(self.buffered_events.pop_front()); - } - } - } - - fn buffer_sse_events(&mut self, finalize: bool) -> Result<()> { - for payload in sse::drain_sse_payloads(&mut self.pending_bytes, finalize) { - self.buffered_events - .push_back(serde_json::from_str(&payload)?); - } - Ok(()) - } -} - -pub(crate) use fabro_store::RunProjection; - -fn client_bundle( - base_url: &str, - http_client: fabro_http::HttpClient, - bearer_token: Option, -) -> ClientBundle { - let client = fabro_api::ApiClient::new_with_client(base_url, http_client.clone()); - ClientBundle { - client, - http_client, - bearer_token, +impl CredentialFallback for CliDevTokenFallback { + fn resolve(&self) -> Option { + load_dev_token_if_available(self.storage_dir.as_deref()).map(Credential::DevToken) } } fn refreshable_oauth( - target: &user_config::ServerTarget, - bearer: Option<&ResolvedBearer>, -) -> Result> { - if matches!(bearer, Some(ResolvedBearer::OAuth(_))) { - return Ok(Some(RefreshableOAuth { - target: target.clone(), - key: ServerTargetKey::new(target)?, - auth_store: AuthStore::default(), - })); + target: &ServerTarget, + credential: Option<&Credential>, +) -> Option { + if matches!(credential, Some(Credential::OAuth(_))) { + let session = OAuthSession::new(target.clone(), AuthStore::default()); + if local_dev_token_fallback(target) { + return Some( + session.with_fallback(Arc::new(CliDevTokenFallback { storage_dir: None })), + ); + } + return Some(session); } - Ok(None) + None } pub(crate) async fn connect_server(storage_dir: &Path) -> Result { - connect_api_client_bundle(storage_dir).await + connect_local_api_client_bundle(storage_dir, &user_config::active_settings_path(None)).await } -pub(crate) async fn connect_server_target(target: &user_config::ServerTarget) -> Result { +pub(crate) async fn connect_server_target(target: &ServerTarget) -> Result { connect_target_api_client_bundle(target).await } pub(crate) async fn connect_server_target_direct(target: &str) -> Result { - if target.starts_with("http://") || target.starts_with("https://") { - connect_server_target(&user_config::ServerTarget::HttpUrl(target.to_string())).await - } else { - let path = Path::new(target); - if !path.is_absolute() { - bail!("server target must be an http(s) URL or absolute Unix socket path"); - } - connect_server_target(&user_config::ServerTarget::UnixSocket(path.to_path_buf())).await - } + let target = target.parse::()?; + connect_server_target(&target).await } pub(crate) async fn connect_server_with_settings( @@ -202,7 +66,7 @@ pub(crate) async fn connect_server_with_settings( base_config_path: &Path, ) -> Result { if let Some(target) = user_config::resolve_nondefault_server_target(args, settings)? { - if let user_config::ServerTarget::UnixSocket(path) = &target { + if let Some(path) = target.as_unix_socket_path() { return connect_managed_unix_socket_api_client_bundle( path, &user_config::storage_dir(settings)?, @@ -221,33 +85,35 @@ async fn connect_managed_unix_socket_api_client_bundle( storage_dir: &Path, active_config_path: &Path, ) -> Result { - let target = user_config::ServerTarget::UnixSocket(path.to_path_buf()); - let bearer = resolve_target_bearer( + let target = ServerTarget::unix_socket_path(path)?; + let credential = resolve_target_credential( &target, Some(storage_dir), local_dev_token_fallback(&target), )?; - let refreshable_oauth = refreshable_oauth(&target, bearer.as_ref())?; - let bearer_token = bearer.as_ref().map(ResolvedBearer::bearer_token); + let oauth_session = refreshable_oauth(&target, credential.as_ref()); + let bearer_token = credential.as_ref().map(Credential::bearer_token); - let bundle = if let Ok(bundle) = - try_connect_unix_socket_api_client_bundle(path, Some(storage_dir), bearer_token).await + let http_client = if let Ok(http_client) = + try_connect_unix_socket_http_client(path, Some(storage_dir), bearer_token).await { - bundle + http_client } else { start::ensure_server_running_on_socket(path, active_config_path, storage_dir) .await .with_context(|| format!("Failed to start fabro server for {}", path.display()))?; - connect_unix_socket_api_client_bundle(path, Some(storage_dir), bearer_token) + connect_unix_socket_http_client(path, Some(storage_dir), bearer_token) .await .with_context(|| format!("Failed to connect to fabro server at {}", path.display()))? }; - Ok(Client::from_bundle( - bundle, - "http://fabro".to_string(), - refreshable_oauth, - )) + build_client( + target, + credential, + oauth_session, + Some(("http://fabro".to_string(), http_client)), + ) + .await } async fn connect_local_api_client_bundle( @@ -259,96 +125,94 @@ async fn connect_local_api_client_bundle( .with_context(|| format!("Failed to start fabro server for {}", storage_dir.display()))?; match bind { Bind::Unix(path) => { - let bundle = - connect_unix_socket_api_client_bundle(&path, Some(storage_dir), None).await?; - Ok(Client::from_bundle( - bundle, - "http://fabro".to_string(), - None, - )) + let http_client = + connect_unix_socket_http_client(&path, Some(storage_dir), None).await?; + Ok(Client::from_http_client("http://fabro", http_client)) } Bind::Tcp(addr) => { let token = wait_for_local_dev_token(storage_dir).await?; let builder = cli_http_client_builder().no_proxy(); let http_client = apply_bearer_token_auth(builder, &token)?.build()?; let base_url = format!("http://{addr}"); - Ok(Client::from_bundle( - client_bundle(&base_url, http_client, Some(token)), - base_url, - None, - )) + Ok(Client::from_http_client(base_url, http_client)) } } } -async fn connect_api_client_bundle(storage_dir: &Path) -> Result { - connect_local_api_client_bundle(storage_dir, &user_config::active_settings_path(None)).await -} - #[allow( dead_code, reason = "Retained for pending storage-backed internal callers and referenced in existing design docs." )] pub(crate) async fn connect_api_client(storage_dir: &Path) -> Result { - connect_api_client_bundle(storage_dir) + connect_local_api_client_bundle(storage_dir, &user_config::active_settings_path(None)) .await - .map(|client| client.client_bundle().client) + .map(|client| client.api_client()) } -async fn connect_target_api_client_bundle(target: &user_config::ServerTarget) -> Result { - match target { - user_config::ServerTarget::HttpUrl(api_url) => { - let bearer = resolve_target_bearer(target, None, local_dev_token_fallback(target))?; - let refreshable_oauth = refreshable_oauth(target, bearer.as_ref())?; - let bundle = connect_remote_api_client_bundle( - api_url, - bearer.as_ref().map(ResolvedBearer::bearer_token), - )?; - Ok(Client::from_bundle( - bundle, - user_config::normalized_http_base_url(api_url).to_string(), - refreshable_oauth, - )) - } - user_config::ServerTarget::UnixSocket(path) => { - let bearer = resolve_target_bearer(target, None, local_dev_token_fallback(target))?; - let refreshable_oauth = refreshable_oauth(target, bearer.as_ref())?; - let bundle = try_connect_unix_socket_api_client_bundle( - path, - None, - bearer.as_ref().map(ResolvedBearer::bearer_token), - ) - .await - .with_context(|| format!("Failed to connect to fabro server at {}", path.display()))?; - Ok(Client::from_bundle( - bundle, - "http://fabro".to_string(), - refreshable_oauth, - )) - } +async fn connect_target_api_client_bundle(target: &ServerTarget) -> Result { + let credential = resolve_target_credential(target, None, local_dev_token_fallback(target))?; + let oauth_session = refreshable_oauth(target, credential.as_ref()); + build_client(target.clone(), credential, oauth_session, None).await +} + +async fn build_client( + target: ServerTarget, + credential: Option, + oauth_session: Option, + transport: Option<(String, fabro_http::HttpClient)>, +) -> Result { + let mut builder = Client::builder() + .target(target.clone()) + .transport_connector(build_cli_transport_connector(target)); + if let Some((base_url, http_client)) = transport { + builder = builder.transport(base_url, http_client); } + if let Some(credential) = credential { + builder = builder.credential(credential); + } + if let Some(oauth_session) = oauth_session { + builder = builder.oauth_session(oauth_session); + } + builder.connect().await } -fn connect_remote_api_client_bundle( - api_url: &str, +fn build_cli_transport_connector(target: ServerTarget) -> TransportConnector { + TransportConnector::new(move |bearer_token| { + let target = target.clone(); + async move { connect_cli_target_transport(&target, bearer_token.as_deref()) } + }) +} + +fn connect_cli_target_transport( + target: &ServerTarget, bearer_token: Option<&str>, -) -> Result { - let normalized = user_config::normalized_http_base_url(api_url); - let mut builder = user_config::cli_http_client_builder(); +) -> Result<(fabro_http::HttpClient, String)> { + if let Some(api_url) = target.as_http_url() { + let mut builder = cli_http_client_builder(); + builder = match bearer_token { + Some(token) => apply_bearer_token_auth(builder, token)?, + None => builder, + }; + let http_client = builder.build()?; + return Ok((http_client, api_url.to_string())); + } + + let Some(path) = target.as_unix_socket_path() else { + bail!("server target must be an http(s) URL or absolute Unix socket path"); + }; + let mut builder = cli_http_client_builder().unix_socket(path).no_proxy(); builder = match bearer_token { Some(token) => apply_bearer_token_auth(builder, token)?, None => builder, }; - let http_client = builder.build()?; - Ok(client_bundle( - normalized, - http_client, - bearer_token.map(ToOwned::to_owned), - )) + let http_client = builder + .build() + .context("Failed to build Unix-socket HTTP client for fabro server")?; + Ok((http_client, "http://fabro".to_string())) } -fn local_dev_token_fallback(target: &user_config::ServerTarget) -> bool { - matches!(target, user_config::ServerTarget::UnixSocket(_)) +fn local_dev_token_fallback(target: &ServerTarget) -> bool { + target.is_unix_socket() } fn load_dev_token_if_available(storage_dir: Option<&Path>) -> Option { @@ -413,52 +277,24 @@ fn apply_bearer_token_auth( Ok(builder.default_headers(headers)) } -fn apply_dev_token_auth( - builder: fabro_http::HttpClientBuilder, - storage_dir: Option<&Path>, -) -> Result { - let Some(token) = load_dev_token_if_available(storage_dir) else { - return Ok(builder); - }; - apply_bearer_token_auth(builder, &token) -} - -fn unix_socket_api_client_bundle( - http_client: fabro_http::HttpClient, - bearer_token: Option, -) -> ClientBundle { - client_bundle("http://fabro", http_client, bearer_token) -} - -async fn build_authed_unix_socket_client( +async fn build_authed_unix_socket_http_client( path: &Path, storage_dir: Option<&Path>, bearer_token: Option<&str>, -) -> Result { - let http_client = if let Some(token) = bearer_token { - apply_bearer_token_auth( - cli_http_client_builder().unix_socket(path).no_proxy(), - token, - )? - .build() - .context("Failed to build Unix-socket HTTP client for fabro server")? +) -> Result { + let builder = cli_http_client_builder().unix_socket(path).no_proxy(); + let builder = if let Some(token) = bearer_token { + apply_bearer_token_auth(builder, token)? } else if let Some(storage_dir) = storage_dir { let token = wait_for_local_dev_token(storage_dir).await?; - apply_bearer_token_auth( - cli_http_client_builder().unix_socket(path).no_proxy(), - &token, - )? - .build() - .context("Failed to build Unix-socket HTTP client for fabro server")? + apply_bearer_token_auth(builder, &token)? } else { - apply_dev_token_auth(cli_http_client_builder().unix_socket(path).no_proxy(), None)? - .build() - .context("Failed to build Unix-socket HTTP client for fabro server")? + builder }; - Ok(unix_socket_api_client_bundle( - http_client, - bearer_token.map(ToOwned::to_owned), - )) + + builder + .build() + .context("Failed to build Unix-socket HTTP client for fabro server") } fn build_unix_socket_probe_client(path: &Path) -> Result { @@ -469,47 +305,46 @@ fn build_unix_socket_probe_client(path: &Path) -> Result .context("Failed to build Unix-socket HTTP client for fabro server") } -async fn try_connect_unix_socket_api_client_bundle( +async fn try_connect_unix_socket_http_client( path: &Path, storage_dir: Option<&Path>, bearer_token: Option<&str>, -) -> Result { +) -> Result { check_server_ready(&build_unix_socket_probe_client(path)?).await?; - build_authed_unix_socket_client(path, storage_dir, bearer_token).await + build_authed_unix_socket_http_client(path, storage_dir, bearer_token).await } -async fn connect_unix_socket_api_client_bundle( +async fn connect_unix_socket_http_client( path: &Path, storage_dir: Option<&Path>, bearer_token: Option<&str>, -) -> Result { +) -> Result { wait_for_server_ready(&build_unix_socket_probe_client(path)?).await?; - build_authed_unix_socket_client(path, storage_dir, bearer_token).await + build_authed_unix_socket_http_client(path, storage_dir, bearer_token).await } -fn resolve_target_bearer( - target: &user_config::ServerTarget, +fn resolve_target_credential( + target: &ServerTarget, storage_dir: Option<&Path>, allow_local_dev_token_fallback: bool, -) -> Result> { +) -> Result> { if let Some(token) = std::env::var("FABRO_DEV_TOKEN") .ok() .filter(|token| validate_dev_token_format(token)) { - return Ok(Some(ResolvedBearer::DevToken(token))); + return Ok(Some(Credential::DevToken(token))); } let store = AuthStore::default(); - let key = ServerTargetKey::new(target)?; - if let Some(entry) = store.get(&key)? { + if let Some(entry) = store.get(target)? { let now = chrono::Utc::now(); if entry.access_token_expires_at > now || entry.refresh_token_expires_at > now { - return Ok(Some(ResolvedBearer::OAuth(entry))); + return Ok(Some(Credential::OAuth(entry))); } } if allow_local_dev_token_fallback { - return Ok(load_dev_token_if_available(storage_dir).map(ResolvedBearer::DevToken)); + return Ok(load_dev_token_if_available(storage_dir).map(Credential::DevToken)); } Ok(None) @@ -530,9 +365,7 @@ async fn wait_for_server_ready(http_client: &fabro_http::HttpClient) -> Result<( while std::time::Instant::now() < deadline { match check_server_ready(http_client).await { Ok(()) => return Ok(()), - Err(err) => { - last_error = Some(err); - } + Err(err) => last_error = Some(err), } sleep(Duration::from_millis(50)).await; } @@ -540,1269 +373,13 @@ async fn wait_for_server_ready(http_client: &fabro_http::HttpClient) -> Result<( Err(last_error.unwrap_or_else(|| anyhow!("server did not become ready in time"))) } -#[derive(Debug, Serialize)] -struct ArtifactBatchUploadManifest { - entries: Vec, -} - -#[derive(Debug, Serialize)] -struct ArtifactBatchUploadEntry { - part: String, - path: String, - #[serde(skip_serializing_if = "Option::is_none")] - sha256: Option, - #[serde(skip_serializing_if = "Option::is_none")] - expected_bytes: Option, - #[serde(skip_serializing_if = "Option::is_none")] - content_type: Option, -} - -impl Client { - fn from_bundle( - bundle: ClientBundle, - base_url: String, - refreshable_oauth: Option, - ) -> Self { - Self { - state: Arc::new(RwLock::new(bundle)), - base_url, - refreshable_oauth, - refresh_lock: Arc::new(Mutex::new(())), - } - } - - fn client_bundle(&self) -> ClientBundle { - self.state - .read() - .expect("server client state lock should not be poisoned") - .clone() - } - - fn replace_client_bundle(&self, bundle: ClientBundle) { - *self - .state - .write() - .expect("server client state lock should not be poisoned") = bundle; - } - - /// Build a client for tests that bypasses proxy discovery. - #[cfg(test)] - pub(crate) fn new_no_proxy(base_url: &str) -> Result { - let http_client = cli_http_client_builder().no_proxy().build()?; - Ok(Self::from_bundle( - client_bundle(base_url, http_client, None), - base_url.to_string(), - None, - )) - } - - pub(crate) fn clone_for_reuse(&self) -> Self { - self.clone() - } - - pub(crate) async fn send_api( - &self, - request: F, - ) -> Result> - where - F: FnOnce(fabro_api::ApiClient) -> Fut + Clone, - Fut: std::future::Future< - Output = std::result::Result< - progenitor_client::ResponseValue, - progenitor_client::Error, - >, - >, - E: serde::Serialize + std::fmt::Debug, - { - let bundle = self.client_bundle(); - match request.clone()(bundle.client.clone()).await { - Ok(response) => Ok(response), - Err(err) => { - let mapped = classify_api_error(err).await; - if self.should_refresh(mapped.failure.as_ref()) { - if let Some(failed_token) = bundle.bearer_token.as_deref() { - self.refresh_access_token(failed_token).await?; - let bundle = self.client_bundle(); - return request(bundle.client.clone()).await.map_err(map_api_error); - } - } - Err(mapped.error) - } - } - } - - fn should_refresh(&self, failure: Option<&ApiFailure>) -> bool { - self.refreshable_oauth.is_some() - && failure.is_some_and(|failure| { - failure.status == fabro_http::StatusCode::UNAUTHORIZED - && failure.code.as_deref() == Some("access_token_expired") - }) - } - - async fn refresh_access_token(&self, failed_access_token: &str) -> Result<()> { - let Some(refreshable) = &self.refreshable_oauth else { - bail!("CLI session has expired. Run `fabro auth login` again."); - }; - let _guard = self.refresh_lock.lock().await; - let current_bundle = self.client_bundle(); - if current_bundle.bearer_token.as_deref() != Some(failed_access_token) { - return Ok(()); - } - - let Some(entry) = refreshable.auth_store.get(&refreshable.key)? else { - let fallback = resolve_target_bearer( - &refreshable.target, - None, - local_dev_token_fallback(&refreshable.target), - )?; - self.rebuild_client_for_target( - &refreshable.target, - fallback.as_ref().map(ResolvedBearer::bearer_token), - ) - .await?; - bail!("CLI session has expired. Run `fabro auth login` again."); - }; - if entry.refresh_token_expires_at <= chrono::Utc::now() { - refreshable.auth_store.remove(&refreshable.key)?; - let fallback = resolve_target_bearer( - &refreshable.target, - None, - local_dev_token_fallback(&refreshable.target), - )?; - self.rebuild_client_for_target( - &refreshable.target, - fallback.as_ref().map(ResolvedBearer::bearer_token), - ) - .await?; - bail!("CLI session has expired. Run `fabro auth login` again."); - } - ensure_refresh_target_transport(&refreshable.target)?; - - let (http_client, base_url) = user_config::build_public_http_client(&refreshable.target)?; - let response = http_client - .post(format!("{base_url}/auth/cli/refresh")) - .header(AUTHORIZATION, format!("Bearer {}", entry.refresh_token)) - .send() - .await?; - - if response.status().is_success() { - let tokens = response - .json::() - .await - .context("failed to parse CLI auth refresh response")?; - let entry = AuthEntry { - access_token: tokens.access_token.clone(), - access_token_expires_at: tokens.access_token_expires_at, - refresh_token: tokens.refresh_token.clone(), - refresh_token_expires_at: tokens.refresh_token_expires_at, - subject: StoredSubject { - idp_issuer: tokens.subject.idp_issuer, - idp_subject: tokens.subject.idp_subject, - login: tokens.subject.login, - name: tokens.subject.name, - email: tokens.subject.email, - }, - logged_in_at: entry.logged_in_at, - }; - refreshable - .auth_store - .put(&refreshable.key, entry.clone()) - .context("failed to persist refreshed CLI auth tokens")?; - self.rebuild_client_for_target(&refreshable.target, Some(&entry.access_token)) - .await?; - return Ok(()); - } - - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - let parsed_error = serde_json::from_str::(&body).ok(); - if parsed_error.as_ref().is_some_and(|error| { - matches!( - error.error.as_str(), - "refresh_token_expired" | "refresh_token_revoked" - ) - }) { - refreshable.auth_store.remove(&refreshable.key)?; - let fallback = resolve_target_bearer( - &refreshable.target, - None, - local_dev_token_fallback(&refreshable.target), - )?; - self.rebuild_client_for_target( - &refreshable.target, - fallback.as_ref().map(ResolvedBearer::bearer_token), - ) - .await?; - } - - if let Some(parsed_error) = parsed_error { - let message = parsed_error - .error_description - .filter(|value| !value.is_empty()) - .unwrap_or_else(|| format!("request failed with status {status}")); - bail!("{message}"); - } - if body.is_empty() { - bail!("request failed with status {status}"); - } - bail!("request failed with status {status}: {body}"); - } - - async fn rebuild_client_for_target( - &self, - target: &user_config::ServerTarget, - bearer_token: Option<&str>, - ) -> Result<()> { - let bundle = match target { - user_config::ServerTarget::HttpUrl(api_url) => { - connect_remote_api_client_bundle(api_url, bearer_token)? - } - user_config::ServerTarget::UnixSocket(path) => { - connect_unix_socket_api_client_bundle(path, None, bearer_token).await? - } - }; - self.replace_client_bundle(bundle); - Ok(()) - } - - pub(crate) async fn send_http_response( - &self, - request: F, - ) -> Result> - where - F: FnOnce(fabro_http::HttpClient) -> Fut + Clone, - Fut: std::future::Future>, - T: Into, - { - let bundle = self.client_bundle(); - let response = request.clone()(bundle.http_client.clone()) - .await - .map_err(Into::into)?; - match classify_http_response(response).await? { - Ok(response) => Ok(Ok(response)), - Err(failure) => { - if self.should_refresh(Some(&failure.failure)) { - if let Some(failed_token) = bundle.bearer_token.as_deref() { - self.refresh_access_token(failed_token).await?; - let bundle = self.client_bundle(); - let response = request(bundle.http_client.clone()) - .await - .map_err(Into::into)?; - return classify_http_response(response).await; - } - } - Ok(Err(failure)) - } - } - } - - async fn send_http(&self, request: F) -> Result - where - F: FnOnce(fabro_http::HttpClient) -> Fut + Clone, - Fut: std::future::Future>, - T: Into, - { - match self.send_http_response(request).await? { - Ok(response) => Ok(response), - Err(failure) => Err(raw_response_failure_error(&failure)), - } - } - - #[allow( - dead_code, - reason = "This accessor is kept for tests and pending callers." - )] - pub(crate) fn http_client(&self) -> fabro_http::HttpClient { - self.client_bundle().http_client - } - - #[allow( - dead_code, - reason = "This accessor is kept for tests and pending callers." - )] - pub(crate) fn base_url(&self) -> &str { - &self.base_url - } - - pub(crate) async fn retrieve_resolved_server_settings(&self) -> Result { - let url = format!("{}/api/v1/settings?view=resolved", self.base_url); - let response = self - .send_http(|http_client| async move { http_client.get(&url).send().await }) - .await?; - - let marker = response - .headers() - .get("x-fabro-settings-view") - .and_then(|value| value.to_str().ok()); - if marker != Some("resolved") { - bail!( - "server does not support resolved settings view; upgrade the server or use --local" - ); - } - - response - .json::() - .await - .context("server returned invalid JSON for the resolved settings view") - } - - pub(crate) async fn create_run_from_manifest( - &self, - manifest: types::RunManifest, - ) -> Result { - let response = self - .send_api( - |client| async move { client.create_run().body(manifest.clone()).send().await }, - ) - .await?; - let status = response.into_inner(); - status - .id - .parse() - .map_err(|err| anyhow!("invalid run ID from server: {err}")) - } - - pub(crate) async fn list_secrets(&self) -> Result> { - let response = self - .send_api(|client| async move { client.list_secrets().send().await }) - .await?; - Ok(response.into_inner().data) - } - - pub(crate) async fn create_secret( - &self, - body: types::CreateSecretRequest, - ) -> Result { - let response = self - .send_api( - |client| async move { client.create_secret().body(body.clone()).send().await }, - ) - .await?; - Ok(response.into_inner()) - } - - pub(crate) async fn delete_secret_by_name(&self, name: &str) -> Result<()> { - self.send_api(|client| async move { - client - .delete_secret_by_name() - .body(types::DeleteSecretRequest { - name: name.to_string(), - }) - .send() - .await - }) - .await?; - Ok(()) - } - - pub(crate) async fn list_models( - &self, - provider: Option<&str>, - query: Option<&str>, - ) -> Result> { - let mut offset = 0u64; - let mut models = Vec::new(); - - loop { - let response = self - .send_api(|client| async move { - let mut request = client.list_models().page_limit(100u64).page_offset(offset); - if let Some(provider) = provider { - request = request.provider(provider.to_string()); - } - if let Some(query) = query { - request = request.query(query.to_string()); - } - request.send().await - }) - .await?; - let parsed = response.into_inner(); - let count = parsed.data.len() as u64; - models.extend(convert_type::<_, Vec>(parsed.data)?); - if !parsed.meta.has_more { - break; - } - offset += count; - } - - Ok(models) - } - - pub(crate) async fn test_model( - &self, - id: &str, - mode: Option, - ) -> Result { - let response = self - .send_api(|client| async move { - let mut request = client.test_model().id(id.to_string()); - if let Some(mode) = mode { - request = request.mode(mode); - } - request.send().await - }) - .await?; - Ok(response.into_inner()) - } - - pub(crate) async fn attach_events( - &self, - run_ids: &[String], - ) -> Result { - let response = self - .send_api(|client| async move { - let mut request = client.attach_events(); - if !run_ids.is_empty() { - request = request.run_id(run_ids.join(",")); - } - request.send().await - }) - .await?; - Ok(response.into_inner()) - } - - pub(crate) async fn get_system_info(&self) -> Result { - let response = self - .send_api(|client| async move { client.get_system_info().send().await }) - .await?; - Ok(response.into_inner()) - } - - pub(crate) async fn get_system_disk_usage( - &self, - verbose: bool, - ) -> Result { - let response = self - .send_api(|client| async move { - client.get_system_disk_usage().verbose(verbose).send().await - }) - .await?; - Ok(response.into_inner()) - } - - pub(crate) async fn prune_runs( - &self, - body: types::PruneRunsRequest, - ) -> Result { - let response = self - .send_api(|client| async move { client.prune_runs().body(body.clone()).send().await }) - .await?; - Ok(response.into_inner()) - } - - pub(crate) async fn get_health(&self) -> Result<()> { - self.send_api(|client| async move { client.get_health().send().await }) - .await?; - Ok(()) - } - - pub(crate) async fn run_diagnostics(&self) -> Result { - let response = self - .send_api(|client| async move { client.run_diagnostics().send().await }) - .await?; - Ok(response.into_inner()) - } - - pub(crate) async fn get_github_repo( - &self, - owner: &str, - name: &str, - ) -> Result { - let response = self - .send_api(|client| async move { - client - .get_github_repo() - .owner(owner.to_string()) - .name(name.to_string()) - .send() - .await - }) - .await?; - Ok(response.into_inner()) - } - - pub(crate) async fn run_preflight( - &self, - manifest: types::RunManifest, - ) -> Result { - self.send_api( - |client| async move { client.run_preflight().body(manifest.clone()).send().await }, - ) - .await - .map(progenitor_client::ResponseValue::into_inner) - } - - pub(crate) async fn render_workflow_graph( - &self, - request: types::RenderWorkflowGraphRequest, - ) -> Result> { - let response = self - .send_api(|client| async move { - client - .render_workflow_graph() - .body(request.clone()) - .send() - .await - }) - .await?; - let mut stream = response.into_inner(); - let mut bytes = Vec::new(); - while let Some(chunk) = stream.next().await { - let chunk = chunk.map_err(|err| anyhow!("{err}"))?; - bytes.extend_from_slice(&chunk); - } - Ok(bytes) - } - - pub(crate) async fn start_run(&self, run_id: &RunId, resume: bool) -> Result<()> { - self.send_api(|client| async move { - client - .start_run() - .id(run_id.to_string()) - .body(types::StartRunRequest { resume }) - .send() - .await - }) - .await?; - Ok(()) - } - - pub(crate) async fn cancel_run(&self, run_id: &RunId) -> Result<()> { - self.send_api( - |client| async move { client.cancel_run().id(run_id.to_string()).send().await }, - ) - .await?; - Ok(()) - } - - pub(crate) async fn archive_run(&self, run_id: &RunId) -> Result<()> { - self.send_api( - |client| async move { client.archive_run().id(run_id.to_string()).send().await }, - ) - .await?; - Ok(()) - } - - pub(crate) async fn unarchive_run(&self, run_id: &RunId) -> Result<()> { - self.send_api(|client| async move { - client.unarchive_run().id(run_id.to_string()).send().await - }) - .await?; - Ok(()) - } - - pub(crate) async fn list_store_runs(&self) -> Result> { - let mut all_runs = Vec::new(); - let mut offset = 0_u64; - let limit = 100_u64; - - loop { - let response = self - .send_api(|client| async move { - client - .list_runs() - .page_limit(limit) - .page_offset(offset) - .include_archived(true) - .send() - .await - }) - .await?; - let parsed = response.into_inner(); - let batch = parsed - .data - .into_iter() - .map(convert_type) - .collect::>>()?; - let batch_len = batch.len() as u64; - all_runs.extend(batch); - - if !parsed.meta.has_more || batch_len == 0 { - break; - } - offset += batch_len; - } - - Ok(all_runs) - } - - pub(crate) async fn retrieve_run(&self, run_id: &RunId) -> Result { - let response = self - .send_api( - |client| async move { client.retrieve_run().id(run_id.to_string()).send().await }, - ) - .await?; - convert_type(response.into_inner()) - } - - pub(crate) async fn resolve_run(&self, selector: &str) -> Result { - let response = self - .send_api(|client| async move { - client - .resolve_run() - .selector(selector.to_string()) - .send() - .await - }) - .await?; - convert_type(response.into_inner()) - } - - pub(crate) async fn get_run_state(&self, run_id: &RunId) -> Result { - let response = self - .send_api( - |client| async move { client.get_run_state().id(run_id.to_string()).send().await }, - ) - .await?; - convert_type(response.into_inner()) - } - - pub(crate) async fn list_run_events( - &self, - run_id: &RunId, - since_seq: Option, - limit: Option, - ) -> Result> { - let mut next_since_seq = since_seq; - let mut all_events = Vec::new(); - - loop { - let response = self - .send_api(|client| async move { - let mut request = client.list_run_events().id(run_id.to_string()); - if let Some(seq) = next_since_seq.and_then(non_zero_u64_from_u32) { - request = request.since_seq(seq); - } - if let Some(limit) = limit.and_then(non_zero_u64_from_usize) { - request = request.limit(limit); - } - request.send().await - }) - .await?; - let parsed = response.into_inner(); - let page_events = parsed - .data - .into_iter() - .map(convert_type::<_, EventEnvelope>) - .collect::>>()?; - let next_page_since_seq = page_events.last().map(|event| event.seq.saturating_add(1)); - all_events.extend(page_events); - - if limit.is_some() || !parsed.meta.has_more || next_page_since_seq.is_none() { - break; - } - next_since_seq = next_page_since_seq; - } - - Ok(all_events) - } - - pub(crate) async fn attach_run_events( - &self, - run_id: &RunId, - since_seq: Option, - ) -> Result { - let response = self - .send_api(|client| async move { - let mut request = client.attach_run_events().id(run_id.to_string()); - if let Some(seq) = since_seq.and_then(non_zero_u64_from_u32) { - request = request.since_seq(seq); - } - request.send().await - }) - .await?; - Ok(RunAttachEventStream::new(response.into_inner())) - } - - pub(crate) async fn list_run_questions( - &self, - run_id: &RunId, - ) -> Result> { - let response = self - .send_api(|client| async move { - client - .list_run_questions() - .id(run_id.to_string()) - .page_limit(100) - .page_offset(0) - .send() - .await - }) - .await?; - Ok(response.into_inner().data) - } - - pub(crate) async fn submit_run_answer( - &self, - run_id: &RunId, - qid: &str, - value: Option, - selected_option_key: Option, - selected_option_keys: Vec, - ) -> Result<()> { - self.send_api(|client| async move { - client - .submit_run_answer() - .id(run_id.to_string()) - .qid(qid) - .body(types::SubmitAnswerRequest { - value: value.clone(), - selected_option_key: selected_option_key.clone(), - selected_option_keys: selected_option_keys.clone(), - }) - .send() - .await - }) - .await?; - Ok(()) - } - - pub(crate) async fn append_run_event(&self, run_id: &RunId, event: &RunEvent) -> Result { - let body: types::RunEvent = convert_type(event)?; - let response = self - .send_api(|client| async move { - client - .append_run_event() - .id(run_id.to_string()) - .body(body.clone()) - .send() - .await - }) - .await?; - u32::try_from(response.into_inner().seq).context("append_run_event returned invalid seq") - } - - pub(crate) async fn write_run_blob(&self, run_id: &RunId, data: &[u8]) -> Result { - let response = self - .send_api(|client| async move { - client - .write_run_blob() - .id(run_id.to_string()) - .body(data.to_vec()) - .send() - .await - }) - .await?; - response - .into_inner() - .id - .parse() - .context("write_run_blob returned invalid blob id") - } - - pub(crate) async fn read_run_blob( - &self, - run_id: &RunId, - blob_id: &RunBlobId, - ) -> Result> { - let response = self - .client_bundle() - .client - .read_run_blob() - .id(run_id.to_string()) - .blob_id(blob_id.to_string()) - .send() - .await; - match response { - Ok(response) => { - let mut stream = response.into_inner(); - let mut bytes = Vec::new(); - while let Some(chunk) = stream.next().await { - let chunk = chunk.map_err(|err| anyhow!("{err}"))?; - bytes.extend_from_slice(&chunk); - } - Ok(Some(Bytes::from(bytes))) - } - Err(err) => { - if is_not_found_error(&err) { - Ok(None) - } else { - Err(map_api_error(err)) - } - } - } - } - - pub(crate) async fn delete_store_run(&self, run_id: &RunId, force: bool) -> Result<()> { - let mut url = fabro_http::Url::parse(&self.base_url) - .with_context(|| format!("invalid server base URL {}", self.base_url))?; - url.path_segments_mut() - .map_err(|()| anyhow!("server base URL cannot accept path segments"))? - .extend(["api", "v1", "runs", &run_id.to_string()]); - if force { - url.query_pairs_mut().append_pair("force", "true"); - } - - self.send_http(|http_client| async move { http_client.delete(url.clone()).send().await }) - .await?; - Ok(()) - } - - pub(crate) async fn list_run_artifacts( - &self, - run_id: &RunId, - ) -> Result> { - let response = self - .send_api(|client| async move { - client - .list_run_artifacts() - .id(run_id.to_string()) - .send() - .await - }) - .await?; - Ok(response.into_inner().data) - } - - pub(crate) async fn download_stage_artifact( - &self, - run_id: &RunId, - stage_id: &StageId, - filename: &str, - ) -> Result> { - let response = self - .send_api(|client| async move { - client - .get_stage_artifact() - .id(run_id.to_string()) - .stage_id(stage_id.to_string()) - .filename(filename) - .send() - .await - }) - .await?; - let mut stream = response.into_inner(); - let mut bytes = Vec::new(); - while let Some(chunk) = stream.next().await { - let chunk = chunk.map_err(|err| anyhow!("{err}"))?; - bytes.extend_from_slice(&chunk); - } - Ok(bytes) - } - - fn stage_artifacts_url(&self, run_id: &RunId, stage_id: &StageId) -> Result { - let mut url = fabro_http::Url::parse(&self.base_url) - .with_context(|| format!("invalid server base URL {}", self.base_url))?; - url.path_segments_mut() - .map_err(|()| anyhow!("server base URL cannot accept path segments"))? - .extend([ - "api", - "v1", - "runs", - &run_id.to_string(), - "stages", - &stage_id.to_string(), - "artifacts", - ]); - Ok(url) - } - - pub(crate) async fn upload_stage_artifact_file( - &self, - run_id: &RunId, - stage_id: &StageId, - filename: &str, - path: &Path, - bearer_token: &str, - ) -> Result<()> { - let mut url = self.stage_artifacts_url(run_id, stage_id)?; - url.query_pairs_mut().append_pair("filename", filename); - - let file = File::open(path) - .await - .with_context(|| format!("failed to open artifact {}", path.display()))?; - let content_length = file - .metadata() - .await - .with_context(|| format!("failed to stat artifact {}", path.display()))? - .len(); - let body = fabro_http::Body::wrap_stream(ReaderStream::new(file)); - - let response = self - .client_bundle() - .http_client - .post(url) - .bearer_auth(bearer_token) - .header(CONTENT_TYPE, "application/octet-stream") - .header(CONTENT_LENGTH, content_length.to_string()) - .body(body) - .send() - .await - .with_context(|| format!("failed to upload artifact {}", path.display()))?; - classify_http_response(response) - .await? - .map(|_| ()) - .map_err(|failure| raw_response_failure_error(&failure)) - } - - pub(crate) async fn upload_stage_artifact_batch( - &self, - run_id: &RunId, - stage_id: &StageId, - artifact_capture_dir: &Path, - artifacts: &[CapturedArtifactInfo], - bearer_token: &str, - ) -> Result<()> { - let url = self.stage_artifacts_url(run_id, stage_id)?; - let mut manifest_entries = Vec::with_capacity(artifacts.len()); - let mut file_parts = Vec::with_capacity(artifacts.len()); - - for (index, artifact) in artifacts.iter().enumerate() { - let part_name = format!("file{}", index + 1); - let path = artifact_capture_dir.join(&artifact.path); - let file = File::open(&path) - .await - .with_context(|| format!("failed to open artifact {}", path.display()))?; - let content_length = file - .metadata() - .await - .with_context(|| format!("failed to stat artifact {}", path.display()))? - .len(); - - manifest_entries.push(ArtifactBatchUploadEntry { - part: part_name.clone(), - path: artifact.path.clone(), - sha256: Some(artifact.content_sha256.clone()), - expected_bytes: Some(artifact.bytes), - content_type: Some(artifact.mime.clone()), - }); - - file_parts.push(( - part_name, - Part::stream_with_length( - fabro_http::Body::wrap_stream(ReaderStream::new(file)), - content_length, - ) - .file_name(artifact.path.clone()), - )); - } - - let manifest = ArtifactBatchUploadManifest { - entries: manifest_entries, - }; - let manifest_part = - Part::text(serde_json::to_string(&manifest)?).mime_str("application/json")?; - let mut form = Form::new().part("manifest", manifest_part); - for (part_name, part) in file_parts { - form = form.part(part_name, part); - } - - let response = self - .client_bundle() - .http_client - .post(url) - .bearer_auth(bearer_token) - .multipart(form) - .send() - .await - .context("failed to upload artifact batch")?; - classify_http_response(response) - .await? - .map(|_| ()) - .map_err(|failure| raw_response_failure_error(&failure)) - } - - pub(crate) async fn generate_preview_url( - &self, - run_id: &RunId, - port: u16, - expires_in_secs: u64, - signed: bool, - ) -> Result { - let expires_in_secs = NonZeroU64::new(expires_in_secs) - .ok_or_else(|| anyhow!("preview expiry must be greater than zero"))?; - let response = self - .send_api(|client| async move { - client - .generate_preview_url() - .id(run_id.to_string()) - .body(types::PreviewUrlRequest { - expires_in_secs, - port: i64::from(port), - signed, - }) - .send() - .await - }) - .await?; - Ok(response.into_inner()) - } - - pub(crate) async fn create_run_ssh_access( - &self, - run_id: &RunId, - ttl_minutes: f64, - ) -> Result { - let response = self - .send_api(|client| async move { - client - .create_run_ssh_access() - .id(run_id.to_string()) - .body(types::SshAccessRequest { ttl_minutes }) - .send() - .await - }) - .await?; - Ok(response.into_inner()) - } - - pub(crate) async fn list_sandbox_files( - &self, - run_id: &RunId, - path: &str, - depth: Option, - ) -> Result> { - let response = self - .send_api(|client| async move { - let mut request = client - .list_sandbox_files() - .id(run_id.to_string()) - .path(path); - if let Some(depth) = depth.and_then(non_zero_u64_from_u32) { - request = request.depth(depth); - } - request.send().await - }) - .await?; - Ok(response.into_inner().data) - } - - pub(crate) async fn get_sandbox_file(&self, run_id: &RunId, path: &str) -> Result> { - let response = self - .send_api(|client| async move { - client - .get_sandbox_file() - .id(run_id.to_string()) - .path(path) - .send() - .await - }) - .await?; - let mut stream = response.into_inner(); - let mut bytes = Vec::new(); - while let Some(chunk) = stream.next().await { - let chunk = chunk.map_err(|err| anyhow!("{err}"))?; - bytes.extend_from_slice(&chunk); - } - Ok(bytes) - } - - pub(crate) async fn put_sandbox_file( - &self, - run_id: &RunId, - path: &str, - bytes: Vec, - ) -> Result<()> { - self.send_api(|client| async move { - client - .put_sandbox_file() - .id(run_id.to_string()) - .path(path) - .body(bytes.clone()) - .send() - .await - }) - .await?; - Ok(()) - } -} - -fn ensure_refresh_target_transport(target: &user_config::ServerTarget) -> Result<()> { - match is_loopback_or_unix_socket(target)? { - LoopbackClassification::Https - | LoopbackClassification::LoopbackHttp - | LoopbackClassification::UnixSocket => Ok(()), - LoopbackClassification::Rejected => bail!(refresh_transport_error(target)), - } -} - -fn refresh_transport_error(target: &user_config::ServerTarget) -> String { - format!( - "Refusing to send refresh-token credentials over plaintext HTTP to a non-loopback host ({target}). Use HTTPS, or bind the server to 127.0.0.1 / ::1." - ) -} - -fn parse_error_response_value(value: &serde_json::Value) -> (Option, Option) { - let first = value - .get("errors") - .and_then(serde_json::Value::as_array) - .and_then(|errors| errors.first()); - let detail = first - .and_then(|entry| entry.get("detail")) - .and_then(serde_json::Value::as_str) - .map(ToOwned::to_owned); - let code = first - .and_then(|entry| entry.get("code")) - .and_then(serde_json::Value::as_str) - .map(ToOwned::to_owned); - (detail, code) -} - -async fn classify_api_error(err: progenitor_client::Error) -> StructuredApiError -where - E: serde::Serialize + std::fmt::Debug, -{ - match err { - progenitor_client::Error::UnexpectedResponse(response) => { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - let mut code = None; - if let Ok(value) = serde_json::from_str::(&body) { - let (detail, parsed_code) = parse_error_response_value(&value); - code = parsed_code; - if let Some(detail) = detail { - return StructuredApiError { - error: anyhow!("{detail}"), - failure: Some(ApiFailure { status, code }), - }; - } - } - let error = if body.is_empty() { - anyhow!("request failed with status {status}") - } else { - anyhow!("request failed with status {status}: {body}") - }; - StructuredApiError { - error, - failure: Some(ApiFailure { status, code }), - } - } - other => map_api_error_structured(other), - } -} - -fn map_api_error_structured(err: progenitor_client::Error) -> StructuredApiError -where - E: serde::Serialize + std::fmt::Debug, -{ - match err { - progenitor_client::Error::ErrorResponse(response) => { - let status = response.status(); - let mut code = None; - if let Ok(value) = serde_json::to_value(response.into_inner()) { - let (detail, parsed_code) = parse_error_response_value(&value); - code = parsed_code; - if let Some(detail) = detail { - return StructuredApiError { - error: anyhow!("{detail}"), - failure: Some(ApiFailure { status, code }), - }; - } - } - StructuredApiError { - error: anyhow!("request failed with status {status}"), - failure: Some(ApiFailure { status, code }), - } - } - progenitor_client::Error::UnexpectedResponse(response) => StructuredApiError { - error: anyhow!("request failed with status {}", response.status()), - failure: Some(ApiFailure { - status: response.status(), - code: None, - }), - }, - other => StructuredApiError { - error: anyhow!("{other}"), - failure: None, - }, - } -} - -pub(crate) fn map_api_error(err: progenitor_client::Error) -> anyhow::Error -where - E: serde::Serialize + std::fmt::Debug, -{ - map_api_error_structured(err).error -} - -pub(crate) struct HttpResponseFailure { - pub(crate) status: fabro_http::StatusCode, - pub(crate) headers: fabro_http::HeaderMap, - pub(crate) body: String, - failure: ApiFailure, -} - -fn raw_response_failure_error(failure: &HttpResponseFailure) -> anyhow::Error { - if let Ok(value) = serde_json::from_str::(&failure.body) { - let (detail, _) = parse_error_response_value(&value); - if let Some(detail) = detail { - return anyhow!("{detail}"); - } - } - - if failure.body.is_empty() { - return anyhow!("request failed with status {}", failure.status); - } - - anyhow!( - "request failed with status {}: {}", - failure.status, - failure.body - ) -} - -async fn classify_http_response( - response: fabro_http::Response, -) -> Result> { - if response.status().is_success() { - return Ok(Ok(response)); - } - let status = response.status(); - let headers = response.headers().clone(); - let body = response.text().await.unwrap_or_default(); - let mut code = None; - if let Ok(value) = serde_json::from_str::(&body) { - let (_, parsed_code) = parse_error_response_value(&value); - code = parsed_code; - } - - Ok(Err(HttpResponseFailure { - status, - headers, - body, - failure: ApiFailure { status, code }, - })) -} - -fn is_not_found_error(err: &progenitor_client::Error) -> bool -where - E: serde::Serialize + std::fmt::Debug, -{ - match err { - progenitor_client::Error::ErrorResponse(response) => { - response.status() == fabro_http::StatusCode::NOT_FOUND - } - progenitor_client::Error::UnexpectedResponse(response) => { - response.status() == fabro_http::StatusCode::NOT_FOUND - } - _ => false, - } -} -fn convert_type(value: TInput) -> Result -where - TInput: serde::Serialize, - TOutput: DeserializeOwned, -{ - serde_json::from_value(serde_json::to_value(value)?).map_err(Into::into) -} - -fn non_zero_u64_from_u32(value: u32) -> Option { - NonZeroU64::new(u64::from(value)) -} - -fn non_zero_u64_from_usize(value: usize) -> Option { - u64::try_from(value).ok().and_then(NonZeroU64::new) -} - #[cfg(test)] #[expect( clippy::disallowed_methods, reason = "server-client tests stage local dev-token fixtures with sync std::fs::write" )] mod tests { - use std::path::PathBuf; - - use chrono::Duration as ChronoDuration; + use chrono::Utc; use super::*; @@ -1859,7 +436,7 @@ mod tests { .server_state() .log_path(), dev_token_path: Some(token_path), - started_at: chrono::Utc::now(), + started_at: Utc::now(), }) .unwrap(); @@ -1874,68 +451,13 @@ mod tests { #[test] fn explicit_http_targets_do_not_allow_local_dev_token_fallback() { - let target = - user_config::ServerTarget::HttpUrl("https://fabro.example.com/api/v1".to_string()); + let target = ServerTarget::http_url("https://fabro.example.com/api/v1").unwrap(); assert!(!local_dev_token_fallback(&target)); } #[test] fn unix_socket_targets_keep_local_dev_token_fallback() { - let target = user_config::ServerTarget::UnixSocket(PathBuf::from("/tmp/fabro.sock")); + let target = ServerTarget::unix_socket_path("/tmp/fabro.sock").unwrap(); assert!(local_dev_token_fallback(&target)); } - - fn oauth_entry(login: &str) -> AuthEntry { - let now = chrono::Utc::now(); - AuthEntry { - access_token: format!("access-{login}"), - access_token_expires_at: now + ChronoDuration::minutes(10), - refresh_token: format!("refresh-{login}"), - refresh_token_expires_at: now + ChronoDuration::days(30), - subject: StoredSubject { - idp_issuer: "https://github.com".to_string(), - idp_subject: "12345".to_string(), - login: login.to_string(), - name: format!("Name {login}"), - email: format!("{login}@example.com"), - }, - logged_in_at: now, - } - } - - #[cfg(unix)] - #[tokio::test] - async fn refresh_access_token_rejects_plain_http_non_loopback_targets() { - let temp = tempfile::tempdir().unwrap(); - let auth_store = AuthStore::new(temp.path().join("auth.json")); - let target = user_config::ServerTarget::HttpUrl("http://fabro.example.com".to_string()); - let key = ServerTargetKey::new(&target).unwrap(); - auth_store.put(&key, oauth_entry("octocat")).unwrap(); - - let http_client = cli_http_client_builder().no_proxy().build().unwrap(); - let client = Client { - state: Arc::new(RwLock::new(client_bundle( - "http://fabro.example.com", - http_client, - Some("access-octocat".to_string()), - ))), - base_url: "http://fabro.example.com".to_string(), - refreshable_oauth: Some(RefreshableOAuth { - target: target.clone(), - key: key.clone(), - auth_store: auth_store.clone(), - }), - refresh_lock: Arc::new(Mutex::new(())), - }; - - let err = client - .refresh_access_token("access-octocat") - .await - .unwrap_err(); - assert!( - err.to_string() - .contains("Refusing to send refresh-token credentials over plaintext HTTP") - ); - assert!(auth_store.get(&key).unwrap().is_some()); - } } diff --git a/lib/crates/fabro-cli/src/server_runs.rs b/lib/crates/fabro-cli/src/server_runs.rs index 99010132f..eae615923 100644 --- a/lib/crates/fabro-cli/src/server_runs.rs +++ b/lib/crates/fabro-cli/src/server_runs.rs @@ -3,8 +3,7 @@ use std::sync::Arc; use anyhow::Result; use chrono::{DateTime, Utc}; -use fabro_store::RunSummary; -use fabro_types::{RunId, RunStatus, StatusReason}; +use fabro_types::{RunId, RunStatus, RunSummary, StatusReason}; use crate::server_client::Client; diff --git a/lib/crates/fabro-cli/src/user_config.rs b/lib/crates/fabro-cli/src/user_config.rs index e82cf759c..c151caa4e 100644 --- a/lib/crates/fabro-cli/src/user_config.rs +++ b/lib/crates/fabro-cli/src/user_config.rs @@ -1,7 +1,8 @@ -use std::fmt; use std::path::{Path, PathBuf}; +use std::str::FromStr; -use anyhow::{Result, bail}; +use anyhow::Result; +pub(crate) use fabro_client::ServerTarget; pub(crate) use fabro_config::user::*; use fabro_types::settings::cli::CliTargetSettings; use fabro_types::settings::{CliSettings, SettingsLayer}; @@ -60,50 +61,10 @@ pub(crate) fn apply_storage_dir_override( layer } -#[derive(Debug, Clone, PartialEq)] -pub(crate) enum ServerTarget { - HttpUrl(String), - UnixSocket(PathBuf), -} - -impl fmt::Display for ServerTarget { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::HttpUrl(api_url) => f.write_str(api_url), - Self::UnixSocket(path) => write!(f, "unix://{}", path.display()), - } - } -} - -pub(crate) fn normalized_http_base_url(api_url: &str) -> &str { - let trimmed = api_url.trim_end_matches('/'); - trimmed.strip_suffix("/api/v1").unwrap_or(trimmed) -} - pub(crate) fn build_public_http_client( target: &ServerTarget, ) -> Result<(fabro_http::HttpClient, String)> { - match target { - ServerTarget::HttpUrl(api_url) => { - let http_client = cli_http_client_builder().build()?; - Ok((http_client, normalized_http_base_url(api_url).to_string())) - } - ServerTarget::UnixSocket(path) => { - #[cfg(unix)] - { - let http_client = cli_http_client_builder() - .unix_socket(path) - .no_proxy() - .build()?; - Ok((http_client, "http://fabro".to_string())) - } - #[cfg(not(unix))] - { - let _ = path; - bail!("Unix-socket HTTP client is not supported on this platform") - } - } - } + target.build_public_http_client() } /// Pull the resolved CLI target configuration out of `[cli.target]`. @@ -125,7 +86,7 @@ fn configured_server_target(settings: &SettingsLayer) -> Result ServerTarget { - ServerTarget::UnixSocket(default_socket_path()) + ServerTarget::unix_socket_path(default_socket_path()).expect("default socket path is absolute") } pub(crate) fn storage_dir(settings: &SettingsLayer) -> anyhow::Result { @@ -153,16 +114,7 @@ pub(crate) fn storage_dir(settings: &SettingsLayer) -> anyhow::Result { } fn parse_server_target(value: &str) -> Result { - if value.starts_with("http://") || value.starts_with("https://") { - return Ok(ServerTarget::HttpUrl(value.to_string())); - } - - let path = Path::new(value); - if path.is_absolute() { - return Ok(ServerTarget::UnixSocket(path.to_path_buf())); - } - - bail!("server target must be an http(s) URL or absolute Unix socket path") + ServerTarget::from_str(value) } fn explicit_server_target(args: &ServerTargetArgs) -> Result> { @@ -219,7 +171,7 @@ mod tests { fn exec_uses_cli_server_target() { assert_eq!( exec_server_target(&server_target_args(Some("https://cli.example.com"))).unwrap(), - Some(ServerTarget::HttpUrl("https://cli.example.com".to_string())) + Some(ServerTarget::http_url("https://cli.example.com").unwrap()) ); } @@ -227,7 +179,7 @@ mod tests { fn exec_supports_explicit_unix_socket_target() { assert_eq!( exec_server_target(&server_target_args(Some("/tmp/fabro.sock"))).unwrap(), - Some(ServerTarget::UnixSocket(PathBuf::from("/tmp/fabro.sock"))) + Some(ServerTarget::unix_socket_path("/tmp/fabro.sock").unwrap()) ); } @@ -249,7 +201,7 @@ url = "https://config.example.com" ); assert_eq!( resolve_server_target(&server_target_args(None), &settings).unwrap(), - ServerTarget::HttpUrl("https://config.example.com".to_string()) + ServerTarget::http_url("https://config.example.com").unwrap() ); } @@ -270,7 +222,7 @@ url = "https://config.example.com" &settings ) .unwrap(), - ServerTarget::HttpUrl("https://cli.example.com".to_string()) + ServerTarget::http_url("https://cli.example.com").unwrap() ); } @@ -279,7 +231,8 @@ url = "https://config.example.com" let settings = SettingsLayer::default(); assert_eq!( resolve_server_target(&server_target_args(None), &settings).unwrap(), - ServerTarget::UnixSocket(dirs::home_dir().unwrap().join(".fabro/fabro.sock")) + ServerTarget::unix_socket_path(dirs::home_dir().unwrap().join(".fabro/fabro.sock")) + .unwrap() ); } @@ -300,7 +253,7 @@ url = "https://config.example.com" &settings ) .unwrap(), - ServerTarget::HttpUrl("https://cli.example.com".to_string()) + ServerTarget::http_url("https://cli.example.com").unwrap() ); } diff --git a/lib/crates/fabro-cli/tests/it/cmd/attach.rs b/lib/crates/fabro-cli/tests/it/cmd/attach.rs index 52b10c838..46f1e53cb 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/attach.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/attach.rs @@ -635,6 +635,7 @@ fn attach_json_errors_without_prompting_for_human_input() { { "event": "run.queued", "id": "[EVENT_ID]", + "properties": {}, "run_id": "[ULID]", "ts": "[TIMESTAMP]" }, diff --git a/lib/crates/fabro-cli/tests/it/cmd/rewind.rs b/lib/crates/fabro-cli/tests/it/cmd/rewind.rs index 3122c81e7..997b16a7f 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/rewind.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/rewind.rs @@ -129,7 +129,7 @@ fn rewind_preserves_event_history_and_clears_terminal_snapshot_state() { assert!( before_events .iter() - .any(|event| event.payload.as_value()["event"] == "run.completed"), + .any(|event| event.event.event_name() == "run.completed"), "setup run should be completed before rewind" ); @@ -153,28 +153,31 @@ fn rewind_preserves_event_history_and_clears_terminal_snapshot_state() { assert_eq!( after_events[..before_events.len()] .iter() - .map(|event| event.payload.as_value()["event"].as_str().unwrap()) + .map(|event| event.event.event_name()) .collect::>(), before_events .iter() - .map(|event| event.payload.as_value()["event"].as_str().unwrap()) + .map(|event| event.event.event_name()) .collect::>(), "rewind should preserve the prior event prefix" ); assert_eq!( - after_events[before_events.len()].payload.as_value()["event"], + after_events[before_events.len()].event.event_name(), "run.rewound" ); assert_eq!( - after_events[before_events.len() + 1].payload.as_value()["event"], + after_events[before_events.len() + 1].event.event_name(), "checkpoint.completed" ); assert_eq!( - after_events[before_events.len() + 2].payload.as_value()["event"], + after_events[before_events.len() + 2].event.event_name(), "run.submitted" ); assert!( - after_events[before_events.len() + 2].payload.as_value()["properties"]["definition_blob"] + after_events[before_events.len() + 2] + .event + .properties() + .unwrap()["definition_blob"] .is_string(), "rewind should re-emit run.submitted with the definition_blob" ); diff --git a/lib/crates/fabro-cli/tests/it/cmd/runner.rs b/lib/crates/fabro-cli/tests/it/cmd/runner.rs index 947e74ab0..8ac91e074 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/runner.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/runner.rs @@ -29,7 +29,7 @@ fn stored_worker_events(run_dir: &std::path::Path) -> Vec { } fn run_event(event: &EventEnvelope) -> RunEvent { - RunEvent::try_from(&event.payload).expect("stored event should parse") + event.event.clone() } fn assert_worker_succeeded(run_dir: &std::path::Path, stdout: &[u8]) { diff --git a/lib/crates/fabro-cli/tests/it/cmd/support.rs b/lib/crates/fabro-cli/tests/it/cmd/support.rs index 679d0d8e4..50499f17a 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/support.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/support.rs @@ -789,14 +789,7 @@ pub(crate) fn wait_for_event_names(run_dir: &Path, expected: &[&str]) { loop { let event_names = run_events(run_dir) .into_iter() - .filter_map(|event| { - event - .payload - .as_value() - .get("event") - .and_then(serde_json::Value::as_str) - .map(ToString::to_string) - }) + .map(|event| event.event.event_name().to_string()) .collect::>(); if expected diff --git a/lib/crates/fabro-cli/tests/it/workflow/mod.rs b/lib/crates/fabro-cli/tests/it/workflow/mod.rs index adcd4103f..d8cac011a 100644 --- a/lib/crates/fabro-cli/tests/it/workflow/mod.rs +++ b/lib/crates/fabro-cli/tests/it/workflow/mod.rs @@ -54,14 +54,9 @@ pub(super) fn completed_nodes(run_dir: &Path) -> Vec { } pub(super) fn has_event(run_dir: &Path, event_name: &str) -> bool { - run_events(run_dir).into_iter().any(|event| { - event - .payload - .as_value() - .get("event") - .and_then(Value::as_str) - == Some(event_name) - }) + run_events(run_dir) + .into_iter() + .any(|event| event.event.event_name() == event_name) } pub(super) fn store_dump_export(context: &TestContext, run_id: &str) -> PathBuf { diff --git a/lib/crates/fabro-client/Cargo.toml b/lib/crates/fabro-client/Cargo.toml new file mode 100644 index 000000000..338496b20 --- /dev/null +++ b/lib/crates/fabro-client/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "fabro-client" +edition.workspace = true +version.workspace = true +publish = false +license.workspace = true +description = "Typed HTTP client for the Fabro API" + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +anyhow.workspace = true +bytes.workspace = true +chrono = { workspace = true, features = ["serde"] } +fabro-api = { path = "../fabro-api" } +fabro-http.workspace = true +fabro-model = { path = "../fabro-model" } +fabro-types = { path = "../fabro-types" } +fabro-util = { path = "../fabro-util" } +fs2.workspace = true +futures.workspace = true +libc = "0.2" +progenitor-client = "0.13" +rand.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +tokio.workspace = true +tokio-util.workspace = true +tracing.workspace = true + +[dev-dependencies] +tempfile = "3" diff --git a/lib/crates/fabro-cli/src/auth_store.rs b/lib/crates/fabro-client/src/auth_store.rs similarity index 68% rename from lib/crates/fabro-cli/src/auth_store.rs rename to lib/crates/fabro-client/src/auth_store.rs index f88291b1c..01185ac72 100644 --- a/lib/crates/fabro-cli/src/auth_store.rs +++ b/lib/crates/fabro-client/src/auth_store.rs @@ -8,9 +8,9 @@ )] use std::collections::BTreeMap; +use std::fs; use std::io::Write as _; use std::path::{Path, PathBuf}; -use std::{fmt, fs}; use chrono::{DateTime, Utc}; use fs2::FileExt; @@ -18,68 +18,31 @@ use rand::Rng; use serde::{Deserialize, Serialize}; use thiserror::Error; -use crate::user_config::{ServerTarget, normalized_http_base_url}; +use crate::target::ServerTarget; const AUTH_FILE_ENV: &str = "FABRO_AUTH_FILE"; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct StoredSubject { - pub(crate) idp_issuer: String, - pub(crate) idp_subject: String, - pub(crate) login: String, - pub(crate) name: String, - pub(crate) email: String, +pub struct StoredSubject { + pub idp_issuer: String, + pub idp_subject: String, + pub login: String, + pub name: String, + pub email: String, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub(crate) struct AuthEntry { - pub(crate) access_token: String, - pub(crate) access_token_expires_at: DateTime, - pub(crate) refresh_token: String, - pub(crate) refresh_token_expires_at: DateTime, - pub(crate) subject: StoredSubject, - pub(crate) logged_in_at: DateTime, -} - -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub(crate) struct ServerTargetKey(String); - -impl ServerTargetKey { - pub(crate) fn new(target: &ServerTarget) -> Result { - match target { - ServerTarget::HttpUrl(api_url) => canonical_http_target(api_url).map(Self), - ServerTarget::UnixSocket(path) => Ok(Self(format!( - "unix://{}", - canonical_socket_path(path)?.display() - ))), - } - } - - fn from_canonical(canonical: String) -> Self { - Self(canonical) - } - - pub(crate) fn as_str(&self) -> &str { - &self.0 - } -} - -impl fmt::Display for ServerTargetKey { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&self.0) - } -} - -impl TryFrom<&ServerTarget> for ServerTargetKey { - type Error = AuthStoreError; - - fn try_from(value: &ServerTarget) -> Result { - Self::new(value) - } +pub struct AuthEntry { + pub access_token: String, + pub access_token_expires_at: DateTime, + pub refresh_token: String, + pub refresh_token_expires_at: DateTime, + pub subject: StoredSubject, + pub logged_in_at: DateTime, } #[derive(Debug, Error)] -pub(crate) enum AuthStoreError { +pub enum AuthStoreError { #[allow( dead_code, reason = "This platform-gated variant is exercised on non-Unix targets." @@ -118,7 +81,7 @@ pub(crate) enum AuthStoreError { } #[derive(Debug, Error)] -pub(crate) enum LockError { +pub enum LockError { #[error( "the filesystem backing {path} does not support file locking; move the auth store to a local filesystem or set {AUTH_FILE_ENV} to a local path" )] @@ -131,7 +94,7 @@ pub(crate) enum LockError { } #[derive(Debug, Clone)] -pub(crate) struct AuthStore { +pub struct AuthStore { path: PathBuf, } @@ -152,47 +115,45 @@ impl Default for AuthStore { } impl AuthStore { - pub(crate) fn new(path: PathBuf) -> Self { + pub fn new(path: PathBuf) -> Self { Self { path } } - pub(crate) fn get(&self, key: &ServerTargetKey) -> Result, AuthStoreError> { + pub fn get(&self, target: &ServerTarget) -> Result, AuthStoreError> { if !self.path.exists() { return Ok(None); } + let key = key_for_target(target); self.with_shared_lock(|| { let file = self.read_auth_file()?; - Ok(file.servers.get(key.as_str()).cloned()) + Ok(file.servers.get(&key).cloned()) }) } - pub(crate) fn put( - &self, - key: &ServerTargetKey, - entry: AuthEntry, - ) -> Result<(), AuthStoreError> { + pub fn put(&self, target: &ServerTarget, entry: AuthEntry) -> Result<(), AuthStoreError> { #[cfg(not(unix))] { - let _ = (key, entry); + let _ = (target, entry); Err(AuthStoreError::UnsupportedPlatform) } #[cfg(unix)] { + let key = key_for_target(target); self.ensure_parent_dir()?; self.with_exclusive_lock(|| { let mut file = self.read_auth_file_if_exists()?; - file.servers.insert(key.to_string(), entry); + file.servers.insert(key, entry); self.write_auth_file(&file) }) } } - pub(crate) fn remove(&self, key: &ServerTargetKey) -> Result { + pub fn remove(&self, target: &ServerTarget) -> Result { #[cfg(not(unix))] { - let _ = key; + let _ = target; Err(AuthStoreError::UnsupportedPlatform) } @@ -202,28 +163,28 @@ impl AuthStore { return Ok(false); } + let key = key_for_target(target); self.ensure_parent_dir()?; self.with_exclusive_lock(|| { let mut file = self.read_auth_file_if_exists()?; - let removed = file.servers.remove(key.as_str()).is_some(); + let removed = file.servers.remove(&key).is_some(); self.write_auth_file(&file)?; Ok(removed) }) } } - pub(crate) fn list(&self) -> Result, AuthStoreError> { + pub fn list(&self) -> Result, AuthStoreError> { if !self.path.exists() { return Ok(Vec::new()); } self.with_shared_lock(|| { let file = self.read_auth_file()?; - Ok(file - .servers + file.servers .into_iter() - .map(|(key, entry)| (ServerTargetKey::from_canonical(key), entry)) - .collect()) + .map(|(key, entry)| Ok((parse_stored_target(&key)?, entry))) + .collect::, AuthStoreError>>() }) } @@ -347,48 +308,26 @@ impl AuthStore { } } -fn canonical_http_target(api_url: &str) -> Result { - let trimmed = api_url.trim(); - let normalized = normalized_http_base_url(trimmed); - let url = - fabro_http::Url::parse(normalized).map_err(|_| AuthStoreError::InvalidServerTarget { - value: api_url.to_string(), - })?; - let scheme = url.scheme().to_ascii_lowercase(); - let Some(host) = url.host_str() else { - return Err(AuthStoreError::InvalidServerTarget { - value: api_url.to_string(), - }); - }; - let host = host.to_ascii_lowercase(); - let Some(port) = url.port_or_known_default() else { - return Err(AuthStoreError::InvalidServerTarget { - value: api_url.to_string(), - }); - }; - let default_port = match scheme.as_str() { - "http" => 80, - "https" => 443, - _ => { - return Err(AuthStoreError::InvalidServerTarget { - value: api_url.to_string(), - }); - } - }; - if port == default_port { - Ok(format!("{scheme}://{host}")) - } else { - Ok(format!("{scheme}://{host}:{port}")) - } +fn key_for_target(target: &ServerTarget) -> String { + target.to_string() } -fn canonical_socket_path(path: &Path) -> Result { - if !path.is_absolute() { - return Err(AuthStoreError::InvalidServerTarget { - value: path.display().to_string(), +fn parse_stored_target(value: &str) -> Result { + if let Some(path) = value.strip_prefix("unix://") { + return ServerTarget::unix_socket_path(path).map_err(|_| { + AuthStoreError::InvalidServerTarget { + value: value.to_string(), + } }); } - Ok(fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf())) + if value.starts_with("http://") || value.starts_with("https://") { + return ServerTarget::http_url(value).map_err(|_| AuthStoreError::InvalidServerTarget { + value: value.to_string(), + }); + } + Err(AuthStoreError::InvalidServerTarget { + value: value.to_string(), + }) } #[cfg(unix)] @@ -438,8 +377,8 @@ mod tests { #[cfg(unix)] use super::{AUTH_FILE_ENV, LockError, classify_lock_error}; - use super::{AuthEntry, AuthStore, ServerTargetKey, StoredSubject}; - use crate::user_config::ServerTarget; + use super::{AuthEntry, AuthStore, StoredSubject, key_for_target}; + use crate::target::ServerTarget; fn entry(login: &str) -> AuthEntry { let now = chrono::Utc::now(); @@ -460,7 +399,7 @@ mod tests { } fn https_target(value: &str) -> ServerTarget { - ServerTarget::HttpUrl(value.to_string()) + ServerTarget::http_url(value).unwrap() } #[cfg(unix)] @@ -468,11 +407,11 @@ mod tests { fn round_trips_https_entry() { let temp = tempfile::tempdir().unwrap(); let store = AuthStore::new(temp.path().join("auth.json")); - let key = ServerTargetKey::new(&https_target("https://fabro.example.com")).unwrap(); + let target = https_target("https://fabro.example.com"); - store.put(&key, entry("octocat")).unwrap(); + store.put(&target, entry("octocat")).unwrap(); - let saved = store.get(&key).unwrap().unwrap(); + let saved = store.get(&target).unwrap().unwrap(); assert_eq!(saved.subject.login, "octocat"); } @@ -481,11 +420,11 @@ mod tests { fn round_trips_loopback_http_entry() { let temp = tempfile::tempdir().unwrap(); let store = AuthStore::new(temp.path().join("auth.json")); - let key = ServerTargetKey::new(&https_target("http://127.0.0.1:3000")).unwrap(); + let target = https_target("http://127.0.0.1:3000"); - store.put(&key, entry("alice")).unwrap(); + store.put(&target, entry("alice")).unwrap(); - let saved = store.get(&key).unwrap().unwrap(); + let saved = store.get(&target).unwrap().unwrap(); assert_eq!(saved.subject.login, "alice"); } @@ -496,19 +435,19 @@ mod tests { let socket = temp.path().join("fabro.sock"); std::fs::write(&socket, "").unwrap(); let store = AuthStore::new(temp.path().join("auth.json")); - let key = ServerTargetKey::new(&ServerTarget::UnixSocket(socket)).unwrap(); + let target = ServerTarget::unix_socket_path(socket).unwrap(); - store.put(&key, entry("unix")).unwrap(); + store.put(&target, entry("unix")).unwrap(); - let saved = store.get(&key).unwrap().unwrap(); + let saved = store.get(&target).unwrap().unwrap(); assert_eq!(saved.subject.login, "unix"); } #[test] fn https_normalization_collapses_equivalent_urls() { - let a = ServerTargetKey::new(&https_target("https://EXAMPLE.COM/")).unwrap(); - let b = ServerTargetKey::new(&https_target("https://example.com:443")).unwrap(); - let c = ServerTargetKey::new(&https_target("https://example.com")).unwrap(); + let a = key_for_target(&https_target("https://EXAMPLE.COM/")); + let b = key_for_target(&https_target("https://example.com:443")); + let c = key_for_target(&https_target("https://example.com")); assert_eq!(a, b); assert_eq!(b, c); @@ -516,36 +455,34 @@ mod tests { #[test] fn distinct_unix_socket_paths_do_not_collide() { - let a = - ServerTargetKey::new(&ServerTarget::UnixSocket(PathBuf::from("/tmp/a.sock"))).unwrap(); - let b = - ServerTargetKey::new(&ServerTarget::UnixSocket(PathBuf::from("/tmp/b.sock"))).unwrap(); + let a = key_for_target(&ServerTarget::unix_socket_path("/tmp/a.sock").unwrap()); + let b = key_for_target(&ServerTarget::unix_socket_path("/tmp/b.sock").unwrap()); assert_ne!(a, b); } #[cfg(unix)] #[test] - fn canonicalizes_symlinked_socket_paths() { + fn preserves_distinct_symlinked_socket_paths() { let temp = tempfile::tempdir().unwrap(); let socket = temp.path().join("fabro.sock"); let link = temp.path().join("fabro-link.sock"); std::fs::write(&socket, "").unwrap(); std::os::unix::fs::symlink(&socket, &link).unwrap(); - let direct = ServerTargetKey::new(&ServerTarget::UnixSocket(socket)).unwrap(); - let via_link = ServerTargetKey::new(&ServerTarget::UnixSocket(link)).unwrap(); + let direct = key_for_target(&ServerTarget::unix_socket_path(socket).unwrap()); + let via_link = key_for_target(&ServerTarget::unix_socket_path(link).unwrap()); - assert_eq!(direct, via_link); + assert_ne!(direct, via_link); } #[test] fn missing_file_returns_empty_results() { let temp = tempfile::tempdir().unwrap(); let store = AuthStore::new(temp.path().join("auth.json")); - let key = ServerTargetKey::new(&https_target("https://fabro.example.com")).unwrap(); + let target = https_target("https://fabro.example.com"); - assert!(store.get(&key).unwrap().is_none()); + assert!(store.get(&target).unwrap().is_none()); assert!(store.list().unwrap().is_empty()); } @@ -555,9 +492,9 @@ mod tests { let path = temp.path().join("auth.json"); std::fs::write(&path, "{not-json").unwrap(); let store = AuthStore::new(path.clone()); - let key = ServerTargetKey::new(&https_target("https://fabro.example.com")).unwrap(); + let target = https_target("https://fabro.example.com"); - let err = store.get(&key).unwrap_err(); + let err = store.get(&target).unwrap_err(); assert!(err.to_string().contains(&path.display().to_string())); } @@ -566,21 +503,21 @@ mod tests { fn concurrent_puts_do_not_corrupt_file() { let temp = tempfile::tempdir().unwrap(); let store = Arc::new(AuthStore::new(temp.path().join("auth.json"))); - let key = ServerTargetKey::new(&https_target("https://fabro.example.com")).unwrap(); + let target = https_target("https://fabro.example.com"); let mut tasks = Vec::new(); for login in ["alice", "bob"] { let store = Arc::clone(&store); - let key = key.clone(); + let target = target.clone(); tasks.push(thread::spawn(move || { - store.put(&key, entry(login)).unwrap(); + store.put(&target, entry(login)).unwrap(); })); } for task in tasks { task.join().unwrap(); } - let saved = store.get(&key).unwrap().unwrap(); + let saved = store.get(&target).unwrap().unwrap(); assert!(matches!(saved.subject.login.as_str(), "alice" | "bob")); } @@ -591,9 +528,9 @@ mod tests { let temp = tempfile::tempdir().unwrap(); let store = AuthStore::new(temp.path().join("auth.json")); - let key = ServerTargetKey::new(&https_target("https://fabro.example.com")).unwrap(); + let target = https_target("https://fabro.example.com"); - store.put(&key, entry("octocat")).unwrap(); + store.put(&target, entry("octocat")).unwrap(); let mode = std::fs::metadata(temp.path().join("auth.json")) .unwrap() @@ -608,9 +545,9 @@ mod tests { fn put_returns_unsupported_platform() { let temp = tempfile::tempdir().unwrap(); let store = AuthStore::new(temp.path().join("auth.json")); - let key = ServerTargetKey::new(&https_target("https://fabro.example.com")).unwrap(); + let target = https_target("https://fabro.example.com"); - let err = store.put(&key, entry("octocat")).unwrap_err(); + let err = store.put(&target, entry("octocat")).unwrap_err(); assert!(err.to_string().contains("not supported on this platform")); } diff --git a/lib/crates/fabro-client/src/client.rs b/lib/crates/fabro-client/src/client.rs new file mode 100644 index 000000000..57098e055 --- /dev/null +++ b/lib/crates/fabro-client/src/client.rs @@ -0,0 +1,1392 @@ +use std::collections::VecDeque; +use std::future::Future; +use std::num::NonZeroU64; +use std::path::Path; +use std::sync::{Arc, RwLock}; + +use anyhow::{Context as _, Result, anyhow, bail}; +use bytes::Bytes; +use fabro_api::types; +use fabro_http::header::{AUTHORIZATION, CONTENT_LENGTH, CONTENT_TYPE}; +use fabro_http::multipart::{Form, Part}; +use fabro_model::Model; +use fabro_types::{ + ArtifactUpload, EventEnvelope, RunBlobId, RunEvent, RunId, RunProjection, RunSummary, StageId, +}; +use futures::StreamExt; +use futures::future::BoxFuture; +use serde::{Deserialize, Serialize}; +use tokio::fs::File; +use tokio::sync::Mutex; +use tokio_util::io::ReaderStream; + +use crate::credential::Credential; +use crate::error::{ + ApiError, ApiFailure, classify_api_error, classify_http_response, convert_type, + is_not_found_error, map_api_error, raw_response_failure_error, +}; +use crate::loopback::LoopbackClassification; +use crate::session::OAuthSession; +use crate::target::ServerTarget; +use crate::{AuthEntry, StoredSubject, sse}; + +type TransportFuture = BoxFuture<'static, Result<(fabro_http::HttpClient, String)>>; + +pub struct RunEventStream { + stream: progenitor_client::ByteStream, + pending_bytes: Vec, + buffered_events: VecDeque, +} + +#[derive(Clone)] +struct ClientState { + client: fabro_api::ApiClient, + http_client: fabro_http::HttpClient, + bearer_token: Option, + base_url: String, +} + +#[derive(Clone)] +pub struct Client { + state: Arc>, + oauth_session: Option, + refresh_lock: Arc>, + transport_connector: Option, +} + +#[derive(Clone)] +pub struct TransportConnector { + connect: Arc) -> TransportFuture + Send + Sync>, +} + +#[derive(Default)] +pub struct ClientBuilder { + target: Option, + credential: Option, + oauth_session: Option, + transport: Option<(String, fabro_http::HttpClient)>, + transport_connector: Option, +} + +#[derive(Debug, Deserialize)] +struct CliTokenResponse { + access_token: String, + access_token_expires_at: chrono::DateTime, + refresh_token: String, + refresh_token_expires_at: chrono::DateTime, + subject: CliTokenSubject, +} + +#[derive(Debug, Deserialize)] +struct CliTokenSubject { + idp_issuer: String, + idp_subject: String, + login: String, + name: String, + email: String, +} + +#[derive(Debug, Deserialize)] +struct OAuthErrorBody { + error: String, + #[serde(default)] + error_description: Option, +} + +#[derive(Debug, Serialize)] +struct ArtifactBatchUploadManifest { + entries: Vec, +} + +#[derive(Debug, Serialize)] +struct ArtifactBatchUploadEntry { + part: String, + path: String, + #[serde(skip_serializing_if = "Option::is_none")] + sha256: Option, + #[serde(skip_serializing_if = "Option::is_none")] + expected_bytes: Option, + #[serde(skip_serializing_if = "Option::is_none")] + content_type: Option, +} + +impl RunEventStream { + #[must_use] + pub fn new(stream: progenitor_client::ByteStream) -> Self { + Self { + stream, + pending_bytes: Vec::new(), + buffered_events: VecDeque::new(), + } + } + + pub async fn next_event(&mut self) -> Result> { + loop { + if let Some(event) = self.buffered_events.pop_front() { + return Ok(Some(event)); + } + + if let Some(chunk) = self.stream.next().await { + let chunk = chunk.map_err(|err| anyhow!("{err}"))?; + self.pending_bytes.extend_from_slice(&chunk); + self.buffer_sse_events(false)?; + } else { + self.buffer_sse_events(true)?; + return Ok(self.buffered_events.pop_front()); + } + } + } + + fn buffer_sse_events(&mut self, finalize: bool) -> Result<()> { + for payload in sse::drain_sse_payloads(&mut self.pending_bytes, finalize) { + self.buffered_events + .push_back(serde_json::from_str(&payload)?); + } + Ok(()) + } +} + +impl TransportConnector { + pub fn new(connect: F) -> Self + where + F: Fn(Option) -> Fut + Send + Sync + 'static, + Fut: Future> + Send + 'static, + { + Self { + connect: Arc::new(move |bearer_token| Box::pin(connect(bearer_token))), + } + } + + pub async fn connect( + &self, + bearer_token: Option, + ) -> Result<(fabro_http::HttpClient, String)> { + (self.connect)(bearer_token).await + } +} + +impl ClientBuilder { + #[must_use] + pub fn target(mut self, target: ServerTarget) -> Self { + self.target = Some(target); + self + } + + #[must_use] + pub fn credential(mut self, credential: Credential) -> Self { + self.credential = Some(credential); + self + } + + #[must_use] + pub fn oauth_session(mut self, oauth_session: OAuthSession) -> Self { + self.oauth_session = Some(oauth_session); + self + } + + #[must_use] + pub fn transport( + mut self, + base_url: impl Into, + http_client: fabro_http::HttpClient, + ) -> Self { + self.transport = Some((base_url.into(), http_client)); + self + } + + #[must_use] + pub fn transport_connector(mut self, transport_connector: TransportConnector) -> Self { + self.transport_connector = Some(transport_connector); + self + } + + pub async fn connect(self) -> Result { + let bearer_token = self + .credential + .as_ref() + .map(Credential::bearer_token) + .map(ToOwned::to_owned); + let target = self.target.clone().or_else(|| { + self.oauth_session + .as_ref() + .map(|session| session.target.clone()) + }); + let transport_connector = self + .transport_connector + .or_else(|| target.map(default_transport_connector)); + + let state = if let Some((base_url, http_client)) = self.transport { + client_state(base_url, http_client, bearer_token.clone()) + } else { + let Some(transport_connector) = transport_connector.clone() else { + bail!("client builder requires a target, transport, or transport connector"); + }; + let (http_client, base_url) = transport_connector.connect(bearer_token.clone()).await?; + client_state(base_url, http_client, bearer_token.clone()) + }; + + Ok(Client { + state: Arc::new(RwLock::new(state)), + oauth_session: self.oauth_session, + refresh_lock: Arc::new(Mutex::new(())), + transport_connector, + }) + } +} + +impl Client { + #[must_use] + pub fn builder() -> ClientBuilder { + ClientBuilder::default() + } + + #[must_use] + pub fn from_http_client( + base_url: impl Into, + http_client: fabro_http::HttpClient, + ) -> Self { + Self { + state: Arc::new(RwLock::new(client_state( + base_url.into(), + http_client, + None, + ))), + oauth_session: None, + refresh_lock: Arc::new(Mutex::new(())), + transport_connector: None, + } + } + + pub fn new_no_proxy(base_url: &str) -> Result { + let http_client = fabro_http::HttpClientBuilder::new().no_proxy().build()?; + Ok(Self::from_http_client(base_url.to_string(), http_client)) + } + + #[must_use] + pub fn clone_for_reuse(&self) -> Self { + self.clone() + } + + #[must_use] + pub fn api_client(&self) -> fabro_api::ApiClient { + self.current_state().client + } + + #[must_use] + pub fn http_client(&self) -> fabro_http::HttpClient { + self.current_state().http_client + } + + #[must_use] + pub fn base_url(&self) -> String { + self.current_state().base_url + } + + fn current_state(&self) -> ClientState { + self.state + .read() + .expect("client state lock should not be poisoned") + .clone() + } + + fn replace_state(&self, state: ClientState) { + *self + .state + .write() + .expect("client state lock should not be poisoned") = state; + } + + async fn send_api( + &self, + request: F, + ) -> Result> + where + F: FnOnce(fabro_api::ApiClient) -> Fut + Clone, + Fut: Future< + Output = std::result::Result< + progenitor_client::ResponseValue, + progenitor_client::Error, + >, + >, + E: serde::Serialize + std::fmt::Debug, + { + let state = self.current_state(); + match request.clone()(state.client.clone()).await { + Ok(response) => Ok(response), + Err(err) => { + let mapped = classify_api_error(err).await; + if self.should_refresh(mapped.failure.as_ref()) { + if let Some(failed_token) = state.bearer_token.as_deref() { + self.refresh_access_token(failed_token).await?; + let state = self.current_state(); + return request(state.client.clone()).await.map_err(map_api_error); + } + } + Err(mapped.error) + } + } + } + + fn should_refresh(&self, failure: Option<&ApiFailure>) -> bool { + self.oauth_session.is_some() + && failure.is_some_and(|failure| { + failure.status == fabro_http::StatusCode::UNAUTHORIZED + && failure.code.as_deref() == Some("access_token_expired") + }) + } + + async fn refresh_access_token(&self, failed_access_token: &str) -> Result<()> { + let Some(oauth_session) = &self.oauth_session else { + bail!("CLI session has expired. Run `fabro auth login` again."); + }; + + let _guard = self.refresh_lock.lock().await; + let current_state = self.current_state(); + if current_state.bearer_token.as_deref() != Some(failed_access_token) { + return Ok(()); + } + + let Some(entry) = oauth_session.auth_store.get(&oauth_session.target)? else { + self.rebuild_with_fallback(oauth_session).await?; + bail!("CLI session has expired. Run `fabro auth login` again."); + }; + if entry.refresh_token_expires_at <= chrono::Utc::now() { + oauth_session.auth_store.remove(&oauth_session.target)?; + self.rebuild_with_fallback(oauth_session).await?; + bail!("CLI session has expired. Run `fabro auth login` again."); + } + ensure_refresh_target_transport(&oauth_session.target)?; + + let (http_client, base_url) = oauth_session.target.build_public_http_client()?; + let response = http_client + .post(format!("{base_url}/auth/cli/refresh")) + .header(AUTHORIZATION, format!("Bearer {}", entry.refresh_token)) + .send() + .await?; + + if response.status().is_success() { + let tokens = response + .json::() + .await + .context("failed to parse CLI auth refresh response")?; + let entry = AuthEntry { + access_token: tokens.access_token.clone(), + access_token_expires_at: tokens.access_token_expires_at, + refresh_token: tokens.refresh_token.clone(), + refresh_token_expires_at: tokens.refresh_token_expires_at, + subject: StoredSubject { + idp_issuer: tokens.subject.idp_issuer, + idp_subject: tokens.subject.idp_subject, + login: tokens.subject.login, + name: tokens.subject.name, + email: tokens.subject.email, + }, + logged_in_at: entry.logged_in_at, + }; + oauth_session + .auth_store + .put(&oauth_session.target, entry.clone()) + .context("failed to persist refreshed CLI auth tokens")?; + self.rebuild_client(Some(entry.access_token)).await?; + return Ok(()); + } + + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + let parsed_error = serde_json::from_str::(&body).ok(); + if parsed_error.as_ref().is_some_and(|error| { + matches!( + error.error.as_str(), + "refresh_token_expired" | "refresh_token_revoked" + ) + }) { + oauth_session.auth_store.remove(&oauth_session.target)?; + self.rebuild_with_fallback(oauth_session).await?; + } + + if let Some(parsed_error) = parsed_error { + let message = parsed_error + .error_description + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| format!("request failed with status {status}")); + bail!("{message}"); + } + if body.is_empty() { + bail!("request failed with status {status}"); + } + bail!("request failed with status {status}: {body}"); + } + + async fn rebuild_with_fallback(&self, oauth_session: &OAuthSession) -> Result<()> { + let credential = oauth_session.resolve_fallback(); + self.rebuild_client( + credential + .as_ref() + .map(Credential::bearer_token) + .map(ToOwned::to_owned), + ) + .await + } + + async fn rebuild_client(&self, bearer_token: Option) -> Result<()> { + let Some(transport_connector) = &self.transport_connector else { + bail!("client transport cannot be rebuilt"); + }; + let (http_client, base_url) = transport_connector.connect(bearer_token.clone()).await?; + self.replace_state(client_state(base_url, http_client, bearer_token)); + Ok(()) + } + + pub async fn send_http_response( + &self, + request: F, + ) -> Result> + where + F: FnOnce(fabro_http::HttpClient) -> Fut + Clone, + Fut: Future>, + T: Into, + { + let state = self.current_state(); + let response = request.clone()(state.http_client.clone()) + .await + .map_err(Into::into)?; + match classify_http_response(response).await? { + Ok(response) => Ok(Ok(response)), + Err(failure) => { + if self.should_refresh(Some(failure.api_failure())) { + if let Some(failed_token) = state.bearer_token.as_deref() { + self.refresh_access_token(failed_token).await?; + let state = self.current_state(); + let response = request(state.http_client.clone()) + .await + .map_err(Into::into)?; + return classify_http_response(response).await; + } + } + Ok(Err(failure)) + } + } + } + + async fn send_http(&self, request: F) -> Result + where + F: FnOnce(fabro_http::HttpClient) -> Fut + Clone, + Fut: Future>, + T: Into, + { + match self.send_http_response(request).await? { + Ok(response) => Ok(response), + Err(failure) => Err(raw_response_failure_error(&failure)), + } + } + + pub async fn retrieve_resolved_server_settings(&self) -> Result { + let url = format!("{}/api/v1/settings?view=resolved", self.base_url()); + let response = self + .send_http(|http_client| async move { http_client.get(&url).send().await }) + .await?; + + let marker = response + .headers() + .get("x-fabro-settings-view") + .and_then(|value| value.to_str().ok()); + if marker != Some("resolved") { + bail!( + "server does not support resolved settings view; upgrade the server or use --local" + ); + } + + response + .json::() + .await + .context("server returned invalid JSON for the resolved settings view") + } + + pub async fn create_run_from_manifest(&self, manifest: types::RunManifest) -> Result { + let response = self + .send_api( + |client| async move { client.create_run().body(manifest.clone()).send().await }, + ) + .await?; + let status = response.into_inner(); + status + .id + .parse() + .map_err(|err| anyhow!("invalid run ID from server: {err}")) + } + + pub async fn list_secrets(&self) -> Result> { + let response = self + .send_api(|client| async move { client.list_secrets().send().await }) + .await?; + Ok(response.into_inner().data) + } + + pub async fn create_secret( + &self, + body: types::CreateSecretRequest, + ) -> Result { + let response = self + .send_api( + |client| async move { client.create_secret().body(body.clone()).send().await }, + ) + .await?; + Ok(response.into_inner()) + } + + pub async fn delete_secret_by_name(&self, name: &str) -> Result<()> { + self.send_api(|client| async move { + client + .delete_secret_by_name() + .body(types::DeleteSecretRequest { + name: name.to_string(), + }) + .send() + .await + }) + .await?; + Ok(()) + } + + pub async fn list_models( + &self, + provider: Option<&str>, + query: Option<&str>, + ) -> Result> { + let mut offset = 0u64; + let mut models = Vec::new(); + + loop { + let response = self + .send_api(|client| async move { + let mut request = client.list_models().page_limit(100u64).page_offset(offset); + if let Some(provider) = provider { + request = request.provider(provider.to_string()); + } + if let Some(query) = query { + request = request.query(query.to_string()); + } + request.send().await + }) + .await?; + let parsed = response.into_inner(); + let count = parsed.data.len() as u64; + models.extend(convert_type::<_, Vec>(parsed.data)?); + if !parsed.meta.has_more { + break; + } + offset += count; + } + + Ok(models) + } + + pub async fn test_model( + &self, + id: &str, + mode: Option, + ) -> Result { + let response = self + .send_api(|client| async move { + let mut request = client.test_model().id(id.to_string()); + if let Some(mode) = mode { + request = request.mode(mode); + } + request.send().await + }) + .await?; + Ok(response.into_inner()) + } + + pub async fn attach_events(&self, run_ids: &[String]) -> Result { + let response = self + .send_api(|client| async move { + let mut request = client.attach_events(); + if !run_ids.is_empty() { + request = request.run_id(run_ids.join(",")); + } + request.send().await + }) + .await?; + Ok(response.into_inner()) + } + + pub async fn get_system_info(&self) -> Result { + let response = self + .send_api(|client| async move { client.get_system_info().send().await }) + .await?; + Ok(response.into_inner()) + } + + pub async fn get_system_disk_usage(&self, verbose: bool) -> Result { + let response = self + .send_api(|client| async move { + client.get_system_disk_usage().verbose(verbose).send().await + }) + .await?; + Ok(response.into_inner()) + } + + pub async fn prune_runs( + &self, + body: types::PruneRunsRequest, + ) -> Result { + let response = self + .send_api(|client| async move { client.prune_runs().body(body.clone()).send().await }) + .await?; + Ok(response.into_inner()) + } + + pub async fn get_health(&self) -> Result<()> { + self.send_api(|client| async move { client.get_health().send().await }) + .await?; + Ok(()) + } + + pub async fn run_diagnostics(&self) -> Result { + let response = self + .send_api(|client| async move { client.run_diagnostics().send().await }) + .await?; + Ok(response.into_inner()) + } + + pub async fn get_github_repo( + &self, + owner: &str, + name: &str, + ) -> Result { + let response = self + .send_api(|client| async move { + client + .get_github_repo() + .owner(owner.to_string()) + .name(name.to_string()) + .send() + .await + }) + .await?; + Ok(response.into_inner()) + } + + pub async fn run_preflight( + &self, + manifest: types::RunManifest, + ) -> Result { + self.send_api( + |client| async move { client.run_preflight().body(manifest.clone()).send().await }, + ) + .await + .map(progenitor_client::ResponseValue::into_inner) + } + + pub async fn render_workflow_graph( + &self, + request: types::RenderWorkflowGraphRequest, + ) -> Result> { + let response = self + .send_api(|client| async move { + client + .render_workflow_graph() + .body(request.clone()) + .send() + .await + }) + .await?; + let mut stream = response.into_inner(); + let mut bytes = Vec::new(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|err| anyhow!("{err}"))?; + bytes.extend_from_slice(&chunk); + } + Ok(bytes) + } + + pub async fn start_run(&self, run_id: &RunId, resume: bool) -> Result<()> { + self.send_api(|client| async move { + client + .start_run() + .id(run_id.to_string()) + .body(types::StartRunRequest { resume }) + .send() + .await + }) + .await?; + Ok(()) + } + + pub async fn cancel_run(&self, run_id: &RunId) -> Result<()> { + self.send_api( + |client| async move { client.cancel_run().id(run_id.to_string()).send().await }, + ) + .await?; + Ok(()) + } + + pub async fn archive_run(&self, run_id: &RunId) -> Result<()> { + self.send_api( + |client| async move { client.archive_run().id(run_id.to_string()).send().await }, + ) + .await?; + Ok(()) + } + + pub async fn unarchive_run(&self, run_id: &RunId) -> Result<()> { + self.send_api(|client| async move { + client.unarchive_run().id(run_id.to_string()).send().await + }) + .await?; + Ok(()) + } + + pub async fn list_store_runs(&self) -> Result> { + let mut all_runs = Vec::new(); + let mut offset = 0_u64; + let limit = 100_u64; + + loop { + let response = self + .send_api(|client| async move { + client + .list_runs() + .page_limit(limit) + .page_offset(offset) + .include_archived(true) + .send() + .await + }) + .await?; + let parsed = response.into_inner(); + let batch = parsed + .data + .into_iter() + .map(convert_type) + .collect::>>()?; + let batch_len = batch.len() as u64; + all_runs.extend(batch); + + if !parsed.meta.has_more || batch_len == 0 { + break; + } + offset += batch_len; + } + + Ok(all_runs) + } + + pub async fn retrieve_run(&self, run_id: &RunId) -> Result { + let response = self + .send_api( + |client| async move { client.retrieve_run().id(run_id.to_string()).send().await }, + ) + .await?; + convert_type(response.into_inner()) + } + + pub async fn resolve_run(&self, selector: &str) -> Result { + let response = self + .send_api(|client| async move { + client + .resolve_run() + .selector(selector.to_string()) + .send() + .await + }) + .await?; + convert_type(response.into_inner()) + } + + pub async fn get_run_state(&self, run_id: &RunId) -> Result { + let response = self + .send_api( + |client| async move { client.get_run_state().id(run_id.to_string()).send().await }, + ) + .await?; + convert_type(response.into_inner()) + } + + pub async fn list_run_events( + &self, + run_id: &RunId, + since_seq: Option, + limit: Option, + ) -> Result> { + let mut next_since_seq = since_seq; + let mut all_events = Vec::new(); + + loop { + let response = self + .send_api(|client| async move { + let mut request = client.list_run_events().id(run_id.to_string()); + if let Some(seq) = next_since_seq.and_then(non_zero_u64_from_u32) { + request = request.since_seq(seq); + } + if let Some(limit) = limit.and_then(non_zero_u64_from_usize) { + request = request.limit(limit); + } + request.send().await + }) + .await?; + let parsed = response.into_inner(); + let page_events = parsed + .data + .into_iter() + .map(convert_type::<_, EventEnvelope>) + .collect::>>()?; + let next_page_since_seq = page_events.last().map(|event| event.seq.saturating_add(1)); + all_events.extend(page_events); + + if limit.is_some() || !parsed.meta.has_more || next_page_since_seq.is_none() { + break; + } + next_since_seq = next_page_since_seq; + } + + Ok(all_events) + } + + pub async fn attach_run_events( + &self, + run_id: &RunId, + since_seq: Option, + ) -> Result { + let response = self + .send_api(|client| async move { + let mut request = client.attach_run_events().id(run_id.to_string()); + if let Some(seq) = since_seq.and_then(non_zero_u64_from_u32) { + request = request.since_seq(seq); + } + request.send().await + }) + .await?; + Ok(RunEventStream::new(response.into_inner())) + } + + pub async fn list_run_questions(&self, run_id: &RunId) -> Result> { + let response = self + .send_api(|client| async move { + client + .list_run_questions() + .id(run_id.to_string()) + .page_limit(100) + .page_offset(0) + .send() + .await + }) + .await?; + Ok(response.into_inner().data) + } + + pub async fn submit_run_answer( + &self, + run_id: &RunId, + qid: &str, + value: Option, + selected_option_key: Option, + selected_option_keys: Vec, + ) -> Result<()> { + self.send_api(|client| async move { + client + .submit_run_answer() + .id(run_id.to_string()) + .qid(qid) + .body(types::SubmitAnswerRequest { + value: value.clone(), + selected_option_key: selected_option_key.clone(), + selected_option_keys: selected_option_keys.clone(), + }) + .send() + .await + }) + .await?; + Ok(()) + } + + pub async fn append_run_event(&self, run_id: &RunId, event: &RunEvent) -> Result { + let body: types::RunEvent = convert_type(event)?; + let response = self + .send_api(|client| async move { + client + .append_run_event() + .id(run_id.to_string()) + .body(body.clone()) + .send() + .await + }) + .await?; + u32::try_from(response.into_inner().seq).context("append_run_event returned invalid seq") + } + + pub async fn write_run_blob(&self, run_id: &RunId, data: &[u8]) -> Result { + let response = self + .send_api(|client| async move { + client + .write_run_blob() + .id(run_id.to_string()) + .body(data.to_vec()) + .send() + .await + }) + .await?; + response + .into_inner() + .id + .parse() + .context("write_run_blob returned invalid blob id") + } + + pub async fn read_run_blob( + &self, + run_id: &RunId, + blob_id: &RunBlobId, + ) -> Result> { + let response = self + .current_state() + .client + .read_run_blob() + .id(run_id.to_string()) + .blob_id(blob_id.to_string()) + .send() + .await; + match response { + Ok(response) => { + let mut stream = response.into_inner(); + let mut bytes = Vec::new(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|err| anyhow!("{err}"))?; + bytes.extend_from_slice(&chunk); + } + Ok(Some(Bytes::from(bytes))) + } + Err(err) => { + if is_not_found_error(&err) { + Ok(None) + } else { + Err(map_api_error(err)) + } + } + } + } + + pub async fn delete_store_run(&self, run_id: &RunId, force: bool) -> Result<()> { + let base_url = self.base_url(); + let mut url = fabro_http::Url::parse(&base_url) + .with_context(|| format!("invalid server base URL {base_url}"))?; + url.path_segments_mut() + .map_err(|()| anyhow!("server base URL cannot accept path segments"))? + .extend(["api", "v1", "runs", &run_id.to_string()]); + if force { + url.query_pairs_mut().append_pair("force", "true"); + } + + self.send_http(|http_client| async move { http_client.delete(url.clone()).send().await }) + .await?; + Ok(()) + } + + pub async fn list_run_artifacts(&self, run_id: &RunId) -> Result> { + let response = self + .send_api(|client| async move { + client + .list_run_artifacts() + .id(run_id.to_string()) + .send() + .await + }) + .await?; + Ok(response.into_inner().data) + } + + pub async fn download_stage_artifact( + &self, + run_id: &RunId, + stage_id: &StageId, + filename: &str, + ) -> Result> { + let response = self + .send_api(|client| async move { + client + .get_stage_artifact() + .id(run_id.to_string()) + .stage_id(stage_id.to_string()) + .filename(filename) + .send() + .await + }) + .await?; + let mut stream = response.into_inner(); + let mut bytes = Vec::new(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|err| anyhow!("{err}"))?; + bytes.extend_from_slice(&chunk); + } + Ok(bytes) + } + + fn stage_artifacts_url(&self, run_id: &RunId, stage_id: &StageId) -> Result { + let base_url = self.base_url(); + let mut url = fabro_http::Url::parse(&base_url) + .with_context(|| format!("invalid server base URL {base_url}"))?; + url.path_segments_mut() + .map_err(|()| anyhow!("server base URL cannot accept path segments"))? + .extend([ + "api", + "v1", + "runs", + &run_id.to_string(), + "stages", + &stage_id.to_string(), + "artifacts", + ]); + Ok(url) + } + + pub async fn upload_stage_artifact_file( + &self, + run_id: &RunId, + stage_id: &StageId, + filename: &str, + path: &Path, + bearer_token: &str, + ) -> Result<()> { + let mut url = self.stage_artifacts_url(run_id, stage_id)?; + url.query_pairs_mut().append_pair("filename", filename); + + let file = File::open(path) + .await + .with_context(|| format!("failed to open artifact {}", path.display()))?; + let content_length = file + .metadata() + .await + .with_context(|| format!("failed to stat artifact {}", path.display()))? + .len(); + let body = fabro_http::Body::wrap_stream(ReaderStream::new(file)); + + let response = self + .current_state() + .http_client + .post(url) + .bearer_auth(bearer_token) + .header(CONTENT_TYPE, "application/octet-stream") + .header(CONTENT_LENGTH, content_length.to_string()) + .body(body) + .send() + .await + .with_context(|| format!("failed to upload artifact {}", path.display()))?; + classify_http_response(response) + .await? + .map(|_| ()) + .map_err(|failure| raw_response_failure_error(&failure)) + } + + pub async fn upload_stage_artifact_batch( + &self, + run_id: &RunId, + stage_id: &StageId, + artifact_capture_dir: &Path, + artifacts: &[ArtifactUpload], + bearer_token: &str, + ) -> Result<()> { + let url = self.stage_artifacts_url(run_id, stage_id)?; + let mut manifest_entries = Vec::with_capacity(artifacts.len()); + let mut file_parts = Vec::with_capacity(artifacts.len()); + + for (index, artifact) in artifacts.iter().enumerate() { + let part_name = format!("file{}", index + 1); + let path = artifact_capture_dir.join(&artifact.path); + let file = File::open(&path) + .await + .with_context(|| format!("failed to open artifact {}", path.display()))?; + let content_length = file + .metadata() + .await + .with_context(|| format!("failed to stat artifact {}", path.display()))? + .len(); + + manifest_entries.push(ArtifactBatchUploadEntry { + part: part_name.clone(), + path: artifact.path.clone(), + sha256: Some(artifact.content_sha256.clone()), + expected_bytes: Some(artifact.bytes), + content_type: Some(artifact.mime.clone()), + }); + + file_parts.push(( + part_name, + Part::stream_with_length( + fabro_http::Body::wrap_stream(ReaderStream::new(file)), + content_length, + ) + .file_name(artifact.path.clone()), + )); + } + + let manifest = ArtifactBatchUploadManifest { + entries: manifest_entries, + }; + let manifest_part = + Part::text(serde_json::to_string(&manifest)?).mime_str("application/json")?; + let mut form = Form::new().part("manifest", manifest_part); + for (part_name, part) in file_parts { + form = form.part(part_name, part); + } + + let response = self + .current_state() + .http_client + .post(url) + .bearer_auth(bearer_token) + .multipart(form) + .send() + .await + .context("failed to upload artifact batch")?; + classify_http_response(response) + .await? + .map(|_| ()) + .map_err(|failure| raw_response_failure_error(&failure)) + } + + pub async fn generate_preview_url( + &self, + run_id: &RunId, + port: u16, + expires_in_secs: u64, + signed: bool, + ) -> Result { + let expires_in_secs = NonZeroU64::new(expires_in_secs) + .ok_or_else(|| anyhow!("preview expiry must be greater than zero"))?; + let response = self + .send_api(|client| async move { + client + .generate_preview_url() + .id(run_id.to_string()) + .body(types::PreviewUrlRequest { + expires_in_secs, + port: i64::from(port), + signed, + }) + .send() + .await + }) + .await?; + Ok(response.into_inner()) + } + + pub async fn create_run_ssh_access( + &self, + run_id: &RunId, + ttl_minutes: f64, + ) -> Result { + let response = self + .send_api(|client| async move { + client + .create_run_ssh_access() + .id(run_id.to_string()) + .body(types::SshAccessRequest { ttl_minutes }) + .send() + .await + }) + .await?; + Ok(response.into_inner()) + } + + pub async fn list_sandbox_files( + &self, + run_id: &RunId, + path: &str, + depth: Option, + ) -> Result> { + let response = self + .send_api(|client| async move { + let mut request = client + .list_sandbox_files() + .id(run_id.to_string()) + .path(path); + if let Some(depth) = depth.and_then(non_zero_u64_from_u32) { + request = request.depth(depth); + } + request.send().await + }) + .await?; + Ok(response.into_inner().data) + } + + pub async fn get_sandbox_file(&self, run_id: &RunId, path: &str) -> Result> { + let response = self + .send_api(|client| async move { + client + .get_sandbox_file() + .id(run_id.to_string()) + .path(path) + .send() + .await + }) + .await?; + let mut stream = response.into_inner(); + let mut bytes = Vec::new(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|err| anyhow!("{err}"))?; + bytes.extend_from_slice(&chunk); + } + Ok(bytes) + } + + pub async fn put_sandbox_file(&self, run_id: &RunId, path: &str, bytes: Vec) -> Result<()> { + self.send_api(|client| async move { + client + .put_sandbox_file() + .id(run_id.to_string()) + .path(path) + .body(bytes.clone()) + .send() + .await + }) + .await?; + Ok(()) + } +} + +fn client_state( + base_url: String, + http_client: fabro_http::HttpClient, + bearer_token: Option, +) -> ClientState { + let client = fabro_api::ApiClient::new_with_client(&base_url, http_client.clone()); + ClientState { + client, + http_client, + bearer_token, + base_url, + } +} + +fn default_transport_connector(target: ServerTarget) -> TransportConnector { + TransportConnector::new(move |bearer_token| { + let target = target.clone(); + async move { connect_target_transport(&target, bearer_token.as_deref()) } + }) +} + +fn connect_target_transport( + target: &ServerTarget, + bearer_token: Option<&str>, +) -> Result<(fabro_http::HttpClient, String)> { + if let Some(api_url) = target.as_http_url() { + let mut builder = fabro_http::HttpClientBuilder::new(); + builder = match bearer_token { + Some(token) => apply_bearer_token_auth(builder, token)?, + None => builder, + }; + let http_client = builder.build()?; + return Ok((http_client, api_url.to_string())); + } + + let Some(path) = target.as_unix_socket_path() else { + bail!("server target must be an http(s) URL or absolute Unix socket path"); + }; + let mut builder = fabro_http::HttpClientBuilder::new() + .unix_socket(path) + .no_proxy(); + builder = match bearer_token { + Some(token) => apply_bearer_token_auth(builder, token)?, + None => builder, + }; + let http_client = builder.build()?; + Ok((http_client, "http://fabro".to_string())) +} + +fn apply_bearer_token_auth( + builder: fabro_http::HttpClientBuilder, + token: &str, +) -> Result { + let mut headers = fabro_http::HeaderMap::new(); + headers.insert( + AUTHORIZATION, + fabro_http::HeaderValue::from_str(&format!("Bearer {token}")) + .context("invalid bearer token header value")?, + ); + Ok(builder.default_headers(headers)) +} + +fn ensure_refresh_target_transport(target: &ServerTarget) -> Result<()> { + match target.loopback_classification()? { + LoopbackClassification::Https + | LoopbackClassification::LoopbackHttp + | LoopbackClassification::UnixSocket => Ok(()), + LoopbackClassification::Rejected => bail!(refresh_transport_error(target)), + } +} + +fn refresh_transport_error(target: &ServerTarget) -> String { + format!( + "Refusing to send refresh-token credentials over plaintext HTTP to a non-loopback host ({target}). Use HTTPS, or bind the server to 127.0.0.1 / ::1." + ) +} + +fn non_zero_u64_from_u32(value: u32) -> Option { + NonZeroU64::new(u64::from(value)) +} + +fn non_zero_u64_from_usize(value: usize) -> Option { + u64::try_from(value).ok().and_then(NonZeroU64::new) +} + +#[cfg(test)] +mod tests { + use chrono::Duration as ChronoDuration; + + use super::*; + use crate::AuthStore; + + fn oauth_entry(login: &str) -> AuthEntry { + let now = chrono::Utc::now(); + AuthEntry { + access_token: format!("access-{login}"), + access_token_expires_at: now + ChronoDuration::minutes(10), + refresh_token: format!("refresh-{login}"), + refresh_token_expires_at: now + ChronoDuration::days(30), + subject: StoredSubject { + idp_issuer: "https://github.com".to_string(), + idp_subject: "12345".to_string(), + login: login.to_string(), + name: format!("Name {login}"), + email: format!("{login}@example.com"), + }, + logged_in_at: now, + } + } + + #[cfg(unix)] + #[tokio::test] + async fn refresh_access_token_rejects_plain_http_non_loopback_targets() { + let temp = tempfile::tempdir().unwrap(); + let auth_store = AuthStore::new(temp.path().join("auth.json")); + let target = ServerTarget::http_url("http://fabro.example.com").unwrap(); + let entry = oauth_entry("octocat"); + auth_store.put(&target, entry.clone()).unwrap(); + + let client = Client::builder() + .target(target.clone()) + .credential(Credential::OAuth(entry)) + .oauth_session(OAuthSession::new(target.clone(), auth_store.clone())) + .transport( + "http://fabro.example.com", + fabro_http::HttpClientBuilder::new() + .no_proxy() + .build() + .unwrap(), + ) + .connect() + .await + .unwrap(); + + let err = client + .refresh_access_token("access-octocat") + .await + .unwrap_err(); + assert!( + err.to_string() + .contains("Refusing to send refresh-token credentials over plaintext HTTP") + ); + assert!(auth_store.get(&target).unwrap().is_some()); + } +} diff --git a/lib/crates/fabro-client/src/credential.rs b/lib/crates/fabro-client/src/credential.rs new file mode 100644 index 000000000..c0a194069 --- /dev/null +++ b/lib/crates/fabro-client/src/credential.rs @@ -0,0 +1,40 @@ +use std::fmt; + +use crate::AuthEntry; + +#[derive(Clone)] +pub enum Credential { + DevToken(String), + OAuth(AuthEntry), +} + +pub trait CredentialFallback: Send + Sync { + fn resolve(&self) -> Option; +} + +impl CredentialFallback for F +where + F: Fn() -> Option + Send + Sync, +{ + fn resolve(&self) -> Option { + self() + } +} + +impl Credential { + pub fn bearer_token(&self) -> &str { + match self { + Self::DevToken(token) => token, + Self::OAuth(entry) => &entry.access_token, + } + } +} + +impl fmt::Debug for Credential { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::DevToken(_) => f.write_str("Credential::DevToken()"), + Self::OAuth(_) => f.write_str("Credential::OAuth()"), + } + } +} diff --git a/lib/crates/fabro-client/src/error.rs b/lib/crates/fabro-client/src/error.rs new file mode 100644 index 000000000..d41d61fa7 --- /dev/null +++ b/lib/crates/fabro-client/src/error.rs @@ -0,0 +1,184 @@ +use anyhow::{Result, anyhow}; +use serde::de::DeserializeOwned; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ApiFailure { + pub status: fabro_http::StatusCode, + pub code: Option, +} + +pub struct StructuredApiError { + pub error: anyhow::Error, + pub failure: Option, +} + +pub struct ApiError { + pub status: fabro_http::StatusCode, + pub headers: fabro_http::HeaderMap, + pub body: String, + failure: ApiFailure, +} + +impl ApiError { + pub fn api_failure(&self) -> &ApiFailure { + &self.failure + } +} + +pub fn parse_error_response_value(value: &serde_json::Value) -> (Option, Option) { + let first = value + .get("errors") + .and_then(serde_json::Value::as_array) + .and_then(|errors| errors.first()); + let detail = first + .and_then(|entry| entry.get("detail")) + .and_then(serde_json::Value::as_str) + .map(ToOwned::to_owned); + let code = first + .and_then(|entry| entry.get("code")) + .and_then(serde_json::Value::as_str) + .map(ToOwned::to_owned); + (detail, code) +} + +pub async fn classify_api_error(err: progenitor_client::Error) -> StructuredApiError +where + E: serde::Serialize + std::fmt::Debug, +{ + match err { + progenitor_client::Error::UnexpectedResponse(response) => { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + let mut code = None; + if let Ok(value) = serde_json::from_str::(&body) { + let (detail, parsed_code) = parse_error_response_value(&value); + code = parsed_code; + if let Some(detail) = detail { + return StructuredApiError { + error: anyhow!("{detail}"), + failure: Some(ApiFailure { status, code }), + }; + } + } + let error = if body.is_empty() { + anyhow!("request failed with status {status}") + } else { + anyhow!("request failed with status {status}: {body}") + }; + StructuredApiError { + error, + failure: Some(ApiFailure { status, code }), + } + } + other => map_api_error_structured(other), + } +} + +fn map_api_error_structured(err: progenitor_client::Error) -> StructuredApiError +where + E: serde::Serialize + std::fmt::Debug, +{ + match err { + progenitor_client::Error::ErrorResponse(response) => { + let status = response.status(); + let mut code = None; + if let Ok(value) = serde_json::to_value(response.into_inner()) { + let (detail, parsed_code) = parse_error_response_value(&value); + code = parsed_code; + if let Some(detail) = detail { + return StructuredApiError { + error: anyhow!("{detail}"), + failure: Some(ApiFailure { status, code }), + }; + } + } + StructuredApiError { + error: anyhow!("request failed with status {status}"), + failure: Some(ApiFailure { status, code }), + } + } + progenitor_client::Error::UnexpectedResponse(response) => StructuredApiError { + error: anyhow!("request failed with status {}", response.status()), + failure: Some(ApiFailure { + status: response.status(), + code: None, + }), + }, + other => StructuredApiError { + error: anyhow!("{other}"), + failure: None, + }, + } +} + +pub fn map_api_error(err: progenitor_client::Error) -> anyhow::Error +where + E: serde::Serialize + std::fmt::Debug, +{ + map_api_error_structured(err).error +} + +pub fn raw_response_failure_error(failure: &ApiError) -> anyhow::Error { + if let Ok(value) = serde_json::from_str::(&failure.body) { + let (detail, _) = parse_error_response_value(&value); + if let Some(detail) = detail { + return anyhow!("{detail}"); + } + } + + if failure.body.is_empty() { + return anyhow!("request failed with status {}", failure.status); + } + + anyhow!( + "request failed with status {}: {}", + failure.status, + failure.body + ) +} + +pub async fn classify_http_response( + response: fabro_http::Response, +) -> Result> { + if response.status().is_success() { + return Ok(Ok(response)); + } + let status = response.status(); + let headers = response.headers().clone(); + let body = response.text().await.unwrap_or_default(); + let mut code = None; + if let Ok(value) = serde_json::from_str::(&body) { + let (_, parsed_code) = parse_error_response_value(&value); + code = parsed_code; + } + + Ok(Err(ApiError { + status, + headers, + body, + failure: ApiFailure { status, code }, + })) +} + +pub fn is_not_found_error(err: &progenitor_client::Error) -> bool +where + E: serde::Serialize + std::fmt::Debug, +{ + match err { + progenitor_client::Error::ErrorResponse(response) => { + response.status() == fabro_http::StatusCode::NOT_FOUND + } + progenitor_client::Error::UnexpectedResponse(response) => { + response.status() == fabro_http::StatusCode::NOT_FOUND + } + _ => false, + } +} + +pub fn convert_type(value: TInput) -> Result +where + TInput: serde::Serialize, + TOutput: DeserializeOwned, +{ + serde_json::from_value(serde_json::to_value(value)?).map_err(Into::into) +} diff --git a/lib/crates/fabro-client/src/lib.rs b/lib/crates/fabro-client/src/lib.rs new file mode 100644 index 000000000..248987500 --- /dev/null +++ b/lib/crates/fabro-client/src/lib.rs @@ -0,0 +1,26 @@ +//! Typed HTTP client for the Fabro API. +//! +//! This crate hosts the reusable client and auth/session plumbing that was +//! previously embedded in `fabro-cli`. + +pub mod auth_store; +pub mod client; +pub mod credential; +pub mod error; +pub mod loopback; +pub mod session; +pub mod sse; +pub mod target; + +pub use auth_store::{AuthEntry, AuthStore, AuthStoreError, LockError, StoredSubject}; +pub use client::{Client, RunEventStream, TransportConnector}; +pub use credential::{Credential, CredentialFallback}; +pub use error::{ + ApiError, ApiFailure, StructuredApiError, classify_api_error, classify_http_response, + convert_type, is_not_found_error, map_api_error, parse_error_response_value, + raw_response_failure_error, +}; +pub use fabro_api::types; +pub use loopback::{LoopbackClassification, TargetSchemeError}; +pub use session::OAuthSession; +pub use target::ServerTarget; diff --git a/lib/crates/fabro-cli/src/loopback_target.rs b/lib/crates/fabro-client/src/loopback.rs similarity index 71% rename from lib/crates/fabro-cli/src/loopback_target.rs rename to lib/crates/fabro-client/src/loopback.rs index 96903111b..20a28b900 100644 --- a/lib/crates/fabro-cli/src/loopback_target.rs +++ b/lib/crates/fabro-client/src/loopback.rs @@ -2,10 +2,10 @@ use std::net::IpAddr; use thiserror::Error; -use crate::user_config::{self, ServerTarget}; +use crate::target::ServerTarget; #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum LoopbackClassification { +pub enum LoopbackClassification { Https, LoopbackHttp, UnixSocket, @@ -13,7 +13,7 @@ pub(crate) enum LoopbackClassification { } #[derive(Debug, Error)] -pub(crate) enum TargetSchemeError { +pub enum TargetSchemeError { #[error("invalid server URL `{value}`: {reason}")] InvalidUrl { value: String, reason: String }, #[error("unsupported server URL scheme `{scheme}`")] @@ -22,22 +22,25 @@ pub(crate) enum TargetSchemeError { MissingHost { value: String }, } -pub(crate) fn is_loopback_or_unix_socket( +pub(crate) fn classify_target( target: &ServerTarget, ) -> Result { - match target { - ServerTarget::UnixSocket(_) => Ok(LoopbackClassification::UnixSocket), - ServerTarget::HttpUrl(api_url) => classify_http_target(api_url), + if target.is_unix_socket() { + Ok(LoopbackClassification::UnixSocket) + } else if let Some(api_url) = target.as_http_url() { + classify_http_target(api_url) + } else { + Err(TargetSchemeError::MissingHost { + value: target.to_string(), + }) } } fn classify_http_target(api_url: &str) -> Result { - let normalized = user_config::normalized_http_base_url(api_url); - let url = - fabro_http::Url::parse(normalized).map_err(|source| TargetSchemeError::InvalidUrl { - value: api_url.to_string(), - reason: source.to_string(), - })?; + let url = fabro_http::Url::parse(api_url).map_err(|source| TargetSchemeError::InvalidUrl { + value: api_url.to_string(), + reason: source.to_string(), + })?; match url.scheme() { "https" => Ok(LoopbackClassification::Https), @@ -50,7 +53,7 @@ fn classify_http_target(api_url: &str) -> Result bool { mod tests { use std::path::PathBuf; - use super::{LoopbackClassification, is_loopback_or_unix_socket}; - use crate::user_config::ServerTarget; + use super::LoopbackClassification; + use crate::target::ServerTarget; #[test] fn classifies_https_loopback_and_unix_targets() { let cases = [ ( - ServerTarget::HttpUrl("https://fabro.example.com".to_string()), + ServerTarget::http_url("https://fabro.example.com").unwrap(), LoopbackClassification::Https, ), ( - ServerTarget::HttpUrl("http://127.0.0.1:3000".to_string()), + ServerTarget::http_url("http://127.0.0.1:3000").unwrap(), LoopbackClassification::LoopbackHttp, ), ( - ServerTarget::HttpUrl("http://[::1]:3000".to_string()), + ServerTarget::http_url("http://[::1]:3000").unwrap(), LoopbackClassification::LoopbackHttp, ), ( - ServerTarget::HttpUrl("http://[::ffff:127.0.0.1]:3000".to_string()), + ServerTarget::http_url("http://[::ffff:127.0.0.1]:3000").unwrap(), LoopbackClassification::LoopbackHttp, ), ( - ServerTarget::UnixSocket(PathBuf::from("/tmp/fabro.sock")), + ServerTarget::unix_socket_path("/tmp/fabro.sock").unwrap(), LoopbackClassification::UnixSocket, ), ]; for (target, expected) in cases { - assert_eq!(is_loopback_or_unix_socket(&target).unwrap(), expected); + assert_eq!(target.loopback_classification().unwrap(), expected); } } @@ -167,18 +170,23 @@ mod tests { ]; for api_url in cases { - let target = ServerTarget::HttpUrl(api_url.to_string()); + let target = ServerTarget::http_url(api_url).unwrap(); assert_eq!( - is_loopback_or_unix_socket(&target).unwrap(), + target.loopback_classification().unwrap(), LoopbackClassification::Rejected ); } } #[test] - fn rejects_unsupported_schemes() { - let target = ServerTarget::HttpUrl("ftp://fabro.example.com".to_string()); - let error = is_loopback_or_unix_socket(&target).unwrap_err(); - assert!(error.to_string().contains("unsupported server URL scheme")); + fn rejects_non_http_server_targets_at_parse_time() { + let error = "ftp://fabro.example.com" + .parse::() + .unwrap_err(); + assert!( + error + .to_string() + .contains("server target must be an http(s) URL or absolute Unix socket path") + ); } } diff --git a/lib/crates/fabro-client/src/session.rs b/lib/crates/fabro-client/src/session.rs new file mode 100644 index 000000000..55435b9e8 --- /dev/null +++ b/lib/crates/fabro-client/src/session.rs @@ -0,0 +1,48 @@ +use std::fmt; +use std::sync::Arc; + +use crate::{AuthStore, Credential, CredentialFallback, ServerTarget}; + +#[derive(Clone)] +pub struct OAuthSession { + pub target: ServerTarget, + pub auth_store: AuthStore, + pub fallback: Option>, +} + +impl OAuthSession { + #[must_use] + pub fn new(target: ServerTarget, auth_store: AuthStore) -> Self { + Self { + target, + auth_store, + fallback: None, + } + } + + #[must_use] + pub fn with_fallback(mut self, fallback: Arc) -> Self { + self.fallback = Some(fallback); + self + } + + #[must_use] + pub fn resolve_fallback(&self) -> Option { + self.fallback + .as_ref() + .and_then(|fallback| fallback.resolve()) + } +} + +impl fmt::Debug for OAuthSession { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("OAuthSession") + .field("target", &self.target) + .field("auth_store", &self.auth_store) + .field( + "fallback", + &self.fallback.as_ref().map(|_| ""), + ) + .finish() + } +} diff --git a/lib/crates/fabro-cli/src/sse.rs b/lib/crates/fabro-client/src/sse.rs similarity index 94% rename from lib/crates/fabro-cli/src/sse.rs rename to lib/crates/fabro-client/src/sse.rs index 7208e625d..10b72f2d3 100644 --- a/lib/crates/fabro-cli/src/sse.rs +++ b/lib/crates/fabro-client/src/sse.rs @@ -1,4 +1,4 @@ -pub(crate) fn drain_sse_payloads(buffer: &mut Vec, finalize: bool) -> Vec { +pub fn drain_sse_payloads(buffer: &mut Vec, finalize: bool) -> Vec { let mut payloads = Vec::new(); while let Some(pos) = buffer.iter().position(|byte| *byte == b'\n') { diff --git a/lib/crates/fabro-client/src/target.rs b/lib/crates/fabro-client/src/target.rs new file mode 100644 index 000000000..e710ca732 --- /dev/null +++ b/lib/crates/fabro-client/src/target.rs @@ -0,0 +1,220 @@ +use std::fmt; +use std::path::{Component, Path, PathBuf}; +use std::str::FromStr; + +use anyhow::{Result, bail}; + +use crate::loopback::{LoopbackClassification, TargetSchemeError, classify_target}; + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum ServerTarget { + HttpUrl(CanonicalHttpUrl), + UnixSocket(CanonicalUnixSocketPath), +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct CanonicalHttpUrl(String); + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct CanonicalUnixSocketPath(PathBuf); + +impl ServerTarget { + pub fn http_url(value: impl AsRef) -> Result { + Ok(Self::HttpUrl(CanonicalHttpUrl::new(value.as_ref())?)) + } + + pub fn unix_socket_path(path: impl AsRef) -> Result { + Ok(Self::UnixSocket(CanonicalUnixSocketPath::new( + path.as_ref(), + )?)) + } + + #[must_use] + pub fn as_http_url(&self) -> Option<&str> { + match self { + Self::HttpUrl(url) => Some(url.as_str()), + Self::UnixSocket(_) => None, + } + } + + #[must_use] + pub fn as_unix_socket_path(&self) -> Option<&Path> { + match self { + Self::HttpUrl(_) => None, + Self::UnixSocket(path) => Some(path.as_path()), + } + } + + #[must_use] + pub fn is_unix_socket(&self) -> bool { + matches!(self, Self::UnixSocket(_)) + } + + pub fn build_public_http_client(&self) -> Result<(fabro_http::HttpClient, String)> { + if let Some(api_url) = self.as_http_url() { + let http_client = fabro_http::HttpClientBuilder::new().build()?; + return Ok((http_client, api_url.to_string())); + } + + let Some(path) = self.as_unix_socket_path() else { + bail!("server target must be an http(s) URL or absolute Unix socket path"); + }; + + #[cfg(unix)] + { + let http_client = fabro_http::HttpClientBuilder::new() + .unix_socket(path) + .no_proxy() + .build()?; + Ok((http_client, "http://fabro".to_string())) + } + #[cfg(not(unix))] + { + let _ = path; + bail!("Unix-socket HTTP client is not supported on this platform") + } + } + + pub fn loopback_classification(&self) -> Result { + classify_target(self) + } +} + +impl fmt::Display for ServerTarget { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if let Some(api_url) = self.as_http_url() { + return f.write_str(api_url); + } + let Some(path) = self.as_unix_socket_path() else { + return Err(fmt::Error); + }; + write!(f, "unix://{}", path.display()) + } +} + +impl FromStr for ServerTarget { + type Err = anyhow::Error; + + fn from_str(value: &str) -> Result { + if value.starts_with("http://") || value.starts_with("https://") { + return Self::http_url(value); + } + + let path = Path::new(value); + if path.is_absolute() { + return Self::unix_socket_path(path); + } + + bail!("server target must be an http(s) URL or absolute Unix socket path") + } +} + +impl CanonicalHttpUrl { + fn new(value: &str) -> Result { + Ok(Self(canonical_http_url(value)?)) + } + + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl CanonicalUnixSocketPath { + fn new(path: &Path) -> Result { + let normalized = lexical_normalize_absolute_path(path)?; + Ok(Self(normalized)) + } + + #[must_use] + pub fn as_path(&self) -> &Path { + &self.0 + } +} + +fn canonical_http_url(value: &str) -> Result { + let trimmed = value.trim(); + let normalized = trim_api_path_suffix(trimmed); + let url = fabro_http::Url::parse(normalized).map_err(|_| { + anyhow::anyhow!("server target must be an http(s) URL or absolute Unix socket path") + })?; + + let scheme = url.scheme().to_ascii_lowercase(); + let default_port = match scheme.as_str() { + "http" => 80, + "https" => 443, + _ => bail!("server target must be an http(s) URL or absolute Unix socket path"), + }; + + let Some(host) = url.host_str() else { + bail!("server target must be an http(s) URL or absolute Unix socket path"); + }; + let host = host.to_ascii_lowercase(); + let Some(port) = url.port_or_known_default() else { + bail!("server target must be an http(s) URL or absolute Unix socket path"); + }; + + if port == default_port { + Ok(format!("{scheme}://{host}")) + } else { + Ok(format!("{scheme}://{host}:{port}")) + } +} + +fn trim_api_path_suffix(value: &str) -> &str { + let trimmed = value.trim_end_matches('/'); + trimmed.strip_suffix("/api/v1").unwrap_or(trimmed) +} + +fn lexical_normalize_absolute_path(path: &Path) -> Result { + if !path.is_absolute() { + bail!("server target must be an http(s) URL or absolute Unix socket path"); + } + + let mut normalized = PathBuf::new(); + for component in path.components() { + match component { + Component::Prefix(prefix) => normalized.push(prefix.as_os_str()), + Component::RootDir => normalized.push(component.as_os_str()), + Component::CurDir => {} + Component::ParentDir => { + let _ = normalized.pop(); + } + Component::Normal(part) => normalized.push(part), + } + } + + Ok(normalized) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use super::ServerTarget; + + #[test] + fn canonicalizes_http_targets_at_construction() { + let target = ServerTarget::http_url("https://EXAMPLE.COM:443/api/v1/").unwrap(); + + assert_eq!(target.as_http_url(), Some("https://example.com")); + assert_eq!(target.to_string(), "https://example.com"); + } + + #[test] + fn canonicalizes_http_targets_by_rebuilding_authority() { + let target = ServerTarget::http_url("http://Example.com:3000/nested/path").unwrap(); + + assert_eq!(target.as_http_url(), Some("http://example.com:3000")); + } + + #[test] + fn lexically_normalizes_unix_socket_paths_without_fs_access() { + let target = ServerTarget::unix_socket_path("/tmp/fabro/../fabro.sock").unwrap(); + + assert_eq!( + target.as_unix_socket_path(), + Some(PathBuf::from("/tmp/fabro.sock").as_path()) + ); + } +} diff --git a/lib/crates/fabro-retro/src/retro_agent.rs b/lib/crates/fabro-retro/src/retro_agent.rs index 2529c436a..7fef673d4 100644 --- a/lib/crates/fabro-retro/src/retro_agent.rs +++ b/lib/crates/fabro-retro/src/retro_agent.rs @@ -308,7 +308,7 @@ async fn upload_data_files( let progress_content = { let lines: Vec = events .iter() - .filter_map(|env| serde_json::to_string(env.payload.as_value()).ok()) + .filter_map(|env| serde_json::to_string(&env.event).ok()) .collect(); if lines.is_empty() { None diff --git a/lib/crates/fabro-server/src/server.rs b/lib/crates/fabro-server/src/server.rs index 81fe6e50f..3a13fd504 100644 --- a/lib/crates/fabro-server/src/server.rs +++ b/lib/crates/fabro-server/src/server.rs @@ -885,9 +885,7 @@ fn start_optional_slack_service(state: &Arc) { loop { match rx.recv().await { Ok(envelope) => { - if let Ok(event) = RunEvent::try_from(&envelope.payload) { - event_service.handle_event(&event).await; - } + event_service.handle_event(&envelope.event).await; } Err(RecvError::Lagged(_)) => {} Err(RecvError::Closed) => break, @@ -1553,7 +1551,7 @@ struct PrunePlan { reason = "sync helper invoked from async handler via spawn_blocking (see callers at :1301 / :1341)" )] fn build_disk_usage_response( - summaries: &[fabro_store::RunSummary], + summaries: &[fabro_types::RunSummary], storage_dir: &std::path::Path, verbose: bool, ) -> anyhow::Result { @@ -1626,7 +1624,7 @@ fn build_disk_usage_response( fn build_prune_plan( request: &PruneRunsRequest, - summaries: &[fabro_store::RunSummary], + summaries: &[fabro_types::RunSummary], storage_dir: &std::path::Path, ) -> anyhow::Result { let scratch_base_dir = scratch_base(storage_dir); @@ -1772,16 +1770,7 @@ fn event_matches_run_filter(event: &EventEnvelope, run_filter: Option<&HashSet().ok()) - else { - return false; - }; - run_filter.contains(&run_id) + run_filter.contains(&event.event.run_id) } fn sse_event_from_store(event: &EventEnvelope) -> Option { @@ -1791,11 +1780,8 @@ fn sse_event_from_store(event: &EventEnvelope) -> Option { } fn attach_event_is_terminal(event: &EventEnvelope) -> bool { - let Ok(run_event) = RunEvent::try_from(&event.payload) else { - return false; - }; matches!( - run_event.body, + &event.event.body, EventBody::RunCompleted(_) | EventBody::RunFailed(_) ) } @@ -2771,7 +2757,7 @@ fn elapsed_secs(duration_ms: Option) -> Option { duration_ms.map(|ms| ms as f64 / 1000.0) } -fn summary_to_api_run_summary(summary: fabro_store::RunSummary) -> serde_json::Value { +fn summary_to_api_run_summary(summary: fabro_types::RunSummary) -> serde_json::Value { let goal = summary.goal.unwrap_or_default(); let title = truncate_goal(&goal); let repository = repository_name(summary.host_repo_path.as_deref()); @@ -3302,9 +3288,13 @@ fn octet_stream_response(bytes: Bytes) -> Response { reason = "Stored event conversion surfaces HTTP errors directly." )] fn api_event_envelope_from_store(event: &EventEnvelope) -> Result { - // The payload is already a serde_json::Value; merge `seq` into it - // instead of serializing the whole envelope and re-parsing. - let mut obj = event.payload.as_value().clone(); + let mut obj = event.event.to_value().map_err(|err| { + ApiError::new( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to serialize stored event: {err}"), + ) + .into_response() + })?; if let serde_json::Value::Object(ref mut map) = obj { map.insert("seq".into(), serde_json::Value::from(event.seq)); } @@ -3578,11 +3568,9 @@ async fn forward_run_events_to_global( loop { match run_events.recv().await { Ok(event) => { - if let Ok(run_event) = RunEvent::try_from(&event.payload) { - let mut runs = state.runs.lock().expect("runs lock poisoned"); - if let Some(managed_run) = runs.get_mut(&run_id) { - reconcile_live_interview_state_for_event(managed_run, &run_event); - } + let mut runs = state.runs.lock().expect("runs lock poisoned"); + if let Some(managed_run) = runs.get_mut(&run_id) { + reconcile_live_interview_state_for_event(managed_run, &event.event); } let _ = state.global_event_tx.send(event); } @@ -8767,8 +8755,8 @@ slug = "fabro" let run_store = state.store.open_run_reader(&run_id).await.unwrap(); let events = run_store.list_events().await.unwrap(); - let created = events[0].payload.as_value(); - let submitted = events[1].payload.as_value(); + let created = events[0].event.to_value().unwrap(); + let submitted = events[1].event.to_value().unwrap(); let manifest_blob = created["properties"]["manifest_blob"] .as_str() .expect("run.created should carry manifest_blob") diff --git a/lib/crates/fabro-store/src/lib.rs b/lib/crates/fabro-store/src/lib.rs index 55ff6a427..42a2cef9f 100644 --- a/lib/crates/fabro-store/src/lib.rs +++ b/lib/crates/fabro-store/src/lib.rs @@ -9,13 +9,15 @@ mod types; pub use artifact_store::{ArtifactStore, NodeArtifact}; pub use error::{Error, Result}; -pub use fabro_types::{RunBlobId, StageId}; -pub use run_state::{NodeState, PendingInterviewRecord, RunProjection}; +pub use fabro_types::{ + EventEnvelope, NodeState, PendingInterviewRecord, RunBlobId, RunProjection, StageId, +}; +pub use run_state::RunProjectionReducer; pub use slate::{ AuthCode, ConsumeOutcome, Database, RefreshToken, RunDatabase, Runs, SlateAuthCodeStore, SlateAuthTokenStore, }; -pub use types::{EventEnvelope, EventPayload, RunSummary}; +pub use types::EventPayload; #[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct ListRunsQuery { diff --git a/lib/crates/fabro-store/src/run_state.rs b/lib/crates/fabro-store/src/run_state.rs index 8ae25dc96..bd8cdbb9d 100644 --- a/lib/crates/fabro-store/src/run_state.rs +++ b/lib/crates/fabro-store/src/run_state.rs @@ -9,56 +9,13 @@ use fabro_types::run_event::{ }; use fabro_types::{ BilledModelUsage, BlockedReason, Checkpoint, Conclusion, EventBody, FailureSignature, - InterviewQuestionRecord, InterviewQuestionType, NodeStatusRecord, Outcome, PullRequestRecord, - Retro, RunControlAction, RunEvent, RunId, RunRecord, RunStatus, RunStatusRecord, SandboxRecord, - StageStatus, StartRecord, StatusReason, + InterviewQuestionRecord, InterviewQuestionType, NodeStatusRecord, Outcome, + PendingInterviewRecord, PullRequestRecord, RunControlAction, RunId, RunProjection, RunRecord, + RunStatus, RunStatusRecord, RunSummary, SandboxRecord, StageStatus, StartRecord, StatusReason, }; use serde_json::Value; -use crate::{Error, EventEnvelope, Result, RunSummary, StageId}; - -#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] -#[serde(default)] -pub struct RunProjection { - pub run: Option, - pub graph_source: Option, - pub start: Option, - pub status: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub prior_status: Option, - pub pending_control: Option, - pub checkpoint: Option, - pub checkpoints: Vec<(u32, Checkpoint)>, - pub conclusion: Option, - pub retro: Option, - pub retro_prompt: Option, - pub retro_response: Option, - pub sandbox: Option, - pub final_patch: Option, - pub pull_request: Option, - pub pending_interviews: BTreeMap, - nodes: HashMap, -} - -#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] -pub struct PendingInterviewRecord { - pub question: InterviewQuestionRecord, - pub started_at: Option>, -} - -#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] -pub struct NodeState { - pub prompt: Option, - pub response: Option, - pub status: Option, - pub provider_used: Option, - pub diff: Option, - pub script_invocation: Option, - pub script_timing: Option, - pub parallel_results: Option, - pub stdout: Option, - pub stderr: Option, -} +use crate::{Error, EventEnvelope, Result}; #[derive(Debug, Clone, Default)] pub(crate) struct EventProjectionCache { @@ -66,8 +23,16 @@ pub(crate) struct EventProjectionCache { pub state: RunProjection, } -impl RunProjection { - pub fn apply_events(events: &[EventEnvelope]) -> Result { +pub trait RunProjectionReducer { + fn apply_events(events: &[EventEnvelope]) -> Result + where + Self: Sized; + + fn apply_event(&mut self, event: &EventEnvelope) -> Result<()>; +} + +impl RunProjectionReducer for RunProjection { + fn apply_events(events: &[EventEnvelope]) -> Result { let mut state = Self::default(); for event in events { state.apply_event(event)?; @@ -75,9 +40,8 @@ impl RunProjection { Ok(state) } - pub fn apply_event(&mut self, event: &EventEnvelope) -> Result<()> { - let stored = RunEvent::from_ref(event.payload.as_value()) - .map_err(|err| Error::InvalidEvent(format!("invalid stored event: {err}")))?; + fn apply_event(&mut self, event: &EventEnvelope) -> Result<()> { + let stored = &event.event; let ts = stored.ts; let run_id = stored.run_id; @@ -420,107 +384,57 @@ impl RunProjection { Ok(()) } +} - pub fn node(&self, node: &StageId) -> Option<&NodeState> { - self.nodes.get(node) - } - - pub fn iter_nodes(&self) -> impl Iterator { - self.nodes.iter() - } - - pub fn is_empty(&self) -> bool { - self.nodes.is_empty() - } - - pub fn set_node(&mut self, node: StageId, state: NodeState) { - self.nodes.insert(node, state); - } - - pub fn list_node_visits(&self, node_id: &str) -> Vec { - let mut visits = self - .nodes - .keys() - .filter(|node| node.node_id() == node_id) - .map(StageId::visit) - .collect::>(); - visits.sort_unstable(); - visits.dedup(); - visits - } - - pub(crate) fn build_summary(&self, run_id: &RunId) -> RunSummary { - let workflow_name = self.run.as_ref().map(|run| { - if run.graph.name.is_empty() { - "unnamed".to_string() - } else { - run.graph.name.clone() - } - }); - let goal = self.run.as_ref().and_then(|run| { - let goal = run.graph.goal(); - (!goal.is_empty()).then(|| goal.to_string()) - }); - RunSummary { - run_id: *run_id, - workflow_name, - workflow_slug: self.run.as_ref().and_then(|run| run.workflow_slug.clone()), - goal, - labels: self - .run - .as_ref() - .map(|run| run.labels.clone()) - .unwrap_or_default(), - host_repo_path: self.run.as_ref().and_then(|run| run.host_repo_path.clone()), - start_time: self.start.as_ref().map(|start| start.start_time), - status: self - .status - .as_ref() - .map_or(RunStatus::Submitted, |status| status.status), - status_reason: self.status.as_ref().and_then(|status| status.status_reason), - blocked_reason: self - .status - .as_ref() - .and_then(|status| status.blocked_reason), - pending_control: self.pending_control, - duration_ms: self - .conclusion - .as_ref() - .map(|conclusion| conclusion.duration_ms), - total_usd_micros: self - .conclusion - .as_ref() - .and_then(|conclusion| conclusion.billing.as_ref()) - .and_then(|billing| billing.total_usd_micros), +pub(crate) fn build_summary(state: &RunProjection, run_id: &RunId) -> RunSummary { + let workflow_name = state.run.as_ref().map(|run| { + if run.graph.name.is_empty() { + "unnamed".to_string() + } else { + run.graph.name.clone() } - } - - fn node_mut(&mut self, node_id: &str, visit: u32) -> &mut NodeState { - self.nodes.entry(StageId::new(node_id, visit)).or_default() - } - - fn current_visit_for(&self, node_id: &str) -> Option { - self.nodes - .keys() - .filter(|node| node.node_id() == node_id) - .map(StageId::visit) - .max() - } - - fn reset_for_rewind(&mut self) { - self.status = None; - self.pending_control = None; - self.checkpoint = None; - self.checkpoints.clear(); - self.conclusion = None; - self.retro = None; - self.retro_prompt = None; - self.retro_response = None; - self.sandbox = None; - self.final_patch = None; - self.pull_request = None; - self.pending_interviews.clear(); - self.nodes.clear(); + }); + let goal = state.run.as_ref().and_then(|run| { + let goal = run.graph.goal(); + (!goal.is_empty()).then(|| goal.to_string()) + }); + RunSummary { + run_id: *run_id, + workflow_name, + workflow_slug: state.run.as_ref().and_then(|run| run.workflow_slug.clone()), + goal, + labels: state + .run + .as_ref() + .map(|run| run.labels.clone()) + .unwrap_or_default(), + host_repo_path: state + .run + .as_ref() + .and_then(|run| run.host_repo_path.clone()), + start_time: state.start.as_ref().map(|start| start.start_time), + status: state + .status + .as_ref() + .map_or(RunStatus::Submitted, |status| status.status), + status_reason: state + .status + .as_ref() + .and_then(|status| status.status_reason), + blocked_reason: state + .status + .as_ref() + .and_then(|status| status.blocked_reason), + pending_control: state.pending_control, + duration_ms: state + .conclusion + .as_ref() + .map(|conclusion| conclusion.duration_ms), + total_usd_micros: state + .conclusion + .as_ref() + .and_then(|conclusion| conclusion.billing.as_ref()) + .and_then(|billing| billing.total_usd_micros), } } @@ -700,13 +614,13 @@ mod tests { }; use fabro_types::settings::SettingsLayer; use fabro_types::{ - Checkpoint, EventBody, InterviewQuestionType, RunBlobId, RunControlAction, RunEvent, - fixtures, + Checkpoint, EventBody, InterviewQuestionType, NodeState, RunBlobId, RunControlAction, + RunEvent, fixtures, }; use serde_json::json; - use super::{NodeState, RunProjection}; - use crate::{EventEnvelope, EventPayload, StageId}; + use super::{RunProjection, RunProjectionReducer, build_summary}; + use crate::{EventEnvelope, StageId}; fn test_event(seq: u32, body: EventBody, node_id: Option<&str>) -> EventEnvelope { let event = RunEvent { @@ -725,11 +639,7 @@ mod tests { body, }; - EventEnvelope { - seq, - payload: EventPayload::new(serde_json::to_value(event).unwrap(), &fixtures::RUN_1) - .unwrap(), - } + EventEnvelope { seq, event } } fn test_raw_event( @@ -740,17 +650,14 @@ mod tests { ) -> EventEnvelope { EventEnvelope { seq, - payload: EventPayload::new( - json!({ - "id": format!("evt-{seq}"), - "ts": Utc::now().to_rfc3339(), - "run_id": fixtures::RUN_1, - "event": event, - "node_id": node_id, - "properties": properties, - }), - &fixtures::RUN_1, - ) + event: RunEvent::from_value(json!({ + "id": format!("evt-{seq}"), + "ts": Utc::now().to_rfc3339(), + "run_id": fixtures::RUN_1, + "event": event, + "node_id": node_id, + "properties": properties, + })) .unwrap(), } } @@ -813,23 +720,21 @@ mod tests { #[test] fn set_node_round_trips_through_json() { - let mut state = RunProjection { - pending_control: Some(RunControlAction::Unpause), - checkpoints: vec![(7, Checkpoint { - timestamp: "2026-04-07T12:00:00Z".parse().unwrap(), - current_node: "build".to_string(), - completed_nodes: vec!["build".to_string()], - node_retries: HashMap::new(), - context_values: HashMap::new(), - node_outcomes: HashMap::new(), - next_node_id: None, - git_commit_sha: None, - loop_failure_signatures: HashMap::new(), - restart_failure_signatures: HashMap::new(), - node_visits: HashMap::from([("build".to_string(), 2usize)]), - })], - ..RunProjection::default() - }; + let mut state = RunProjection::default(); + state.pending_control = Some(RunControlAction::Unpause); + state.checkpoints = vec![(7, Checkpoint { + timestamp: "2026-04-07T12:00:00Z".parse().unwrap(), + current_node: "build".to_string(), + completed_nodes: vec!["build".to_string()], + node_retries: HashMap::new(), + context_values: HashMap::new(), + node_outcomes: HashMap::new(), + next_node_id: None, + git_commit_sha: None, + loop_failure_signatures: HashMap::new(), + restart_failure_signatures: HashMap::new(), + node_visits: HashMap::from([("build".to_string(), 2usize)]), + })]; state.set_node(StageId::new("build", 2), NodeState { stdout: Some("done".to_string()), ..NodeState::default() @@ -955,7 +860,7 @@ mod tests { assert_eq!(status_json["status_reason"], serde_json::Value::Null); assert_eq!(status_json["blocked_reason"], "human_input_required"); - let summary = state.build_summary(&fixtures::RUN_1); + let summary = build_summary(&state, &fixtures::RUN_1); let summary_json = serde_json::to_value(summary).unwrap(); assert_eq!(summary_json["status"], "paused"); assert_eq!(summary_json["status_reason"], serde_json::Value::Null); @@ -1053,25 +958,23 @@ mod tests { #[test] fn summary_synthesizes_submitted_when_run_exists_without_status() { - let state = RunProjection { - run: Some(fabro_types::RunRecord { - run_id: fixtures::RUN_1, - settings: SettingsLayer::default(), - graph: fabro_types::Graph::new("test"), - workflow_slug: Some("test".to_string()), - working_directory: std::path::PathBuf::from("/tmp/run"), - host_repo_path: Some("/tmp/repo".to_string()), - repo_origin_url: None, - base_branch: None, - labels: HashMap::new(), - provenance: None, - manifest_blob: None, - definition_blob: None, - }), - ..RunProjection::default() - }; + let mut state = RunProjection::default(); + state.run = Some(fabro_types::RunRecord { + run_id: fixtures::RUN_1, + settings: SettingsLayer::default(), + graph: fabro_types::Graph::new("test"), + workflow_slug: Some("test".to_string()), + working_directory: std::path::PathBuf::from("/tmp/run"), + host_repo_path: Some("/tmp/repo".to_string()), + repo_origin_url: None, + base_branch: None, + labels: HashMap::new(), + provenance: None, + manifest_blob: None, + definition_blob: None, + }); - let summary_json = serde_json::to_value(state.build_summary(&fixtures::RUN_1)).unwrap(); + let summary_json = serde_json::to_value(build_summary(&state, &fixtures::RUN_1)).unwrap(); assert_eq!(summary_json["status"], "submitted"); } @@ -1082,45 +985,39 @@ mod tests { RunBlobId::new(br#"{"version":1,"workflow_path":"workflow.fabro"}"#).to_string(); let events = vec![ EventEnvelope { - seq: 1, - payload: EventPayload::new( - json!({ - "id": "evt-run-created", - "ts": "2026-04-07T12:00:00Z", - "run_id": fixtures::RUN_1, - "event": "run.created", - "properties": { - "settings": SettingsLayer::default(), - "graph": { - "name": "test", - "nodes": {}, - "edges": [], - "attrs": {} - }, - "labels": {}, - "run_dir": "/tmp/run", - "working_directory": "/tmp/run", - "manifest_blob": manifest_blob - } - }), - &fixtures::RUN_1, - ) + seq: 1, + event: RunEvent::from_value(json!({ + "id": "evt-run-created", + "ts": "2026-04-07T12:00:00Z", + "run_id": fixtures::RUN_1, + "event": "run.created", + "properties": { + "settings": SettingsLayer::default(), + "graph": { + "name": "test", + "nodes": {}, + "edges": [], + "attrs": {} + }, + "labels": {}, + "run_dir": "/tmp/run", + "working_directory": "/tmp/run", + "manifest_blob": manifest_blob + } + })) .unwrap(), }, EventEnvelope { - seq: 2, - payload: EventPayload::new( - json!({ - "id": "evt-run-submitted", - "ts": "2026-04-07T12:00:01Z", - "run_id": fixtures::RUN_1, - "event": "run.submitted", - "properties": { - "definition_blob": definition_blob - } - }), - &fixtures::RUN_1, - ) + seq: 2, + event: RunEvent::from_value(json!({ + "id": "evt-run-submitted", + "ts": "2026-04-07T12:00:01Z", + "run_id": fixtures::RUN_1, + "event": "run.submitted", + "properties": { + "definition_blob": definition_blob + } + })) .unwrap(), }, ]; @@ -1130,11 +1027,11 @@ mod tests { assert_eq!( value["run"]["manifest_blob"], - events[0].payload.as_value()["properties"]["manifest_blob"] + events[0].event.properties().unwrap()["manifest_blob"] ); assert_eq!( value["run"]["definition_blob"], - events[1].payload.as_value()["properties"]["definition_blob"] + events[1].event.properties().unwrap()["definition_blob"] ); } diff --git a/lib/crates/fabro-store/src/slate/mod.rs b/lib/crates/fabro-store/src/slate/mod.rs index e368970b0..d5e02d668 100644 --- a/lib/crates/fabro-store/src/slate/mod.rs +++ b/lib/crates/fabro-store/src/slate/mod.rs @@ -10,14 +10,15 @@ use std::time::Duration; pub use auth_codes::{AuthCode, SlateAuthCodeStore}; pub use auth_tokens::{ConsumeOutcome, RefreshToken, SlateAuthTokenStore}; -use fabro_types::RunId; +use fabro_types::{RunId, RunSummary}; use object_store::ObjectStore; pub use run_store::RunDatabase; use run_store::RunDatabaseInner; use slatedb::config::{CompressionCodec, Settings}; use tokio::sync::{Mutex, OnceCell}; -use crate::{Error, ListRunsQuery, Result, RunSummary, keys}; +use crate::run_state::build_summary; +use crate::{Error, ListRunsQuery, Result, keys}; #[derive(Clone)] pub struct Database { @@ -170,7 +171,7 @@ impl Database { let mut summaries = Vec::new(); for run_id in run_ids { if let Some(active) = self.get_active_run(&run_id).await { - summaries.push(active.state().await?.build_summary(&run_id)); + summaries.push(build_summary(&active.state().await?, &run_id)); continue; } if !RunDatabase::has_any_events(&db, &run_id).await? { @@ -245,7 +246,7 @@ impl Runs { pub async fn find(&self, run_id: &RunId) -> Result> { match self.db.open_run_reader(run_id).await { - Ok(run_db) => Ok(Some(run_db.state().await?.build_summary(run_id))), + Ok(run_db) => Ok(Some(build_summary(&run_db.state().await?, run_id))), Err(Error::RunNotFound(_)) => Ok(None), Err(err) => Err(err), } diff --git a/lib/crates/fabro-store/src/slate/run_store.rs b/lib/crates/fabro-store/src/slate/run_store.rs index 388e5a188..16f21e3e2 100644 --- a/lib/crates/fabro-store/src/slate/run_store.rs +++ b/lib/crates/fabro-store/src/slate/run_store.rs @@ -4,14 +4,14 @@ use std::sync::atomic::{AtomicU32, Ordering}; use bytes::Bytes; use chrono::Utc; -use fabro_types::{RunBlobId, RunId}; +use fabro_types::{RunBlobId, RunEvent, RunId, RunSummary}; use futures::Stream; use slatedb::{Db, DbRead}; use tokio::sync::{Mutex, broadcast, mpsc}; use tokio_stream::wrappers::UnboundedReceiverStream; -use crate::run_state::EventProjectionCache; -use crate::{Error, EventEnvelope, EventPayload, Result, RunProjection, RunSummary, keys}; +use crate::run_state::{EventProjectionCache, RunProjectionReducer, build_summary}; +use crate::{Error, EventEnvelope, EventPayload, Result, RunProjection, keys}; const DEFAULT_EVENT_TAIL_LIMIT: usize = 1024; #[derive(Clone)] @@ -131,7 +131,7 @@ impl RunDatabase { { let events = list_events_from(db, run_id, 1).await?; let state = RunProjection::apply_events(&events)?; - Ok(state.build_summary(run_id)) + Ok(build_summary(&state, run_id)) } async fn projected_state(&self) -> Result { @@ -193,7 +193,7 @@ impl RunDatabase { let seq = self.inner.event_seq.fetch_add(1, Ordering::SeqCst); let event = EventEnvelope { seq, - payload: payload.clone(), + event: RunEvent::try_from(payload)?, }; self.inner .db @@ -341,7 +341,7 @@ where } events.push(EventEnvelope { seq, - payload: serde_json::from_slice(&entry.value)?, + event: serde_json::from_slice(&entry.value)?, }); } events.sort_by_key(|event| event.seq); diff --git a/lib/crates/fabro-store/src/types.rs b/lib/crates/fabro-store/src/types.rs index 6023d49d6..65235a55b 100644 --- a/lib/crates/fabro-store/src/types.rs +++ b/lib/crates/fabro-store/src/types.rs @@ -1,28 +1,8 @@ -use std::collections::HashMap; - -use chrono::{DateTime, Utc}; -use fabro_types::{BlockedReason, RunControlAction, RunEvent, RunId, RunStatus, StatusReason}; +use fabro_types::{RunEvent, RunId}; use serde::{Deserialize, Serialize}; use crate::{Error, Result}; -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct RunSummary { - pub run_id: RunId, - pub workflow_name: Option, - pub workflow_slug: Option, - pub goal: Option, - pub labels: HashMap, - pub host_repo_path: Option, - pub start_time: Option>, - pub status: RunStatus, - pub status_reason: Option, - pub blocked_reason: Option, - pub pending_control: Option, - pub duration_ms: Option, - pub total_usd_micros: Option, -} - #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(transparent)] pub struct EventPayload(serde_json::Value); @@ -81,111 +61,3 @@ impl TryFrom<&EventPayload> for RunEvent { .map_err(|err| Error::InvalidEvent(format!("invalid stored event: {err}"))) } } - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct EventEnvelope { - pub seq: u32, - #[serde(flatten)] - pub payload: EventPayload, -} - -#[cfg(test)] -mod tests { - use chrono::{TimeZone, Utc}; - use fabro_types::run_event::RunCompletedProps; - use fabro_types::{ActorRef, EventBody, ParallelBranchId, RunEvent, StageId, fixtures}; - - use super::{EventEnvelope, EventPayload}; - - #[test] - fn wire_event_envelope_round_trips() { - let event = RunEvent { - id: "evt_1".to_string(), - ts: Utc.with_ymd_and_hms(2026, 4, 9, 12, 0, 0).unwrap(), - run_id: fixtures::RUN_1, - node_id: Some("code".to_string()), - node_label: Some("Code".to_string()), - stage_id: Some(StageId::new("code", 1)), - parallel_group_id: None, - parallel_branch_id: None, - session_id: None, - parent_session_id: None, - tool_call_id: None, - actor: None, - body: EventBody::RunCompleted(RunCompletedProps { - duration_ms: 42, - artifact_count: 0, - status: "success".to_string(), - reason: None, - total_usd_micros: None, - final_git_commit_sha: None, - final_patch: None, - billing: None, - }), - }; - let payload = EventPayload::new(event.to_value().unwrap(), &fixtures::RUN_1).unwrap(); - let envelope = EventEnvelope { seq: 7, payload }; - - let wire = serde_json::to_value(&envelope).unwrap(); - assert_eq!(wire["seq"], 7); - assert_eq!(wire["id"], "evt_1"); - assert_eq!(wire["event"], "run.completed"); - assert!(wire.get("payload").is_none(), "wire shape must be flat"); - - let parsed: EventEnvelope = serde_json::from_value(wire).unwrap(); - assert_eq!(parsed, envelope); - } - - #[test] - fn wire_event_envelope_round_trips_with_all_envelope_fields() { - let group = StageId::new("review", 2); - let branch = ParallelBranchId::new(group.clone(), 3); - let event = RunEvent { - id: "evt_2".to_string(), - ts: Utc.with_ymd_and_hms(2026, 4, 9, 13, 0, 0).unwrap(), - run_id: fixtures::RUN_1, - node_id: Some("review".to_string()), - node_label: Some("Review".to_string()), - stage_id: Some(StageId::new("review", 2)), - parallel_group_id: Some(group), - parallel_branch_id: Some(branch), - session_id: Some("ses_42".to_string()), - parent_session_id: Some("ses_root".to_string()), - tool_call_id: Some("tool_call_xyz".to_string()), - actor: Some(ActorRef::agent( - Some("ses_42".to_string()), - Some("claude-sonnet".to_string()), - )), - body: EventBody::RunCompleted(RunCompletedProps { - duration_ms: 100, - artifact_count: 1, - status: "success".to_string(), - reason: None, - total_usd_micros: None, - final_git_commit_sha: None, - final_patch: None, - billing: None, - }), - }; - let payload = EventPayload::new(event.to_value().unwrap(), &fixtures::RUN_1).unwrap(); - let envelope = EventEnvelope { seq: 99, payload }; - - let wire = serde_json::to_value(&envelope).unwrap(); - assert_eq!(wire["seq"], 99); - assert_eq!(wire["id"], "evt_2"); - assert_eq!(wire["stage_id"], "review@2"); - assert_eq!(wire["parallel_group_id"], "review@2"); - assert_eq!(wire["parallel_branch_id"], "review@2:3"); - assert_eq!(wire["session_id"], "ses_42"); - assert_eq!(wire["parent_session_id"], "ses_root"); - assert_eq!(wire["tool_call_id"], "tool_call_xyz"); - assert_eq!(wire["actor"]["kind"], "agent"); - assert_eq!(wire["actor"]["id"], "ses_42"); - assert_eq!(wire["actor"]["display"], "claude-sonnet"); - assert_eq!(wire["event"], "run.completed"); - assert!(wire.get("payload").is_none(), "wire shape must be flat"); - - let parsed: EventEnvelope = serde_json::from_value(wire).unwrap(); - assert_eq!(parsed, envelope); - } -} diff --git a/lib/crates/fabro-types/src/artifact.rs b/lib/crates/fabro-types/src/artifact.rs new file mode 100644 index 000000000..0cd6af42a --- /dev/null +++ b/lib/crates/fabro-types/src/artifact.rs @@ -0,0 +1,30 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ArtifactUpload { + pub path: String, + pub mime: String, + pub content_md5: String, + pub content_sha256: String, + pub bytes: u64, +} + +#[cfg(test)] +mod tests { + use super::ArtifactUpload; + + #[test] + fn round_trips_through_serde_json() { + let artifact = ArtifactUpload { + path: "artifacts/log.txt".to_string(), + mime: "text/plain".to_string(), + content_md5: "md5".to_string(), + content_sha256: "sha256".to_string(), + bytes: 42, + }; + + let value = serde_json::to_value(&artifact).unwrap(); + let parsed: ArtifactUpload = serde_json::from_value(value).unwrap(); + assert_eq!(parsed, artifact); + } +} diff --git a/lib/crates/fabro-types/src/event_envelope.rs b/lib/crates/fabro-types/src/event_envelope.rs new file mode 100644 index 000000000..16de63da5 --- /dev/null +++ b/lib/crates/fabro-types/src/event_envelope.rs @@ -0,0 +1,135 @@ +use serde::{Deserialize, Serialize}; + +use crate::RunEvent; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct EventEnvelope { + pub seq: u32, + #[serde(flatten)] + pub event: RunEvent, +} + +#[cfg(test)] +mod tests { + use chrono::{TimeZone, Utc}; + + use super::EventEnvelope; + use crate::run_event::RunCompletedProps; + use crate::{ActorRef, EventBody, ParallelBranchId, RunEvent, StageId, fixtures}; + + #[test] + fn wire_event_envelope_round_trips() { + let event = RunEvent { + id: "evt_1".to_string(), + ts: Utc.with_ymd_and_hms(2026, 4, 9, 12, 0, 0).unwrap(), + run_id: fixtures::RUN_1, + node_id: Some("code".to_string()), + node_label: Some("Code".to_string()), + stage_id: Some(StageId::new("code", 1)), + parallel_group_id: None, + parallel_branch_id: None, + session_id: None, + parent_session_id: None, + tool_call_id: None, + actor: None, + body: EventBody::RunCompleted(RunCompletedProps { + duration_ms: 42, + artifact_count: 0, + status: "success".to_string(), + reason: None, + total_usd_micros: None, + final_git_commit_sha: None, + final_patch: None, + billing: None, + }), + }; + let envelope = EventEnvelope { seq: 7, event }; + + let wire = serde_json::to_value(&envelope).unwrap(); + assert_eq!(wire["seq"], 7); + assert_eq!(wire["id"], "evt_1"); + assert_eq!(wire["event"], "run.completed"); + + let parsed: EventEnvelope = serde_json::from_value(wire).unwrap(); + assert_eq!(parsed, envelope); + } + + #[test] + fn wire_event_envelope_round_trips_with_all_envelope_fields() { + let group = StageId::new("review", 2); + let branch = ParallelBranchId::new(group.clone(), 3); + let event = RunEvent { + id: "evt_2".to_string(), + ts: Utc.with_ymd_and_hms(2026, 4, 9, 13, 0, 0).unwrap(), + run_id: fixtures::RUN_1, + node_id: Some("review".to_string()), + node_label: Some("Review".to_string()), + stage_id: Some(StageId::new("review", 2)), + parallel_group_id: Some(group), + parallel_branch_id: Some(branch), + session_id: Some("ses_42".to_string()), + parent_session_id: Some("ses_root".to_string()), + tool_call_id: Some("tool_call_xyz".to_string()), + actor: Some(ActorRef::agent( + Some("ses_42".to_string()), + Some("claude-sonnet".to_string()), + )), + body: EventBody::RunCompleted(RunCompletedProps { + duration_ms: 100, + artifact_count: 1, + status: "success".to_string(), + reason: None, + total_usd_micros: None, + final_git_commit_sha: None, + final_patch: None, + billing: None, + }), + }; + let envelope = EventEnvelope { seq: 99, event }; + + let wire = serde_json::to_value(&envelope).unwrap(); + assert_eq!(wire["seq"], 99); + assert_eq!(wire["id"], "evt_2"); + assert_eq!(wire["stage_id"], "review@2"); + assert_eq!(wire["parallel_group_id"], "review@2"); + assert_eq!(wire["parallel_branch_id"], "review@2:3"); + assert_eq!(wire["session_id"], "ses_42"); + assert_eq!(wire["parent_session_id"], "ses_root"); + assert_eq!(wire["tool_call_id"], "tool_call_xyz"); + assert_eq!(wire["actor"]["kind"], "agent"); + assert_eq!(wire["actor"]["id"], "ses_42"); + assert_eq!(wire["actor"]["display"], "claude-sonnet"); + assert_eq!(wire["event"], "run.completed"); + + let parsed: EventEnvelope = serde_json::from_value(wire).unwrap(); + assert_eq!(parsed, envelope); + } + + #[test] + fn preserves_unknown_event_names_and_properties() { + let wire = serde_json::json!({ + "seq": 7, + "id": "evt_unknown", + "ts": "2026-04-20T12:00:00.000Z", + "run_id": fixtures::RUN_1, + "event": "vendor.custom.event", + "properties": { + "answer": 42, + "nested": { "ok": true } + } + }); + + let parsed: EventEnvelope = serde_json::from_value(wire.clone()).unwrap(); + let serialized = serde_json::to_value(&parsed).unwrap(); + + assert_eq!(serialized["seq"], wire["seq"]); + assert_eq!(serialized["id"], wire["id"]); + assert_eq!(serialized["run_id"], wire["run_id"]); + assert_eq!(serialized["event"], wire["event"]); + assert_eq!(serialized["properties"], wire["properties"]); + assert_eq!( + chrono::DateTime::parse_from_rfc3339(serialized["ts"].as_str().unwrap()).unwrap(), + chrono::DateTime::parse_from_rfc3339(wire["ts"].as_str().unwrap()).unwrap(), + ); + } +} diff --git a/lib/crates/fabro-types/src/lib.rs b/lib/crates/fabro-types/src/lib.rs index 6ff85e5b1..99d4ae9c4 100644 --- a/lib/crates/fabro-types/src/lib.rs +++ b/lib/crates/fabro-types/src/lib.rs @@ -1,10 +1,12 @@ extern crate self as fabro_types; +pub mod artifact; pub mod auth; pub mod billing; pub mod blob_ref; pub mod checkpoint; pub mod conclusion; +pub mod event_envelope; pub mod failure_signature; pub mod graph; pub mod interview; @@ -16,12 +18,15 @@ pub mod run; pub mod run_blob_id; pub mod run_event; pub mod run_id; +pub mod run_projection; +pub mod run_summary; pub mod sandbox_record; pub mod settings; pub mod stage_id; pub mod start; pub mod status; +pub use artifact::ArtifactUpload; pub use auth::{IdpIdentity, IdpIdentityError}; pub use billing::{ AnthropicBillingFacts, AnthropicModelPricing, BilledModelUsage, BilledTokenCounts, @@ -34,6 +39,7 @@ pub use blob_ref::{ }; pub use checkpoint::Checkpoint; pub use conclusion::{Conclusion, StageSummary}; +pub use event_envelope::EventEnvelope; pub use failure_signature::FailureSignature; pub use graph::{AttrValue, Edge, Graph, Node, is_llm_handler_type, shape_to_handler_type}; pub use interview::{InterviewQuestionRecord, InterviewQuestionType}; @@ -51,6 +57,8 @@ pub use run::{ pub use run_blob_id::RunBlobId; pub use run_event::{ActorKind, ActorRef, EventBody, RunEvent, RunNoticeLevel}; pub use run_id::{RunId, fixtures}; +pub use run_projection::{NodeState, PendingInterviewRecord, RunProjection}; +pub use run_summary::RunSummary; pub use sandbox_record::SandboxRecord; pub use stage_id::{ParallelBranchId, StageId}; pub use start::StartRecord; diff --git a/lib/crates/fabro-types/src/run_projection.rs b/lib/crates/fabro-types/src/run_projection.rs new file mode 100644 index 000000000..4fcebdc10 --- /dev/null +++ b/lib/crates/fabro-types/src/run_projection.rs @@ -0,0 +1,109 @@ +use std::collections::{BTreeMap, HashMap}; + +use chrono::{DateTime, Utc}; + +use crate::{ + Checkpoint, Conclusion, InterviewQuestionRecord, NodeStatusRecord, PullRequestRecord, Retro, + RunControlAction, RunRecord, RunStatus, RunStatusRecord, SandboxRecord, StageId, StartRecord, +}; + +#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] +#[serde(default)] +pub struct RunProjection { + pub run: Option, + pub graph_source: Option, + pub start: Option, + pub status: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prior_status: Option, + pub pending_control: Option, + pub checkpoint: Option, + pub checkpoints: Vec<(u32, Checkpoint)>, + pub conclusion: Option, + pub retro: Option, + pub retro_prompt: Option, + pub retro_response: Option, + pub sandbox: Option, + pub final_patch: Option, + pub pull_request: Option, + pub pending_interviews: BTreeMap, + nodes: HashMap, +} + +#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] +pub struct PendingInterviewRecord { + pub question: InterviewQuestionRecord, + pub started_at: Option>, +} + +#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] +pub struct NodeState { + pub prompt: Option, + pub response: Option, + pub status: Option, + pub provider_used: Option, + pub diff: Option, + pub script_invocation: Option, + pub script_timing: Option, + pub parallel_results: Option, + pub stdout: Option, + pub stderr: Option, +} + +impl RunProjection { + pub fn node(&self, node: &StageId) -> Option<&NodeState> { + self.nodes.get(node) + } + + pub fn iter_nodes(&self) -> impl Iterator { + self.nodes.iter() + } + + pub fn is_empty(&self) -> bool { + self.nodes.is_empty() + } + + pub fn set_node(&mut self, node: StageId, state: NodeState) { + self.nodes.insert(node, state); + } + + pub fn list_node_visits(&self, node_id: &str) -> Vec { + let mut visits = self + .nodes + .keys() + .filter(|node| node.node_id() == node_id) + .map(StageId::visit) + .collect::>(); + visits.sort_unstable(); + visits.dedup(); + visits + } + + pub fn node_mut(&mut self, node_id: &str, visit: u32) -> &mut NodeState { + self.nodes.entry(StageId::new(node_id, visit)).or_default() + } + + pub fn current_visit_for(&self, node_id: &str) -> Option { + self.nodes + .keys() + .filter(|node| node.node_id() == node_id) + .map(StageId::visit) + .max() + } + + pub fn reset_for_rewind(&mut self) { + self.status = None; + self.pending_control = None; + self.checkpoint = None; + self.checkpoints.clear(); + self.conclusion = None; + self.retro = None; + self.retro_prompt = None; + self.retro_response = None; + self.sandbox = None; + self.final_patch = None; + self.pull_request = None; + self.pending_interviews.clear(); + self.nodes.clear(); + } +} diff --git a/lib/crates/fabro-types/src/run_summary.rs b/lib/crates/fabro-types/src/run_summary.rs new file mode 100644 index 000000000..0e025ae6b --- /dev/null +++ b/lib/crates/fabro-types/src/run_summary.rs @@ -0,0 +1,56 @@ +use std::collections::HashMap; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +use crate::{BlockedReason, RunControlAction, RunId, RunStatus, StatusReason}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RunSummary { + pub run_id: RunId, + pub workflow_name: Option, + pub workflow_slug: Option, + pub goal: Option, + pub labels: HashMap, + pub host_repo_path: Option, + pub start_time: Option>, + pub status: RunStatus, + pub status_reason: Option, + pub blocked_reason: Option, + pub pending_control: Option, + pub duration_ms: Option, + pub total_usd_micros: Option, +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use chrono::{TimeZone, Utc}; + + use super::RunSummary; + use crate::{BlockedReason, RunControlAction, RunStatus, StatusReason, fixtures}; + + #[test] + fn round_trips_through_serde_json() { + let summary = RunSummary { + run_id: fixtures::RUN_1, + workflow_name: Some("workflow".to_string()), + workflow_slug: Some("workflow".to_string()), + goal: Some("ship it".to_string()), + labels: HashMap::from([("team".to_string(), "core".to_string())]), + host_repo_path: Some("/tmp/repo".to_string()), + start_time: Some(Utc.with_ymd_and_hms(2026, 4, 20, 12, 0, 0).unwrap()), + status: RunStatus::Blocked, + status_reason: Some(StatusReason::SandboxInitializing), + blocked_reason: Some(BlockedReason::HumanInputRequired), + pending_control: Some(RunControlAction::Pause), + duration_ms: Some(42), + total_usd_micros: Some(123), + }; + + let value = serde_json::to_value(&summary).unwrap(); + let parsed: RunSummary = serde_json::from_value(value).unwrap(); + assert_eq!(parsed, summary); + } +} diff --git a/lib/crates/fabro-workflow/src/artifact_snapshot.rs b/lib/crates/fabro-workflow/src/artifact_snapshot.rs index 78f8ac5df..ce9abeb47 100644 --- a/lib/crates/fabro-workflow/src/artifact_snapshot.rs +++ b/lib/crates/fabro-workflow/src/artifact_snapshot.rs @@ -2,7 +2,7 @@ use std::path::Path; use fabro_agent::Sandbox; use fabro_sandbox::shell_quote; -use serde::{Deserialize, Serialize}; +use fabro_types::ArtifactUpload; use sha2::{Digest, Sha256}; use tokio::fs; use tracing::{debug, warn}; @@ -15,25 +15,15 @@ pub struct DiscoveredFile { pub mtime_epoch_secs: f64, } -/// Metadata for a single captured artifact file. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CapturedArtifactInfo { - pub path: String, - pub mime: String, - pub content_md5: String, - pub content_sha256: String, - pub bytes: u64, -} - /// Summary of an artifact collection run. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct ArtifactCollectionSummary { pub files_copied: usize, pub total_bytes: u64, pub files_skipped: usize, pub download_errors: usize, pub hash_errors: usize, - pub captured_assets: Vec, + pub captured_assets: Vec, } /// Directories to exclude from the find search and checkpoint commits. @@ -261,7 +251,7 @@ fn normalize_paths(discovered: Vec, root: &str) -> Vec std::result::Result { +) -> std::result::Result { let mime = mime_guess::from_path(relative_path) .first_or_octet_stream() .to_string(); @@ -271,7 +261,7 @@ async fn compute_artifact_info( let bytes = u64::try_from(data.len()).unwrap_or(u64::MAX); let content_md5 = format!("{:x}", md5::compute(&data)); let content_sha256 = hex::encode(Sha256::digest(&data)); - Ok(CapturedArtifactInfo { + Ok(ArtifactUpload { path: relative_path.to_string(), mime, content_md5, @@ -308,7 +298,7 @@ pub async fn collect_artifacts( let mut total_bytes: u64 = 0; let mut download_errors: usize = 0; let mut hash_errors: usize = 0; - let mut captured_assets: Vec = Vec::new(); + let mut captured_assets: Vec = Vec::new(); for file in &to_collect { let dest = artifact_capture_dir.join(&file.relative_path); diff --git a/lib/crates/fabro-workflow/src/artifact_upload.rs b/lib/crates/fabro-workflow/src/artifact_upload.rs index e6f28d59e..2fb9d3c9a 100644 --- a/lib/crates/fabro-workflow/src/artifact_upload.rs +++ b/lib/crates/fabro-workflow/src/artifact_upload.rs @@ -4,9 +4,7 @@ use std::sync::Arc; use anyhow::Result; use async_trait::async_trait; use fabro_store::ArtifactStore; -use fabro_types::StageId; - -use crate::artifact_snapshot::CapturedArtifactInfo; +use fabro_types::{ArtifactUpload, StageId}; #[async_trait] pub trait StageArtifactUploader: Send + Sync { @@ -14,7 +12,7 @@ pub trait StageArtifactUploader: Send + Sync { &self, stage_id: &StageId, artifact_capture_dir: &Path, - artifacts: &[CapturedArtifactInfo], + artifacts: &[ArtifactUpload], ) -> Result<()>; } diff --git a/lib/crates/fabro-workflow/src/event.rs b/lib/crates/fabro-workflow/src/event.rs index 31116566c..947ffd593 100644 --- a/lib/crates/fabro-workflow/src/event.rs +++ b/lib/crates/fabro-workflow/src/event.rs @@ -3166,7 +3166,7 @@ mod tests { let line = events .into_iter() .next() - .map(|event| event.payload.as_value().clone()) + .map(|event| event.event.to_value().unwrap()) .unwrap(); assert!(line.get("id").is_some()); assert_eq!(line["event"], "run.notice"); diff --git a/lib/crates/fabro-workflow/src/lib.rs b/lib/crates/fabro-workflow/src/lib.rs index ceada6f1a..1d05277b3 100644 --- a/lib/crates/fabro-workflow/src/lib.rs +++ b/lib/crates/fabro-workflow/src/lib.rs @@ -92,19 +92,19 @@ pub fn build_completed_stages(cp: &records::Checkpoint, run_failed: bool) -> Vec pub fn extract_stage_durations_from_events(events: &[EventEnvelope]) -> HashMap { let mut durations = HashMap::new(); for envelope in events { - let value = envelope.payload.as_value(); - let event_name = value.get("event").and_then(serde_json::Value::as_str); - if event_name != Some("stage.completed") && event_name != Some("stage.failed") { + let event = &envelope.event; + let event_name = event.event_name(); + if event_name != "stage.completed" && event_name != "stage.failed" { continue; } - let Some(node_id) = value.get("node_id").and_then(serde_json::Value::as_str) else { + let Some(node_id) = event.node_id.as_deref() else { continue; }; - let Some(duration_ms) = value - .get("properties") - .and_then(serde_json::Value::as_object) - .and_then(|properties| properties.get("duration_ms")) - .and_then(serde_json::Value::as_u64) + let Some(duration_ms) = event + .properties() + .ok() + .and_then(|properties| properties.get("duration_ms").cloned()) + .and_then(|duration| duration.as_u64()) else { continue; }; diff --git a/lib/crates/fabro-workflow/src/lifecycle/artifact.rs b/lib/crates/fabro-workflow/src/lifecycle/artifact.rs index 8b5e70037..88e35ab6a 100644 --- a/lib/crates/fabro-workflow/src/lifecycle/artifact.rs +++ b/lib/crates/fabro-workflow/src/lifecycle/artifact.rs @@ -9,12 +9,12 @@ use fabro_core::lifecycle::{AttemptContext, AttemptResultContext, NodeDecision, use fabro_core::outcome::NodeResult; use fabro_core::state::ExecutionState; use fabro_store::ArtifactStore; -use fabro_types::{RunId, StageId}; +use fabro_types::{ArtifactUpload, RunId, StageId}; use tokio::fs; use tokio::time::sleep; use crate::artifact::{normalize_durable_updates, offload_large_values, sync_artifacts_to_env}; -use crate::artifact_snapshot::{CapturedArtifactInfo, collect_artifacts}; +use crate::artifact_snapshot::collect_artifacts; use crate::artifact_upload::ArtifactSink; use crate::event::{Emitter, Event, RunNoticeLevel}; use crate::graph::{WorkflowGraph, WorkflowNode}; @@ -209,7 +209,7 @@ impl ArtifactLifecycle { &self, stage_id: &StageId, artifact_capture_dir: &std::path::Path, - artifacts: &[CapturedArtifactInfo], + artifacts: &[ArtifactUpload], ) -> Result<(), String> { let Some(sink) = self.artifact_sink.as_ref() else { return Err("artifact sink is not configured".to_string()); @@ -238,7 +238,7 @@ impl ArtifactLifecycle { sink: &ArtifactSink, stage_id: &StageId, artifact_capture_dir: &std::path::Path, - artifacts: &[CapturedArtifactInfo], + artifacts: &[ArtifactUpload], ) -> Result<(), String> { match sink { ArtifactSink::Store(store) => { @@ -257,7 +257,7 @@ impl ArtifactLifecycle { store: &ArtifactStore, stage_id: &StageId, artifact_capture_dir: &std::path::Path, - artifacts: &[CapturedArtifactInfo], + artifacts: &[ArtifactUpload], ) -> Result<(), String> { for artifact in artifacts { let local_path = artifact_capture_dir.join(&artifact.path); diff --git a/lib/crates/fabro-workflow/src/operations/create.rs b/lib/crates/fabro-workflow/src/operations/create.rs index a00695024..0ea901856 100644 --- a/lib/crates/fabro-workflow/src/operations/create.rs +++ b/lib/crates/fabro-workflow/src/operations/create.rs @@ -1022,10 +1022,7 @@ mod tests { let run_store = store.open_run_reader(&created.run_id).await.unwrap(); let events = run_store.list_events().await.unwrap(); - assert_eq!( - events.first().unwrap().payload.as_value()["event"], - "run.created" - ); + assert_eq!(events.first().unwrap().event.event_name(), "run.created"); } #[tokio::test] diff --git a/lib/crates/fabro-workflow/src/run_lookup.rs b/lib/crates/fabro-workflow/src/run_lookup.rs index 86bc556b6..8d9ae49e5 100644 --- a/lib/crates/fabro-workflow/src/run_lookup.rs +++ b/lib/crates/fabro-workflow/src/run_lookup.rs @@ -11,8 +11,8 @@ use anyhow::{Context, Result, bail}; use chrono::{DateTime, Utc}; use fabro_config::Storage; use fabro_config::user::default_storage_dir; -use fabro_store::{Database, RunSummary}; -use fabro_types::RunId; +use fabro_store::Database; +use fabro_types::{RunId, RunSummary}; use serde::Serialize; use crate::operations::make_run_dir; From eb8ea317ec69062991e69a35ef1dca4c3ae2eec6 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Mon, 20 Apr 2026 21:44:09 -0400 Subject: [PATCH 03/12] refactor(client): dedupe helpers and fix TOCTOU in auth store - Expose apply_bearer_token_auth and ensure_refresh_target_transport from fabro-client; drop the CLI's duplicate copies. - Collapse AuthStore's two read paths into one NotFound-tolerant reader and drop the pre-existence checks in get/remove/list. - Avoid rewriting auth.json when remove found nothing. - Inline the one-line user_config::build_public_http_client wrapper. - Trim unused pub use fabro_api::types re-export and the narrating doc comment in fabro-client/src/lib.rs. - Clean up pre-existing unused imports in run/create.rs and loopback.rs tests. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../fabro-cli/src/commands/auth/login.rs | 21 ++------- .../fabro-cli/src/commands/auth/logout.rs | 2 +- .../fabro-cli/src/commands/run/create.rs | 2 +- lib/crates/fabro-cli/src/server_client.rs | 15 +----- lib/crates/fabro-cli/src/user_config.rs | 6 --- lib/crates/fabro-client/src/auth_store.rs | 46 +++++++------------ lib/crates/fabro-client/src/client.rs | 4 +- lib/crates/fabro-client/src/lib.rs | 9 ++-- lib/crates/fabro-client/src/loopback.rs | 2 - 9 files changed, 30 insertions(+), 77 deletions(-) diff --git a/lib/crates/fabro-cli/src/commands/auth/login.rs b/lib/crates/fabro-cli/src/commands/auth/login.rs index 227ff8936..c7c85f42d 100644 --- a/lib/crates/fabro-cli/src/commands/auth/login.rs +++ b/lib/crates/fabro-cli/src/commands/auth/login.rs @@ -3,7 +3,7 @@ use std::time::Duration; use anyhow::{Context as _, Result, anyhow, bail}; use chrono::{DateTime, Utc}; use fabro_api::types; -use fabro_client::{AuthEntry, AuthStore, StoredSubject}; +use fabro_client::{AuthEntry, AuthStore, StoredSubject, ensure_refresh_target_transport}; use fabro_http::header::CONTENT_TYPE; use fabro_types::settings::CliSettings; use fabro_types::settings::cli::CliLayer; @@ -96,14 +96,7 @@ pub(super) async fn login_command( } }; - match target.loopback_classification()? { - fabro_client::LoopbackClassification::Https - | fabro_client::LoopbackClassification::LoopbackHttp - | fabro_client::LoopbackClassification::UnixSocket => {} - fabro_client::LoopbackClassification::Rejected => { - bail!("{}", token_transport_error(&target)); - } - } + ensure_refresh_target_transport(&target)?; let tokens = exchange_cli_token(&target, &code, &pkce.verifier, &redirect_uri).await?; let entry = AuthEntry { @@ -129,7 +122,7 @@ pub(super) async fn login_command( #[cfg(unix)] async fn fetch_cli_auth_config(target: &ServerTarget) -> Result { - let (http_client, base_url) = user_config::build_public_http_client(target)?; + let (http_client, base_url) = target.build_public_http_client()?; let client = fabro_api::ApiClient::new_with_client(&base_url, http_client); client .get_cli_auth_config() @@ -146,7 +139,7 @@ async fn exchange_cli_token( code_verifier: &str, redirect_uri: &str, ) -> Result { - let (http_client, base_url) = user_config::build_public_http_client(target)?; + let (http_client, base_url) = target.build_public_http_client()?; let response = http_client .post(format!("{base_url}/auth/cli/token")) .header(CONTENT_TYPE, "application/json") @@ -241,12 +234,6 @@ fn login_failure_message(error_code: &str, error_description: Option<&str>) -> S } } -fn token_transport_error(target: &ServerTarget) -> String { - format!( - "Refusing to send refresh-token credentials over plaintext HTTP to a non-loopback host ({target}). Use HTTPS, or bind the server to 127.0.0.1 / ::1." - ) -} - fn identity_summary(subject: &StoredSubject) -> String { if !subject.name.is_empty() && !subject.email.is_empty() { format!("{} ({} <{}>)", subject.login, subject.name, subject.email) diff --git a/lib/crates/fabro-cli/src/commands/auth/logout.rs b/lib/crates/fabro-cli/src/commands/auth/logout.rs index 4743eb42b..4b72ef5c1 100644 --- a/lib/crates/fabro-cli/src/commands/auth/logout.rs +++ b/lib/crates/fabro-cli/src/commands/auth/logout.rs @@ -58,7 +58,7 @@ pub(super) async fn logout_command( } async fn revoke_remote_session(target: &ServerTarget, entry: &AuthEntry) -> Result<()> { - let (http_client, base_url) = user_config::build_public_http_client(target)?; + let (http_client, base_url) = target.build_public_http_client()?; let response = http_client .post(format!("{base_url}/auth/cli/logout")) .header(AUTHORIZATION, format!("Bearer {}", entry.refresh_token)) diff --git a/lib/crates/fabro-cli/src/commands/run/create.rs b/lib/crates/fabro-cli/src/commands/run/create.rs index 6ee251680..e1c93496c 100644 --- a/lib/crates/fabro-cli/src/commands/run/create.rs +++ b/lib/crates/fabro-cli/src/commands/run/create.rs @@ -13,7 +13,7 @@ use super::overrides::run_args_layer; use crate::args::RunArgs; use crate::command_context::CommandContext; use crate::manifest_builder::{ManifestBuildInput, build_run_manifest, run_manifest_args}; -use crate::user_config::{self, ServerTarget}; +use crate::user_config; pub(crate) struct CreatedRun { pub(crate) run_id: RunId, diff --git a/lib/crates/fabro-cli/src/server_client.rs b/lib/crates/fabro-cli/src/server_client.rs index 4f12f2c0c..337a34530 100644 --- a/lib/crates/fabro-cli/src/server_client.rs +++ b/lib/crates/fabro-cli/src/server_client.rs @@ -5,10 +5,10 @@ use std::time::Duration; use anyhow::{Context as _, Result, anyhow, bail}; use fabro_client::{ AuthStore, Credential, CredentialFallback, OAuthSession, ServerTarget, TransportConnector, + apply_bearer_token_auth, }; pub(crate) use fabro_client::{Client, RunEventStream}; use fabro_config::Storage; -use fabro_http::header::AUTHORIZATION; use fabro_server::bind::Bind; pub(crate) use fabro_types::RunProjection; use fabro_types::settings::SettingsLayer; @@ -264,19 +264,6 @@ async fn wait_for_local_dev_token(storage_dir: &Path) -> Result { ); } -fn apply_bearer_token_auth( - builder: fabro_http::HttpClientBuilder, - token: &str, -) -> Result { - let mut headers = fabro_http::HeaderMap::new(); - headers.insert( - AUTHORIZATION, - fabro_http::HeaderValue::from_str(&format!("Bearer {token}")) - .context("invalid dev token header value")?, - ); - Ok(builder.default_headers(headers)) -} - async fn build_authed_unix_socket_http_client( path: &Path, storage_dir: Option<&Path>, diff --git a/lib/crates/fabro-cli/src/user_config.rs b/lib/crates/fabro-cli/src/user_config.rs index c151caa4e..3fa3710da 100644 --- a/lib/crates/fabro-cli/src/user_config.rs +++ b/lib/crates/fabro-cli/src/user_config.rs @@ -61,12 +61,6 @@ pub(crate) fn apply_storage_dir_override( layer } -pub(crate) fn build_public_http_client( - target: &ServerTarget, -) -> Result<(fabro_http::HttpClient, String)> { - target.build_public_http_client() -} - /// Pull the resolved CLI target configuration out of `[cli.target]`. /// Returns either an http(s) URL or a unix socket path. fn cli_target_from_settings(settings: &CliSettings) -> Option { diff --git a/lib/crates/fabro-client/src/auth_store.rs b/lib/crates/fabro-client/src/auth_store.rs index 01185ac72..e436e3e78 100644 --- a/lib/crates/fabro-client/src/auth_store.rs +++ b/lib/crates/fabro-client/src/auth_store.rs @@ -120,10 +120,6 @@ impl AuthStore { } pub fn get(&self, target: &ServerTarget) -> Result, AuthStoreError> { - if !self.path.exists() { - return Ok(None); - } - let key = key_for_target(target); self.with_shared_lock(|| { let file = self.read_auth_file()?; @@ -143,7 +139,7 @@ impl AuthStore { let key = key_for_target(target); self.ensure_parent_dir()?; self.with_exclusive_lock(|| { - let mut file = self.read_auth_file_if_exists()?; + let mut file = self.read_auth_file()?; file.servers.insert(key, entry); self.write_auth_file(&file) }) @@ -159,26 +155,20 @@ impl AuthStore { #[cfg(unix)] { - if !self.path.exists() { - return Ok(false); - } - let key = key_for_target(target); self.ensure_parent_dir()?; self.with_exclusive_lock(|| { - let mut file = self.read_auth_file_if_exists()?; + let mut file = self.read_auth_file()?; let removed = file.servers.remove(&key).is_some(); - self.write_auth_file(&file)?; + if removed { + self.write_auth_file(&file)?; + } Ok(removed) }) } } pub fn list(&self) -> Result, AuthStoreError> { - if !self.path.exists() { - return Ok(Vec::new()); - } - self.with_shared_lock(|| { let file = self.read_auth_file()?; file.servers @@ -189,21 +179,19 @@ impl AuthStore { } fn read_auth_file(&self) -> Result { - let contents = fs::read_to_string(&self.path).map_err(|source| AuthStoreError::Read { - path: self.path.clone(), - source, - })?; - serde_json::from_str(&contents).map_err(|source| AuthStoreError::Corrupt { - path: self.path.clone(), - source, - }) - } - - fn read_auth_file_if_exists(&self) -> Result { - if !self.path.exists() { - return Ok(AuthFile::default()); + match fs::read_to_string(&self.path) { + Ok(contents) => { + serde_json::from_str(&contents).map_err(|source| AuthStoreError::Corrupt { + path: self.path.clone(), + source, + }) + } + Err(source) if source.kind() == std::io::ErrorKind::NotFound => Ok(AuthFile::default()), + Err(source) => Err(AuthStoreError::Read { + path: self.path.clone(), + source, + }), } - self.read_auth_file() } #[cfg(unix)] diff --git a/lib/crates/fabro-client/src/client.rs b/lib/crates/fabro-client/src/client.rs index 57098e055..3cc165c28 100644 --- a/lib/crates/fabro-client/src/client.rs +++ b/lib/crates/fabro-client/src/client.rs @@ -1294,7 +1294,7 @@ fn connect_target_transport( Ok((http_client, "http://fabro".to_string())) } -fn apply_bearer_token_auth( +pub fn apply_bearer_token_auth( builder: fabro_http::HttpClientBuilder, token: &str, ) -> Result { @@ -1307,7 +1307,7 @@ fn apply_bearer_token_auth( Ok(builder.default_headers(headers)) } -fn ensure_refresh_target_transport(target: &ServerTarget) -> Result<()> { +pub fn ensure_refresh_target_transport(target: &ServerTarget) -> Result<()> { match target.loopback_classification()? { LoopbackClassification::Https | LoopbackClassification::LoopbackHttp diff --git a/lib/crates/fabro-client/src/lib.rs b/lib/crates/fabro-client/src/lib.rs index 248987500..e8a78d380 100644 --- a/lib/crates/fabro-client/src/lib.rs +++ b/lib/crates/fabro-client/src/lib.rs @@ -1,7 +1,4 @@ //! Typed HTTP client for the Fabro API. -//! -//! This crate hosts the reusable client and auth/session plumbing that was -//! previously embedded in `fabro-cli`. pub mod auth_store; pub mod client; @@ -13,14 +10,16 @@ pub mod sse; pub mod target; pub use auth_store::{AuthEntry, AuthStore, AuthStoreError, LockError, StoredSubject}; -pub use client::{Client, RunEventStream, TransportConnector}; +pub use client::{ + Client, RunEventStream, TransportConnector, apply_bearer_token_auth, + ensure_refresh_target_transport, +}; pub use credential::{Credential, CredentialFallback}; pub use error::{ ApiError, ApiFailure, StructuredApiError, classify_api_error, classify_http_response, convert_type, is_not_found_error, map_api_error, parse_error_response_value, raw_response_failure_error, }; -pub use fabro_api::types; pub use loopback::{LoopbackClassification, TargetSchemeError}; pub use session::OAuthSession; pub use target::ServerTarget; diff --git a/lib/crates/fabro-client/src/loopback.rs b/lib/crates/fabro-client/src/loopback.rs index 20a28b900..f980c9f43 100644 --- a/lib/crates/fabro-client/src/loopback.rs +++ b/lib/crates/fabro-client/src/loopback.rs @@ -122,8 +122,6 @@ fn ip_is_loopback(ip: &IpAddr) -> bool { #[cfg(test)] mod tests { - use std::path::PathBuf; - use super::LoopbackClassification; use crate::target::ServerTarget; From 1214bebd1410e6867d4257adc26c5243a74969f8 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Mon, 20 Apr 2026 21:49:09 -0400 Subject: [PATCH 04/12] fix(client): reject obfuscated IPv4 literals at target parse time url::Url normalizes decimal (http://2130706433) and hex (http://0x7f000001) IPv4 host forms into 127.0.0.1, so after canonical_http_url rewrites the target the loopback classifier cannot tell them apart from a legitimate http://127.0.0.1 and lets a refresh token ride plaintext HTTP. Detect these forms on the raw input string and bail out before the url crate can hide them, and split the loopback test to cover the parse-time rejection path. Co-Authored-By: Claude Opus 4.7 (1M context) --- lib/crates/fabro-client/src/loopback.rs | 12 ++++++++-- lib/crates/fabro-client/src/target.rs | 30 +++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/lib/crates/fabro-client/src/loopback.rs b/lib/crates/fabro-client/src/loopback.rs index f980c9f43..bda6f0b25 100644 --- a/lib/crates/fabro-client/src/loopback.rs +++ b/lib/crates/fabro-client/src/loopback.rs @@ -163,8 +163,6 @@ mod tests { "http://127.0.0.1:1@attacker.com", "http://localhost", "http://localhost.evil.com", - "http://2130706433", - "http://0x7f000001", ]; for api_url in cases { @@ -176,6 +174,16 @@ mod tests { } } + #[test] + fn rejects_obfuscated_ipv4_literals_at_parse_time() { + for api_url in ["http://2130706433", "http://0x7f000001"] { + assert!( + ServerTarget::http_url(api_url).is_err(), + "{api_url} should not parse as a server target" + ); + } + } + #[test] fn rejects_non_http_server_targets_at_parse_time() { let error = "ftp://fabro.example.com" diff --git a/lib/crates/fabro-client/src/target.rs b/lib/crates/fabro-client/src/target.rs index e710ca732..4ad663ac5 100644 --- a/lib/crates/fabro-client/src/target.rs +++ b/lib/crates/fabro-client/src/target.rs @@ -146,6 +146,10 @@ fn canonical_http_url(value: &str) -> Result { _ => bail!("server target must be an http(s) URL or absolute Unix socket path"), }; + if raw_url_host(normalized).is_some_and(is_obfuscated_ipv4_literal) { + bail!("server target must be an http(s) URL or absolute Unix socket path"); + } + let Some(host) = url.host_str() else { bail!("server target must be an http(s) URL or absolute Unix socket path"); }; @@ -166,6 +170,32 @@ fn trim_api_path_suffix(value: &str) -> &str { trimmed.strip_suffix("/api/v1").unwrap_or(trimmed) } +/// Extract the host substring from `value` without going through +/// [`fabro_http::Url`]. `Url` normalizes decimal/hex IPv4 literals into dotted +/// form, which hides the original input from later inspection. +fn raw_url_host(value: &str) -> Option<&str> { + let (_, remainder) = value.split_once("://")?; + let authority_end = remainder.find(['/', '?', '#']).unwrap_or(remainder.len()); + let authority = &remainder[..authority_end]; + let after_userinfo = authority + .rsplit_once('@') + .map_or(authority, |(_, host)| host); + if after_userinfo.starts_with('[') { + return None; + } + let host = after_userinfo + .split_once(':') + .map_or(after_userinfo, |(host, _)| host); + (!host.is_empty()).then_some(host) +} + +fn is_obfuscated_ipv4_literal(host: &str) -> bool { + if host.starts_with("0x") || host.starts_with("0X") { + return true; + } + !host.contains('.') && !host.is_empty() && host.bytes().all(|b| b.is_ascii_digit()) +} + fn lexical_normalize_absolute_path(path: &Path) -> Result { if !path.is_absolute() { bail!("server target must be an http(s) URL or absolute Unix socket path"); From 7dd058cc40314df654ed7e02535c0283bd88f029 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Mon, 20 Apr 2026 22:17:23 -0400 Subject: [PATCH 05/12] refactor: unify run vocabulary and metadata snapshot layout Implements the plan at docs/plans/2026-04-20-003-refactor-unify-run-vocabulary-metadata-plan.md. - Rename RunRecord to RunSpec and RunProjection.run to .spec everywhere in Rust source, tests, helpers, test names, and error messages. - Introduce SerializableProjection wrapper that trims bulky node text fields (prompt, response, diff, stdout, stderr) for run.json snapshots. - Collapse metadata-branch and CLI export to one RunDump::from_projection builder emitting run.json + graph.fabro + stages/{stage_id}/... and drop legacy top-level start/status/checkpoint/sandbox/retro/conclusion split files. - Replace MetadataStore::write_checkpoint with write_snapshot returning the commit SHA; add read_run_projection/read_run_spec; demote read_checkpoint/read_start_record to projection-field extractors. - Switch fork, rewind, rebuild_meta, CLI rewind recovery, and retro upload to read the unified projection layout. - Add additive query methods on RunSpec and RunProjection. Serde-level `alias = "spec"` shim dropped; `rename = "run"` retained to keep the server API wire format stable per the plan's scope boundary. Co-Authored-By: Claude Opus 4.7 (1M context) --- Cargo.lock | 1 + docs/agents/outputs.mdx | 6 +- docs/agents/prompts.mdx | 2 +- docs/api-reference/fabro-api.yaml | 2 +- docs/execution/checkpoints.mdx | 14 +- docs/execution/retros.mdx | 2 +- ...ctor-unify-run-vocabulary-metadata-plan.md | 354 +++++++++++++++++ docs/reference/cli.mdx | 4 +- docs/reference/run-directory.mdx | 17 +- lib/crates/fabro-checkpoint/Cargo.toml | 1 + lib/crates/fabro-checkpoint/src/metadata.rs | 220 ++++++----- .../fabro-cli/src/commands/pr/create.rs | 12 +- .../fabro-cli/src/commands/run/attach.rs | 6 +- .../fabro-cli/src/commands/run/create.rs | 2 +- lib/crates/fabro-cli/src/commands/run/fork.rs | 4 +- .../fabro-cli/src/commands/run/rewind.rs | 17 +- .../fabro-cli/src/commands/run/runner.rs | 8 +- .../fabro-cli/src/commands/runs/inspect.rs | 6 +- .../fabro-cli/src/commands/store/dump.rs | 101 +++-- .../src/commands/store/run_export.rs | 373 +----------------- lib/crates/fabro-cli/src/server_runs.rs | 2 +- lib/crates/fabro-cli/tests/it/cmd/config.rs | 14 +- lib/crates/fabro-cli/tests/it/cmd/create.rs | 18 +- lib/crates/fabro-cli/tests/it/cmd/fork.rs | 20 +- lib/crates/fabro-cli/tests/it/cmd/inspect.rs | 12 +- .../fabro-cli/tests/it/cmd/pr_create.rs | 2 +- lib/crates/fabro-cli/tests/it/cmd/run.rs | 2 +- lib/crates/fabro-cli/tests/it/cmd/runner.rs | 10 +- .../fabro-cli/tests/it/cmd/store_dump.rs | 33 +- lib/crates/fabro-cli/tests/it/cmd/support.rs | 40 +- .../fabro-cli/tests/it/scenario/lifecycle.rs | 16 +- .../fabro-cli/tests/it/scenario/recovery.rs | 28 +- .../fabro-cli/tests/it/scenario/smoke.rs | 2 +- .../tests/it/workflow/command_agent_mixed.rs | 2 +- .../tests/it/workflow/command_pipeline.rs | 2 +- .../fabro-cli/tests/it/workflow/full_stack.rs | 18 +- lib/crates/fabro-cli/tests/it/workflow/mod.rs | 8 +- .../fabro-config/src/effective_settings.rs | 2 +- lib/crates/fabro-retro/src/retro_agent.rs | 253 ++++++++++-- lib/crates/fabro-server/src/server.rs | 61 ++- lib/crates/fabro-store/src/lib.rs | 2 + lib/crates/fabro-store/src/run_state.rs | 77 +++- .../src/serializable_projection.rs | 34 ++ lib/crates/fabro-store/src/slate/mod.rs | 26 +- .../tests/serializable_projection.rs | 156 ++++++++ lib/crates/fabro-types/src/lib.rs | 2 +- lib/crates/fabro-types/src/run.rs | 51 ++- .../fabro-types/tests/run_spec_methods.rs | 45 +++ ...{run_record_serde.rs => run_spec_serde.rs} | 8 +- lib/crates/fabro-workflow/src/error.rs | 4 +- lib/crates/fabro-workflow/src/git.rs | 16 +- .../fabro-workflow/src/lifecycle/git.rs | 74 ++-- .../fabro-workflow/src/operations/create.rs | 33 +- .../fabro-workflow/src/operations/fork.rs | 191 +++++---- .../src/operations/rebuild_meta.rs | 292 +++++++------- .../fabro-workflow/src/operations/resume.rs | 2 +- .../fabro-workflow/src/operations/rewind.rs | 83 ++-- .../fabro-workflow/src/operations/start.rs | 6 +- .../src/pipeline/execute/tests.rs | 4 +- .../fabro-workflow/src/pipeline/finalize.rs | 9 +- .../fabro-workflow/src/pipeline/initialize.rs | 6 +- .../fabro-workflow/src/pipeline/persist.rs | 52 +-- .../src/pipeline/pull_request.rs | 72 ++-- .../fabro-workflow/src/pipeline/retro.rs | 14 +- .../fabro-workflow/src/pipeline/types.rs | 24 +- lib/crates/fabro-workflow/src/records/mod.rs | 2 +- lib/crates/fabro-workflow/src/records/run.rs | 2 +- lib/crates/fabro-workflow/src/run_dump.rs | 366 ++++++++++------- lib/crates/fabro-workflow/src/run_lookup.rs | 28 +- .../fabro-workflow/src/runtime_store.rs | 10 +- .../tests/it/daytona_integration.rs | 2 +- .../fabro-workflow/tests/it/integration.rs | 12 +- .../src/models/run-projection.ts | 2 +- 73 files changed, 2075 insertions(+), 1329 deletions(-) create mode 100644 docs/plans/2026-04-20-003-refactor-unify-run-vocabulary-metadata-plan.md create mode 100644 lib/crates/fabro-store/src/serializable_projection.rs create mode 100644 lib/crates/fabro-store/tests/serializable_projection.rs create mode 100644 lib/crates/fabro-types/tests/run_spec_methods.rs rename lib/crates/fabro-types/tests/{run_record_serde.rs => run_spec_serde.rs} (95%) diff --git a/Cargo.lock b/Cargo.lock index c779b34bf..566cb9a11 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1583,6 +1583,7 @@ name = "fabro-checkpoint" version = "0.208.0-nightly.1" dependencies = [ "chrono", + "fabro-store", "fabro-types", "git2", "serde", diff --git a/docs/agents/outputs.mdx b/docs/agents/outputs.mdx index 0b240e14f..e375ff405 100644 --- a/docs/agents/outputs.mdx +++ b/docs/agents/outputs.mdx @@ -7,7 +7,7 @@ When an agent or prompt node finishes, Fabro captures its response text and prod ## Response capture -After an agent or prompt node completes, Fabro captures the full response text and writes it to the run logs at `{run_dir}/nodes/{node_id}/response.md`. It also writes the final outcome (status, context updates, routing directives) to `{run_dir}/nodes/{node_id}/status.json`. +After an agent or prompt node completes, Fabro captures the full response text and persists it to `stages/{node_id}@{visit}/response.md` in metadata snapshots and `fabro store dump` output. It also writes the final outcome (status, context updates, routing directives) to `stages/{node_id}@{visit}/status.json`. ## Context updates @@ -92,7 +92,7 @@ review -> approve [label="Approve"] ## Output logging -Fabro writes several files per stage to `{run_dir}/nodes/{node_id}/`: +Fabro writes several files per stage to `stages/{node_id}@{visit}/` in metadata snapshots and `fabro store dump` output: | File | Contents | |---|---| @@ -100,7 +100,7 @@ Fabro writes several files per stage to `{run_dir}/nodes/{node_id}/`: | `response.md` | The full LLM response text | | `status.json` | The outcome: status, context updates, routing directives, usage stats | -These files are written for every agent and prompt node execution, including retries (visit count is appended to the directory name for repeat visits). Use them for debugging unexpected agent behavior or verifying that routing directives were extracted correctly. +These files are written for every agent and prompt node execution, including retries. Use them for debugging unexpected agent behavior or verifying that routing directives were extracted correctly. ## File tracking diff --git a/docs/agents/prompts.mdx b/docs/agents/prompts.mdx index 4a7df5d93..69404aa79 100644 --- a/docs/agents/prompts.mdx +++ b/docs/agents/prompts.mdx @@ -295,4 +295,4 @@ Use prompt nodes for analysis, classification, and summarization tasks where too ## Prompt logging -Fabro writes the assembled prompt to `{run_dir}/nodes/{node_id}/prompt.md` for every agent and prompt stage. This includes the preamble (if any) and the expanded prompt text. Use these files for debugging when an agent behaves unexpectedly. +Fabro persists the assembled prompt to `stages/{node_id}@{visit}/prompt.md` in metadata snapshots and `fabro store dump` output for every agent and prompt stage. This includes the preamble (if any) and the expanded prompt text. Use these files for debugging when an agent behaves unexpectedly. diff --git a/docs/api-reference/fabro-api.yaml b/docs/api-reference/fabro-api.yaml index 6db9b4b44..4310b499b 100644 --- a/docs/api-reference/fabro-api.yaml +++ b/docs/api-reference/fabro-api.yaml @@ -3714,7 +3714,7 @@ components: required: - nodes properties: - run: + spec: type: ["object", "null"] additionalProperties: true graph_source: diff --git a/docs/execution/checkpoints.mdx b/docs/execution/checkpoints.mdx index 4c8ae13aa..deed1b105 100644 --- a/docs/execution/checkpoints.mdx +++ b/docs/execution/checkpoints.mdx @@ -43,18 +43,18 @@ The `Fabro-Checkpoint` trailer links each run branch commit to its metadata bran The metadata branch (`fabro/meta/{run_id}`) is an orphan branch that stores structured run data using Git's object storage directly (via `git2`). It is initialized at run start with: -- **`run.json`** — Run record: run ID, created_at, config, graph, workflow slug, working directory, host repo path, base branch, labels -- **`start.json`** — Start record: run ID, start time, run branch, base SHA +- **`run.json`** — Current projection snapshot: run spec, start/status records, current checkpoint, conclusion, sandbox, retro state, and other run-level metadata +- **`graph.fabro`** — Workflow source for the run After each node, the metadata branch is updated with: -- **`checkpoint.json`** — Full execution state (see below) -- **`artifacts/*.json`** — Any offloaded artifact data (large context values over 100KB) -- **`nodes/{node_id}/`** — Per-node execution trace files (prompts, responses, status, diffs — files under 512KB from an allowlist) +- **`run.json`** — Refreshed projection snapshot with the new current checkpoint +- **`stages/{node_id}@{visit}/...`** — Per-stage execution trace files (prompts, responses, status, diffs, stdout/stderr, and tool metadata) +- **`retro/*.md`** — Retro prompt/response text when present ## What's in a checkpoint -The `checkpoint.json` captures everything needed to resume a run: +The `run.json.checkpoint` snapshot captures everything needed to resume a run: | Field | Description | |---|---| @@ -134,7 +134,7 @@ git show fabro/run/01JKXYZ... git diff main..fabro/run/01JKXYZ... # Read checkpoint data from the metadata branch -git show fabro/meta/01JKXYZ...:checkpoint.json | jq .current_node +git show fabro/meta/01JKXYZ...:run.json | jq .checkpoint.current_node ``` ## Rewinding to an earlier checkpoint diff --git a/docs/execution/retros.mdx b/docs/execution/retros.mdx index 200bd52c8..8aae4be1e 100644 --- a/docs/execution/retros.mdx +++ b/docs/execution/retros.mdx @@ -101,7 +101,7 @@ Retro generation happens in two phases after a run completes: 1. **Derive** — Fabro extracts stage durations from durable run events and builds a retro from the checkpoint data. This is deterministic, fast, and produces the quantitative layer. -2. **Narrate** — An LLM agent session analyzes the run data. The agent receives temp files named `progress.jsonl`, `checkpoint.json`, `run.json`, and `start.json` inside its sandbox so it can grep and read the event stream and run state. The narrative fields are merged back into durable retro state. +2. **Narrate** — An LLM agent session analyzes the run data. The agent receives `progress.jsonl`, `run.json`, `graph.fabro`, and per-stage files under `stages/{node_id}@{visit}/...` inside its sandbox so it can grep and read the event stream, run snapshot, workflow source, and full stage payloads. The narrative fields are merged back into durable retro state. Both phases run automatically at the end of every CLI run. The API server derives the quantitative layer but does not currently run the narrative agent. diff --git a/docs/plans/2026-04-20-003-refactor-unify-run-vocabulary-metadata-plan.md b/docs/plans/2026-04-20-003-refactor-unify-run-vocabulary-metadata-plan.md new file mode 100644 index 000000000..fe8819d25 --- /dev/null +++ b/docs/plans/2026-04-20-003-refactor-unify-run-vocabulary-metadata-plan.md @@ -0,0 +1,354 @@ +--- +title: "refactor: unify run vocabulary and metadata snapshot layout" +type: refactor +status: active +date: 2026-04-20 +origin: /Users/bhelmkamp/.claude/plans/make-a-full-plan-pure-wombat.md +deepened: 2026-04-20 +--- + +# refactor: unify run vocabulary and metadata snapshot layout + +## Overview + +Align the run domain vocabulary and metadata-branch layout around the event-sourced projection the code already maintains in memory. The refactor renames `RunRecord` to `RunSpec`, collapses metadata snapshots into a trimmed `RunProjection` in `run.json`, normalizes per-stage files to `stages/{node_id}@{visit}/...`, and updates every metadata-branch consumer to read that unified shape. + +## Problem Frame + +The durable write path is already projection-oriented, but the git metadata branch still presents the same run through multiple accidental shapes: + +- `run.json` is only the spec slice (`RunRecord`) +- run lifecycle state is split across `start.json`, `status.json`, `checkpoint.json`, `sandbox.json`, `retro.json`, and `conclusion.json` +- node payloads use multiple incompatible path conventions under `nodes/` +- fork, rewind, rebuild, retro upload, and CLI dump/export all read or write those legacy files directly + +That mismatch leaks implementation history into the domain model and makes every consumer reason about special cases. The current tree on `main` still shows the old design in `RunDump`, `MetadataStore`, fork/rewind/rebuild operations, CLI rewind recovery, and `fabro store dump`. This plan makes the metadata branch a true serialized projection snapshot and removes the split-file vocabulary drift. + +## Requirements Trace + +- R1. Rename `RunRecord` to `RunSpec` and rename `RunProjection.run` to `RunProjection.spec` everywhere in Rust code and tests. Do not ship alias shims. +- R2. Introduce a metadata-only serializer that writes `run.json` as a trimmed `RunProjection`, stripping bulky `NodeState` text fields (`prompt`, `response`, `diff`, `stdout`, `stderr`) while preserving all other projection data. +- R3. Standardize metadata-branch and export layout around `run.json`, `graph.fabro`, `retro/*.md`, `events.jsonl`, `checkpoints/*.json`, artifact exports, and `stages/{node_id}@{visit}/...`. Stop writing top-level `start.json`, `status.json`, `checkpoint.json`, `sandbox.json`, `retro.json`, and `conclusion.json`. +- R4. Replace metadata helpers and writers that special-case `checkpoint.json` with snapshot-oriented helpers that can write a full projection commit and still return the metadata-branch commit SHA when checkpoint flows need it. +- R5. Update metadata consumers (`fork`, `rewind`, `rebuild_meta`, CLI rewind recovery, retro upload, store dump/export) to read the unified projection layout without changing user-visible behavior. +- R6. Add additive query methods on `RunSpec` and `RunProjection` for common reads while keeping existing public-field access valid. +- R7. Update crate tests, CLI integration tests, and snapshots to the new layout with no coverage regression. + +## Scope Boundaries + +- No backward-compatibility read path for old metadata branches. This repo is still pre-launch greenfield. +- No OpenAPI or generated TypeScript client change. Server APIs expose run state independently of metadata-branch file layout. +- Keep `graph.fabro`, `retro/prompt.md`, `retro/response.md`, `events.jsonl`, `checkpoints/*.json`, and artifact export support. +- Do not move artifact exports away from `artifacts/nodes/{node_id}/visit-{n}/...` unless implementation proves a hard blocker; artifact path cleanup is not the point of this refactor. +- Do not convert fork/rewind to read events directly from durable storage; they continue to operate from metadata branches. + +## Context & Research + +### Relevant Code and Patterns + +- `lib/crates/fabro-types/src/run.rs` and `lib/crates/fabro-store/src/run_state.rs` define the core vocabulary and projection shape that this refactor renames and extends. +- `lib/crates/fabro-workflow/src/run_dump.rs` and `lib/crates/fabro-cli/src/commands/store/run_export.rs` currently duplicate layout/serialization logic and already drift on node path format. +- `lib/crates/fabro-workflow/src/lifecycle/git.rs` and `lib/crates/fabro-workflow/src/pipeline/finalize.rs` still use phase-specific `RunDump` constructors and a `checkpoint.json`-oriented metadata helper. +- `lib/crates/fabro-workflow/src/operations/{fork.rs,rewind.rs,rebuild_meta.rs}` plus `lib/crates/fabro-cli/src/commands/run/rewind.rs` are the critical metadata readers/writers that must switch from standalone `checkpoint.json` and `start.json` reads to projection reads. +- `lib/crates/fabro-types/src/stage_id.rs` already defines `Display` as `{node_id}@{visit}`, which should become the on-disk stage directory name. +- `files-internal/testing-strategy.md` says CLI integration tests should remain command-driven and black-box; layout-specific assertions belong in the right layer rather than by planting run internals by hand. + +### Institutional Learnings + +- No matching `docs/solutions/` entries were present in this repo at planning time, so this plan is grounded in current code and test patterns rather than prior internal solution notes. + +### External References + +- None. This is an internal Rust refactor with sufficient local context. + +## Key Technical Decisions + +- Hard rename `RunRecord` to `RunSpec` and `RunProjection.run` to `RunProjection.spec`. + Rationale: the current names are the main source of spec/projection confusion, and a greenfield codebase does not benefit from preserving legacy aliases. +- `run.json` becomes the single top-level serialized projection snapshot for metadata branches and exports, including `conclusion`. + Rationale: leaving `conclusion.json` behind would preserve the accidental fragmentation this refactor is trying to remove. +- Use a dedicated metadata serializer wrapper instead of changing `RunProjection`'s canonical serde implementation. + Rationale: ordinary projection serde remains valuable for tests and internal round-trips, while metadata snapshots need one specific trimmed representation. +- Normalize per-stage paths to `stages/{stage_id}/{filename}` using `StageId::Display`. + Rationale: this removes visit-1 special cases and aligns the on-disk layout with the stage identifier already exposed in APIs and logs. +- Replace `MetadataStore::write_checkpoint` with a snapshot-oriented commit helper rather than passing renamed data through a stale `checkpoint_json` API. + Rationale: checkpoint commits still need a returned SHA, but the helper should describe the new snapshot semantics instead of the deleted file. +- Delete the CLI-only `StoreRunExport` duplication and reuse the workflow dump builder. + Rationale: this refactor changes layout semantics in one place; keeping two near-identical serializers would make future drift likely. +- Keep artifact exports under `artifacts/nodes/{node_id}/visit-{n}/...` in this unit. + Rationale: artifact lookup is already keyed by `StageId` at API boundaries, but changing artifact paths would widen scope without addressing the metadata-vocabulary problem. +- Query methods remain additive. + Rationale: field privacy is a follow-up concern, and this refactor already changes many call sites. + +## Open Questions + +### Resolved During Planning + +- Should `conclusion.json` survive as a separate top-level file? + No. It should collapse into `run.json` with the rest of the projection. +- Should CLI export keep its own serializer? + No. Reuse the workflow dump/export builder so metadata branches and `fabro store dump` cannot diverge again. +- Does the layout change need to cover CLI rewind recovery as well as workflow operations? + Yes. `lib/crates/fabro-cli/src/commands/run/rewind.rs` currently reads `checkpoint.json` from the metadata branch and must switch with the rest of the readers. + +### Deferred to Implementation + +- Exact helper names for the new metadata commit writer (`write_snapshot`, `write_projection_commit`, etc.). The plan fixes the API shape and intent, but the final Rust name can be chosen during implementation. +- Whether the shared export builder stays in `lib/crates/fabro-workflow/src/run_dump.rs` or moves to a nearby module. The key constraint is one authoritative layout builder, not a specific file name. +- Whether any low-value tests should move layers while being updated. Follow `files-internal/testing-strategy.md` if implementation reveals a better layer, but do not turn this refactor into a broad test reorganization. + +## High-Level Technical Design + +> *This illustrates the intended approach and is directional guidance for review, not implementation specification. The implementing agent should treat it as context, not code to reproduce.* + +```text +Durable event store + -> RunProjection { spec, start, status, checkpoint, conclusion, retro, sandbox, nodes, ... } + -> Metadata serializer (trim bulky node text fields) + -> Metadata/export tree: + run.json # trimmed RunProjection snapshot + graph.fabro # readable workflow source + stages//... # prompt.md, response.md, status.json, provider_used.json, + # diff.patch, script_invocation.json, script_timing.json, + # parallel_results.json, stdout.log, stderr.log + retro/prompt.md + retro/response.md + events.jsonl + checkpoints/.json + artifacts/nodes//visit-/... +``` + +## Implementation Units + +- [ ] **Unit 1: Rename run vocabulary to spec/projection** + +**Goal:** Replace the legacy `RunRecord`/`run` vocabulary with `RunSpec`/`spec` across the domain model and its consumers. + +**Requirements:** R1 + +**Dependencies:** None + +**Files:** +- Modify: `lib/crates/fabro-types/src/run.rs` +- Modify: `lib/crates/fabro-types/src/lib.rs` +- Modify: `lib/crates/fabro-workflow/src/records/{run.rs,mod.rs}` +- Modify: `lib/crates/fabro-store/src/run_state.rs` +- Modify: `lib/crates/fabro-workflow/src/{runtime_store.rs,run_lookup.rs}` +- Modify: `lib/crates/fabro-workflow/src/pipeline/{pull_request.rs,retro.rs,types.rs,execute/tests.rs}` +- Modify: `lib/crates/fabro-workflow/src/operations/{create.rs,start.rs,fork.rs,rebuild_meta.rs}` +- Modify: `lib/crates/fabro-cli/src/commands/{run/create.rs,run/fork.rs,run/rewind.rs,runs/inspect.rs,pr/create.rs,store/dump.rs}` +- Modify: `lib/crates/fabro-server/src/server.rs` +- Test: `lib/crates/fabro-types/tests/run_record_serde.rs` (rename to `run_spec_serde.rs`) +- Test: `lib/crates/fabro-cli/tests/it/cmd/create.rs` + +**Approach:** +- Make this a pure mechanical rename first so later layout changes can focus on behavior rather than symbol churn. +- Rename `Persisted::run_record` and other outward-facing internal helpers to `spec`-oriented names in the same pass. +- Keep the data shape unchanged in this unit; only names move. + +**Execution note:** Land as a mechanical rename before touching metadata serialization or file layout. + +**Patterns to follow:** +- `lib/crates/fabro-types/src/stage_id.rs` accessor style for the later query-method unit. + +**Test scenarios:** +- Happy path: `run_spec_serde.rs` round-trips a `RunSpec` with templated settings and blob refs exactly as the old `RunRecord` test did. +- Happy path: `RunProjection::apply_event` stores `spec` on `RunCreated` and updates the spec's `definition_blob` on `RunSubmitted`. +- Edge case: workspace code compiles with no lingering `RunRecord` or `run_record` identifiers in Rust source. + +**Verification:** +- The workspace compiles after the rename with no alias shims. +- Rust source no longer contains `RunRecord` or `run_record` identifiers. + +- [ ] **Unit 2: Add trimmed projection serialization and additive query methods** + +**Goal:** Define the metadata snapshot serialization contract and expose additive readers on `RunSpec` and `RunProjection`. + +**Requirements:** R2, R6 + +**Dependencies:** Unit 1 + +**Files:** +- Create: `lib/crates/fabro-store/src/serializable_projection.rs` +- Modify: `lib/crates/fabro-store/src/lib.rs` +- Modify: `lib/crates/fabro-store/src/run_state.rs` +- Modify: `lib/crates/fabro-types/src/run.rs` +- Test: `lib/crates/fabro-store/src/serializable_projection.rs` +- Test: `lib/crates/fabro-store/src/run_state.rs` +- Test: `lib/crates/fabro-types/tests/run_spec_methods.rs` + +**Approach:** +- Add a metadata-only serializer wrapper around `RunProjection` that strips `NodeState.prompt`, `response`, `diff`, `stdout`, and `stderr` from `run.json` while preserving top-level fields and the non-bulky node metadata. +- Keep ordinary `RunProjection` serde untouched so existing test helpers and internal round-trips keep working. +- Add query methods such as `RunSpec::id()`, `RunSpec::graph()`, `RunProjection::spec()`, `RunProjection::status()`, and `RunProjection::current_checkpoint()` without changing field visibility. + +**Execution note:** Start with failing round-trip tests before wiring the new serializer into metadata writers. + +**Patterns to follow:** +- Existing `RunProjection::node`, `iter_nodes`, and `list_node_visits` helpers in `lib/crates/fabro-store/src/run_state.rs` +- `StageId` accessor methods in `lib/crates/fabro-types/src/stage_id.rs` + +**Test scenarios:** +- Happy path: a projection with full top-level state and one populated node round-trips through the metadata serializer, deserializes back, and keeps all non-bulky fields intact while clearing the bulky text fields. +- Happy path: `RunSpec` getters expose `run_id`, `graph`, `settings`, `workflow_slug`, `working_directory`, and labels from a representative fixture. +- Edge case: an empty projection round-trips unchanged. +- Edge case: projections containing `foo@1` and `foo@2` nodes preserve both `StageId` keys across the round-trip. +- Edge case: `RunProjection::status()` returns `None` when no status record exists and the correct enum when one does. + +**Verification:** +- The metadata serializer can round-trip a projection into the trimmed wire shape and back. +- New accessors compile without forcing existing field access call sites to change. + +- [ ] **Unit 3: Unify metadata and export writers around one snapshot layout** + +**Goal:** Make one authoritative dump/export builder produce the unified `run.json` + `stages/` layout for both metadata branches and CLI export. + +**Requirements:** R2, R3, R4 + +**Dependencies:** Unit 2 + +**Files:** +- Modify: `lib/crates/fabro-workflow/src/run_dump.rs` +- Modify: `lib/crates/fabro-workflow/src/lifecycle/git.rs` +- Modify: `lib/crates/fabro-workflow/src/pipeline/finalize.rs` +- Modify: `lib/crates/fabro-workflow/src/git.rs` +- Modify: `lib/crates/fabro-cli/src/commands/store/{dump.rs,run_export.rs}` +- Test: `lib/crates/fabro-workflow/src/git.rs` +- Test: `lib/crates/fabro-workflow/src/pipeline/finalize.rs` +- Test: `lib/crates/fabro-cli/tests/it/cmd/store_dump.rs` + +**Approach:** +- Replace `RunDump::metadata_init`, `metadata_checkpoint`, `metadata_finalize`, and the CLI-only `StoreRunExport::from_store_state_and_events` path with one authoritative builder that starts from a `RunProjection`. +- Have metadata snapshots always emit `run.json` through the trimmed serializer, `graph.fabro` when present, and stage files under `stages/{stage_id}/...`. +- Keep export-only concerns (`events.jsonl`, `checkpoints/*.json`, hydrated blobs, artifact bytes) as opt-in helpers on the shared builder rather than as a second serializer. +- Remove top-level split JSON files, including `conclusion.json`, from both metadata branches and CLI export. +- Update checkpoint persistence in lifecycle code to use the new generic snapshot commit helper instead of a `checkpoint.json`-specific API. + +**Patterns to follow:** +- Existing `RunDumpEntry` helpers in `lib/crates/fabro-workflow/src/run_dump.rs` +- `StageId::Display` in `lib/crates/fabro-types/src/stage_id.rs` + +**Test scenarios:** +- Happy path: an init-state projection writes only `run.json` and `graph.fabro` when no stages or retro data exist. +- Happy path: a checkpoint-state projection writes `run.json` plus `stages/@1/` files for prompt, response, status, provider, diff, script metadata, stdout, and stderr when present. +- Happy path: CLI dump/export uses the same builder and still emits `events.jsonl`, `checkpoints/*.json`, `retro/*.md`, and artifact payloads. +- Edge case: a node with multiple visits writes both `stages/build@1/...` and `stages/build@2/...` with no visit-1 special case. +- Edge case: `run.json` contains `start`, `status`, `checkpoint`, `sandbox`, `retro`, and `conclusion`, but not bulky node text payloads. + +**Verification:** +- Writer/export code no longer contains legacy `nodes/` metadata stage paths or top-level split-file emission logic. +- Shared writer tests prove metadata branches and CLI export emit the same projection layout. + +- [ ] **Unit 4: Update metadata readers, recovery flows, and rebuild logic** + +**Goal:** Move every metadata-branch consumer from standalone file reads to projection reads, including the rebuild and rewind recovery paths. + +**Requirements:** R4, R5 + +**Dependencies:** Unit 3 + +**Files:** +- Modify: `lib/crates/fabro-checkpoint/src/metadata.rs` +- Modify: `lib/crates/fabro-workflow/src/operations/{fork.rs,rewind.rs,rebuild_meta.rs}` +- Modify: `lib/crates/fabro-cli/src/commands/run/rewind.rs` +- Test: `lib/crates/fabro-checkpoint/src/metadata.rs` +- Test: `lib/crates/fabro-workflow/src/operations/{fork.rs,rewind.rs,rebuild_meta.rs}` +- Test: `lib/crates/fabro-cli/tests/it/cmd/fork.rs` +- Test: `lib/crates/fabro-cli/tests/it/scenario/recovery.rs` +- Test: `lib/crates/fabro-workflow/tests/it/{integration.rs,daytona_integration.rs}` + +**Approach:** +- Add `MetadataStore::read_run_projection` and `read_run_spec`; either delete `read_checkpoint`/`read_start_record` or demote them to projection-field extractors after callers switch. +- Update `fork` to read the source projection, clone the spec/start/sandbox slices it intentionally carries forward, inject the new run ID, and write the new run's metadata branch through the unified snapshot writer. +- Update rewind parallel detection to read `projection.spec.graph`, and update CLI rewind recovery to pull the restored checkpoint from the projection snapshot instead of `checkpoint.json`. +- Rewrite `rebuild_meta` to emit one snapshot commit per metadata commit (init/checkpoint/finalize) through the shared writer while preserving `git_commit_sha` backfill semantics inside `projection.checkpoint`. + +**Patterns to follow:** +- Existing timeline and run-SHA backfill helpers in `lib/crates/fabro-workflow/src/operations/{rewind.rs,rebuild_meta.rs}` +- `RunStoreHandle::state()` projection access in `lib/crates/fabro-workflow/src/runtime_store.rs` + +**Test scenarios:** +- Happy path: a forked run gets a new `run.json` projection with the new run ID, inherited sandbox/start context, and no top-level split JSON files. +- Error path: forking still fails cleanly when the source metadata branch lacks `run.json` or the target checkpoint lacks a run commit SHA. +- Happy path: rewind parallel detection still recognizes interior parallel groups from `projection.spec.graph`. +- Happy path: CLI rewind recovery reads the checkpoint from the projection snapshot and replays `RunRewound` plus restored checkpoint events correctly. +- Integration: rebuild-meta emits one `run.json` snapshot per metadata commit, and each snapshot contains the expected checkpoint payload and backfilled `git_commit_sha`. +- Error path: rebuild-meta remains atomic on failure and still refuses to overwrite an existing metadata branch. + +**Verification:** +- Metadata consumers no longer require top-level `checkpoint.json`, `start.json`, or `sandbox.json`. +- Fork, rewind, and rebuild tests pass against the unified layout. + +- [ ] **Unit 5: Sweep downstream docs, retro prompts, and snapshots** + +**Goal:** Align retro tooling, integration tests, and snapshots with the unified metadata vocabulary and file layout. + +**Requirements:** R3, R5, R7 + +**Dependencies:** Unit 4 + +**Files:** +- Modify: `lib/crates/fabro-retro/src/retro_agent.rs` +- Modify: `lib/crates/fabro-cli/tests/it/cmd/{store_dump.rs,start.rs,fork.rs}` +- Modify: `lib/crates/fabro-cli/tests/it/scenario/recovery.rs` +- Modify: `lib/crates/fabro-workflow/src/git.rs` +- Modify: `lib/crates/fabro-workflow/src/pipeline/finalize.rs` +- Modify: `lib/crates/fabro-workflow/tests/it/{integration.rs,daytona_integration.rs}` +- Test: the files above + +**Approach:** +- Update retro agent instructions and sandbox uploads so the agent reads `run.json` projection data plus `graph.fabro` and stage files instead of `checkpoint.json` and `start.json`. +- Rename or replace tests that currently assert `conclusion.json` or old `nodes/...` layouts so they assert conclusion presence inside `run.json` and stage files under `stages/`. +- Keep CLI integration tests black-box per `files-internal/testing-strategy.md`; layout assertions should come from public command behavior or crate-level tests, not hand-planted run internals. +- Review snapshot diffs before accepting them because this refactor intentionally changes many file paths and exported filenames. + +**Patterns to follow:** +- Snapshot discipline in `files-internal/testing-strategy.md` +- Existing retro upload flow in `lib/crates/fabro-retro/src/retro_agent.rs` + +**Test scenarios:** +- Happy path: retro sandbox upload includes `run.json` projection data, and the prompt tells the retro agent to inspect `run.json` plus `graph.fabro`/stage files rather than `checkpoint.json`. +- Happy path: `fabro store dump` snapshots show `run.json`, `graph.fabro`, `stages/...`, `retro/*.md`, `events.jsonl`, and `checkpoints/*.json`, with no legacy split JSON files. +- Happy path: integration and Daytona tests read run spec and checkpoint data through the new projection helpers and still observe correct `git_commit_sha` behavior. +- Edge case: tests that previously referred to missing `status.json` or `sandbox.json` continue to assert the public command behavior without relying on those internal filenames existing. + +**Verification:** +- Snapshot and integration tests reference only the new layout. +- Retro tooling and test names no longer describe deleted files such as `conclusion.json` or `checkpoint.json` as metadata-branch invariants. + +## System-Wide Impact + +- **Interaction graph:** metadata snapshots are written from lifecycle init/checkpoint/finalize and rebuild-meta; they are read by fork, rewind, CLI rewind recovery, retro upload, store dump/export, and metadata-focused tests. +- **Error propagation:** deserialization errors shift from file-specific entities (`checkpoint`, `run record`) to projection parsing plus field-extraction errors; reader helpers should preserve branch/path context so failures stay diagnosable. +- **State lifecycle risks:** partial migration of writers/readers would silently break metadata-driven flows; the refactor must switch readers and writers in the same series and preserve checkpoint commit SHA capture. +- **API surface parity:** `StageId` already uses `node@visit`, so metadata paths, CLI exports, and test fixtures should align on the same identifier format. Artifact exports are the deliberate exception in this unit and remain on `artifacts/nodes/...`. +- **Integration coverage:** the highest-value end-to-end paths are checkpoint persistence, fork from checkpoint, rewind + resume recovery, rebuild metadata from durable state, and `fabro store dump`. +- **Unchanged invariants:** event semantics, durable-store state accumulation, `graph.fabro` export, and artifact export support remain intact; the refactor changes metadata serialization shape, not workflow execution behavior. + +## Risks & Dependencies + +| Risk | Mitigation | +|------|------------| +| A reader still depends on `checkpoint.json`, `start.json`, or `sandbox.json` after those files stop being written | Exhaustively update metadata helper call sites and keep dedicated fork/rewind/recovery integration coverage in the same series | +| `run.json` trimming accidentally drops state consumers still need | Add round-trip tests that prove all non-bulky top-level and node metadata survives the trimmed serializer | +| Shared writer migration leaves CLI export and metadata branches on subtly different layouts | Delete or subsume `StoreRunExport` in the same series rather than maintaining parallel serializers | +| `git_commit_sha` handling regresses during fork or rebuild | Preserve dedicated tests for missing-SHA errors, backfilled SHAs, and forked checkpoint snapshots | +| Large mechanical rename obscures behavioral regressions in review | Land the rename first, keep later units behavior-focused, and use targeted tests for each behavior-bearing unit | + +## Documentation / Operational Notes + +- Update inline comments, test names, and docstrings that still describe `run.json` as a run record or refer to `checkpoint.json`, `status.json`, `sandbox.json`, or `nodes/...` as metadata-branch invariants. +- No rollout or migration plan is needed for existing branches because the repo is still pre-launch; local stale metadata branches can be regenerated or discarded. +- Snapshot updates should follow the repo's `cargo insta pending-snapshots` discipline rather than bulk-accepting blindly. + +## Sources & References + +- **Source plan:** `/Users/bhelmkamp/.claude/plans/make-a-full-plan-pure-wombat.md` +- Related code: + - `lib/crates/fabro-types/src/run.rs` + - `lib/crates/fabro-store/src/run_state.rs` + - `lib/crates/fabro-workflow/src/run_dump.rs` + - `lib/crates/fabro-checkpoint/src/metadata.rs` + - `lib/crates/fabro-workflow/src/operations/{fork.rs,rewind.rs,rebuild_meta.rs}` + - `lib/crates/fabro-cli/src/commands/{store/dump.rs,store/run_export.rs,run/rewind.rs}` +- Related guidance: `files-internal/testing-strategy.md` diff --git a/docs/reference/cli.mdx b/docs/reference/cli.mdx index de9f5a5fc..6a091803c 100644 --- a/docs/reference/cli.mdx +++ b/docs/reference/cli.mdx @@ -444,7 +444,7 @@ Manage GitHub pull requests created by workflow runs. Requires GitHub access to ### `fabro pr create` -Create a GitHub pull request from a completed workflow run. Uses the run's persisted run record, conclusion, and diff. +Create a GitHub pull request from a completed workflow run. Uses the run's persisted run spec, conclusion, and diff. ```bash fabro pr create @@ -619,7 +619,7 @@ fabro logs -f my-workflow -p ## `fabro inspect` -Show detailed JSON data for a workflow run, including its run record, start record, conclusion, checkpoint, and sandbox record. +Show detailed JSON data for a workflow run, including its run spec, start record, conclusion, checkpoint, and sandbox record. ```bash fabro inspect diff --git a/docs/reference/run-directory.mdx b/docs/reference/run-directory.mdx index 6bab5df6a..e88d11a91 100644 --- a/docs/reference/run-directory.mdx +++ b/docs/reference/run-directory.mdx @@ -24,7 +24,7 @@ The naming format is `YYYYMMDD-{run_id}`, where `run_id` is the ULID assigned to ## Local-only directories -These paths are local runtime state and caches, not the canonical run record. +These paths are local runtime state and caches, not the canonical run state. - **`worktree/`** — When running in worktree mode, Fabro creates a Git worktree here as the working directory for agents and commands. - **`runtime/`** — Local runtime files. Today this is mainly materialized blob payloads under `runtime/blobs/`. @@ -34,11 +34,18 @@ Large durable values, event streams, checkpoints, diffs, conclusions, and retros ## Reconstructed and export-only layouts -Some file names you may have seen in older runs or older docs still exist in reconstructed metadata branches or `fabro store dump` exports: +Reconstructed metadata branches and `fabro store dump` exports now use the same core layout: -- `run.json`, `start.json`, and `checkpoint.json` on metadata branches for rewind and fork -- `run.json`, `start.json`, `checkpoint.json`, `conclusion.json`, `retro.json`, and `events.jsonl` in `fabro store dump` output -- Per-node prompt, response, status, stdout, and stderr files in `fabro store dump` output and metadata rebuilds +- `run.json` for the current projection snapshot, including the current checkpoint +- `graph.fabro` for workflow source +- `retro/*.md` for retro prompt/response text +- `stages/{node_id}@{visit}/...` for per-stage prompt, response, status, diff, stdout, and stderr files + +`fabro store dump` adds export-only history surfaces on top of that shared layout: + +- `events.jsonl` for the durable event stream +- `checkpoints/*.json` for checkpoint history snapshots +- `artifacts/nodes/{node_id}/visit-{n}/...` for exported artifact payloads ## Browsing runs diff --git a/lib/crates/fabro-checkpoint/Cargo.toml b/lib/crates/fabro-checkpoint/Cargo.toml index 02a87e246..b443ab206 100644 --- a/lib/crates/fabro-checkpoint/Cargo.toml +++ b/lib/crates/fabro-checkpoint/Cargo.toml @@ -14,6 +14,7 @@ doctest = false workspace = true [dependencies] +fabro-store = { path = "../fabro-store" } fabro-types = { path = "../fabro-types" } git2.workspace = true serde.workspace = true diff --git a/lib/crates/fabro-checkpoint/src/metadata.rs b/lib/crates/fabro-checkpoint/src/metadata.rs index a25d6f41f..8347f7b35 100644 --- a/lib/crates/fabro-checkpoint/src/metadata.rs +++ b/lib/crates/fabro-checkpoint/src/metadata.rs @@ -1,6 +1,7 @@ use std::path::{Path, PathBuf}; -use fabro_types::{Checkpoint, RunRecord, StartRecord}; +use fabro_store::RunProjection; +use fabro_types::{Checkpoint, RunSpec, StartRecord}; use git2::{Repository, Signature}; use crate::META_BRANCH_PREFIX; @@ -11,7 +12,7 @@ use crate::git::Store; /// Git-native metadata storage for pipeline runs. /// -/// Stores checkpoint data, run records, and metadata on an orphan branch +/// Stores checkpoint data, run specs, and metadata on an orphan branch /// (`fabro/meta/{run_id}`) so that runs can be resumed from git alone. pub struct MetadataStore { repo_path: PathBuf, @@ -46,9 +47,6 @@ impl MetadataStore { } /// Initialize a run's metadata branch with the given files. - /// - /// Callers pass all files (run.json, start.json, sandbox.json, etc.) - /// via the `files` slice. pub fn init_run(&self, run_id: &str, files: &[(&str, &[u8])]) -> Result<(), MetadataError> { let (store, sig) = self.open_store()?; let branch = Self::branch_name(run_id); @@ -59,8 +57,7 @@ impl MetadataStore { Ok(()) } - /// Write arbitrary files to the metadata branch without overwriting - /// checkpoint.json. + /// Write arbitrary files to the metadata branch. pub fn write_files( &self, run_id: &str, @@ -75,21 +72,19 @@ impl MetadataStore { Ok(()) } - /// Write checkpoint data (and optional artifacts) to the metadata branch. - /// Returns the SHA of the new commit on the shadow branch. - pub fn write_checkpoint( + /// Write a projection snapshot commit to the metadata branch and return + /// the new commit SHA. + pub fn write_snapshot( &self, run_id: &str, - checkpoint_json: &[u8], - artifacts: &[(&str, &[u8])], + entries: &[(&str, &[u8])], + message: &str, ) -> Result { let (store, sig) = self.open_store()?; let branch = Self::branch_name(run_id); let branch_store = BranchStore::new(&store, &branch, &sig); - let mut entries: Vec<(&str, &[u8])> = vec![("checkpoint.json", checkpoint_json)]; - entries.extend_from_slice(artifacts); - let message = self.commit_message("checkpoint"); - let oid = branch_store.write_entries(&entries, &message)?; + let message = self.commit_message(message); + let oid = branch_store.write_entries(entries, &message)?; Ok(oid.to_string()) } @@ -110,42 +105,62 @@ impl MetadataStore { Ok(branch_store.read_entry(path)?) } + /// Read the projection snapshot from the metadata branch tip. Returns + /// `None` if branch or file doesn't exist. + pub fn read_run_projection( + repo_path: &Path, + run_id: &str, + ) -> Result, MetadataError> { + let branch = Self::branch_name(run_id); + match Self::read_file(repo_path, run_id, "run.json")? { + Some(bytes) => { + let projection: RunProjection = + serde_json::from_slice(&bytes).map_err(|source| { + MetadataError::Deserialize { + entity: "run projection", + branch: branch.clone(), + source, + } + })?; + let has_projection_data = projection.spec.is_some() + || projection.start.is_some() + || projection.status.is_some() + || projection.checkpoint.is_some() + || projection.conclusion.is_some() + || projection.sandbox.is_some() + || projection.retro.is_some() + || projection.graph_source.is_some() + || projection.iter_nodes().next().is_some(); + if !has_projection_data { + return Err(MetadataError::Deserialize { + entity: "run projection", + branch, + source: serde_json::Error::io(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "run.json does not contain a serialized projection snapshot", + )), + }); + } + Ok(Some(projection)) + } + None => Ok(None), + } + } + /// Read a checkpoint from the metadata branch. Returns `None` if branch or /// file doesn't exist. pub fn read_checkpoint( repo_path: &Path, run_id: &str, ) -> Result, MetadataError> { - let branch = Self::branch_name(run_id); - match Self::read_file(repo_path, run_id, "checkpoint.json")? { - Some(bytes) => serde_json::from_slice(&bytes).map(Some).map_err(|source| { - MetadataError::Deserialize { - entity: "checkpoint", - branch, - source, - } - }), - None => Ok(None), - } + Ok(Self::read_run_projection(repo_path, run_id)? + .and_then(|projection| projection.checkpoint)) } - /// Read the run record from the metadata branch. Returns `None` if not + /// Read the run spec from the metadata branch. Returns `None` if not /// found. - pub fn read_run_record( - repo_path: &Path, - run_id: &str, - ) -> Result, MetadataError> { - let branch = Self::branch_name(run_id); - match Self::read_file(repo_path, run_id, "run.json")? { - Some(bytes) => serde_json::from_slice(&bytes).map(Some).map_err(|source| { - MetadataError::Deserialize { - entity: "run record", - branch, - source, - } - }), - None => Ok(None), - } + pub fn read_run_spec(repo_path: &Path, run_id: &str) -> Result, MetadataError> { + Ok(Self::read_run_projection(repo_path, run_id)?.and_then(|projection| projection.spec)) } /// Read the start record from the metadata branch. Returns `None` if not @@ -154,17 +169,7 @@ impl MetadataStore { repo_path: &Path, run_id: &str, ) -> Result, MetadataError> { - let branch = Self::branch_name(run_id); - match Self::read_file(repo_path, run_id, "start.json")? { - Some(bytes) => serde_json::from_slice(&bytes).map(Some).map_err(|source| { - MetadataError::Deserialize { - entity: "start record", - branch, - source, - } - }), - None => Ok(None), - } + Ok(Self::read_run_projection(repo_path, run_id)?.and_then(|projection| projection.start)) } /// Read an artifact from the metadata branch. Returns `None` if not found. @@ -215,8 +220,8 @@ mod tests { .unwrap(); } - fn test_run_record(run_id: fabro_types::RunId) -> RunRecord { - RunRecord { + fn test_run_spec(run_id: fabro_types::RunId) -> RunSpec { + RunSpec { run_id, settings: SettingsLayer::default(), graph: Graph::new("test"), @@ -252,6 +257,16 @@ mod tests { } } + fn test_projection(run_id: fabro_types::RunId) -> RunProjection { + let mut projection = RunProjection::default(); + projection.spec = Some(test_run_spec(run_id)); + projection + } + + fn projection_bytes(projection: &RunProjection) -> Vec { + serde_json::to_vec_pretty(projection).unwrap() + } + fn branch_entry(repo_dir: &Path, run_id: &str, path: &str) -> Vec { let repo = Repository::discover(repo_dir).unwrap(); let store = Store::new(repo); @@ -268,16 +283,16 @@ mod tests { let store = MetadataStore::new(dir.path(), &GitAuthor::default()); let run_id = fixtures::RUN_1.to_string(); - let run_record = serde_json::to_vec_pretty(&test_run_record(fixtures::RUN_1)).unwrap(); + let projection = projection_bytes(&test_projection(fixtures::RUN_1)); store - .init_run(&run_id, &[("run.json", &run_record)]) + .init_run(&run_id, &[("run.json", &projection)]) .unwrap(); - let read_record = MetadataStore::read_run_record(dir.path(), &run_id) + let read_spec = MetadataStore::read_run_spec(dir.path(), &run_id) .unwrap() .unwrap(); - assert_eq!(read_record.run_id, fixtures::RUN_1); - assert_eq!(read_record.graph.name, "test"); + assert_eq!(read_spec.run_id, fixtures::RUN_1); + assert_eq!(read_spec.graph.name, "test"); } #[test] @@ -287,7 +302,10 @@ mod tests { let run_id = fixtures::RUN_2.to_string(); let store = MetadataStore::new(dir.path(), &GitAuthor::default()); - store.init_run(&run_id, &[]).unwrap(); + let init_projection = projection_bytes(&test_projection(fixtures::RUN_2)); + store + .init_run(&run_id, &[("run.json", &init_projection)]) + .unwrap(); let mut checkpoint = test_checkpoint( "node_a", @@ -297,9 +315,11 @@ mod tests { checkpoint .context_values .insert("goal".to_string(), serde_json::json!("test")); - let checkpoint_json = serde_json::to_vec_pretty(&checkpoint).unwrap(); + let mut snapshot = test_projection(fixtures::RUN_2); + snapshot.checkpoint = Some(checkpoint); + let snapshot_json = projection_bytes(&snapshot); store - .write_checkpoint(&run_id, &checkpoint_json, &[]) + .write_snapshot(&run_id, &[("run.json", &snapshot_json)], "checkpoint") .unwrap(); let loaded = MetadataStore::read_checkpoint(dir.path(), &run_id) @@ -321,23 +341,27 @@ mod tests { let run_id = fixtures::RUN_3.to_string(); let store = MetadataStore::new(dir.path(), &GitAuthor::default()); - store.init_run(&run_id, &[]).unwrap(); - - let checkpoint_one = - serde_json::to_vec_pretty(&test_checkpoint("node_a", vec!["start".to_string()], None)) - .unwrap(); + let init_projection = projection_bytes(&test_projection(fixtures::RUN_3)); store - .write_checkpoint(&run_id, &checkpoint_one, &[]) + .init_run(&run_id, &[("run.json", &init_projection)]) .unwrap(); - let checkpoint_two = serde_json::to_vec_pretty(&test_checkpoint( + let mut snapshot_one = test_projection(fixtures::RUN_3); + snapshot_one.checkpoint = Some(test_checkpoint("node_a", vec!["start".to_string()], None)); + let checkpoint_one = projection_bytes(&snapshot_one); + store + .write_snapshot(&run_id, &[("run.json", &checkpoint_one)], "checkpoint") + .unwrap(); + + let mut snapshot_two = test_projection(fixtures::RUN_3); + snapshot_two.checkpoint = Some(test_checkpoint( "node_b", vec!["start".to_string(), "node_a".to_string()], Some("node_c".to_string()), - )) - .unwrap(); + )); + let checkpoint_two = projection_bytes(&snapshot_two); store - .write_checkpoint(&run_id, &checkpoint_two, &[]) + .write_snapshot(&run_id, &[("run.json", &checkpoint_two)], "checkpoint") .unwrap(); let loaded = MetadataStore::read_checkpoint(dir.path(), &run_id) @@ -363,16 +387,24 @@ mod tests { let run_id = fixtures::RUN_4.to_string(); let store = MetadataStore::new(dir.path(), &GitAuthor::default()); - store.init_run(&run_id, &[]).unwrap(); + let init_projection = projection_bytes(&test_projection(fixtures::RUN_4)); + store + .init_run(&run_id, &[("run.json", &init_projection)]) + .unwrap(); let artifact_data = br#"{"large_output":"some data"}"#; - let checkpoint_json = - serde_json::to_vec_pretty(&test_checkpoint("node_a", Vec::new(), None)).unwrap(); + let mut snapshot = test_projection(fixtures::RUN_4); + snapshot.checkpoint = Some(test_checkpoint("node_a", Vec::new(), None)); + let snapshot_json = projection_bytes(&snapshot); store - .write_checkpoint(&run_id, &checkpoint_json, &[( - "artifacts/response.plan.json", - artifact_data.as_slice(), - )]) + .write_snapshot( + &run_id, + &[ + ("run.json", &snapshot_json), + ("artifacts/response.plan.json", artifact_data.as_slice()), + ], + "checkpoint", + ) .unwrap(); let read_back = MetadataStore::read_artifact(dir.path(), &run_id, "response.plan") @@ -388,26 +420,26 @@ mod tests { let run_id = fixtures::RUN_5.to_string(); let store = MetadataStore::new(dir.path(), &GitAuthor::default()); - let run_record = serde_json::to_vec_pretty(&test_run_record(fixtures::RUN_5)).unwrap(); + let projection = projection_bytes(&test_projection(fixtures::RUN_5)); store - .init_run(&run_id, &[("run.json", &run_record)]) + .init_run(&run_id, &[("run.json", &projection)]) .unwrap(); store .write_files( &run_id, - &[("retro.json", b"{\"status\":\"ok\"}")], + &[("retro/prompt.md", b"how did it go?")], "finalize run", ) .unwrap(); - let data = branch_entry(dir.path(), &run_id, "retro.json"); - assert_eq!(data, b"{\"status\":\"ok\"}"); + let data = branch_entry(dir.path(), &run_id, "retro/prompt.md"); + assert_eq!(data, b"how did it go?"); - let record = MetadataStore::read_run_record(dir.path(), &run_id) + let spec = MetadataStore::read_run_spec(dir.path(), &run_id) .unwrap() .unwrap(); - assert_eq!(record.run_id, fixtures::RUN_5); + assert_eq!(spec.run_id, fixtures::RUN_5); } #[test] @@ -418,11 +450,11 @@ mod tests { let run_id = fixtures::RUN_6.to_string(); let store = MetadataStore::new(dir.path(), &GitAuthor::default()); store - .init_run(&run_id, &[("sandbox.json", b"{\"type\":\"local\"}")]) + .init_run(&run_id, &[("graph.fabro", b"digraph Test {}")]) .unwrap(); - let data = branch_entry(dir.path(), &run_id, "sandbox.json"); - assert_eq!(data, b"{\"type\":\"local\"}"); + let data = branch_entry(dir.path(), &run_id, "graph.fabro"); + assert_eq!(data, b"digraph Test {}"); } #[test] @@ -438,8 +470,10 @@ mod tests { run_branch: Some("fabro/run/test".to_string()), base_sha: None, }; - let bytes = serde_json::to_vec_pretty(&start_record).unwrap(); - store.init_run(&run_id, &[("start.json", &bytes)]).unwrap(); + let mut projection = test_projection(fixtures::RUN_6); + projection.start = Some(start_record); + let bytes = projection_bytes(&projection); + store.init_run(&run_id, &[("run.json", &bytes)]).unwrap(); let loaded = MetadataStore::read_start_record(dir.path(), &run_id) .unwrap() diff --git a/lib/crates/fabro-cli/src/commands/pr/create.rs b/lib/crates/fabro-cli/src/commands/pr/create.rs index 5d880f8e6..b0d10a56e 100644 --- a/lib/crates/fabro-cli/src/commands/pr/create.rs +++ b/lib/crates/fabro-cli/src/commands/pr/create.rs @@ -34,9 +34,9 @@ pub(super) async fn create_command( let run_store = rebuild_run_store(&run_id, &events).await?; let state = run_store.state().await?; - let record = state.run.context("Failed to load run record from store")?; + let run_spec = state.spec.context("Failed to load run spec from store")?; ensure_matching_repo_origin( - record.repo_origin_url.as_deref(), + run_spec.repo_origin_url.as_deref(), "create a pull request for", )?; @@ -72,7 +72,7 @@ pub(super) async fn create_command( let (origin_url, detected_branch) = detect_repo_info(&cwd).map_err(|err| anyhow::anyhow!("{err}"))?; - let base_branch = record + let base_branch = run_spec .base_branch .as_deref() .or(detected_branch.as_deref()) @@ -113,12 +113,12 @@ pub(super) async fn create_command( .clone() }); - let record = maybe_open_pull_request( + let pull_request = maybe_open_pull_request( &creds, &origin_url, base_branch, run_branch, - record.graph.goal(), + run_spec.graph.goal(), &diff, &model, true, @@ -129,7 +129,7 @@ pub(super) async fn create_command( .await .map_err(|err| anyhow::anyhow!("{err}"))?; - match record { + match pull_request { Some(record) => { info!(pr_url = %record.html_url, "Pull request created"); if cli.output.format == OutputFormat::Json { diff --git a/lib/crates/fabro-cli/src/commands/run/attach.rs b/lib/crates/fabro-cli/src/commands/run/attach.rs index c39fc605c..49251fef6 100644 --- a/lib/crates/fabro-cli/src/commands/run/attach.rs +++ b/lib/crates/fabro-cli/src/commands/run/attach.rs @@ -82,11 +82,11 @@ pub(crate) async fn attach_run_with_client( printer: Printer, ) -> Result { let state = client.get_run_state(run_id).await?; - let auto_approve = state.run.as_ref().is_some_and(|record| { + let auto_approve = state.spec.as_ref().is_some_and(|record| { fabro_config::resolve_run_from_file(&record.settings) .is_ok_and(|settings| settings.execution.approval == ApprovalMode::Auto) }); - let verbose = state.run.as_ref().is_some_and(|record| { + let verbose = state.spec.as_ref().is_some_and(|record| { fabro_config::resolve_cli_from_file(&record.settings) .is_ok_and(|settings| settings.output.verbosity == OutputVerbosity::Verbose) }); @@ -502,7 +502,7 @@ mod tests { fn terminal_run_state_response() -> serde_json::Value { serde_json::json!({ - "run": null, + "spec": null, "graph_source": null, "start": null, "status": { diff --git a/lib/crates/fabro-cli/src/commands/run/create.rs b/lib/crates/fabro-cli/src/commands/run/create.rs index 48685ee1d..0a936ad6b 100644 --- a/lib/crates/fabro-cli/src/commands/run/create.rs +++ b/lib/crates/fabro-cli/src/commands/run/create.rs @@ -20,7 +20,7 @@ pub(crate) struct CreatedRun { pub(crate) local_run_dir: Option, } -/// Create a workflow run: allocate run directory, persist RunRecord, return +/// Create a workflow run: allocate run directory, persist RunSpec, return /// (run_id, run_dir). /// /// This does NOT execute the workflow — it only prepares the run directory. diff --git a/lib/crates/fabro-cli/src/commands/run/fork.rs b/lib/crates/fabro-cli/src/commands/run/fork.rs index 1d186f1da..b4f8c81e4 100644 --- a/lib/crates/fabro-cli/src/commands/run/fork.rs +++ b/lib/crates/fabro-cli/src/commands/run/fork.rs @@ -25,8 +25,8 @@ pub(crate) async fn run( let client = ctx.server().await?; let run_id = client.resolve_run(&args.run_id).await?.run_id; let state = client.get_run_state(&run_id).await?; - let record = state.run.context("Failed to load run record from store")?; - ensure_matching_repo_origin(record.repo_origin_url.as_deref(), "fork")?; + let run_spec = state.spec.context("Failed to load run spec from store")?; + ensure_matching_repo_origin(run_spec.repo_origin_url.as_deref(), "fork")?; let store = Store::new(repo); let events = client.list_run_events(&run_id, None, None).await?; let run_store = rebuild_run_store(&run_id, &events).await?; diff --git a/lib/crates/fabro-cli/src/commands/run/rewind.rs b/lib/crates/fabro-cli/src/commands/run/rewind.rs index 9486b6756..1645dacfa 100644 --- a/lib/crates/fabro-cli/src/commands/run/rewind.rs +++ b/lib/crates/fabro-cli/src/commands/run/rewind.rs @@ -47,8 +47,8 @@ pub(crate) async fn run( .as_ref() .map(|record| record.status) .context("run has no recorded status — cannot rewind")?; - let record = state.run.context("Failed to load run record from store")?; - ensure_matching_repo_origin(record.repo_origin_url.as_deref(), "rewind")?; + let run_spec = state.spec.context("Failed to load run spec from store")?; + ensure_matching_repo_origin(run_spec.repo_origin_url.as_deref(), "rewind")?; let store = Store::new(repo); let events = client.list_run_events(&run_id, None, None).await?; let run_store = rebuild_run_store(&run_id, &events).await?; @@ -120,12 +120,13 @@ async fn reset_rewound_run_state( anyhow::anyhow!("failed to load durable store state before rewind: {err}") })?; - let definition_blob = state.run.as_ref().and_then(|run| run.definition_blob); - let _run_record = state - .run - .context("failed to restore run record after rewind: missing run metadata")?; - let checkpoint = MetadataStore::read_checkpoint(git_store.repo_dir(), &run_id.to_string())? - .context("rewound metadata branch is missing checkpoint.json")?; + let definition_blob = state.spec.as_ref().and_then(|run| run.definition_blob); + if state.spec.is_none() { + anyhow::bail!("failed to restore run spec after rewind: missing run metadata"); + } + let checkpoint = MetadataStore::read_run_projection(git_store.repo_dir(), &run_id.to_string())? + .and_then(|projection| projection.checkpoint) + .context("rewound metadata branch is missing run.json checkpoint state")?; let previous_status = state.status.map(|status| status.status.to_string()); client diff --git a/lib/crates/fabro-cli/src/commands/run/runner.rs b/lib/crates/fabro-cli/src/commands/run/runner.rs index bfc4e60be..290aef5c5 100644 --- a/lib/crates/fabro-cli/src/commands/run/runner.rs +++ b/lib/crates/fabro-cli/src/commands/run/runner.rs @@ -70,10 +70,10 @@ pub(crate) async fn execute( .state() .await .with_context(|| format!("failed to load run state for {run_id}"))?; - let run_record = run_state - .run + let run_spec = run_state + .spec .as_ref() - .ok_or_else(|| anyhow!("Run {run_id} has no run record in store"))?; + .ok_or_else(|| anyhow!("Run {run_id} has no run spec in store"))?; let artifact_sink = Some(ArtifactSink::Uploader(build_artifact_uploader( run_id, client.clone_for_reuse(), @@ -90,7 +90,7 @@ pub(crate) async fn execute( Some(arc) => Some(arc.read().await), None => None, }; - maybe_build_github_credentials(&run_record.settings, vault_guard.as_deref())? + maybe_build_github_credentials(&run_spec.settings, vault_guard.as_deref())? }; let services = StartServices { run_id, diff --git a/lib/crates/fabro-cli/src/commands/runs/inspect.rs b/lib/crates/fabro-cli/src/commands/runs/inspect.rs index 9abaf309b..2c3b7911d 100644 --- a/lib/crates/fabro-cli/src/commands/runs/inspect.rs +++ b/lib/crates/fabro-cli/src/commands/runs/inspect.rs @@ -14,7 +14,7 @@ use crate::server_runs::ServerRunSummaryInfo; pub(crate) struct InspectOutput { pub run_id: String, pub status: RunStatus, - pub run_record: Option, + pub run_spec: Option, pub start_record: Option, pub conclusion: Option, pub checkpoint: Option, @@ -45,8 +45,8 @@ fn inspect_run_state(run: &ServerRunSummaryInfo, state: RunProjection) -> Inspec .status .as_ref() .map_or(run.status(), |record| record.status), - run_record: state - .run + run_spec: state + .spec .and_then(|record| serde_json::to_value(record).ok()), start_record: state .start diff --git a/lib/crates/fabro-cli/src/commands/store/dump.rs b/lib/crates/fabro-cli/src/commands/store/dump.rs index 9cd3cc2cd..3e27dbcde 100644 --- a/lib/crates/fabro-cli/src/commands/store/dump.rs +++ b/lib/crates/fabro-cli/src/commands/store/dump.rs @@ -63,7 +63,7 @@ pub(crate) async fn export_run( ) -> Result { let state = run_store.state().await?; let run_id = state - .run + .spec .as_ref() .map(|run| run.run_id) .context("run has no data in the store")?; @@ -317,7 +317,7 @@ mod tests { use fabro_types::settings::SettingsLayer; use fabro_types::{ AggregateStats, AttrValue, BilledTokenCounts, Checkpoint, Conclusion, Graph, - NodeStatusRecord, Retro, RunId, RunRecord, RunStatus, RunStatusRecord, SandboxRecord, + NodeStatusRecord, Retro, RunId, RunSpec, RunStatus, RunStatusRecord, SandboxRecord, StageStatus, StartRecord, StatusReason, fixtures, }; use fabro_workflow::event::{Event, append_event}; @@ -348,13 +348,13 @@ mod tests { (store, artifact_store) } - fn sample_run_record(run_id: RunId, _created_at: DateTime) -> RunRecord { + fn sample_run_spec(run_id: RunId, _created_at: DateTime) -> RunSpec { let mut graph = Graph::new("night-sky"); graph.attrs.insert( "goal".to_string(), AttrValue::String("map the constellations".to_string()), ); - RunRecord { + RunSpec { run_id, settings: SettingsLayer::default(), graph, @@ -478,7 +478,7 @@ mod tests { let created_at = dt("2026-03-27T12:00:00Z"); let run_id = test_run_id(); let run = store.create_run(&run_id).await.unwrap(); - let run_record = sample_run_record(run_id, created_at); + let run_spec = sample_run_spec(run_id, created_at); let start_record = sample_start_record(run_id, created_at); let status_record = sample_status(); let mut first_checkpoint = sample_checkpoint("plan", 1); @@ -500,19 +500,19 @@ mod tests { let node = StageId::new("code", 2); append_event(&run, &run_id, &Event::RunCreated { run_id, - settings: serde_json::to_value(&run_record.settings).unwrap(), - graph: serde_json::to_value(&run_record.graph).unwrap(), + settings: serde_json::to_value(&run_spec.settings).unwrap(), + graph: serde_json::to_value(&run_spec.graph).unwrap(), workflow_source: Some("digraph night_sky {}".to_string()), workflow_config: None, - labels: run_record.labels.clone().into_iter().collect(), + labels: run_spec.labels.clone().into_iter().collect(), run_dir: "/tmp/night-sky-run".to_string(), - working_directory: run_record.working_directory.display().to_string(), - host_repo_path: run_record.host_repo_path.clone(), - repo_origin_url: run_record.repo_origin_url.clone(), - base_branch: run_record.base_branch.clone(), - workflow_slug: run_record.workflow_slug.clone(), + working_directory: run_spec.working_directory.display().to_string(), + host_repo_path: run_spec.host_repo_path.clone(), + repo_origin_url: run_spec.repo_origin_url.clone(), + base_branch: run_spec.base_branch.clone(), + workflow_slug: run_spec.workflow_slug.clone(), db_prefix: None, - provenance: run_record.provenance.clone(), + provenance: run_spec.provenance.clone(), manifest_blob: None, }) .await @@ -520,7 +520,7 @@ mod tests { append_event(&run, &run_id, &Event::WorkflowRunStarted { name: "night-sky".to_string(), run_id, - base_branch: run_record.base_branch.clone(), + base_branch: run_spec.base_branch.clone(), base_sha: start_record.base_sha.clone(), run_branch: start_record.run_branch.clone(), worktree_dir: None, @@ -707,47 +707,80 @@ mod tests { let file_count = export_run(&run, &artifact_store, output.path()) .await .unwrap(); - assert_eq!(file_count, 20); + assert_eq!(file_count, 16); - let exported_run: RunRecord = read_json(&output.path().join("run.json")); - assert_eq!(exported_run.run_id, run_id); - - let exported_start: StartRecord = read_json(&output.path().join("start.json")); - assert_eq!(exported_start.run_id, run_id); - - let exported_status: RunStatusRecord = read_json(&output.path().join("status.json")); - assert_eq!(exported_status.status, RunStatus::Succeeded); - - let exported_checkpoint: Checkpoint = read_json(&output.path().join("checkpoint.json")); - assert_eq!(exported_checkpoint.current_node, "code"); + let exported_run: RunProjection = read_json(&output.path().join("run.json")); assert_eq!( - exported_checkpoint.context_values.get("artifact"), + exported_run.spec.as_ref().map(|run| run.run_id), + Some(run_id) + ); + assert_eq!( + exported_run.start.as_ref().map(|start| start.run_id), + Some(run_id) + ); + assert_eq!( + exported_run.status.as_ref().map(|status| status.status), + Some(RunStatus::Succeeded) + ); + assert_eq!( + exported_run + .checkpoint + .as_ref() + .map(|checkpoint| checkpoint.current_node.as_str()), + Some("code") + ); + assert_eq!( + exported_run + .checkpoint + .as_ref() + .and_then(|checkpoint| checkpoint.context_values.get("artifact")), Some(&serde_json::json!({"done": true})) ); + assert!(exported_run.conclusion.is_some()); + assert!(exported_run.sandbox.is_some()); + assert!(exported_run.retro.is_some()); + assert!(!output.path().join("start.json").exists()); + assert!(!output.path().join("status.json").exists()); + assert!(!output.path().join("checkpoint.json").exists()); + assert!(!output.path().join("sandbox.json").exists()); + assert!(!output.path().join("retro.json").exists()); + assert!(!output.path().join("conclusion.json").exists()); assert_eq!( std::fs::read_to_string(output.path().join("graph.fabro")).unwrap(), "digraph night_sky {}" ); assert_eq!( - std::fs::read_to_string(output.path().join("nodes/code/visit-2/prompt.md")).unwrap(), + std::fs::read_to_string(output.path().join("stages/code@2/prompt.md")).unwrap(), "Plan the fix" ); assert_eq!( - std::fs::read_to_string(output.path().join("nodes/code/visit-2/response.md")).unwrap(), + std::fs::read_to_string(output.path().join("stages/code@2/response.md")).unwrap(), "Implemented" ); let node_status: NodeStatusRecord = - read_json(&output.path().join("nodes/code/visit-2/status.json")); + read_json(&output.path().join("stages/code@2/status.json")); assert_eq!(node_status.status, StageStatus::Success); assert_eq!( - std::fs::read_to_string(output.path().join("nodes/code/visit-2/stdout.log")).unwrap(), + std::fs::read_to_string(output.path().join("stages/code@2/stdout.log")).unwrap(), "stdout line" ); assert_eq!( - std::fs::read_to_string(output.path().join("nodes/code/visit-2/stderr.log")).unwrap(), + std::fs::read_to_string(output.path().join("stages/code@2/stderr.log")).unwrap(), "" ); + assert!( + output + .path() + .join("stages/code@2/script_invocation.json") + .is_file() + ); + assert!( + output + .path() + .join("stages/code@2/script_timing.json") + .is_file() + ); assert_eq!( std::fs::read_to_string(output.path().join("retro/prompt.md")).unwrap(), @@ -799,7 +832,7 @@ mod tests { .unwrap(), b"hello" ); - assert!(!output.path().join("nodes/artifact-only").exists()); + assert!(!output.path().join("stages/artifact-only@7").exists()); } #[test] diff --git a/lib/crates/fabro-cli/src/commands/store/run_export.rs b/lib/crates/fabro-cli/src/commands/store/run_export.rs index b3a507a48..42738384d 100644 --- a/lib/crates/fabro-cli/src/commands/store/run_export.rs +++ b/lib/crates/fabro-cli/src/commands/store/run_export.rs @@ -1,372 +1 @@ -#![expect( - clippy::disallowed_methods, - reason = "CLI-owned export writer uses sync std::fs for final local materialization" -)] - -use std::collections::HashMap; -#[expect( - clippy::disallowed_types, - reason = "in-memory Vec::write_all for jsonl serialization; no filesystem or network I/O" -)] -use std::io::Write; -use std::path::{Component, Path, PathBuf}; - -use anyhow::{Context, Result, bail}; -use bytes::Bytes; -use fabro_store::{EventEnvelope, RunProjection, StageId}; -use fabro_types::{RunBlobId, parse_blob_ref, parse_legacy_blob_file_ref}; -use futures::future::BoxFuture; - -#[derive(Debug, Clone)] -pub(super) struct StoreRunExport { - entries: Vec, -} - -#[derive(Debug, Clone)] -struct StoreRunExportEntry { - path: String, - contents: StoreRunExportContents, -} - -#[derive(Debug, Clone)] -enum StoreRunExportContents { - Text(String), - Json(serde_json::Value), - Bytes(Vec), -} - -impl StoreRunExport { - pub(super) fn from_store_state_and_events( - state: &RunProjection, - events: &[EventEnvelope], - ) -> Result { - let mut entries = Vec::new(); - - if let Some(record) = state.run.as_ref() { - push_json_entry(&mut entries, "run.json", record); - } - if let Some(record) = state.start.as_ref() { - push_json_entry(&mut entries, "start.json", record); - } - if let Some(record) = state.status.as_ref() { - push_json_entry(&mut entries, "status.json", record); - } - if let Some(record) = state.checkpoint.as_ref() { - push_json_entry(&mut entries, "checkpoint.json", record); - } - if let Some(record) = state.conclusion.as_ref() { - push_json_entry(&mut entries, "conclusion.json", record); - } - if let Some(record) = state.retro.as_ref() { - push_json_entry(&mut entries, "retro.json", record); - } - if let Some(graph_source) = state.graph_source.as_ref() { - entries.push(StoreRunExportEntry::text( - "graph.fabro", - graph_source.clone(), - )); - } - if let Some(record) = state.sandbox.as_ref() { - push_json_entry(&mut entries, "sandbox.json", record); - } - - let mut node_keys: Vec<_> = state.iter_nodes().map(|(node, _)| node.clone()).collect(); - node_keys.sort(); - for node_key in &node_keys { - let node = state - .node(node_key) - .with_context(|| format!("missing node {node_key:?} in projection"))?; - let node_id_segment = validate_single_path_segment("node id", node_key.node_id())?; - let base = PathBuf::from("nodes") - .join(node_id_segment) - .join(format!("visit-{}", node_key.visit())); - - if let Some(prompt) = node.prompt.as_ref() { - entries.push(StoreRunExportEntry::text_path( - &base.join("prompt.md"), - prompt.clone(), - )); - } - if let Some(response) = node.response.as_ref() { - entries.push(StoreRunExportEntry::text_path( - &base.join("response.md"), - response.clone(), - )); - } - if let Some(status) = node.status.as_ref() { - push_json_entry_path(&mut entries, &base.join("status.json"), status); - } - if let Some(stdout) = node.stdout.as_ref() { - entries.push(StoreRunExportEntry::text_path( - &base.join("stdout.log"), - stdout.clone(), - )); - } - if let Some(stderr) = node.stderr.as_ref() { - entries.push(StoreRunExportEntry::text_path( - &base.join("stderr.log"), - stderr.clone(), - )); - } - } - - if let Some(prompt) = state.retro_prompt.as_ref() { - entries.push(StoreRunExportEntry::text("retro/prompt.md", prompt.clone())); - } - if let Some(response) = state.retro_response.as_ref() { - entries.push(StoreRunExportEntry::text( - "retro/response.md", - response.clone(), - )); - } - - let mut events_jsonl = Vec::new(); - for event in events { - serde_json::to_writer(&mut events_jsonl, event)?; - events_jsonl.write_all(b"\n")?; - } - entries.push(StoreRunExportEntry::bytes("events.jsonl", events_jsonl)); - - for (seq, checkpoint) in &state.checkpoints { - push_json_entry_path( - &mut entries, - &PathBuf::from("checkpoints").join(format!("{seq:04}.json")), - checkpoint, - ); - } - - Ok(Self { entries }) - } - - pub(super) fn add_artifact_bytes( - &mut self, - stage_id: &StageId, - filename: &str, - data: Vec, - ) -> Result<()> { - let path = artifact_dump_path(stage_id, filename)?; - self.entries - .push(StoreRunExportEntry::bytes_path(&path, data)); - Ok(()) - } - - pub(super) async fn hydrate_referenced_blobs_with_reader<'a, F>( - &mut self, - mut read_blob: F, - ) -> Result<()> - where - F: FnMut(RunBlobId) -> BoxFuture<'a, Result>>, - { - let mut cache = HashMap::new(); - for entry in &mut self.entries { - if let StoreRunExportContents::Json(value) = &mut entry.contents { - let mut blob_ids = Vec::new(); - collect_blob_refs_in_value(value, &mut blob_ids); - for blob_id in blob_ids { - if cache.contains_key(&blob_id) { - continue; - } - let blob = read_blob(blob_id) - .await? - .with_context(|| format!("blob {blob_id:?} is missing from the store"))?; - let hydrated: serde_json::Value = serde_json::from_slice(&blob) - .with_context(|| format!("blob {blob_id:?} is not valid JSON"))?; - cache.insert(blob_id, hydrated); - } - replace_blob_refs_in_value(value, &cache)?; - } - } - Ok(()) - } - - pub(super) fn write_to_dir(&self, root: &Path) -> Result { - for entry in &self.entries { - entry.write_to_dir(root)?; - } - Ok(self.entries.len()) - } -} - -impl StoreRunExportEntry { - fn text(path: impl Into, contents: String) -> Self { - Self { - path: path.into(), - contents: StoreRunExportContents::Text(contents), - } - } - - fn text_path(path: &Path, contents: String) -> Self { - Self { - path: path_to_string(path), - contents: StoreRunExportContents::Text(contents), - } - } - - fn json(path: impl Into, contents: serde_json::Value) -> Self { - Self { - path: path.into(), - contents: StoreRunExportContents::Json(contents), - } - } - - fn json_path(path: &Path, contents: serde_json::Value) -> Self { - Self { - path: path_to_string(path), - contents: StoreRunExportContents::Json(contents), - } - } - - fn bytes(path: impl Into, contents: Vec) -> Self { - Self { - path: path.into(), - contents: StoreRunExportContents::Bytes(contents), - } - } - - fn bytes_path(path: &Path, contents: Vec) -> Self { - Self { - path: path_to_string(path), - contents: StoreRunExportContents::Bytes(contents), - } - } - - fn write_to_dir(&self, root: &Path) -> Result<()> { - let relative = validate_relative_path("run dump path", &self.path)?; - let path = root.join(relative); - ensure_parent_dir(&path)?; - std::fs::write(&path, self.contents.to_bytes()?) - .with_context(|| format!("failed to write {}", path.display()))?; - Ok(()) - } -} - -impl StoreRunExportContents { - fn to_bytes(&self) -> Result> { - match self { - Self::Text(value) => Ok(value.as_bytes().to_vec()), - Self::Json(value) => Ok(serde_json::to_vec_pretty(value)?), - Self::Bytes(value) => Ok(value.clone()), - } - } -} - -fn push_json_entry(entries: &mut Vec, path: &str, value: &T) -where - T: serde::Serialize, -{ - if let Ok(value) = serde_json::to_value(value) { - entries.push(StoreRunExportEntry::json(path, value)); - } -} - -fn push_json_entry_path(entries: &mut Vec, path: &Path, value: &T) -where - T: serde::Serialize, -{ - if let Ok(value) = serde_json::to_value(value) { - entries.push(StoreRunExportEntry::json_path(path, value)); - } -} - -fn path_to_string(path: &Path) -> String { - path.to_string_lossy().into_owned() -} - -fn validate_single_path_segment(kind: &str, value: &str) -> Result { - let path = validate_relative_path(kind, value)?; - if path.components().count() != 1 { - bail!("{kind} {value:?} must be a single path segment"); - } - Ok(path) -} - -fn validate_relative_path(kind: &str, value: &str) -> Result { - let mut normalized = PathBuf::new(); - for component in Path::new(value).components() { - match component { - Component::Normal(part) => normalized.push(part), - Component::CurDir => {} - Component::ParentDir | Component::RootDir | Component::Prefix(_) => { - bail!("{kind} {value:?} must be a relative path without '..'"); - } - } - } - if normalized.as_os_str().is_empty() { - bail!("{kind} {value:?} must not be empty"); - } - Ok(normalized) -} - -fn collect_blob_refs_in_value(value: &serde_json::Value, blob_ids: &mut Vec) { - match value { - serde_json::Value::String(current) => { - if let Some(blob_id) = - parse_blob_ref(current).or_else(|| parse_legacy_blob_file_ref(current)) - { - blob_ids.push(blob_id); - } - } - serde_json::Value::Array(items) => { - for item in items { - collect_blob_refs_in_value(item, blob_ids); - } - } - serde_json::Value::Object(map) => { - for item in map.values() { - collect_blob_refs_in_value(item, blob_ids); - } - } - serde_json::Value::Null | serde_json::Value::Bool(_) | serde_json::Value::Number(_) => {} - } -} - -fn replace_blob_refs_in_value( - value: &mut serde_json::Value, - cache: &HashMap, -) -> Result<()> { - match value { - serde_json::Value::String(current) => { - let Some(blob_id) = - parse_blob_ref(current).or_else(|| parse_legacy_blob_file_ref(current)) - else { - return Ok(()); - }; - let hydrated = cache - .get(&blob_id) - .cloned() - .with_context(|| format!("blob {blob_id:?} is missing from the hydration cache"))?; - *value = hydrated; - } - serde_json::Value::Array(items) => { - for item in items { - replace_blob_refs_in_value(item, cache)?; - } - } - serde_json::Value::Object(map) => { - for item in map.values_mut() { - replace_blob_refs_in_value(item, cache)?; - } - } - serde_json::Value::Null | serde_json::Value::Bool(_) | serde_json::Value::Number(_) => {} - } - Ok(()) -} - -fn artifact_dump_path(stage_id: &StageId, filename: &str) -> Result { - let node_id_segment = validate_single_path_segment("node id", stage_id.node_id())?; - let filename_path = validate_relative_path("artifact filename", filename)?; - Ok(PathBuf::from("artifacts") - .join("nodes") - .join(node_id_segment) - .join(format!("visit-{}", stage_id.visit())) - .join(filename_path)) -} - -fn ensure_parent_dir(path: &Path) -> Result<()> { - let parent = path - .parent() - .with_context(|| format!("path {} has no parent", path.display()))?; - std::fs::create_dir_all(parent) - .with_context(|| format!("failed to create {}", parent.display()))?; - Ok(()) -} +pub(super) use fabro_workflow::run_dump::RunDump as StoreRunExport; diff --git a/lib/crates/fabro-cli/src/server_runs.rs b/lib/crates/fabro-cli/src/server_runs.rs index 99010132f..d335892a5 100644 --- a/lib/crates/fabro-cli/src/server_runs.rs +++ b/lib/crates/fabro-cli/src/server_runs.rs @@ -26,7 +26,7 @@ impl ServerRunSummaryInfo { self.summary .workflow_name .clone() - .unwrap_or_else(|| "[no run record]".to_string()) + .unwrap_or_else(|| "[no run spec]".to_string()) } pub(crate) fn workflow_slug(&self) -> Option<&str> { diff --git a/lib/crates/fabro-cli/tests/it/cmd/config.rs b/lib/crates/fabro-cli/tests/it/cmd/config.rs index 28d1f1e05..d80fa1169 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/config.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/config.rs @@ -573,27 +573,27 @@ fn create_explicit_workflow_path_uses_project_config_relative_to_workflow() { }); let state = run_state(&run_dir); - let run_record = - serde_json::to_value(state.run.as_ref().expect("run record should exist")).unwrap(); + let run_spec = + serde_json::to_value(state.spec.as_ref().expect("run spec should exist")).unwrap(); assert_eq!( - run_record["settings"]["run"]["execution"]["approval"].as_str(), + run_spec["settings"]["run"]["execution"]["approval"].as_str(), Some("auto") ); assert_eq!( - run_record["settings"]["server"]["storage"]["root"].as_str(), + run_spec["settings"]["server"]["storage"]["root"].as_str(), Some(storage_dir.to_str().unwrap()) ); assert_eq!( - run_record["settings"]["run"]["sandbox"]["preserve"].as_bool(), + run_spec["settings"]["run"]["sandbox"]["preserve"].as_bool(), Some(true) ); assert_eq!( - run_record["settings"]["run"]["model"]["name"].as_str(), + run_spec["settings"]["run"]["model"]["name"].as_str(), Some("gpt-5.2") ); // v2 R30: run.prepare.steps replaces the whole ordered list across layers. assert_eq!( - run_record["settings"]["run"]["prepare"]["steps"], + run_spec["settings"]["run"]["prepare"]["steps"], serde_json::json!([{"script": "workflow-setup"}]) ); } diff --git a/lib/crates/fabro-cli/tests/it/cmd/create.rs b/lib/crates/fabro-cli/tests/it/cmd/create.rs index 5a5ba1f6a..05386a24d 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/create.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/create.rs @@ -228,7 +228,7 @@ digraph BarBaz { let run_dir = context.find_run_dir(&run_id); let state = run_state(&run_dir); - let run = state.run.as_ref().expect("run record should exist"); + let run = state.spec.as_ref().expect("run spec should exist"); fabro_json_snapshot!( context, serde_json::json!({ @@ -284,7 +284,7 @@ digraph FooWorkflow { let run_dir = context.find_run_dir(&run_id); let state = run_state(&run_dir); - let run = state.run.as_ref().expect("run record should exist"); + let run = state.spec.as_ref().expect("run spec should exist"); fabro_json_snapshot!( context, serde_json::json!({ @@ -351,16 +351,16 @@ fn create_persists_requested_overrides_into_store() { .to_string(); let run = resolve_run(&context, &run_id); let state = run_state(&run.run_dir); - let run_record = state.run.as_ref().expect("run record should exist"); + let run_spec = state.spec.as_ref().expect("run spec should exist"); let labels = json!({ - "env": run_record.labels.get("env"), - "team": run_record.labels.get("team"), + "env": run_spec.labels.get("env"), + "team": run_spec.labels.get("team"), }); - let settings = &run_record.settings; + let settings = &run_spec.settings; let resolved_run = resolved_run(settings); let cli_settings = fabro_config::resolve_cli_from_file(settings).expect("cli settings"); let compact = json!({ - "workflow_slug": run_record.workflow_slug, + "workflow_slug": run_spec.workflow_slug, "settings": { "goal": match resolved_run.goal.as_ref() { Some(fabro_types::settings::run::RunGoal::Inline(value)) => Some(value.as_source()), @@ -435,9 +435,9 @@ fn create_json_does_not_imply_auto_approve() { assert!( resolved_run( &run_state(&run.run_dir) - .run + .spec .as_ref() - .expect("run record should exist") + .expect("run spec should exist") .settings, ) .execution diff --git a/lib/crates/fabro-cli/tests/it/cmd/fork.rs b/lib/crates/fabro-cli/tests/it/cmd/fork.rs index 162bdf6a5..03ab5e9aa 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/fork.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/fork.rs @@ -134,20 +134,22 @@ fn fork_from_earlier_checkpoint_uses_expected_sha() { ]); assert_eq!(new_head.trim(), expected_head); - let checkpoint = git_show_json( + let run_snapshot = git_show_json( &setup.repo_dir, - &format!("fabro/meta/{new_run_id}:checkpoint.json"), + &format!("fabro/meta/{new_run_id}:run.json"), ); - assert_eq!(checkpoint["current_node"].as_str(), Some("step_one")); assert_eq!( - checkpoint["git_commit_sha"].as_str(), + run_snapshot["checkpoint"]["current_node"].as_str(), + Some("step_one") + ); + assert_eq!( + run_snapshot["checkpoint"]["git_commit_sha"].as_str(), Some(expected_head.as_str()) ); - let start = git_show_json( - &setup.repo_dir, - &format!("fabro/meta/{new_run_id}:start.json"), - ); let expected_branch = format!("fabro/run/{new_run_id}"); - assert_eq!(start["run_branch"].as_str(), Some(expected_branch.as_str())); + assert_eq!( + run_snapshot["start"]["run_branch"].as_str(), + Some(expected_branch.as_str()) + ); } diff --git a/lib/crates/fabro-cli/tests/it/cmd/inspect.rs b/lib/crates/fabro-cli/tests/it/cmd/inspect.rs index e8345019b..7b35d48af 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/inspect.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/inspect.rs @@ -98,7 +98,7 @@ fn inspect_resolves_selector_via_server_endpoint() { { "run_id": "[ULID]", "status": "succeeded", - "run_record": null, + "run_spec": null, "start_record": null, "conclusion": null, "checkpoint": null, @@ -113,7 +113,7 @@ fn inspect_resolves_selector_via_server_endpoint() { } #[test] -fn inspect_created_run_shows_run_record_without_start_or_conclusion() { +fn inspect_created_run_shows_run_spec_without_start_or_conclusion() { let context = test_context!(); let run = setup_created_fast_dry_run(&context); let output = run_success(&context, &["inspect", &run.run_id]); @@ -123,7 +123,7 @@ fn inspect_created_run_shows_run_record_without_start_or_conclusion() { { "run_id": "[ULID]", "status": "submitted", - "run_record": { + "run_spec": { "goal": "Run tests and report results", "workflow_name": "Simple", "workflow_slug": "simple", @@ -156,7 +156,7 @@ fn inspect_completed_run_shows_run_start_conclusion_checkpoint() { { "run_id": "[ULID]", "status": "succeeded", - "run_record": { + "run_spec": { "goal": "Run tests and report results", "workflow_name": "Simple", "workflow_slug": "simple", @@ -222,7 +222,7 @@ fn inspect_completed_run_reads_store_without_disk_metadata_files() { { "run_id": "[ULID]", "status": "succeeded", - "run_record": { + "run_spec": { "goal": "Run tests and report results", "workflow_name": "Simple", "workflow_slug": "simple", @@ -273,7 +273,7 @@ fn inspect_git_backed_run_exposes_checkpoint_and_sandbox_state() { { "run_id": "[ULID]", "status": "succeeded", - "run_record": { + "run_spec": { "goal": "Edit a tracked file", "workflow_name": "Flow", "workflow_slug": "flow", diff --git a/lib/crates/fabro-cli/tests/it/cmd/pr_create.rs b/lib/crates/fabro-cli/tests/it/cmd/pr_create.rs index f80603aa7..9b76b4eae 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/pr_create.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/pr_create.rs @@ -65,7 +65,7 @@ fn pr_create_completed_dry_run_without_run_branch_errors() { } #[test] -fn pr_create_uses_store_run_record_without_run_json() { +fn pr_create_uses_store_run_spec_without_run_json() { let context = test_context!(); let run = setup_completed_fast_dry_run(&context); diff --git a/lib/crates/fabro-cli/tests/it/cmd/run.rs b/lib/crates/fabro-cli/tests/it/cmd/run.rs index 38fb975bd..ddcb9e400 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/run.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/run.rs @@ -42,7 +42,7 @@ fn preflight_response() -> serde_json::Value { fn remote_run_state_response() -> serde_json::Value { serde_json::json!({ - "run": null, + "spec": null, "graph_source": null, "start": null, "status": null, diff --git a/lib/crates/fabro-cli/tests/it/cmd/runner.rs b/lib/crates/fabro-cli/tests/it/cmd/runner.rs index 947e74ab0..f3c7314e1 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/runner.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/runner.rs @@ -275,7 +275,7 @@ digraph GitHubApp { let run_dir = context.find_run_dir(&run_id); let state = run_state(&run_dir); - let run = state.run.as_ref().expect("run record should exist"); + let run = state.spec.as_ref().expect("run spec should exist"); let resolved_server = fabro_config::resolve_server_from_file(&run.settings).unwrap(); fabro_json_snapshot!( context, @@ -465,7 +465,7 @@ digraph Test { } #[test] -fn runner_reports_missing_run_record_without_prefetching_events() { +fn runner_reports_missing_run_spec_without_prefetching_events() { let context = test_context!(); let server = MockServer::start(); let run_id = unique_run_id(); @@ -478,7 +478,7 @@ fn runner_reports_missing_run_record_without_prefetching_events() { .header("Content-Type", "application/json") .body( serde_json::json!({ - "run": null, + "spec": null, "graph_source": null, "start": null, "status": null, @@ -523,14 +523,14 @@ fn runner_reports_missing_run_record_without_prefetching_events() { assert!( !output.status.success(), - "worker should fail when run record is missing:\nstdout:\n{}\nstderr:\n{}", + "worker should fail when run spec is missing:\nstdout:\n{}\nstderr:\n{}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); state_mock.assert(); events_mock.assert_calls(0); assert!( - output_stderr(&output).contains("has no run record in store"), + output_stderr(&output).contains("has no run spec in store"), "{}", output_stderr(&output) ); diff --git a/lib/crates/fabro-cli/tests/it/cmd/store_dump.rs b/lib/crates/fabro-cli/tests/it/cmd/store_dump.rs index 1bf94bea2..6f01b2b50 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/store_dump.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/store_dump.rs @@ -70,7 +70,7 @@ fn store_dump_accepts_server_target_from_separate_home() { String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); - assert!(output_dir.join("checkpoint.json").is_file()); + assert!(output_dir.join("run.json").is_file()); } #[test] @@ -145,10 +145,10 @@ fn store_dump_exports_large_command_output_backed_by_blob_refs() { String::from_utf8_lossy(&dump_output.stderr) ); - let checkpoint = fs::read_to_string(output_dir.join("checkpoint.json")).unwrap(); + let run_json = fs::read_to_string(output_dir.join("run.json")).unwrap(); assert!( - !checkpoint.contains("blob://sha256/"), - "checkpoint export should hydrate blob refs\n{checkpoint}" + !run_json.contains("blob://sha256/"), + "run export should hydrate blob refs\n{run_json}" ); } @@ -248,10 +248,10 @@ include = ["assets/**"] String::from_utf8_lossy(&dump_output.stderr) ); - let checkpoint = fs::read_to_string(output_dir.join("checkpoint.json")).unwrap(); + let run_json = fs::read_to_string(output_dir.join("run.json")).unwrap(); assert!( - !checkpoint.contains("blob://sha256/"), - "checkpoint export should hydrate blob refs\n{checkpoint}" + !run_json.contains("blob://sha256/"), + "run export should hydrate blob refs\n{run_json}" ); assert_eq!( fs::read_to_string(output_dir.join("artifacts/nodes/big/visit-1/assets/shared/report.txt")) @@ -278,28 +278,23 @@ fn store_dump_exports_completed_run_snapshot() { success: true exit_code: 0 ----- stdout ----- - Exported 17 files for run [ULID] to [TEMP_DIR]/export + Exported 12 files for run [ULID] to [TEMP_DIR]/export ----- stderr ----- "); assert_snapshot!(dump_file_summary(&output_dir), @" - checkpoint.json checkpoints/0013.json checkpoints/0017.json checkpoints/0021.json - conclusion.json events.jsonl graph.fabro - nodes/exit/visit-1/status.json - nodes/report/visit-1/response.md - nodes/report/visit-1/status.json - nodes/run_tests/visit-1/response.md - nodes/run_tests/visit-1/status.json - nodes/start/visit-1/status.json run.json - sandbox.json - start.json - status.json + stages/exit@1/status.json + stages/report@1/response.md + stages/report@1/status.json + stages/run_tests@1/response.md + stages/run_tests@1/status.json + stages/start@1/status.json "); } diff --git a/lib/crates/fabro-cli/tests/it/cmd/support.rs b/lib/crates/fabro-cli/tests/it/cmd/support.rs index 679d0d8e4..43240a6f7 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/support.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/support.rs @@ -904,29 +904,29 @@ pub(crate) fn compact_inspect(output: &Output) -> Value { Value::Array( items.into_iter() .map(|item| { - let run_record = item["run_record"].clone(); + let run_spec = item["run_spec"].clone(); let checkpoint = item["checkpoint"].clone(); let conclusion = item["conclusion"].clone(); let sandbox = item["sandbox"].clone(); - let dry_run = run_record + let dry_run = run_spec .pointer("/settings/run/execution/mode") .and_then(Value::as_str) .map(|mode| Value::Bool(mode == "dry_run")); serde_json::json!({ "run_id": "[ULID]", "status": item["status"], - "run_record": { - "goal": run_record.pointer("/settings/run/goal"), - "workflow_name": run_record.pointer("/graph/name"), - "workflow_slug": run_record.pointer("/workflow_slug"), - "sandbox_provider": run_record.pointer("/settings/run/sandbox/provider"), + "run_spec": { + "goal": run_spec.pointer("/settings/run/goal"), + "workflow_name": run_spec.pointer("/graph/name"), + "workflow_slug": run_spec.pointer("/workflow_slug"), + "sandbox_provider": run_spec.pointer("/settings/run/sandbox/provider"), "dry_run": dry_run, - "provenance": run_record.pointer("/provenance").as_ref().map(|_| { + "provenance": run_spec.pointer("/provenance").as_ref().map(|_| { serde_json::json!({ "server_version": "[VERSION]", - "client_name": run_record.pointer("/provenance/client/name"), + "client_name": run_spec.pointer("/provenance/client/name"), "client_version": "[VERSION]", - "subject_auth_method": run_record.pointer("/provenance/subject/auth_method"), + "subject_auth_method": run_spec.pointer("/provenance/subject/auth_method"), }) }), }, @@ -966,7 +966,7 @@ pub(crate) fn compact_git_inspect(output: &Output) -> Value { Value::Array( items.into_iter() .map(|item| { - let run_record = item["run_record"].clone(); + let run_spec = item["run_spec"].clone(); let start_record = item["start_record"].clone(); let checkpoint = item["checkpoint"].clone(); let conclusion = item["conclusion"].clone(); @@ -974,18 +974,18 @@ pub(crate) fn compact_git_inspect(output: &Output) -> Value { serde_json::json!({ "run_id": "[ULID]", "status": item["status"], - "run_record": { - "goal": run_record.pointer("/settings/run/goal"), - "workflow_name": run_record.pointer("/graph/name"), - "workflow_slug": run_record.pointer("/workflow_slug"), - "llm_provider": run_record.pointer("/settings/run/model/provider"), - "sandbox_provider": run_record.pointer("/settings/run/sandbox/provider"), - "provenance": run_record.pointer("/provenance").as_ref().map(|_| { + "run_spec": { + "goal": run_spec.pointer("/settings/run/goal"), + "workflow_name": run_spec.pointer("/graph/name"), + "workflow_slug": run_spec.pointer("/workflow_slug"), + "llm_provider": run_spec.pointer("/settings/run/model/provider"), + "sandbox_provider": run_spec.pointer("/settings/run/sandbox/provider"), + "provenance": run_spec.pointer("/provenance").as_ref().map(|_| { serde_json::json!({ "server_version": "[VERSION]", - "client_name": run_record.pointer("/provenance/client/name"), + "client_name": run_spec.pointer("/provenance/client/name"), "client_version": "[VERSION]", - "subject_auth_method": run_record.pointer("/provenance/subject/auth_method"), + "subject_auth_method": run_spec.pointer("/provenance/subject/auth_method"), }) }), }, diff --git a/lib/crates/fabro-cli/tests/it/scenario/lifecycle.rs b/lib/crates/fabro-cli/tests/it/scenario/lifecycle.rs index 3be397fd8..7a64dd807 100644 --- a/lib/crates/fabro-cli/tests/it/scenario/lifecycle.rs +++ b/lib/crates/fabro-cli/tests/it/scenario/lifecycle.rs @@ -53,15 +53,15 @@ fn local_run_lifecycle() { "workflow_name should be CommandPipeline" ); - // 3. inspect — JSON array with run_record and conclusion + // 3. inspect — JSON array with run_spec and conclusion let inspect_out = cmd(&["inspect", &run_id]).success(); let inspect_stdout = String::from_utf8(inspect_out.get_output().stdout.clone()).unwrap(); let items: Vec = serde_json::from_str(&inspect_stdout).expect("inspect should produce a JSON array"); assert!(!items.is_empty(), "inspect should return at least one item"); assert!( - items[0]["run_record"].is_object(), - "inspect should include run_record" + items[0]["run_spec"].is_object(), + "inspect should include run_spec" ); assert!( items[0]["conclusion"].is_object(), @@ -317,14 +317,14 @@ digraph FooWorkflow { .assert() .success(); - let run_record = run_state(&context.find_run_dir(&run_id)) - .run - .expect("run record should exist"); + let run_spec = run_state(&context.find_run_dir(&run_id)) + .spec + .expect("run spec should exist"); fabro_json_snapshot!( context, serde_json::json!({ - "graph_name": run_record.graph.name, - "workflow_slug": run_record.workflow_slug, + "graph_name": run_spec.graph.name, + "workflow_slug": run_spec.workflow_slug, }), @r#" { diff --git a/lib/crates/fabro-cli/tests/it/scenario/recovery.rs b/lib/crates/fabro-cli/tests/it/scenario/recovery.rs index 04ee4127b..28e8f55b4 100644 --- a/lib/crates/fabro-cli/tests/it/scenario/recovery.rs +++ b/lib/crates/fabro-cli/tests/it/scenario/recovery.rs @@ -8,6 +8,7 @@ use std::path::Path; use fabro_checkpoint::branch::BranchStore; use fabro_checkpoint::git::Store as GitStore; +use fabro_store::RunProjection; use fabro_test::{fabro_snapshot, test_context}; use fabro_types::Checkpoint; use fabro_workflow::operations::{RunTimeline, build_timeline}; @@ -42,12 +43,14 @@ fn metadata_checkpoints(repo_dir: &Path, run_id: &str) -> Vec { .rev() .filter(|commit| commit.message.starts_with("checkpoint")) .map(|commit| { - let checkpoint_blob = store - .read_blob_at(commit.oid, "checkpoint.json") - .expect("checkpoint blob should load") - .expect("checkpoint blob should exist"); - serde_json::from_slice::(&checkpoint_blob) - .expect("checkpoint blob should deserialize") + let projection_blob = store + .read_blob_at(commit.oid, "run.json") + .expect("projection blob should load") + .expect("projection blob should exist"); + serde_json::from_slice::(&projection_blob) + .expect("projection blob should deserialize") + .checkpoint + .expect("projection checkpoint should exist") }) .collect() } @@ -59,11 +62,14 @@ fn latest_metadata_checkpoint(repo_dir: &Path, run_id: &str) -> Checkpoint { .resolve_ref(&format!("fabro/meta/{run_id}")) .expect("metadata branch should resolve") .expect("metadata branch tip should exist"); - let checkpoint_blob = store - .read_blob_at(tip, "checkpoint.json") - .expect("latest checkpoint blob should load") - .expect("latest checkpoint blob should exist"); - serde_json::from_slice(&checkpoint_blob).expect("latest checkpoint blob should deserialize") + let projection_blob = store + .read_blob_at(tip, "run.json") + .expect("latest projection blob should load") + .expect("latest projection blob should exist"); + serde_json::from_slice::(&projection_blob) + .expect("latest projection blob should deserialize") + .checkpoint + .expect("latest projection checkpoint should exist") } fn timeline_run_shas(repo_dir: &Path, run_id: &str) -> Vec> { diff --git a/lib/crates/fabro-cli/tests/it/scenario/smoke.rs b/lib/crates/fabro-cli/tests/it/scenario/smoke.rs index a93036ea4..13160b955 100644 --- a/lib/crates/fabro-cli/tests/it/scenario/smoke.rs +++ b/lib/crates/fabro-cli/tests/it/scenario/smoke.rs @@ -5,7 +5,7 @@ use crate::support::{LightweightCli, unique_run_id}; fn live_run_state_response() -> serde_json::Value { serde_json::json!({ - "run": null, + "spec": null, "graph_source": null, "start": null, "status": { diff --git a/lib/crates/fabro-cli/tests/it/workflow/command_agent_mixed.rs b/lib/crates/fabro-cli/tests/it/workflow/command_agent_mixed.rs index 287465b65..ee9ba5be8 100644 --- a/lib/crates/fabro-cli/tests/it/workflow/command_agent_mixed.rs +++ b/lib/crates/fabro-cli/tests/it/workflow/command_agent_mixed.rs @@ -49,7 +49,7 @@ fn scenario_command_agent_mixed(sandbox: &str) { ); let export_dir = store_dump_export(&context, &run_id_for(&run_dir)); - let stdout = std::fs::read_to_string(export_dir.join("nodes/verify/visit-1/stdout.log")) + let stdout = std::fs::read_to_string(export_dir.join("stages/verify@1/stdout.log")) .expect("verify stdout.log should exist"); assert!( stdout.contains("SCENARIO_FLAG_42"), diff --git a/lib/crates/fabro-cli/tests/it/workflow/command_pipeline.rs b/lib/crates/fabro-cli/tests/it/workflow/command_pipeline.rs index 8050455d9..5a46d92e4 100644 --- a/lib/crates/fabro-cli/tests/it/workflow/command_pipeline.rs +++ b/lib/crates/fabro-cli/tests/it/workflow/command_pipeline.rs @@ -48,7 +48,7 @@ fn scenario_command_pipeline(sandbox: &str) { ); let export_dir = store_dump_export(&context, &run_id_for(&run_dir)); - let stdout1 = std::fs::read_to_string(export_dir.join("nodes/step1/visit-1/stdout.log")) + let stdout1 = std::fs::read_to_string(export_dir.join("stages/step1@1/stdout.log")) .expect("step1 stdout.log should exist"); assert!( stdout1.contains("hello-from-step1"), diff --git a/lib/crates/fabro-cli/tests/it/workflow/full_stack.rs b/lib/crates/fabro-cli/tests/it/workflow/full_stack.rs index f243cd5f0..4b03f1155 100644 --- a/lib/crates/fabro-cli/tests/it/workflow/full_stack.rs +++ b/lib/crates/fabro-cli/tests/it/workflow/full_stack.rs @@ -6,8 +6,8 @@ use fabro_test::test_context; use super::{ - completed_nodes, find_run_dir, fixture, has_event, read_conclusion, read_run_record, - run_id_for, sandbox_tests, store_dump_export, timeout_for, + completed_nodes, find_run_dir, fixture, has_event, read_conclusion, read_run_spec, run_id_for, + sandbox_tests, store_dump_export, timeout_for, }; sandbox_tests!(full_stack, keys = ["ANTHROPIC_API_KEY"]); @@ -42,15 +42,15 @@ fn scenario_full_stack(sandbox: &str) { "duration_ms should be > 0" ); - // RunRecord should have key fields - let run_record = read_run_record(&run_dir); + // RunSpec should have key fields + let run_spec = read_run_spec(&run_dir); assert!( - run_record["run_id"].as_str().is_some(), - "run record should have run_id" + run_spec["run_id"].as_str().is_some(), + "run spec should have run_id" ); assert!( - run_record["graph"]["name"].as_str().is_some(), - "run record should have graph.name" + run_spec["graph"]["name"].as_str().is_some(), + "run spec should have graph.name" ); // Progress events @@ -74,7 +74,7 @@ fn scenario_full_stack(sandbox: &str) { // Verify node stdout should contain PASS let export_dir = store_dump_export(&context, &run_id_for(&run_dir)); - let stdout = std::fs::read_to_string(export_dir.join("nodes/verify/visit-1/stdout.log")) + let stdout = std::fs::read_to_string(export_dir.join("stages/verify@1/stdout.log")) .expect("verify stdout.log should exist"); assert!( stdout.contains("PASS"), diff --git a/lib/crates/fabro-cli/tests/it/workflow/mod.rs b/lib/crates/fabro-cli/tests/it/workflow/mod.rs index adcd4103f..33413aa9a 100644 --- a/lib/crates/fabro-cli/tests/it/workflow/mod.rs +++ b/lib/crates/fabro-cli/tests/it/workflow/mod.rs @@ -37,13 +37,13 @@ pub(super) fn read_conclusion(run_dir: &Path) -> Value { .expect("conclusion should serialize") } -pub(super) fn read_run_record(run_dir: &Path) -> Value { +pub(super) fn read_run_spec(run_dir: &Path) -> Value { serde_json::to_value( run_state(run_dir) - .run - .expect("run store run record should exist"), + .spec + .expect("run store run spec should exist"), ) - .expect("run record should serialize") + .expect("run spec should serialize") } pub(super) fn completed_nodes(run_dir: &Path) -> Vec { diff --git a/lib/crates/fabro-config/src/effective_settings.rs b/lib/crates/fabro-config/src/effective_settings.rs index 8fb12a8f2..685bf9bae 100644 --- a/lib/crates/fabro-config/src/effective_settings.rs +++ b/lib/crates/fabro-config/src/effective_settings.rs @@ -117,7 +117,7 @@ fn strip_owner_domains(file: &mut SettingsLayer) { /// the server's local `~/.fabro/settings.toml` when the corresponding client /// value is absent. Run-shaped defaults (model, prepare, sandbox, checkpoint, /// hooks, agent mcps, etc.) also flow from server to client so the persisted -/// run record matches the server's local configuration. +/// run spec matches the server's local configuration. fn apply_server_defaults(mut settings: SettingsLayer, server: &SettingsLayer) -> SettingsLayer { // Server-owned domains: server-side always wins when client left blank. // Use the v2 merge matrix with the server layer in lower precedence so diff --git a/lib/crates/fabro-retro/src/retro_agent.rs b/lib/crates/fabro-retro/src/retro_agent.rs index 2529c436a..859964d2e 100644 --- a/lib/crates/fabro-retro/src/retro_agent.rs +++ b/lib/crates/fabro-retro/src/retro_agent.rs @@ -1,16 +1,16 @@ -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use std::time::Duration; use fabro_agent::tool_registry::RegisteredTool; use fabro_agent::{ AgentProfile, AnthropicProfile, GeminiProfile, OpenAiProfile, Sandbox, Session, SessionEvent, - SessionOptions, Turn, + SessionOptions, Turn, shell_quote, }; use fabro_llm::client::Client; use fabro_llm::provider::Provider; use fabro_llm::types::ToolDefinition; -use fabro_store::{EventEnvelope, RunProjection}; +use fabro_store::{EventEnvelope, RunProjection, SerializableProjection}; use tokio::task::JoinHandle; use crate::retro::{RetroNarrative, SmoothnessRating}; @@ -19,9 +19,9 @@ const RETRO_SYSTEM_PROMPT: &str = r"You are a workflow run retrospective analyst You have access to the run's data files: - `progress.jsonl` — the full event stream (stage starts/completions, agent tool calls, errors, retries) -- `checkpoint.json` — final execution state with node outcomes -- `run.json` — run record with config, graph, and metadata -- `start.json` — start record with start time and git info +- `run.json` — serialized run projection with the run spec, checkpoint state, conclusion, retro data, and other metadata +- `graph.fabro` — the workflow source for the run +- `stages/{node_id}@{visit}/...` — per-stage prompt, response, status, diff, stdout/stderr, and tool metadata files ## Your task @@ -30,6 +30,7 @@ You have access to the run's data files: - Check agent tool call patterns for wrong approaches or pivots - Note which stages took longest or had issues - Look for patterns indicating friction (repeated similar tool calls, error recovery) + - Use `run.json` for the run-level snapshot, `graph.fabro` for workflow intent, and `stages/` for full per-stage payloads 2. **Call the `submit_retro` tool** with your structured analysis. @@ -124,7 +125,8 @@ pub fn build_retro_prompt(retro_data_dir: &str) -> String { format!( "Analyze the workflow run data at `{retro_data_dir}/` and generate a retrospective. \ The key file is `{retro_data_dir}/progress.jsonl` which contains the full event stream. \ - Also check `{retro_data_dir}/checkpoint.json` for stage outcomes. \ + Use `{retro_data_dir}/run.json` for the run-level snapshot, `{retro_data_dir}/graph.fabro` \ + for the workflow source, and `{retro_data_dir}/stages/` for full per-stage payloads. \ Use grep to search for interesting signals (failures, retries, errors, approach changes) \ rather than reading the entire file. When done, call the `submit_retro` tool with your analysis." ) @@ -299,12 +301,6 @@ async fn upload_data_files( _run_dir: &Path, target_dir: &str, ) -> anyhow::Result<()> { - // Create target directory - sandbox - .exec_command(&format!("mkdir -p {target_dir}"), 10_000, None, None, None) - .await - .map_err(|e| anyhow::anyhow!("Failed to create retro data dir: {e}"))?; - let progress_content = { let lines: Vec = events .iter() @@ -316,33 +312,88 @@ async fn upload_data_files( Some(lines.join("\n") + "\n") } }; - if let Some(content) = progress_content { - sandbox - .write_file(&format!("{target_dir}/progress.jsonl"), &content) - .await - .map_err(|e| anyhow::anyhow!("Failed to upload progress.jsonl: {e}"))?; - } + upload_file(sandbox, target_dir, "progress.jsonl", progress_content).await?; - let checkpoint_content = state - .checkpoint - .clone() - .map(|cp| serde_json::to_string_pretty(&cp)) - .transpose()?; - upload_file(sandbox, target_dir, "checkpoint.json", checkpoint_content).await?; - - let run_content = state - .run - .clone() - .map(|run| serde_json::to_string_pretty(&run)) - .transpose()?; + let run_content = Some(serde_json::to_string_pretty(&SerializableProjection( + state, + ))?); upload_file(sandbox, target_dir, "run.json", run_content).await?; + upload_file( + sandbox, + target_dir, + "graph.fabro", + state.graph_source.clone(), + ) + .await?; - let start_content = state - .start - .clone() - .map(|start| serde_json::to_string_pretty(&start)) - .transpose()?; - upload_file(sandbox, target_dir, "start.json", start_content).await?; + let mut stage_ids: Vec<_> = state + .iter_nodes() + .map(|(stage_id, _)| stage_id.clone()) + .collect(); + stage_ids.sort(); + + for stage_id in stage_ids { + let Some(node) = state.node(&stage_id) else { + continue; + }; + let base = PathBuf::from("stages").join(stage_id.to_string()); + let prompt_path = base.join("prompt.md").to_string_lossy().into_owned(); + let response_path = base.join("response.md").to_string_lossy().into_owned(); + let status_path = base.join("status.json").to_string_lossy().into_owned(); + let provider_used_path = base + .join("provider_used.json") + .to_string_lossy() + .into_owned(); + let diff_path = base.join("diff.patch").to_string_lossy().into_owned(); + let script_invocation_path = base + .join("script_invocation.json") + .to_string_lossy() + .into_owned(); + let script_timing_path = base + .join("script_timing.json") + .to_string_lossy() + .into_owned(); + let parallel_results_path = base + .join("parallel_results.json") + .to_string_lossy() + .into_owned(); + let stdout_path = base.join("stdout.log").to_string_lossy().into_owned(); + let stderr_path = base.join("stderr.log").to_string_lossy().into_owned(); + upload_file(sandbox, target_dir, &prompt_path, node.prompt.clone()).await?; + upload_file(sandbox, target_dir, &response_path, node.response.clone()).await?; + upload_json_file(sandbox, target_dir, &status_path, node.status.as_ref()).await?; + upload_json_file( + sandbox, + target_dir, + &provider_used_path, + node.provider_used.as_ref(), + ) + .await?; + upload_file(sandbox, target_dir, &diff_path, node.diff.clone()).await?; + upload_json_file( + sandbox, + target_dir, + &script_invocation_path, + node.script_invocation.as_ref(), + ) + .await?; + upload_json_file( + sandbox, + target_dir, + &script_timing_path, + node.script_timing.as_ref(), + ) + .await?; + upload_json_file( + sandbox, + target_dir, + ¶llel_results_path, + node.parallel_results.as_ref(), + ) + .await?; + upload_file(sandbox, target_dir, &stdout_path, node.stdout.clone()).await?; + upload_file(sandbox, target_dir, &stderr_path, node.stderr.clone()).await?; + } Ok(()) } @@ -354,16 +405,58 @@ async fn upload_file( content: Option, ) -> anyhow::Result<()> { if let Some(content) = content { + let path = Path::new(target_dir).join(filename); + let remote_path = path.to_string_lossy().into_owned(); + ensure_remote_dir(sandbox, &path).await?; sandbox - .write_file(&format!("{target_dir}/{filename}"), &content) + .write_file(&remote_path, &content) .await .map_err(|e| anyhow::anyhow!("Failed to upload {filename}: {e}"))?; } Ok(()) } +async fn upload_json_file( + sandbox: &Arc, + target_dir: &str, + filename: &str, + value: Option<&T>, +) -> anyhow::Result<()> +where + T: serde::Serialize, +{ + let content = value.map(serde_json::to_string_pretty).transpose()?; + upload_file(sandbox, target_dir, filename, content).await +} + +async fn ensure_remote_dir(sandbox: &Arc, path: &Path) -> anyhow::Result<()> { + let parent = path + .parent() + .ok_or_else(|| anyhow::anyhow!("Retro upload path has no parent: {}", path.display()))?; + let command = format!("mkdir -p {}", shell_quote(&parent.to_string_lossy())); + let result = sandbox + .exec_command(&command, 10_000, None, None, None) + .await + .map_err(|e| anyhow::anyhow!("Failed to create retro upload dir: {e}"))?; + if result.exit_code != 0 { + return Err(anyhow::anyhow!( + "Failed to create retro upload dir {}: {}", + parent.display(), + result.stderr + )); + } + Ok(()) +} + #[cfg(test)] mod tests { + use std::sync::Arc; + + use chrono::{TimeZone, Utc}; + use fabro_agent::LocalSandbox; + use fabro_store::{NodeState, StageId}; + use fabro_types::{NodeStatusRecord, StageStatus}; + use super::*; #[test] @@ -414,4 +507,88 @@ mod tests { assert!(narrative.friction_points.is_empty()); assert!(narrative.open_items.is_empty()); } + + #[test] + fn retro_prompt_mentions_graph_and_stage_files() { + let prompt = build_retro_prompt(RETRO_DATA_DIR); + + assert!(prompt.contains("run.json")); + assert!(prompt.contains("graph.fabro")); + assert!(prompt.contains("stages/")); + } + + #[tokio::test] + async fn upload_data_files_writes_projection_graph_and_stage_files() { + let sandbox_root = tempfile::tempdir().expect("sandbox tempdir should exist"); + let sandbox: Arc = + Arc::new(LocalSandbox::new(sandbox_root.path().to_path_buf())); + let output_dir = tempfile::tempdir().expect("retro tempdir should exist"); + let target_dir = output_dir.path().join("retro"); + let target_dir_str = target_dir.to_string_lossy().to_string(); + + let stage_id = StageId::new("build", 2); + let mut state = RunProjection::default(); + state.graph_source = Some("digraph Ship {}".to_string()); + state.set_node(stage_id, NodeState { + prompt: Some("plan".to_string()), + response: Some("done".to_string()), + status: Some(NodeStatusRecord { + status: StageStatus::Success, + notes: Some("ok".to_string()), + failure_reason: None, + timestamp: Utc + .with_ymd_and_hms(2026, 4, 20, 12, 1, 0) + .single() + .unwrap(), + }), + provider_used: Some(serde_json::json!({ "provider": "openai" })), + diff: Some("diff --git a/a b/a".to_string()), + script_invocation: Some(serde_json::json!({ "command": "cargo test" })), + script_timing: Some(serde_json::json!({ "duration_ms": 10 })), + parallel_results: Some(serde_json::json!([{ "stage": "fanout@1" }])), + stdout: Some("stdout".to_string()), + stderr: Some("stderr".to_string()), + }); + + upload_data_files(&sandbox, &state, &[], output_dir.path(), &target_dir_str) + .await + .expect("retro files should upload"); + + let run_json: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string(target_dir.join("run.json")).expect("run.json should exist"), + ) + .expect("run.json should parse"); + assert!(run_json.get("spec").is_some()); + assert!(run_json.get("run").is_none()); + assert!(run_json["nodes"]["build@2"]["prompt"].is_null()); + assert!(run_json["nodes"]["build@2"]["diff"].is_null()); + assert_eq!( + std::fs::read_to_string(target_dir.join("graph.fabro")) + .expect("graph.fabro should exist"), + "digraph Ship {}" + ); + assert_eq!( + std::fs::read_to_string(target_dir.join("stages/build@2/prompt.md")) + .expect("prompt file should exist"), + "plan" + ); + assert_eq!( + std::fs::read_to_string(target_dir.join("stages/build@2/response.md")) + .expect("response file should exist"), + "done" + ); + assert_eq!( + std::fs::read_to_string(target_dir.join("stages/build@2/stdout.log")) + .expect("stdout file should exist"), + "stdout" + ); + assert!( + target_dir.join("stages/build@2/status.json").exists(), + "status file should exist" + ); + assert!( + !target_dir.join("progress.jsonl").exists(), + "progress file should be omitted when there are no events" + ); + } } diff --git a/lib/crates/fabro-server/src/server.rs b/lib/crates/fabro-server/src/server.rs index 81fe6e50f..a6e14a95d 100644 --- a/lib/crates/fabro-server/src/server.rs +++ b/lib/crates/fabro-server/src/server.rs @@ -4378,14 +4378,14 @@ async fn start_run( } } - let Some(run_record) = run_state.run.as_ref() else { + let Some(run_spec) = run_state.spec.as_ref() else { return ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, - "run record missing from store", + "run spec missing from store", ) .into_response(); }; - let run_dir = match resolved_storage_dir(&run_record.settings) { + let run_dir = match resolved_storage_dir(&run_spec.settings) { Ok(storage_dir) => Storage::new(storage_dir) .run_scratch(&id) .root() @@ -4562,7 +4562,7 @@ async fn execute_run_in_process(state: Arc, run_id: RunId) { return; } }; - let github_settings = match resolved_github_settings(&persisted.run_record().settings) { + let github_settings = match resolved_github_settings(&persisted.run_spec().settings) { Ok(settings) => settings, Err(err) => { tracing::error!(run_id = %run_id, error = %err, "Invalid GitHub integration config"); @@ -4577,7 +4577,7 @@ async fn execute_run_in_process(state: Arc, run_id: RunId) { } }; let github_app_result = match fabro_config::resolve_run_from_file( - &persisted.run_record().settings, + &persisted.run_spec().settings, ) { Ok(settings) => { let required_github_credentials = (settings.execution.mode != RunMode::DryRun @@ -5101,10 +5101,10 @@ async fn get_run_settings( .into_response(); } }; - let Some(run_record) = run_state.run else { + let Some(run_spec) = run_state.spec else { return ApiError::not_found("Run not found.").into_response(); }; - let redacted = settings_view::redact_for_api(&run_record.settings); + let redacted = settings_view::redact_for_api(&run_spec.settings); let mut value = match serde_json::to_value(&redacted) { Ok(value) => value, Err(err) => { @@ -5503,10 +5503,7 @@ async fn read_run_blob( } } -async fn load_run_record( - state: &AppState, - run_id: &RunId, -) -> Result { +async fn load_run_spec(state: &AppState, run_id: &RunId) -> Result { let run_store = state .store .open_run_reader(run_id) @@ -5515,10 +5512,10 @@ async fn load_run_record( let run_state = run_store.state().await.map_err(|err| { ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() })?; - run_state.run.ok_or_else(|| { + run_state.spec.ok_or_else(|| { ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, - "run record missing from store", + "run spec missing from store", ) .into_response() }) @@ -5533,7 +5530,7 @@ async fn list_run_artifacts( Ok(id) => id, Err(response) => return response, }; - if let Err(response) = load_run_record(state.as_ref(), &id).await { + if let Err(response) = load_run_spec(state.as_ref(), &id).await { return response; } @@ -5570,7 +5567,7 @@ async fn list_stage_artifacts( Ok(stage_id) => stage_id, Err(response) => return response, }; - if let Err(response) = load_run_record(state.as_ref(), &id).await { + if let Err(response) = load_run_spec(state.as_ref(), &id).await { return response; } @@ -5962,7 +5959,7 @@ async fn put_stage_artifact( if let Some(response) = reject_if_archived(state.as_ref(), &id).await { return response; } - if let Err(response) = load_run_record(state.as_ref(), &id).await.map(|_| ()) { + if let Err(response) = load_run_spec(state.as_ref(), &id).await.map(|_| ()) { return response; } @@ -6020,7 +6017,7 @@ async fn get_stage_artifact( Ok(path) => path, Err(response) => return response, }; - if let Err(response) = load_run_record(state.as_ref(), &id).await { + if let Err(response) = load_run_spec(state.as_ref(), &id).await { return response; } @@ -8655,20 +8652,20 @@ slug = "fabro" let response = app.oneshot(req).await.unwrap(); let body = response_json!(response, StatusCode::OK).await; assert_eq!( - body["run"]["provenance"]["server"]["version"], + body["spec"]["provenance"]["server"]["version"], FABRO_VERSION ); assert_eq!( - body["run"]["provenance"]["client"]["user_agent"], + body["spec"]["provenance"]["client"]["user_agent"], "fabro-cli/1.2.3" ); - assert_eq!(body["run"]["provenance"]["client"]["name"], "fabro-cli"); - assert_eq!(body["run"]["provenance"]["client"]["version"], "1.2.3"); + assert_eq!(body["spec"]["provenance"]["client"]["name"], "fabro-cli"); + assert_eq!(body["spec"]["provenance"]["client"]["version"], "1.2.3"); assert_eq!( - body["run"]["provenance"]["subject"]["auth_method"], + body["spec"]["provenance"]["subject"]["auth_method"], "disabled" ); - assert!(body["run"]["provenance"]["subject"]["login"].is_null()); + assert!(body["spec"]["provenance"]["subject"]["login"].is_null()); } #[tokio::test] @@ -8741,10 +8738,10 @@ slug = "fabro" .unwrap(); let state_body = response_json!(state_response, StatusCode::OK).await; assert_eq!( - state_body["run"]["provenance"]["subject"]["auth_method"], + state_body["spec"]["provenance"]["subject"]["auth_method"], "dev_token" ); - assert_eq!(state_body["run"]["provenance"]["subject"]["login"], "dev"); + assert_eq!(state_body["spec"]["provenance"]["subject"]["login"], "dev"); } #[tokio::test] @@ -9009,7 +9006,7 @@ slug = "fabro" } #[tokio::test] - async fn create_run_persists_run_record() { + async fn create_run_persists_run_spec() { let state = create_app_state(); let app = build_router(Arc::clone(&state), AuthMode::Disabled); @@ -9026,7 +9023,7 @@ slug = "fabro" .await .unwrap(); - assert!(run_state.run.is_some()); + assert!(run_state.spec.is_some()); } #[tokio::test] @@ -9873,7 +9870,7 @@ level = "debug" .and_then(|run| run.run_dir.clone()) .expect("run_dir should be recorded") }; - let run_record = state + let run_spec = state .store .open_run_reader(&run_id) .await @@ -9881,10 +9878,10 @@ level = "debug" .state() .await .unwrap() - .run - .expect("run record should exist"); - let resolved_run = fabro_config::resolve_run_from_file(&run_record.settings).unwrap(); - let resolved_server = fabro_config::resolve_server_from_file(&run_record.settings).unwrap(); + .spec + .expect("run spec should exist"); + let resolved_run = fabro_config::resolve_run_from_file(&run_spec.settings).unwrap(); + let resolved_server = fabro_config::resolve_server_from_file(&run_spec.settings).unwrap(); // Verify a sampling of the persisted v2 settings, including inherited // run execution mode from server settings. diff --git a/lib/crates/fabro-store/src/lib.rs b/lib/crates/fabro-store/src/lib.rs index 55ff6a427..0a07a85b1 100644 --- a/lib/crates/fabro-store/src/lib.rs +++ b/lib/crates/fabro-store/src/lib.rs @@ -4,6 +4,7 @@ mod artifact_store; mod error; mod keys; mod run_state; +mod serializable_projection; mod slate; mod types; @@ -11,6 +12,7 @@ pub use artifact_store::{ArtifactStore, NodeArtifact}; pub use error::{Error, Result}; pub use fabro_types::{RunBlobId, StageId}; pub use run_state::{NodeState, PendingInterviewRecord, RunProjection}; +pub use serializable_projection::SerializableProjection; pub use slate::{ AuthCode, ConsumeOutcome, Database, RefreshToken, RunDatabase, Runs, SlateAuthCodeStore, SlateAuthTokenStore, diff --git a/lib/crates/fabro-store/src/run_state.rs b/lib/crates/fabro-store/src/run_state.rs index 8ae25dc96..0ad23d276 100644 --- a/lib/crates/fabro-store/src/run_state.rs +++ b/lib/crates/fabro-store/src/run_state.rs @@ -10,7 +10,7 @@ use fabro_types::run_event::{ use fabro_types::{ BilledModelUsage, BlockedReason, Checkpoint, Conclusion, EventBody, FailureSignature, InterviewQuestionRecord, InterviewQuestionType, NodeStatusRecord, Outcome, PullRequestRecord, - Retro, RunControlAction, RunEvent, RunId, RunRecord, RunStatus, RunStatusRecord, SandboxRecord, + Retro, RunControlAction, RunEvent, RunId, RunSpec, RunStatus, RunStatusRecord, SandboxRecord, StageStatus, StartRecord, StatusReason, }; use serde_json::Value; @@ -20,7 +20,7 @@ use crate::{Error, EventEnvelope, Result, RunSummary, StageId}; #[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] #[serde(default)] pub struct RunProjection { - pub run: Option, + pub spec: Option, pub graph_source: Option, pub start: Option, pub status: Option, @@ -85,7 +85,7 @@ impl RunProjection { EventBody::RunCreated(props) => { let working_directory = PathBuf::from(&props.working_directory); let labels = props.labels.clone().into_iter().collect::>(); - self.run = Some(RunRecord { + self.spec = Some(RunSpec { run_id, settings: props.settings.clone(), graph: props.graph.clone(), @@ -110,8 +110,8 @@ impl RunProjection { }); } EventBody::RunSubmitted(props) => { - if let Some(run) = self.run.as_mut() { - run.definition_blob = props.definition_blob; + if let Some(spec) = self.spec.as_mut() { + spec.definition_blob = props.definition_blob; } self.status = Some(run_status_record(RunStatus::Submitted, props.reason, ts)); } @@ -449,29 +449,55 @@ impl RunProjection { visits } + pub fn spec(&self) -> Option<&RunSpec> { + self.spec.as_ref() + } + + pub fn status(&self) -> Option { + self.status.as_ref().map(|status| status.status) + } + + pub fn is_terminal(&self) -> bool { + self.status().is_some_and(RunStatus::is_terminal) + } + + pub fn current_checkpoint(&self) -> Option<&Checkpoint> { + self.checkpoint.as_ref() + } + + pub fn pending_interviews(&self) -> &BTreeMap { + &self.pending_interviews + } + pub(crate) fn build_summary(&self, run_id: &RunId) -> RunSummary { - let workflow_name = self.run.as_ref().map(|run| { - if run.graph.name.is_empty() { + let workflow_name = self.spec.as_ref().map(|spec| { + if spec.graph.name.is_empty() { "unnamed".to_string() } else { - run.graph.name.clone() + spec.graph.name.clone() } }); - let goal = self.run.as_ref().and_then(|run| { - let goal = run.graph.goal(); + let goal = self.spec.as_ref().and_then(|spec| { + let goal = spec.graph.goal(); (!goal.is_empty()).then(|| goal.to_string()) }); RunSummary { run_id: *run_id, workflow_name, - workflow_slug: self.run.as_ref().and_then(|run| run.workflow_slug.clone()), + workflow_slug: self + .spec + .as_ref() + .and_then(|spec| spec.workflow_slug.clone()), goal, labels: self - .run + .spec .as_ref() - .map(|run| run.labels.clone()) + .map(|spec| spec.labels.clone()) .unwrap_or_default(), - host_repo_path: self.run.as_ref().and_then(|run| run.host_repo_path.clone()), + host_repo_path: self + .spec + .as_ref() + .and_then(|spec| spec.host_repo_path.clone()), start_time: self.start.as_ref().map(|start| start.start_time), status: self .status @@ -770,6 +796,20 @@ mod tests { #[test] fn deserialize_and_round_trip_projection_preserves_stage_ids_and_pending_control() { let state: RunProjection = serde_json::from_value(serde_json::json!({ + "spec": { + "run_id": "01JW6A7VNFZSFF0SKXJG29Z2M3", + "settings": { "_version": 1 }, + "graph": { "name": "ship", "nodes": {}, "edges": [], "attrs": {} }, + "workflow_slug": "demo", + "working_directory": "/tmp/project", + "host_repo_path": null, + "repo_origin_url": null, + "base_branch": null, + "labels": {}, + "provenance": null, + "manifest_blob": null, + "definition_blob": null + }, "pending_control": "cancel", "checkpoints": [[ 0, @@ -802,6 +842,7 @@ mod tests { let round_tripped: RunProjection = serde_json::from_value(serde_json::to_value(&state).unwrap()).unwrap(); + let serialized = serde_json::to_value(&state).unwrap(); let round_tripped_node = round_tripped.node(&stage_id).unwrap(); assert_eq!(round_tripped_node.stdout.as_deref(), Some("done")); assert_eq!(round_tripped.list_node_visits("build"), vec![2]); @@ -809,6 +850,8 @@ mod tests { round_tripped.pending_control, Some(RunControlAction::Cancel) ); + assert!(serialized.get("spec").is_some()); + assert!(serialized.get("run").is_none()); } #[test] @@ -1054,7 +1097,7 @@ mod tests { #[test] fn summary_synthesizes_submitted_when_run_exists_without_status() { let state = RunProjection { - run: Some(fabro_types::RunRecord { + spec: Some(fabro_types::RunSpec { run_id: fixtures::RUN_1, settings: SettingsLayer::default(), graph: fabro_types::Graph::new("test"), @@ -1129,11 +1172,11 @@ mod tests { let value = serde_json::to_value(&state).unwrap(); assert_eq!( - value["run"]["manifest_blob"], + value["spec"]["manifest_blob"], events[0].payload.as_value()["properties"]["manifest_blob"] ); assert_eq!( - value["run"]["definition_blob"], + value["spec"]["definition_blob"], events[1].payload.as_value()["properties"]["definition_blob"] ); } diff --git a/lib/crates/fabro-store/src/serializable_projection.rs b/lib/crates/fabro-store/src/serializable_projection.rs new file mode 100644 index 000000000..073a21790 --- /dev/null +++ b/lib/crates/fabro-store/src/serializable_projection.rs @@ -0,0 +1,34 @@ +use serde::{Serialize, Serializer}; + +use crate::RunProjection; + +pub struct SerializableProjection<'a>(pub &'a RunProjection); + +impl Serialize for SerializableProjection<'_> { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut projection = self.0.clone(); + let stage_ids: Vec<_> = projection + .iter_nodes() + .map(|(stage_id, _)| stage_id.clone()) + .collect(); + + for stage_id in stage_ids { + let Some(node) = projection.node(&stage_id).cloned() else { + continue; + }; + projection.set_node(stage_id, crate::NodeState { + prompt: None, + response: None, + diff: None, + stdout: None, + stderr: None, + ..node + }); + } + + projection.serialize(serializer) + } +} diff --git a/lib/crates/fabro-store/src/slate/mod.rs b/lib/crates/fabro-store/src/slate/mod.rs index e368970b0..a7e33bebb 100644 --- a/lib/crates/fabro-store/src/slate/mod.rs +++ b/lib/crates/fabro-store/src/slate/mod.rs @@ -273,7 +273,7 @@ mod tests { use chrono::{DateTime, Utc}; use fabro_types::settings::SettingsLayer; - use fabro_types::{AttrValue, Graph, RunControlAction, RunRecord, RunStatus, StatusReason}; + use fabro_types::{AttrValue, Graph, RunControlAction, RunSpec, RunStatus, StatusReason}; use futures::TryStreamExt; use object_store::memory::InMemory; use object_store::path::Path; @@ -315,13 +315,13 @@ mod tests { (object_store, store) } - fn sample_run_record(label: &str) -> RunRecord { + fn sample_run_spec(label: &str) -> RunSpec { let mut graph = Graph::new("night-sky"); graph.attrs.insert( "goal".to_string(), AttrValue::String("map the constellations".to_string()), ); - RunRecord { + RunSpec { run_id: test_run_id(label), settings: SettingsLayer::default(), graph, @@ -357,20 +357,20 @@ mod tests { } async fn append_created(run: &RunDatabase, label: &str, created_at: DateTime) { - let run_record = sample_run_record(label); + let run_spec = sample_run_spec(label); run.append_event(&event_payload( label, &created_at.to_rfc3339(), "run.created", &serde_json::json!({ - "settings": run_record.settings, - "graph": run_record.graph, - "workflow_slug": run_record.workflow_slug, - "working_directory": run_record.working_directory, + "settings": run_spec.settings, + "graph": run_spec.graph, + "workflow_slug": run_spec.workflow_slug, + "working_directory": run_spec.working_directory, "run_dir": format!("/tmp/{label}"), - "host_repo_path": run_record.host_repo_path, - "base_branch": run_record.base_branch, - "labels": run_record.labels, + "host_repo_path": run_spec.host_repo_path, + "base_branch": run_spec.base_branch, + "labels": run_spec.labels, }), )) .await @@ -436,7 +436,7 @@ mod tests { assert_eq!(summary[1].status_reason, Some(StatusReason::Completed)); let reopened = store.open_run(&test_run_id("run-1")).await.unwrap(); - let stored = reopened.state().await.unwrap().run.unwrap(); + let stored = reopened.state().await.unwrap().spec.unwrap(); assert_eq!(stored.run_id, test_run_id("run-1")); store.delete_run(&test_run_id("run-1")).await.unwrap(); @@ -579,7 +579,7 @@ mod tests { let reader = store.open_run_reader(&test_run_id("run-1")).await.unwrap(); let state = reader.state().await.unwrap(); - assert_eq!(state.run.unwrap().run_id, test_run_id("run-1")); + assert_eq!(state.spec.unwrap().run_id, test_run_id("run-1")); run.append_event(&event_payload( "run-1", diff --git a/lib/crates/fabro-store/tests/serializable_projection.rs b/lib/crates/fabro-store/tests/serializable_projection.rs new file mode 100644 index 000000000..0e6f5dc6b --- /dev/null +++ b/lib/crates/fabro-store/tests/serializable_projection.rs @@ -0,0 +1,156 @@ +use std::collections::{BTreeMap, HashMap}; +use std::path::PathBuf; + +use chrono::{TimeZone, Utc}; +use fabro_store::{NodeState, RunProjection, SerializableProjection, StageId}; +use fabro_types::graph::Graph; +use fabro_types::run::RunSpec; +use fabro_types::settings::SettingsLayer; +use fabro_types::{ + Checkpoint, NodeStatusRecord, RunStatus, RunStatusRecord, SandboxRecord, StageStatus, + StartRecord, fixtures, +}; +use serde_json::json; + +fn sample_run_spec() -> RunSpec { + RunSpec { + run_id: fixtures::RUN_1, + settings: SettingsLayer::default(), + graph: Graph::new("ship"), + workflow_slug: Some("demo".to_string()), + working_directory: PathBuf::from("/tmp/project"), + host_repo_path: Some("/tmp/project".to_string()), + repo_origin_url: Some("https://github.com/fabro-sh/fabro.git".to_string()), + base_branch: Some("main".to_string()), + labels: HashMap::from([("team".to_string(), "platform".to_string())]), + provenance: None, + manifest_blob: None, + definition_blob: None, + } +} + +fn sample_checkpoint() -> Checkpoint { + Checkpoint { + timestamp: Utc + .with_ymd_and_hms(2026, 4, 20, 12, 0, 0) + .single() + .expect("timestamp should be representable"), + current_node: "build".to_string(), + completed_nodes: vec!["build".to_string()], + node_retries: HashMap::new(), + context_values: HashMap::new(), + node_outcomes: HashMap::new(), + next_node_id: Some("ship".to_string()), + git_commit_sha: Some("abc123".to_string()), + loop_failure_signatures: HashMap::new(), + restart_failure_signatures: HashMap::new(), + node_visits: HashMap::from([("build".to_string(), 2usize)]), + } +} + +#[test] +fn serializable_projection_round_trips_and_trims_bulky_node_fields() { + let stage_id = StageId::new("build", 2); + let mut projection = RunProjection::default(); + projection.spec = Some(sample_run_spec()); + projection.start = Some(StartRecord { + run_id: fixtures::RUN_1, + start_time: Utc + .with_ymd_and_hms(2026, 4, 20, 12, 0, 0) + .single() + .expect("start_time should be representable"), + run_branch: Some("fabro/run/demo".to_string()), + base_sha: Some("deadbeef".to_string()), + }); + projection.status = Some(RunStatusRecord::new(RunStatus::Running, None)); + projection.checkpoint = Some(sample_checkpoint()); + projection.sandbox = Some(SandboxRecord { + provider: "local".to_string(), + working_directory: "/tmp/project".to_string(), + identifier: Some("sandbox-1".to_string()), + host_working_directory: None, + container_mount_point: None, + }); + projection.pending_interviews = BTreeMap::new(); + projection.set_node(stage_id.clone(), NodeState { + prompt: Some("plan the work".to_string()), + response: Some("done".to_string()), + status: Some(NodeStatusRecord { + status: StageStatus::Success, + notes: Some("ok".to_string()), + failure_reason: None, + timestamp: Utc + .with_ymd_and_hms(2026, 4, 20, 12, 1, 0) + .single() + .expect("timestamp should be representable"), + }), + provider_used: Some(json!({ "provider": "openai", "model": "gpt-5.4" })), + diff: Some("diff --git a/a b/a".to_string()), + script_invocation: Some(json!({ "command": "cargo test" })), + script_timing: Some(json!({ "duration_ms": 10 })), + parallel_results: Some(json!([{ "stage": "fanout@1" }])), + stdout: Some("stdout".to_string()), + stderr: Some("stderr".to_string()), + }); + + let serialized = serde_json::to_value(SerializableProjection(&projection)) + .expect("projection should serialize"); + let round_tripped: RunProjection = + serde_json::from_value(serialized).expect("serialized projection should deserialize"); + let node = round_tripped.node(&stage_id).expect("node should remain"); + + assert_eq!(round_tripped.spec().map(RunSpec::id), Some(fixtures::RUN_1)); + assert_eq!( + round_tripped + .current_checkpoint() + .expect("checkpoint should remain") + .current_node, + "build" + ); + assert_eq!(round_tripped.status(), Some(RunStatus::Running)); + assert!(!round_tripped.is_terminal()); + assert_eq!(node.prompt, None); + assert_eq!(node.response, None); + assert_eq!(node.diff, None); + assert_eq!(node.stdout, None); + assert_eq!(node.stderr, None); + assert_eq!( + node.provider_used, + Some(json!({ "provider": "openai", "model": "gpt-5.4" })) + ); + assert_eq!( + node.script_invocation, + Some(json!({ "command": "cargo test" })) + ); + assert_eq!(node.script_timing, Some(json!({ "duration_ms": 10 }))); + assert_eq!( + node.parallel_results, + Some(json!([{ "stage": "fanout@1" }])) + ); +} + +#[test] +fn projection_query_methods_expose_common_state() { + let mut projection = RunProjection::default(); + projection.spec = Some(sample_run_spec()); + projection.status = Some(RunStatusRecord::new(RunStatus::Archived, None)); + projection.checkpoint = Some(sample_checkpoint()); + projection.pending_interviews = BTreeMap::from([( + "q-1".to_string(), + fabro_store::PendingInterviewRecord::default(), + )]); + + assert_eq!( + projection.spec().map(RunSpec::workflow_slug), + Some(Some("demo")) + ); + assert_eq!(projection.status(), Some(RunStatus::Archived)); + assert!(projection.is_terminal()); + assert_eq!( + projection + .current_checkpoint() + .map(|checkpoint| checkpoint.current_node.as_str()), + Some("build") + ); + assert!(projection.pending_interviews().contains_key("q-1")); +} diff --git a/lib/crates/fabro-types/src/lib.rs b/lib/crates/fabro-types/src/lib.rs index 6ff85e5b1..c10b41009 100644 --- a/lib/crates/fabro-types/src/lib.rs +++ b/lib/crates/fabro-types/src/lib.rs @@ -45,7 +45,7 @@ pub use retro::{ OpenItemKind, Retro, RetroNarrative, SmoothnessRating, StageRetro, }; pub use run::{ - RunAuthMethod, RunClientProvenance, RunProvenance, RunRecord, RunServerProvenance, + RunAuthMethod, RunClientProvenance, RunProvenance, RunServerProvenance, RunSpec, RunSubjectProvenance, }; pub use run_blob_id::RunBlobId; diff --git a/lib/crates/fabro-types/src/run.rs b/lib/crates/fabro-types/src/run.rs index ec9fcd6b7..7a3e9ff4c 100644 --- a/lib/crates/fabro-types/src/run.rs +++ b/lib/crates/fabro-types/src/run.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; @@ -49,7 +49,7 @@ pub struct RunProvenance { } #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RunRecord { +pub struct RunSpec { pub run_id: RunId, pub settings: SettingsLayer, pub graph: Graph, @@ -71,3 +71,50 @@ pub struct RunRecord { #[serde(default, skip_serializing_if = "Option::is_none")] pub definition_blob: Option, } + +impl RunSpec { + #[must_use] + pub fn id(&self) -> RunId { + self.run_id + } + + #[must_use] + pub fn graph(&self) -> &Graph { + &self.graph + } + + #[must_use] + pub fn settings(&self) -> &SettingsLayer { + &self.settings + } + + #[must_use] + pub fn workflow_slug(&self) -> Option<&str> { + self.workflow_slug.as_deref() + } + + #[must_use] + pub fn working_directory(&self) -> &Path { + &self.working_directory + } + + #[must_use] + pub fn labels(&self) -> &HashMap { + &self.labels + } + + #[must_use] + pub fn host_repo_path(&self) -> Option<&str> { + self.host_repo_path.as_deref() + } + + #[must_use] + pub fn repo_origin_url(&self) -> Option<&str> { + self.repo_origin_url.as_deref() + } + + #[must_use] + pub fn base_branch(&self) -> Option<&str> { + self.base_branch.as_deref() + } +} diff --git a/lib/crates/fabro-types/tests/run_spec_methods.rs b/lib/crates/fabro-types/tests/run_spec_methods.rs new file mode 100644 index 000000000..20a4f2942 --- /dev/null +++ b/lib/crates/fabro-types/tests/run_spec_methods.rs @@ -0,0 +1,45 @@ +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +use fabro_types::fixtures; +use fabro_types::graph::Graph; +use fabro_types::run::RunSpec; +use fabro_types::settings::SettingsLayer; + +fn sample_run_spec() -> RunSpec { + RunSpec { + run_id: fixtures::RUN_1, + settings: SettingsLayer::default(), + graph: Graph::new("ship"), + workflow_slug: Some("demo".to_string()), + working_directory: PathBuf::from("/tmp/project"), + host_repo_path: Some("/tmp/project".to_string()), + repo_origin_url: Some("https://github.com/fabro-sh/fabro.git".to_string()), + base_branch: Some("main".to_string()), + labels: HashMap::from([("team".to_string(), "platform".to_string())]), + provenance: None, + manifest_blob: None, + definition_blob: None, + } +} + +#[test] +fn run_spec_getters_return_declared_fields() { + let run_spec = sample_run_spec(); + + assert_eq!(run_spec.id(), fixtures::RUN_1); + assert_eq!(run_spec.graph().name, "ship"); + assert_eq!(run_spec.settings(), &SettingsLayer::default()); + assert_eq!(run_spec.workflow_slug(), Some("demo")); + assert_eq!(run_spec.working_directory(), Path::new("/tmp/project")); + assert_eq!( + run_spec.labels().get("team").map(String::as_str), + Some("platform") + ); + assert_eq!(run_spec.host_repo_path(), Some("/tmp/project")); + assert_eq!( + run_spec.repo_origin_url(), + Some("https://github.com/fabro-sh/fabro.git") + ); + assert_eq!(run_spec.base_branch(), Some("main")); +} diff --git a/lib/crates/fabro-types/tests/run_record_serde.rs b/lib/crates/fabro-types/tests/run_spec_serde.rs similarity index 95% rename from lib/crates/fabro-types/tests/run_record_serde.rs rename to lib/crates/fabro-types/tests/run_spec_serde.rs index 9212dc640..8b9e59d05 100644 --- a/lib/crates/fabro-types/tests/run_record_serde.rs +++ b/lib/crates/fabro-types/tests/run_spec_serde.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use fabro_types::fixtures; use fabro_types::graph::Graph; -use fabro_types::run::RunRecord; +use fabro_types::run::RunSpec; use fabro_types::settings::run::{RunGoalLayer, RunLayer}; use fabro_types::settings::server::{ GithubIntegrationLayer, ServerIntegrationsLayer, ServerLayer, ServerStorageLayer, @@ -37,8 +37,8 @@ fn templated_settings() -> SettingsLayer { } #[test] -fn run_record_round_trips_templated_settings() { - let record = RunRecord { +fn run_spec_round_trips_templated_settings() { + let record = RunSpec { run_id: fixtures::RUN_1, settings: templated_settings(), graph: Graph::new("ship"), @@ -54,7 +54,7 @@ fn run_record_round_trips_templated_settings() { }; let json = serde_json::to_value(&record).expect("record should serialize"); - let round_trip: RunRecord = + let round_trip: RunSpec = serde_json::from_value(json.clone()).expect("record should deserialize"); assert_eq!( diff --git a/lib/crates/fabro-workflow/src/error.rs b/lib/crates/fabro-workflow/src/error.rs index 2959fba1a..cd957786b 100644 --- a/lib/crates/fabro-workflow/src/error.rs +++ b/lib/crates/fabro-workflow/src/error.rs @@ -474,14 +474,14 @@ mod tests { let source = serde_json::from_str::("not json").unwrap_err(); let source_message = source.to_string(); let fabro_error = Error::from(MetadataError::Deserialize { - entity: "run record", + entity: "run spec", branch: "fabro/meta/run-1".to_string(), source, }); assert!(matches!(fabro_error, Error::Engine { .. })); let message = fabro_error.to_string(); - assert!(message.contains("deserialize run record on branch fabro/meta/run-1")); + assert!(message.contains("deserialize run spec on branch fabro/meta/run-1")); assert!(message.contains(&source_message)); } diff --git a/lib/crates/fabro-workflow/src/git.rs b/lib/crates/fabro-workflow/src/git.rs index 3c2bee10d..213ce94c0 100644 --- a/lib/crates/fabro-workflow/src/git.rs +++ b/lib/crates/fabro-workflow/src/git.rs @@ -531,15 +531,15 @@ mod tests { .unwrap(); let state = run.state().await.unwrap(); - let files = RunDump::metadata_checkpoint(&state).git_entries().unwrap(); + let files = RunDump::from_projection(&state).git_entries().unwrap(); let paths: Vec<&str> = files.iter().map(|(path, _)| path.as_str()).collect(); - assert!(paths.contains(&"nodes/work-visit_2/prompt.md")); - assert!(paths.contains(&"nodes/work-visit_2/response.md")); - assert!(paths.contains(&"nodes/work-visit_2/status.json")); - assert!(paths.contains(&"nodes/work-visit_2/provider_used.json")); - assert!(paths.contains(&"nodes/work-visit_2/script_invocation.json")); - assert!(paths.contains(&"nodes/work-visit_2/script_timing.json")); - assert!(paths.contains(&"nodes/work-visit_2/parallel_results.json")); + assert!(paths.contains(&"stages/work@2/prompt.md")); + assert!(paths.contains(&"stages/work@2/response.md")); + assert!(paths.contains(&"stages/work@2/status.json")); + assert!(paths.contains(&"stages/work@2/provider_used.json")); + assert!(paths.contains(&"stages/work@2/script_invocation.json")); + assert!(paths.contains(&"stages/work@2/script_timing.json")); + assert!(paths.contains(&"stages/work@2/parallel_results.json")); } #[test] diff --git a/lib/crates/fabro-workflow/src/lifecycle/git.rs b/lib/crates/fabro-workflow/src/lifecycle/git.rs index 3900a545b..aaf468b10 100644 --- a/lib/crates/fabro-workflow/src/lifecycle/git.rs +++ b/lib/crates/fabro-workflow/src/lifecycle/git.rs @@ -93,7 +93,7 @@ impl RunLifecycle for GitLifecycle { let git_author = self.run_options.git_author(); let store = MetadataStore::new(repo_path, &git_author); let state = self.run_store.state().await.ok(); - let init_dump = state.as_ref().map(RunDump::metadata_init); + let init_dump = state.as_ref().map(RunDump::from_projection); let init_entries = init_dump .as_ref() .and_then(|dump| dump.git_entries().ok()) @@ -148,34 +148,56 @@ impl RunLifecycle for GitLifecycle { HashMap::new(), None, ); - if let Ok(cp_json) = serde_json::to_vec_pretty(&checkpoint) { - let mut extra_entries: Vec<(String, Vec)> = Vec::new(); - if let Ok(store_state) = self.run_store.state().await { - if let Ok(mut dump_entries) = - RunDump::metadata_checkpoint(&store_state).git_entries() - { - extra_entries.append(&mut dump_entries); + match self.run_store.state().await { + Ok(mut snapshot_state) => { + snapshot_state.checkpoint = Some(checkpoint); + let dump = RunDump::from_projection(&snapshot_state); + match dump.git_entries() { + Ok(dump_entries) => { + let refs: Vec<(&str, &[u8])> = dump_entries + .iter() + .map(|(path, bytes)| (path.as_str(), bytes.as_slice())) + .collect(); + match store.write_snapshot( + &self.run_id.to_string(), + &refs, + "checkpoint", + ) { + Ok(sha) => Some(sha), + Err(e) => { + self.emitter.emit(&Event::RunNotice { + level: RunNoticeLevel::Warn, + code: "checkpoint_metadata_write_failed".to_string(), + message: format!( + "[node: {node_id}] metadata checkpoint write failed: {e}" + ), + }); + None + } + } + } + Err(e) => { + self.emitter.emit(&Event::RunNotice { + level: RunNoticeLevel::Warn, + code: "checkpoint_metadata_write_failed".to_string(), + message: format!( + "[node: {node_id}] metadata checkpoint serialization failed: {e}" + ), + }); + None + } } } - let extra_refs: Vec<(&str, &[u8])> = extra_entries - .iter() - .map(|(k, v)| (k.as_str(), v.as_slice())) - .collect(); - match store.write_checkpoint(&self.run_id.to_string(), &cp_json, &extra_refs) { - Ok(sha) => Some(sha), - Err(e) => { - self.emitter.emit(&Event::RunNotice { - level: RunNoticeLevel::Warn, - code: "checkpoint_metadata_write_failed".to_string(), - message: format!( - "[node: {node_id}] metadata checkpoint write failed: {e}" - ), - }); - None - } + Err(e) => { + self.emitter.emit(&Event::RunNotice { + level: RunNoticeLevel::Warn, + code: "checkpoint_metadata_write_failed".to_string(), + message: format!( + "[node: {node_id}] failed to load run state for metadata snapshot: {e}" + ), + }); + None } - } else { - None } } else { None diff --git a/lib/crates/fabro-workflow/src/operations/create.rs b/lib/crates/fabro-workflow/src/operations/create.rs index a00695024..33c4f167b 100644 --- a/lib/crates/fabro-workflow/src/operations/create.rs +++ b/lib/crates/fabro-workflow/src/operations/create.rs @@ -27,7 +27,7 @@ use crate::event::{Event, append_event, to_run_event_at}; use crate::file_resolver::FileResolver; use crate::pipeline::types::PersistOptions; use crate::pipeline::{self, Persisted, TransformOptions, Validated}; -use crate::records::RunRecord; +use crate::records::RunSpec; use crate::run_lookup::default_scratch_base; use crate::run_materialization::materialize_run; use crate::transforms::Transform; @@ -202,7 +202,7 @@ async fn persist_created_run( submitted_manifest_bytes: Option<&[u8]>, accepted_definition: Option<&RunDefinition>, ) -> Result<(), Error> { - let record = persisted.run_record(); + let record = persisted.run_spec(); let run_store = match store.create_run(&record.run_id).await { Ok(run_store) => run_store, Err(err) => store @@ -409,7 +409,7 @@ fn persist_validated( let run_id = run_id.unwrap_or_else(RunId::new); let run_dir = run_dir.unwrap_or_else(|| default_run_dir(&run_id)); - let run_record = RunRecord { + let run_spec = RunSpec { run_id, settings, graph: validated.graph().clone(), @@ -424,10 +424,7 @@ fn persist_validated( definition_blob: None, }; - pipeline::persist(validated, PersistOptions { - run_dir, - run_record, - }) + pipeline::persist(validated, PersistOptions { run_dir, run_spec }) } pub(crate) fn default_run_dir(run_id: &RunId) -> PathBuf { @@ -812,9 +809,9 @@ mod tests { .unwrap(); assert_eq!(created.run_id, fixtures::RUN_1); - assert_eq!(created.persisted.run_record().graph.goal(), "override goal"); + assert_eq!(created.persisted.run_spec().graph.goal(), "override goal"); assert_eq!( - fabro_config::resolve_run_from_file(&created.persisted.run_record().settings) + fabro_config::resolve_run_from_file(&created.persisted.run_spec().settings) .unwrap() .model .name @@ -824,7 +821,7 @@ mod tests { Some("claude-sonnet-4-6") ); assert_eq!( - fabro_config::resolve_run_from_file(&created.persisted.run_record().settings) + fabro_config::resolve_run_from_file(&created.persisted.run_spec().settings) .unwrap() .model .provider @@ -834,7 +831,7 @@ mod tests { Some("anthropic") ); assert_eq!( - match fabro_config::resolve_run_from_file(&created.persisted.run_record().settings) + match fabro_config::resolve_run_from_file(&created.persisted.run_spec().settings) .unwrap() .goal { @@ -847,13 +844,13 @@ mod tests { Some("override goal") ); assert!( - fabro_config::resolve_run_from_file(&created.persisted.run_record().settings) + fabro_config::resolve_run_from_file(&created.persisted.run_spec().settings) .unwrap() .pull_request .is_none() ); assert_eq!( - created.persisted.run_record().workflow_slug.as_deref(), + created.persisted.run_spec().workflow_slug.as_deref(), Some("slug") ); let run_store = store.open_run(&fixtures::RUN_1).await.unwrap(); @@ -906,13 +903,13 @@ mod tests { .await .unwrap(); - assert_eq!(created.persisted.run_record().working_directory, workspace); + assert_eq!(created.persisted.run_spec().working_directory, workspace); assert_eq!( - created.persisted.run_record().host_repo_path.as_deref(), + created.persisted.run_spec().host_repo_path.as_deref(), Some( created .persisted - .run_record() + .run_spec() .working_directory .to_string_lossy() .as_ref() @@ -946,7 +943,7 @@ mod tests { .unwrap(); assert_eq!( - created.persisted.run_record().repo_origin_url.as_deref(), + created.persisted.run_spec().repo_origin_url.as_deref(), Some("https://github.com/acme/widgets") ); } @@ -1077,7 +1074,7 @@ mod tests { let run_store = store.open_run_reader(&created.run_id).await.unwrap(); let state = run_store.state().await.unwrap(); - let run = state.run.expect("run should be projected"); + let run = state.spec.expect("run should be projected"); let provenance = run.provenance.expect("provenance should be projected"); assert_eq!(provenance.server.unwrap().version, "0.9.0"); diff --git a/lib/crates/fabro-workflow/src/operations/fork.rs b/lib/crates/fabro-workflow/src/operations/fork.rs index 8f039fa04..32037a39c 100644 --- a/lib/crates/fabro-workflow/src/operations/fork.rs +++ b/lib/crates/fabro-workflow/src/operations/fork.rs @@ -1,12 +1,14 @@ use anyhow::{Context, Result}; use fabro_checkpoint::branch::BranchStore; use fabro_checkpoint::git::Store; +use fabro_store::RunProjection; use fabro_types::RunId; use git2::{Oid, Signature}; use super::rewind::{RewindTarget, TimelineEntry, build_timeline}; use crate::git::{MetadataStore, RUN_BRANCH_PREFIX, push_run_branches}; -use crate::records::{Checkpoint, RunRecord, StartRecord}; +use crate::records::{Checkpoint, RunSpec, StartRecord}; +use crate::run_dump::RunDump; #[derive(Debug, Clone)] pub struct ForkRunInput { @@ -65,27 +67,20 @@ fn fork_from_entry( .ensure_branch() .map_err(|e| anyhow::anyhow!("failed to create metadata branch: {e}"))?; - let source_entries = source_bs - .read_entries(&["run.json", "start.json", "sandbox.json"]) - .map_err(|e| anyhow::anyhow!("failed to read source metadata: {e}"))?; + let source_projection = source_bs + .read_entry("run.json") + .map_err(|e| anyhow::anyhow!("failed to read source metadata: {e}"))? + .context("source run has no run.json") + .and_then(|bytes| { + serde_json::from_slice::(&bytes) + .context("failed to parse source run.json") + })?; - let mut run_record_bytes = None; - let mut sandbox_bytes = None; - for (path, data) in source_entries { - match path { - "run.json" => run_record_bytes = Some(data), - "sandbox.json" => sandbox_bytes = Some(data), - _ => {} - } - } - let run_record_bytes = - run_record_bytes.ok_or_else(|| anyhow::anyhow!("source run has no run.json"))?; - - let mut run_record: RunRecord = - serde_json::from_slice(&run_record_bytes).context("failed to parse source run.json")?; - run_record.run_id = new_run_id; - let new_run_record_bytes = - serde_json::to_vec_pretty(&run_record).context("failed to serialize new run.json")?; + let mut run_spec: RunSpec = source_projection + .spec + .clone() + .context("source run projection has no spec")?; + run_spec.run_id = new_run_id; let now = new_run_id.created_at(); let start_record = StartRecord { @@ -94,38 +89,60 @@ fn fork_from_entry( run_branch: Some(new_run_branch.clone()), base_sha: None, }; - let new_start_record_bytes = - serde_json::to_vec_pretty(&start_record).context("failed to serialize new start.json")?; + + let mut init_projection = RunProjection::default(); + init_projection.spec = Some(run_spec.clone()); + init_projection + .graph_source + .clone_from(&source_projection.graph_source); + init_projection.start = Some(start_record.clone()); + init_projection + .sandbox + .clone_from(&source_projection.sandbox); let checkpoint_bytes = store - .read_blob_at(entry.metadata_commit_oid, "checkpoint.json") - .map_err(|e| anyhow::anyhow!("failed to read checkpoint blob: {e}"))? + .read_blob_at(entry.metadata_commit_oid, "run.json") + .map_err(|e| anyhow::anyhow!("failed to read checkpoint snapshot: {e}"))? .ok_or_else(|| { anyhow::anyhow!( - "no checkpoint.json at metadata commit {}", + "no run.json at metadata commit {}", entry.metadata_commit_oid ) })?; - let mut checkpoint: Checkpoint = serde_json::from_slice(&checkpoint_bytes) - .context("failed to parse source checkpoint.json")?; + let mut checkpoint_projection: RunProjection = serde_json::from_slice(&checkpoint_bytes) + .context("failed to parse source checkpoint snapshot")?; + let mut checkpoint: Checkpoint = checkpoint_projection + .checkpoint + .clone() + .context("source checkpoint snapshot has no checkpoint")?; checkpoint.git_commit_sha.clone_from(&entry.run_commit_sha); - let checkpoint_bytes = - serde_json::to_vec_pretty(&checkpoint).context("failed to serialize checkpoint.json")?; + checkpoint_projection.spec = Some(run_spec); + checkpoint_projection.graph_source = source_projection.graph_source; + checkpoint_projection.start = Some(start_record); + checkpoint_projection.sandbox = source_projection.sandbox; + checkpoint_projection.checkpoint = Some(checkpoint); - let mut init_entries: Vec<(&str, &[u8])> = vec![("run.json", &new_run_record_bytes)]; - init_entries.push(("start.json", &new_start_record_bytes)); - if let Some(ref sandbox) = sandbox_bytes { - init_entries.push(("sandbox.json", sandbox)); - } + let init_entries = RunDump::from_projection(&init_projection) + .git_entries() + .context("failed to build init metadata snapshot")?; + let init_refs: Vec<(&str, &[u8])> = init_entries + .iter() + .map(|(path, bytes)| (path.as_str(), bytes.as_slice())) + .collect(); + new_bs + .write_entries(&init_refs, "init run") + .map_err(|e| anyhow::anyhow!("failed to write init metadata snapshot: {e}"))?; + let checkpoint_entries = RunDump::from_projection(&checkpoint_projection) + .git_entries() + .context("failed to build checkpoint metadata snapshot")?; + let checkpoint_refs: Vec<(&str, &[u8])> = checkpoint_entries + .iter() + .map(|(path, bytes)| (path.as_str(), bytes.as_slice())) + .collect(); new_bs - .write_entries(&init_entries, "init run") - .map_err(|e| anyhow::anyhow!("failed to write init metadata entries: {e}"))?; - let mut checkpoint_entries: Vec<(&str, &[u8])> = vec![("checkpoint.json", &checkpoint_bytes)]; - checkpoint_entries.extend(init_entries.iter().copied()); - new_bs - .write_entries(&checkpoint_entries, "checkpoint") - .map_err(|e| anyhow::anyhow!("failed to write metadata entries: {e}"))?; + .write_entries(&checkpoint_refs, "checkpoint") + .map_err(|e| anyhow::anyhow!("failed to write metadata snapshot: {e}"))?; if push { let source_run_branch = format!("{RUN_BRANCH_PREFIX}{source_run_id}"); @@ -147,6 +164,7 @@ fn fork_from_entry( mod tests { use std::str::FromStr; + use fabro_store::RunProjection; use fabro_types::RunId; use git2::Oid; @@ -158,27 +176,31 @@ mod tests { value.parse().unwrap() } - fn make_run_record_json(run_id: &RunId) -> Vec { - let record = serde_json::json!({ - "run_id": run_id.to_string(), - "created_at": "2025-01-01T00:00:00Z", - "settings": {}, - "graph": { - "name": "test_workflow", - "nodes": { - "start": {"id": "start", "attrs": {}}, - "build": {"id": "build", "attrs": {}}, - "test": {"id": "test", "attrs": {}} + fn make_run_projection(run_id: &RunId) -> RunProjection { + let mut projection = RunProjection::default(); + projection.spec = Some( + serde_json::from_value(serde_json::json!({ + "run_id": run_id.to_string(), + "created_at": "2025-01-01T00:00:00Z", + "settings": {}, + "graph": { + "name": "test_workflow", + "nodes": { + "start": {"id": "start", "attrs": {}}, + "build": {"id": "build", "attrs": {}}, + "test": {"id": "test", "attrs": {}} + }, + "edges": [ + {"from": "start", "to": "build", "attrs": {}}, + {"from": "build", "to": "test", "attrs": {}} + ], + "attrs": {} }, - "edges": [ - {"from": "start", "to": "build", "attrs": {}}, - {"from": "build", "to": "test", "attrs": {}} - ], - "attrs": {} - }, - "working_directory": "/tmp/test", - }); - serde_json::to_vec_pretty(&record).unwrap() + "working_directory": "/tmp/test", + })) + .unwrap(), + ); + projection } fn make_start_record_json(run_id: &RunId) -> Vec { @@ -220,17 +242,25 @@ mod tests { let bs = BranchStore::new(store, &meta_branch, &sig); bs.ensure_branch().unwrap(); - let run_record = make_run_record_json(run_id); - let start_record = make_start_record_json(run_id); - bs.write_entries( - &[("run.json", &run_record), ("start.json", &start_record)], - "init run", - ) - .unwrap(); + let mut init_projection = make_run_projection(run_id); + init_projection.start = + Some(serde_json::from_slice(&make_start_record_json(run_id)).unwrap()); + let init_json = serde_json::to_vec_pretty(&init_projection).unwrap(); + bs.write_entries(&[("run.json", &init_json)], "init run") + .unwrap(); for (i, node) in nodes.iter().enumerate() { - let cp = make_checkpoint_json(node, 1, Some(&run_oids[i].to_string())); - bs.write_entry("checkpoint.json", &cp, "checkpoint") + let mut projection = init_projection.clone(); + projection.checkpoint = Some( + serde_json::from_slice(&make_checkpoint_json( + node, + 1, + Some(&run_oids[i].to_string()), + )) + .unwrap(), + ); + let checkpoint_json = serde_json::to_vec_pretty(&projection).unwrap(); + bs.write_entry("run.json", &checkpoint_json, "checkpoint") .unwrap(); } @@ -259,8 +289,11 @@ mod tests { let sig = test_sig(); let bs = BranchStore::new(&store, &new_meta_branch, &sig); let run_json = bs.read_entry("run.json").unwrap().unwrap(); - let run_record: RunRecord = serde_json::from_slice(&run_json).unwrap(); - assert_eq!(run_record.run_id, new_run_id); + let run_spec: RunProjection = serde_json::from_slice(&run_json).unwrap(); + assert_eq!( + run_spec.spec.as_ref().map(|run| run.run_id), + Some(new_run_id) + ); let timeline = build_timeline(&store, &new_run_id.to_string()).unwrap(); assert_eq!(timeline.entries.len(), 1); @@ -279,13 +312,15 @@ mod tests { let meta_branch = MetadataStore::branch_name(&run_id.to_string()); let bs = BranchStore::new(&store, &meta_branch, &sig); bs.ensure_branch().unwrap(); - bs.write_entry("run.json", &make_run_record_json(&run_id), "init") + let init_projection = serde_json::to_vec_pretty(&make_run_projection(&run_id)).unwrap(); + bs.write_entry("run.json", &init_projection, "init") .unwrap(); - let cp = make_checkpoint_json("start", 1, None); - let oid = bs - .write_entry("checkpoint.json", &cp, "checkpoint") - .unwrap(); + let mut checkpoint_projection = make_run_projection(&run_id); + checkpoint_projection.checkpoint = + Some(serde_json::from_slice(&make_checkpoint_json("start", 1, None)).unwrap()); + let cp = serde_json::to_vec_pretty(&checkpoint_projection).unwrap(); + let oid = bs.write_entry("run.json", &cp, "checkpoint").unwrap(); let entry = TimelineEntry { ordinal: 1, node_name: "start".to_string(), diff --git a/lib/crates/fabro-workflow/src/operations/rebuild_meta.rs b/lib/crates/fabro-workflow/src/operations/rebuild_meta.rs index 0ce2a8fe1..50408dc78 100644 --- a/lib/crates/fabro-workflow/src/operations/rebuild_meta.rs +++ b/lib/crates/fabro-workflow/src/operations/rebuild_meta.rs @@ -5,8 +5,8 @@ use std::path::PathBuf; use anyhow::{Context, Result, bail}; use fabro_checkpoint::branch::BranchStore; use fabro_checkpoint::git::Store as GitStore; -use fabro_store::{Database as DurableStore, RunDatabase as DurableRunStore}; -use fabro_types::{RunId, StageId}; +use fabro_store::{Database as DurableStore, RunDatabase as DurableRunStore, RunProjection}; +use fabro_types::{EventBody, RunEvent, RunId}; use git2::{Repository, Signature}; use tokio::task::spawn_blocking; use ulid::Ulid; @@ -14,6 +14,7 @@ use ulid::Ulid; use super::rewind::{self, RunTimeline, build_timeline}; use crate::git::MetadataStore; use crate::records::Checkpoint; +use crate::run_dump::RunDump; pub async fn rebuild_metadata_branch( git_store: &GitStore, @@ -25,11 +26,10 @@ pub async fn rebuild_metadata_branch( bail!("metadata branch already exists for run {run_id}"); } - let state = run_store.state().await?; - let run_record = state - .run - .clone() - .ok_or_else(|| anyhow::anyhow!("run record not found for {run_id}"))?; + let events = run_store.list_events().await?; + if events.is_empty() { + bail!("run spec not found for {run_id}"); + } let sig = Signature::now("Fabro", "noreply@fabro.sh")?; let scratch_branch = format!("fabro/meta-rebuild/{run_id}/{}", Ulid::new()); @@ -37,99 +37,68 @@ pub async fn rebuild_metadata_branch( let result = async { bs.ensure_branch()?; + let mut projection = RunProjection::default(); + let mut latest_init_snapshot = None; + let mut init_written = false; + let mut checkpoint_snapshots: Vec<(u32, RunProjection)> = Vec::new(); - let mut init_entries = Vec::new(); - init_entries.push(( - "run.json".to_string(), - serde_json::to_vec_pretty(&run_record)?, - )); - if let Some(start) = state.start.clone() { - init_entries.push(("start.json".to_string(), serde_json::to_vec_pretty(&start)?)); - } - if let Some(sandbox) = state.sandbox.clone() { - init_entries.push(( - "sandbox.json".to_string(), - serde_json::to_vec_pretty(&sandbox)?, - )); - } - write_entries(&bs, &init_entries, "init run")?; + for event in &events { + let stored = RunEvent::from_ref(event.payload.as_value()) + .map_err(|err| anyhow::anyhow!("invalid stored event: {err}"))?; + let is_checkpoint = matches!(stored.body, EventBody::CheckpointCompleted(_)); - let mut checkpoints = state.checkpoints.clone(); - backfill_missing_checkpoint_shas(git_store, run_id, &mut checkpoints); - - for (_seq, checkpoint) in checkpoints { - let mut entries = Vec::new(); - entries.push(( - "checkpoint.json".to_string(), - serde_json::to_vec_pretty(&checkpoint)?, - )); - - for node_id in &checkpoint.completed_nodes { - let max_visit = checkpoint.node_visits.get(node_id).copied().unwrap_or(1); - for visit in 1..=max_visit { - let visit = u32::try_from(visit) - .with_context(|| format!("visit {visit} for node {node_id} exceeds u32"))?; - let Some(node) = state.node(&StageId::new(node_id, visit)).cloned() else { - continue; - }; - - if let Some(prompt) = node.prompt { - entries.push(( - node_file_path(node_id, visit, "prompt.md"), - prompt.into_bytes(), - )); - } - if let Some(response) = node.response { - entries.push(( - node_file_path(node_id, visit, "response.md"), - response.into_bytes(), - )); - } - if let Some(status) = node.status { - entries.push(( - node_file_path(node_id, visit, "status.json"), - serde_json::to_vec_pretty(&status)?, - )); - } - if let Some(provider_used) = node.provider_used { - entries.push(( - node_file_path(node_id, visit, "provider_used.json"), - serde_json::to_vec_pretty(&provider_used)?, - )); - } - if let Some(diff) = node.diff { - entries.push(( - node_file_path(node_id, visit, "diff.patch"), - diff.into_bytes(), - )); - } - if let Some(script_invocation) = node.script_invocation { - entries.push(( - node_file_path(node_id, visit, "script_invocation.json"), - serde_json::to_vec_pretty(&script_invocation)?, - )); - } - if let Some(script_timing) = node.script_timing { - entries.push(( - node_file_path(node_id, visit, "script_timing.json"), - serde_json::to_vec_pretty(&script_timing)?, - )); - } - if let Some(parallel_results) = node.parallel_results { - entries.push(( - node_file_path(node_id, visit, "parallel_results.json"), - serde_json::to_vec_pretty(¶llel_results)?, - )); - } - } + if !is_checkpoint && projection.spec.is_some() { + latest_init_snapshot = Some(projection.clone()); } - write_entries(&bs, &entries, "checkpoint")?; + projection.apply_event(event)?; + + if is_checkpoint { + if !init_written { + let init_snapshot = latest_init_snapshot.clone().unwrap_or_else(|| { + let mut snapshot = projection.clone(); + snapshot.checkpoint = None; + snapshot.checkpoints.clear(); + snapshot + }); + write_projection_snapshot(&bs, &init_snapshot, "init run")?; + init_written = true; + } + checkpoint_snapshots.push((event.seq, projection.clone())); + } } - if let Some(retro) = state.retro.clone() { - let entries = vec![("retro.json".to_string(), serde_json::to_vec_pretty(&retro)?)]; - write_entries(&bs, &entries, "finalize run")?; + if projection.spec.is_none() { + bail!("run spec not found for {run_id}"); + } + + if !init_written { + write_projection_snapshot(&bs, &projection, "init run")?; + } + + let mut checkpoints: Vec<(u32, Checkpoint)> = checkpoint_snapshots + .iter() + .map(|(seq, snapshot)| { + let checkpoint = snapshot + .checkpoint + .clone() + .expect("checkpoint snapshots must include projection.checkpoint"); + (*seq, checkpoint) + }) + .collect(); + backfill_missing_checkpoint_shas(git_store, run_id, &mut checkpoints); + + for ((_, snapshot), (_, checkpoint)) in checkpoint_snapshots.iter_mut().zip(checkpoints) { + snapshot.checkpoint = Some(checkpoint); + write_projection_snapshot(&bs, snapshot, "checkpoint")?; + } + + if projection.conclusion.is_some() + || projection.retro.is_some() + || projection.retro_prompt.is_some() + || projection.retro_response.is_some() + { + write_projection_snapshot(&bs, &projection, "finalize run")?; } Ok::<(), anyhow::Error>(()) @@ -248,6 +217,17 @@ fn write_entries( Ok(()) } +fn write_projection_snapshot( + branch_store: &BranchStore<'_>, + projection: &RunProjection, + message: &str, +) -> Result<()> { + let entries = RunDump::from_projection(projection) + .git_entries() + .context("failed to serialize metadata projection snapshot")?; + write_entries(branch_store, &entries, message) +} + fn backfill_missing_checkpoint_shas( git_store: &GitStore, run_id: &RunId, @@ -280,14 +260,6 @@ fn backfill_missing_checkpoint_shas( } } -fn node_file_path(node_id: &str, visit: u32, filename: &str) -> String { - if visit <= 1 { - format!("nodes/{node_id}/{filename}") - } else { - format!("nodes/{node_id}-visit_{visit}/{filename}") - } -} - fn find_run_id_by_prefix_in_refs(repo: &Repository, prefix: &str) -> Result> { let refs = repo.references()?; let pattern = "refs/heads/fabro/meta/"; @@ -367,14 +339,14 @@ mod tests { use chrono::{TimeZone, Utc}; use fabro_graphviz::graph::Graph; - use fabro_store::{Database, StageId}; + use fabro_store::{Database, RunProjection, StageId}; use fabro_types::settings::SettingsLayer; - use fabro_types::{RunId, RunRecord, SandboxRecord, StartRecord, fixtures}; + use fabro_types::{RunId, RunSpec, SandboxRecord, StartRecord, fixtures}; use object_store::memory::InMemory; use super::*; use crate::event::{Event, append_event}; - use crate::operations::test_support::{make_checkpoint_json, temp_repo, test_sig}; + use crate::operations::test_support::{temp_repo, test_sig}; use crate::records::Checkpoint; fn created_at() -> chrono::DateTime { @@ -398,8 +370,8 @@ mod tests { )) } - fn sample_run_record(run_id: RunId, host_repo_path: Option<&str>) -> RunRecord { - RunRecord { + fn sample_run_spec(run_id: RunId, host_repo_path: Option<&str>) -> RunSpec { + RunSpec { run_id, settings: SettingsLayer::default(), graph: Graph::new("test"), @@ -467,22 +439,22 @@ mod tests { host_repo_path: Option<&str>, ) -> DurableRunStore { let run_store = store.create_run(&run_id).await.unwrap(); - let run_record = sample_run_record(run_id, host_repo_path); + let run_spec = sample_run_spec(run_id, host_repo_path); append_event(&run_store, &run_id, &Event::RunCreated { run_id, - settings: serde_json::to_value(&run_record.settings).unwrap(), - graph: serde_json::to_value(&run_record.graph).unwrap(), + settings: serde_json::to_value(&run_spec.settings).unwrap(), + graph: serde_json::to_value(&run_spec.graph).unwrap(), workflow_source: None, workflow_config: None, - labels: run_record.labels.clone().into_iter().collect(), + labels: run_spec.labels.clone().into_iter().collect(), run_dir: String::new(), - working_directory: run_record.working_directory.display().to_string(), - host_repo_path: run_record.host_repo_path.clone(), - repo_origin_url: run_record.repo_origin_url.clone(), - base_branch: run_record.base_branch.clone(), - workflow_slug: run_record.workflow_slug.clone(), + working_directory: run_spec.working_directory.display().to_string(), + host_repo_path: run_spec.host_repo_path.clone(), + repo_origin_url: run_spec.repo_origin_url.clone(), + base_branch: run_spec.base_branch.clone(), + workflow_slug: run_spec.workflow_slug.clone(), db_prefix: None, - provenance: run_record.provenance.clone(), + provenance: run_spec.provenance.clone(), manifest_blob: None, }) .await @@ -693,31 +665,55 @@ mod tests { assert_eq!(checkpoint_commits.len(), 2); assert_eq!( git_store - .read_blob_at(checkpoint_commits[0], "nodes/build/prompt.md") + .read_blob_at(checkpoint_commits[0], "stages/build@1/prompt.md") .unwrap() .as_deref(), Some("visit one".as_bytes()) ); - assert!( - git_store - .read_blob_at(checkpoint_commits[0], "nodes/build-visit_2/prompt.md") + let first_projection: RunProjection = serde_json::from_slice( + &git_store + .read_blob_at(checkpoint_commits[0], "run.json") .unwrap() - .is_none() + .unwrap(), + ) + .unwrap(); + assert_eq!( + first_projection + .checkpoint + .as_ref() + .and_then(|checkpoint| checkpoint.node_visits.get("build")) + .copied(), + Some(1) ); assert_eq!( git_store - .read_blob_at(checkpoint_commits[1], "nodes/build/prompt.md") + .read_blob_at(checkpoint_commits[1], "stages/build@1/prompt.md") .unwrap() .as_deref(), Some("visit one".as_bytes()) ); assert_eq!( git_store - .read_blob_at(checkpoint_commits[1], "nodes/build-visit_2/prompt.md") + .read_blob_at(checkpoint_commits[1], "stages/build@2/prompt.md") .unwrap() .as_deref(), Some("visit two".as_bytes()) ); + let second_projection: RunProjection = serde_json::from_slice( + &git_store + .read_blob_at(checkpoint_commits[1], "run.json") + .unwrap() + .unwrap(), + ) + .unwrap(); + assert_eq!( + second_projection + .checkpoint + .as_ref() + .and_then(|checkpoint| checkpoint.node_visits.get("build")) + .copied(), + Some(2) + ); } #[tokio::test] @@ -796,16 +792,40 @@ mod tests { let branch = MetadataStore::branch_name(&test_run_id().to_string()); let bs = BranchStore::new(&git_store, &branch, &sig); bs.ensure_branch().unwrap(); - bs.write_entry("run.json", b"{}", "init run").unwrap(); + let mut init_projection = RunProjection::default(); + init_projection.spec = Some(sample_run_spec(test_run_id(), None)); + init_projection.start = Some(sample_start_record(test_run_id())); bs.write_entry( - "checkpoint.json", - &make_checkpoint_json("start", 1, Some("aaa")), + "run.json", + &serde_json::to_vec_pretty(&init_projection).unwrap(), + "init run", + ) + .unwrap(); + + let mut first_checkpoint_projection = init_projection.clone(); + first_checkpoint_projection.checkpoint = Some(sample_checkpoint( + "start", + &["start"], + &[("start", 1)], + Some("aaa"), + )); + bs.write_entry( + "run.json", + &serde_json::to_vec_pretty(&first_checkpoint_projection).unwrap(), "checkpoint", ) .unwrap(); + + let mut second_checkpoint_projection = init_projection; + second_checkpoint_projection.checkpoint = Some(sample_checkpoint( + "build", + &["start", "build"], + &[("start", 1), ("build", 1)], + Some("bbb"), + )); bs.write_entry( - "checkpoint.json", - &make_checkpoint_json("build", 1, Some("bbb")), + "run.json", + &serde_json::to_vec_pretty(&second_checkpoint_projection).unwrap(), "checkpoint", ) .unwrap(); @@ -830,7 +850,7 @@ mod tests { } #[tokio::test] - async fn rebuild_metadata_branch_errors_when_run_record_is_missing() { + async fn rebuild_metadata_branch_errors_when_run_spec_is_missing() { let (_dir, git_store) = temp_repo(); let durable_store = memory_store(); let run_store = durable_store.create_run(&test_run_id()).await.unwrap(); @@ -838,7 +858,7 @@ mod tests { let err = rebuild_metadata_branch(&git_store, &run_store, &test_run_id()) .await .unwrap_err(); - assert!(err.to_string().contains("run record not found")); + assert!(err.to_string().contains("run spec not found")); } #[tokio::test] @@ -984,20 +1004,22 @@ mod tests { .map(|commit| commit.oid) .collect(); - let first: Checkpoint = serde_json::from_slice( + let first_projection: RunProjection = serde_json::from_slice( &git_store - .read_blob_at(checkpoint_commits[0], "checkpoint.json") + .read_blob_at(checkpoint_commits[0], "run.json") .unwrap() .unwrap(), ) .unwrap(); - let second: Checkpoint = serde_json::from_slice( + let second_projection: RunProjection = serde_json::from_slice( &git_store - .read_blob_at(checkpoint_commits[1], "checkpoint.json") + .read_blob_at(checkpoint_commits[1], "run.json") .unwrap() .unwrap(), ) .unwrap(); + let first = first_projection.checkpoint.unwrap(); + let second = second_projection.checkpoint.unwrap(); assert_eq!( first.git_commit_sha.as_deref(), diff --git a/lib/crates/fabro-workflow/src/operations/resume.rs b/lib/crates/fabro-workflow/src/operations/resume.rs index c2098b775..dd523e4e8 100644 --- a/lib/crates/fabro-workflow/src/operations/resume.rs +++ b/lib/crates/fabro-workflow/src/operations/resume.rs @@ -36,7 +36,7 @@ pub async fn resume(run_dir: &Path, services: StartServices) -> Result Result { if !commit.message.starts_with("checkpoint") { continue; } - let blob = store - .read_blob_at(commit.oid, "checkpoint.json") - .map_err(|e| anyhow::anyhow!("failed to read checkpoint blob: {e}"))?; - let Some(bytes) = blob else { continue }; - let cp: Checkpoint = serde_json::from_slice(&bytes) - .with_context(|| format!("failed to parse checkpoint at {}", commit.oid))?; + let Some(projection) = read_projection_at_commit(store, commit.oid)? else { + continue; + }; + let cp = projection.checkpoint.with_context(|| { + format!( + "metadata checkpoint {} is missing projection.checkpoint", + commit.oid + ) + })?; ordinal += 1; let visit = cp.node_visits.get(&cp.current_node).copied().unwrap_or(1); @@ -354,34 +357,38 @@ pub fn find_run_id_by_prefix(repo: &Repository, prefix: &str) -> Result { } fn load_parallel_map(store: &Store, run_id: &str) -> HashMap { - let branch = MetadataStore::branch_name(run_id); - let Ok(sig) = Signature::now("Fabro", "noreply@fabro.sh") else { + let Ok(Some(projection)) = MetadataStore::read_run_projection(store.repo_dir(), run_id) else { return HashMap::new(); }; - let bs = BranchStore::new(store, &branch, &sig); - if let Ok(Some(run_bytes)) = bs.read_entry("run.json") { - if let Ok(record) = serde_json::from_slice::(&run_bytes) { - return detect_parallel_interior(&record.graph); - } + if let Some(spec) = projection.spec { + return detect_parallel_interior(&spec.graph); } - let graph_bytes = match bs.read_entry("workflow.fabro") { - Ok(Some(bytes)) => bytes, - _ => match bs.read_entry("graph.fabro") { - Ok(Some(bytes)) => bytes, - _ => return HashMap::new(), - }, + let Some(dot_source) = projection.graph_source else { + return HashMap::new(); }; - let dot_source = String::from_utf8_lossy(&graph_bytes); let Ok(graph) = parser::parse(&dot_source) else { return HashMap::new(); }; detect_parallel_interior(&graph) } +fn read_projection_at_commit(store: &Store, oid: Oid) -> Result> { + let blob = store + .read_blob_at(oid, "run.json") + .map_err(|e| anyhow::anyhow!("failed to read projection blob: {e}"))?; + let Some(bytes) = blob else { + return Ok(None); + }; + let projection = serde_json::from_slice(&bytes) + .with_context(|| format!("failed to parse projection at {oid}"))?; + Ok(Some(projection)) +} + #[cfg(test)] mod tests { + use fabro_store::RunProjection; use fabro_types::{RunId, fixtures}; use super::super::test_support::*; @@ -391,6 +398,19 @@ mod tests { value.parse().unwrap() } + fn checkpoint_projection_json( + current_node: &str, + visit: usize, + git_commit_sha: Option<&str>, + ) -> Vec { + let mut projection = RunProjection::default(); + projection.checkpoint = Some( + serde_json::from_slice(&make_checkpoint_json(current_node, visit, git_commit_sha)) + .unwrap(), + ); + serde_json::to_vec_pretty(&projection).unwrap() + } + #[test] fn parse_target_ordinal() { assert_eq!( @@ -416,12 +436,10 @@ mod tests { bs.ensure_branch().unwrap(); bs.write_entry("run.json", b"{}", "init run").unwrap(); - let cp1 = make_checkpoint_json("start", 1, Some("aaa")); - bs.write_entry("checkpoint.json", &cp1, "checkpoint") - .unwrap(); - let cp2 = make_checkpoint_json("build", 1, Some("bbb")); - bs.write_entry("checkpoint.json", &cp2, "checkpoint") - .unwrap(); + let cp1 = checkpoint_projection_json("start", 1, Some("aaa")); + bs.write_entry("run.json", &cp1, "checkpoint").unwrap(); + let cp2 = checkpoint_projection_json("build", 1, Some("bbb")); + bs.write_entry("run.json", &cp2, "checkpoint").unwrap(); let timeline = build_timeline(&store, "test-run-1").unwrap(); assert_eq!(timeline.entries.len(), 2); @@ -513,13 +531,10 @@ mod tests { bs.ensure_branch().unwrap(); bs.write_entry("run.json", b"{}", "init run").unwrap(); - let cp1 = make_checkpoint_json("start", 1, None); - let oid1 = bs - .write_entry("checkpoint.json", &cp1, "checkpoint") - .unwrap(); - let cp2 = make_checkpoint_json("build", 1, None); - bs.write_entry("checkpoint.json", &cp2, "checkpoint") - .unwrap(); + let cp1 = checkpoint_projection_json("start", 1, None); + let oid1 = bs.write_entry("run.json", &cp1, "checkpoint").unwrap(); + let cp2 = checkpoint_projection_json("build", 1, None); + bs.write_entry("run.json", &cp2, "checkpoint").unwrap(); rewind(&store, &RewindInput { run_id: fixtures::RUN_1, diff --git a/lib/crates/fabro-workflow/src/operations/start.rs b/lib/crates/fabro-workflow/src/operations/start.rs index 5494a9ad1..c065dee81 100644 --- a/lib/crates/fabro-workflow/src/operations/start.rs +++ b/lib/crates/fabro-workflow/src/operations/start.rs @@ -277,7 +277,7 @@ async fn persist_terminal_engine_failure( impl RunSession { async fn new(persisted: &Persisted, services: StartServices) -> Result { - let record = persisted.run_record(); + let record = persisted.run_spec(); let settings = &record.settings; let working_directory = record.working_directory.clone(); let state = services @@ -292,7 +292,7 @@ impl RunSession { meta_branch: Some(MetadataStore::branch_name(&record.run_id.to_string())), }) }); - let definition_blob = state.run.as_ref().and_then(|run| run.definition_blob); + let definition_blob = state.spec.as_ref().and_then(|run| run.definition_blob); let accepted_definition = match definition_blob { Some(blob_id) => { Some(load_accepted_run_definition(&services.run_store, blob_id).await?) @@ -669,7 +669,7 @@ impl RunSession { let preserve_sandbox = self.preserve_sandbox; let on_node = self.on_node.clone(); - let record = persisted.run_record(); + let record = persisted.run_spec(); let run_options = RunOptions { settings: record.settings.clone(), run_dir: persisted.run_dir().to_path_buf(), diff --git a/lib/crates/fabro-workflow/src/pipeline/execute/tests.rs b/lib/crates/fabro-workflow/src/pipeline/execute/tests.rs index bac21f96f..16c824d01 100644 --- a/lib/crates/fabro-workflow/src/pipeline/execute/tests.rs +++ b/lib/crates/fabro-workflow/src/pipeline/execute/tests.rs @@ -30,7 +30,7 @@ use crate::handler::{Handler as HandlerTrait, HandlerRegistry}; use crate::outcome::{Outcome, OutcomeExt, StageStatus}; use crate::pipeline::initialize; use crate::pipeline::types::{InitOptions, LlmSpec, Persisted, SandboxEnvSpec}; -use crate::records::RunRecord; +use crate::records::RunSpec; use crate::run_options::{GitCheckpointOptions, LifecycleOptions, RunOptions}; use crate::run_status::{RunStatus, StatusReason}; use crate::test_support::run_graph; @@ -135,7 +135,7 @@ fn persisted_workflow(graph: Graph, source: String, run_dir: &Path, run_id: RunI source, vec![], run_dir.to_path_buf(), - RunRecord { + RunSpec { run_id, settings: SettingsLayer::default(), graph, diff --git a/lib/crates/fabro-workflow/src/pipeline/finalize.rs b/lib/crates/fabro-workflow/src/pipeline/finalize.rs index bd9ed3d34..188b3fd02 100644 --- a/lib/crates/fabro-workflow/src/pipeline/finalize.rs +++ b/lib/crates/fabro-workflow/src/pipeline/finalize.rs @@ -149,11 +149,10 @@ fn build_conclusion_from_parts( } } -/// Write a finalize commit to the shadow branch with retro.json and final node -/// files. +/// Write a finalize projection snapshot commit to the metadata branch. /// -/// This captures the last diff.patch (written after the final checkpoint) and -/// retro.json. Best-effort: errors are logged as warnings. +/// This captures the final `run.json` projection state, including conclusion +/// and retro data. Best-effort: errors are logged as warnings. pub async fn write_finalize_commit(run_options: &RunOptions, run_store: &RunStoreHandle) { let (Some(meta_branch), Some(repo_path)) = ( run_options @@ -170,7 +169,7 @@ pub async fn write_finalize_commit(run_options: &RunOptions, run_store: &RunStor let Ok(store_state) = run_store.state().await else { return; }; - let dump = RunDump::metadata_finalize(&store_state); + let dump = RunDump::from_projection(&store_state); if let Err(e) = dump.write_to_metadata_store(&store, &run_options.run_id.to_string(), "finalize run") { diff --git a/lib/crates/fabro-workflow/src/pipeline/initialize.rs b/lib/crates/fabro-workflow/src/pipeline/initialize.rs index 71cb57aac..82d53bf82 100644 --- a/lib/crates/fabro-workflow/src/pipeline/initialize.rs +++ b/lib/crates/fabro-workflow/src/pipeline/initialize.rs @@ -465,7 +465,7 @@ pub async fn initialize( persisted: Persisted, mut options: InitOptions, ) -> Result { - let (graph, source, _diagnostics, run_dir, _run_record) = persisted.into_parts(); + let (graph, source, _diagnostics, run_dir, _run_spec) = persisted.into_parts(); options.run_options.run_dir = run_dir.clone(); options.run_options.git = options.git.clone(); @@ -770,7 +770,7 @@ mod tests { use super::*; use crate::event::StoreProgressLogger; use crate::pipeline::types::InitOptions; - use crate::records::RunRecord; + use crate::records::RunSpec; use crate::run_options::RunOptions; fn test_run_id() -> RunId { @@ -864,7 +864,7 @@ mod tests { source, vec![], run_dir.to_path_buf(), - RunRecord { + RunSpec { run_id: test_run_id(), settings: SettingsLayer::default(), graph, diff --git a/lib/crates/fabro-workflow/src/pipeline/persist.rs b/lib/crates/fabro-workflow/src/pipeline/persist.rs index 5b14f80f3..db513c7d8 100644 --- a/lib/crates/fabro-workflow/src/pipeline/persist.rs +++ b/lib/crates/fabro-workflow/src/pipeline/persist.rs @@ -11,7 +11,7 @@ pub(crate) fn persist( mut options: PersistOptions, ) -> Result { let (graph, source, diagnostics) = validated.into_parts(); - options.run_record.graph = graph.clone(); + options.run_spec.graph = graph.clone(); std::fs::create_dir_all(&options.run_dir).map_err(|err| { Error::Io(format!( @@ -25,7 +25,7 @@ pub(crate) fn persist( source, diagnostics, options.run_dir, - options.run_record, + options.run_spec, )) } @@ -37,10 +37,10 @@ pub(crate) async fn load_from_store( .state() .await .map_err(|err| Error::engine(err.to_string()))?; - let run_record = state - .run - .ok_or_else(|| Error::Precondition("run record missing from store".to_string()))?; - let graph = run_record.graph.clone(); + let run_spec = state + .spec + .ok_or_else(|| Error::Precondition("run spec missing from store".to_string()))?; + let graph = run_spec.graph.clone(); let source = state.graph_source.unwrap_or_default(); Ok(Persisted::new( @@ -48,7 +48,7 @@ pub(crate) async fn load_from_store( source, Vec::new(), run_dir.to_path_buf(), - run_record, + run_spec, )) } @@ -70,7 +70,7 @@ mod tests { use super::*; use crate::event::{Event, append_event}; - use crate::records::RunRecord; + use crate::records::RunSpec; fn memory_store() -> Arc { Arc::new(Database::new( @@ -125,8 +125,8 @@ mod tests { graph } - fn sample_record(graph: Graph) -> RunRecord { - RunRecord { + fn sample_record(graph: Graph) -> RunSpec { + RunSpec { run_id: fixtures::RUN_1, settings: SettingsLayer { run: Some(RunLayer { @@ -161,7 +161,7 @@ mod tests { } } - async fn seeded_store(run_dir: &Path, record: &RunRecord, source: Option<&str>) -> RunDatabase { + async fn seeded_store(run_dir: &Path, record: &RunSpec, source: Option<&str>) -> RunDatabase { let store = memory_store(); let run_store = store.create_run(&record.run_id).await.unwrap(); append_event(&run_store, &record.run_id, &Event::RunCreated { @@ -194,8 +194,8 @@ mod tests { let persisted = persist( Validated::new(graph.clone(), source, vec![]), PersistOptions { - run_dir: run_dir.clone(), - run_record: sample_record(different_graph()), + run_dir: run_dir.clone(), + run_spec: sample_record(different_graph()), }, ) .unwrap(); @@ -207,13 +207,13 @@ mod tests { ); assert_eq!(persisted.run_dir(), run_dir.as_path()); assert_eq!( - serde_json::to_value(persisted.run_record().graph.clone()).unwrap(), + serde_json::to_value(persisted.run_spec().graph.clone()).unwrap(), serde_json::to_value(graph).unwrap() ); } #[test] - fn persist_overwrites_run_record_graph_with_validated_graph() { + fn persist_overwrites_run_spec_graph_with_validated_graph() { let temp = tempfile::tempdir().unwrap(); let run_dir = temp.path().join("run"); let (graph, source) = graph_and_source(); @@ -221,22 +221,22 @@ mod tests { let persisted = persist( Validated::new(graph.clone(), source, vec![]), PersistOptions { - run_dir: run_dir.clone(), - run_record: sample_record(different_graph()), + run_dir: run_dir.clone(), + run_spec: sample_record(different_graph()), }, ) .unwrap(); - assert_eq!(persisted.run_record().graph.name, graph.name); - assert!(persisted.run_record().graph.nodes.contains_key("exit")); + assert_eq!(persisted.run_spec().graph.name, graph.name); + assert!(persisted.run_spec().graph.nodes.contains_key("exit")); assert_eq!( - serde_json::to_value(persisted.run_record().graph.clone()).unwrap(), + serde_json::to_value(persisted.run_spec().graph.clone()).unwrap(), serde_json::to_value(graph).unwrap() ); } #[tokio::test] - async fn load_from_store_roundtrips_full_run_record_fields() { + async fn load_from_store_roundtrips_full_run_spec_fields() { let temp = tempfile::tempdir().unwrap(); let run_dir = temp.path().join("run"); let (graph, source) = graph_and_source(); @@ -246,8 +246,8 @@ mod tests { persist( Validated::new(graph, source.clone(), vec![]), PersistOptions { - run_dir: run_dir.clone(), - run_record: expected.clone(), + run_dir: run_dir.clone(), + run_spec: expected.clone(), }, ) .unwrap(); @@ -257,7 +257,7 @@ mod tests { .await .unwrap(); - let loaded_record = loaded.run_record(); + let loaded_record = loaded.run_spec(); assert_eq!(loaded_record.run_id, expected.run_id); assert!( (loaded_record.run_id.created_at().timestamp_millis() @@ -288,7 +288,7 @@ mod tests { let err = persist(Validated::new(graph, source, vec![]), PersistOptions { run_dir, - run_record: sample_record(different_graph()), + run_spec: sample_record(different_graph()), }) .unwrap_err(); @@ -313,7 +313,7 @@ mod tests { } #[tokio::test] - async fn load_from_store_reads_graph_from_run_record_and_source_from_store() { + async fn load_from_store_reads_graph_from_run_spec_and_source_from_store() { let temp = tempfile::tempdir().unwrap(); let run_dir = temp.path().join("run"); std::fs::create_dir_all(&run_dir).unwrap(); diff --git a/lib/crates/fabro-workflow/src/pipeline/pull_request.rs b/lib/crates/fabro-workflow/src/pipeline/pull_request.rs index d052fd7a0..6f7140fab 100644 --- a/lib/crates/fabro-workflow/src/pipeline/pull_request.rs +++ b/lib/crates/fabro-workflow/src/pipeline/pull_request.rs @@ -11,7 +11,7 @@ use tracing::{debug, info}; use super::types::{Concluded, Finalized, PullRequestOptions}; use crate::event::{Emitter, Event, RunNoticeLevel}; use crate::outcome::{StageStatus, format_cost as outcome_format_cost}; -use crate::records::{Conclusion, RunRecord}; +use crate::records::{Conclusion, RunSpec}; use crate::runtime_store::RunStoreHandle; /// Derive a PR title from the workflow goal. @@ -107,7 +107,7 @@ fn format_retro_section(retro: &Retro) -> String { /// optionally a workflow graph summary in another `
` block. fn format_arc_details_section( conclusion: &Conclusion, - run_record: Option<&RunRecord>, + run_spec: Option<&RunSpec>, dot_source: Option<&str>, ) -> String { let mut parts = Vec::new(); @@ -143,8 +143,8 @@ fn format_arc_details_section( parts.push(String::new()); parts.push("
".to_string()); - // Workflow graph summary — prefer RunRecord's graph, fall back to DOT parsing - if let Some(record) = run_record { + // Workflow graph summary — prefer RunSpec's graph, fall back to DOT parsing + if let Some(record) = run_spec { let workflow_name = if record.graph.name.is_empty() { "unnamed" } else { @@ -328,7 +328,7 @@ pub async fn build_pr_body( .ok(); let plan_text = run_state.as_ref().and_then(read_plan_text); let retro = run_state.as_ref().and_then(|state| state.retro.clone()); - let run_record = run_state.as_ref().and_then(|state| state.run.clone()); + let run_spec = run_state.as_ref().and_then(|state| state.spec.clone()); let dot_source = run_state .as_ref() .and_then(|state| state.graph_source.clone()); @@ -374,7 +374,7 @@ pub async fn build_pr_body( let retro_section = retro.as_ref().map(format_retro_section).unwrap_or_default(); let arc_details_section = conclusion .as_ref() - .map(|c| format_arc_details_section(c, run_record.as_ref(), dot_source.as_deref())) + .map(|c| format_arc_details_section(c, run_spec.as_ref(), dot_source.as_deref())) .unwrap_or_default(); let body = assemble_pr_body( @@ -597,7 +597,7 @@ mod tests { }; use fabro_store::Database; use fabro_types::settings::SettingsLayer; - use fabro_types::{BilledTokenCounts, RunRecord, fixtures}; + use fabro_types::{BilledTokenCounts, RunSpec, fixtures}; use futures::stream; use object_store::memory::InMemory; @@ -1086,7 +1086,7 @@ mod tests { let store = test_store(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); - let run_record = RunRecord { + let run_spec = RunSpec { run_id: fixtures::RUN_1, settings: SettingsLayer::default(), graph: Graph::new("test"), @@ -1102,19 +1102,19 @@ mod tests { }; append_event(&run_store, &fixtures::RUN_1, &Event::RunCreated { run_id: fixtures::RUN_1, - settings: serde_json::to_value(&run_record.settings).unwrap(), - graph: serde_json::to_value(&run_record.graph).unwrap(), + settings: serde_json::to_value(&run_spec.settings).unwrap(), + graph: serde_json::to_value(&run_spec.graph).unwrap(), workflow_source: Some("digraph test { plan -> code }".to_string()), workflow_config: None, - labels: run_record.labels.clone().into_iter().collect(), - run_dir: run_record.working_directory.display().to_string(), - working_directory: run_record.working_directory.display().to_string(), - host_repo_path: run_record.host_repo_path.clone(), - repo_origin_url: run_record.repo_origin_url.clone(), - base_branch: run_record.base_branch.clone(), - workflow_slug: run_record.workflow_slug.clone(), + labels: run_spec.labels.clone().into_iter().collect(), + run_dir: run_spec.working_directory.display().to_string(), + working_directory: run_spec.working_directory.display().to_string(), + host_repo_path: run_spec.host_repo_path.clone(), + repo_origin_url: run_spec.repo_origin_url.clone(), + base_branch: run_spec.base_branch.clone(), + workflow_slug: run_spec.workflow_slug.clone(), db_prefix: None, - provenance: run_record.provenance.clone(), + provenance: run_spec.provenance.clone(), manifest_blob: None, }) .await @@ -1151,7 +1151,7 @@ mod tests { let store = test_store(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); - let run_record = RunRecord { + let run_spec = RunSpec { run_id: fixtures::RUN_1, settings: SettingsLayer::default(), graph: Graph::new("test"), @@ -1167,19 +1167,19 @@ mod tests { }; append_event(&run_store, &fixtures::RUN_1, &Event::RunCreated { run_id: fixtures::RUN_1, - settings: serde_json::to_value(&run_record.settings).unwrap(), - graph: serde_json::to_value(&run_record.graph).unwrap(), + settings: serde_json::to_value(&run_spec.settings).unwrap(), + graph: serde_json::to_value(&run_spec.graph).unwrap(), workflow_source: Some("digraph test { plan -> code }".to_string()), workflow_config: None, - labels: run_record.labels.clone().into_iter().collect(), - run_dir: run_record.working_directory.display().to_string(), - working_directory: run_record.working_directory.display().to_string(), - host_repo_path: run_record.host_repo_path.clone(), - repo_origin_url: run_record.repo_origin_url.clone(), - base_branch: run_record.base_branch.clone(), - workflow_slug: run_record.workflow_slug.clone(), + labels: run_spec.labels.clone().into_iter().collect(), + run_dir: run_spec.working_directory.display().to_string(), + working_directory: run_spec.working_directory.display().to_string(), + host_repo_path: run_spec.host_repo_path.clone(), + repo_origin_url: run_spec.repo_origin_url.clone(), + base_branch: run_spec.base_branch.clone(), + workflow_slug: run_spec.workflow_slug.clone(), db_prefix: None, - provenance: run_record.provenance.clone(), + provenance: run_spec.provenance.clone(), manifest_blob: None, }) .await @@ -1369,7 +1369,7 @@ mod tests { let tmp = tempfile::tempdir().unwrap(); let store = test_store(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); - let run_record = RunRecord { + let run_spec = RunSpec { run_id: fixtures::RUN_1, settings: SettingsLayer::default(), graph: Graph::new("test"), @@ -1385,19 +1385,19 @@ mod tests { }; append_event(&run_store, &fixtures::RUN_1, &Event::RunCreated { run_id: fixtures::RUN_1, - settings: serde_json::to_value(&run_record.settings).unwrap(), - graph: serde_json::to_value(&run_record.graph).unwrap(), + settings: serde_json::to_value(&run_spec.settings).unwrap(), + graph: serde_json::to_value(&run_spec.graph).unwrap(), workflow_source: None, workflow_config: None, - labels: run_record.labels.clone().into_iter().collect(), - run_dir: run_record.working_directory.display().to_string(), + labels: run_spec.labels.clone().into_iter().collect(), + run_dir: run_spec.working_directory.display().to_string(), working_directory: tmp.path().display().to_string(), host_repo_path: None, - repo_origin_url: run_record.repo_origin_url.clone(), + repo_origin_url: run_spec.repo_origin_url.clone(), base_branch: None, workflow_slug: None, db_prefix: None, - provenance: run_record.provenance.clone(), + provenance: run_spec.provenance.clone(), manifest_blob: None, }) .await diff --git a/lib/crates/fabro-workflow/src/pipeline/retro.rs b/lib/crates/fabro-workflow/src/pipeline/retro.rs index a6def63d4..300f53afd 100644 --- a/lib/crates/fabro-workflow/src/pipeline/retro.rs +++ b/lib/crates/fabro-workflow/src/pipeline/retro.rs @@ -192,7 +192,7 @@ mod tests { use crate::context::Context; use crate::event::{Emitter, Event, StoreProgressLogger, append_event}; use crate::pipeline::types::Executed; - use crate::records::{Checkpoint, CheckpointExt, RunRecord}; + use crate::records::{Checkpoint, CheckpointExt, RunSpec}; use crate::run_options::RunOptions; fn test_run_id() -> RunId { @@ -232,7 +232,7 @@ mod tests { ) -> fabro_store::RunDatabase { let inner = test_store().create_run(&test_run_id()).await.unwrap(); let run_store = inner; - let run_record = RunRecord { + let run_spec = RunSpec { run_id: test_run_id(), settings: SettingsLayer::default(), graph: Graph::new("test"), @@ -248,19 +248,19 @@ mod tests { }; append_event(&run_store, &test_run_id(), &Event::RunCreated { run_id: test_run_id(), - settings: serde_json::to_value(&run_record.settings).unwrap(), - graph: serde_json::to_value(&run_record.graph).unwrap(), + settings: serde_json::to_value(&run_spec.settings).unwrap(), + graph: serde_json::to_value(&run_spec.graph).unwrap(), workflow_source: None, workflow_config: None, - labels: run_record.labels.clone().into_iter().collect(), + labels: run_spec.labels.clone().into_iter().collect(), run_dir: run_dir.to_string_lossy().to_string(), working_directory: run_dir.to_string_lossy().to_string(), host_repo_path: None, - repo_origin_url: run_record.repo_origin_url.clone(), + repo_origin_url: run_spec.repo_origin_url.clone(), base_branch: None, workflow_slug: None, db_prefix: None, - provenance: run_record.provenance.clone(), + provenance: run_spec.provenance.clone(), manifest_blob: None, }) .await diff --git a/lib/crates/fabro-workflow/src/pipeline/types.rs b/lib/crates/fabro-workflow/src/pipeline/types.rs index 1c69b3fe3..3f78fb826 100644 --- a/lib/crates/fabro-workflow/src/pipeline/types.rs +++ b/lib/crates/fabro-workflow/src/pipeline/types.rs @@ -26,7 +26,7 @@ use crate::event::Emitter; use crate::file_resolver::FileResolver; use crate::handler::HandlerRegistry; use crate::outcome::Outcome; -use crate::records::{Checkpoint, Conclusion, RunRecord}; +use crate::records::{Checkpoint, Conclusion, RunSpec}; use crate::run_control::RunControlState; use crate::run_options::{GitCheckpointOptions, LifecycleOptions, RunOptions}; use crate::runtime_store::RunStoreHandle; @@ -113,12 +113,12 @@ impl Validated { /// Options for the PERSIST phase. pub(crate) struct PersistOptions { - pub run_dir: PathBuf, - pub run_record: RunRecord, + pub run_dir: PathBuf, + pub run_spec: RunSpec, } /// Output of the PERSIST phase. Run directory created and the validated -/// workflow is persisted into the durable run record. +/// workflow is persisted into the durable run spec. #[derive(Debug)] #[non_exhaustive] pub struct Persisted { @@ -126,7 +126,7 @@ pub struct Persisted { source: String, diagnostics: Vec, run_dir: PathBuf, - run_record: RunRecord, + run_spec: RunSpec, } impl Persisted { @@ -136,14 +136,14 @@ impl Persisted { source: String, diagnostics: Vec, run_dir: PathBuf, - run_record: RunRecord, + run_spec: RunSpec, ) -> Self { Self { graph, source, diagnostics, run_dir, - run_record, + run_spec, } } @@ -163,8 +163,8 @@ impl Persisted { &self.run_dir } - pub fn run_record(&self) -> &RunRecord { - &self.run_record + pub fn run_spec(&self) -> &RunSpec { + &self.run_spec } /// True if any diagnostic has Error severity. @@ -191,14 +191,14 @@ impl Persisted { Ok(()) } - /// Consume into owned graph, source, diagnostics, run dir, and run record. - pub fn into_parts(self) -> (Graph, String, Vec, PathBuf, RunRecord) { + /// Consume into owned graph, source, diagnostics, run dir, and run spec. + pub fn into_parts(self) -> (Graph, String, Vec, PathBuf, RunSpec) { ( self.graph, self.source, self.diagnostics, self.run_dir, - self.run_record, + self.run_spec, ) } diff --git a/lib/crates/fabro-workflow/src/records/mod.rs b/lib/crates/fabro-workflow/src/records/mod.rs index 9c2af89c1..836a4c309 100644 --- a/lib/crates/fabro-workflow/src/records/mod.rs +++ b/lib/crates/fabro-workflow/src/records/mod.rs @@ -5,5 +5,5 @@ mod start; pub use checkpoint::{Checkpoint, CheckpointExt}; pub use conclusion::{Conclusion, StageSummary}; -pub use run::RunRecord; +pub use run::RunSpec; pub use start::StartRecord; diff --git a/lib/crates/fabro-workflow/src/records/run.rs b/lib/crates/fabro-workflow/src/records/run.rs index eb2dd8789..6bbe14f21 100644 --- a/lib/crates/fabro-workflow/src/records/run.rs +++ b/lib/crates/fabro-workflow/src/records/run.rs @@ -1 +1 @@ -pub use fabro_types::run::RunRecord; +pub use fabro_types::run::RunSpec; diff --git a/lib/crates/fabro-workflow/src/run_dump.rs b/lib/crates/fabro-workflow/src/run_dump.rs index f1997653b..68773e2e5 100644 --- a/lib/crates/fabro-workflow/src/run_dump.rs +++ b/lib/crates/fabro-workflow/src/run_dump.rs @@ -13,7 +13,7 @@ use std::path::{Component, Path, PathBuf}; use anyhow::{Context, Result, bail}; use bytes::Bytes; -use fabro_store::{EventEnvelope, RunProjection, StageId}; +use fabro_store::{EventEnvelope, RunProjection, SerializableProjection, StageId}; use fabro_types::{RunBlobId, parse_blob_ref, parse_legacy_blob_file_ref}; use futures::future::BoxFuture; @@ -39,137 +39,26 @@ pub enum RunDumpContents { impl RunDump { #[must_use] - pub fn metadata_init(state: &RunProjection) -> Self { - let mut entries = Vec::new(); - if let Some(record) = state.run.as_ref() { - push_json_entry(&mut entries, "run.json", record); - } - if let Some(record) = state.start.as_ref() { - push_json_entry(&mut entries, "start.json", record); - } - if let Some(record) = state.sandbox.as_ref() { - push_json_entry(&mut entries, "sandbox.json", record); - } - Self { entries } - } - - #[must_use] - pub fn metadata_checkpoint(state: &RunProjection) -> Self { - let mut entries = Vec::new(); - let mut keys: Vec<_> = state.iter_nodes().map(|(node, _)| node.clone()).collect(); - keys.sort(); - - for node_key in keys { - let Some(node) = state.node(&node_key) else { - continue; - }; - let node_id = node_key.node_id(); - let visit = node_key.visit(); - - if let Some(prompt) = node.prompt.as_ref() { - entries.push(RunDumpEntry::text( - metadata_node_file_path(node_id, visit, "prompt.md"), - prompt.clone(), - )); - } - if let Some(response) = node.response.as_ref() { - entries.push(RunDumpEntry::text( - metadata_node_file_path(node_id, visit, "response.md"), - response.clone(), - )); - } - if let Some(status) = node.status.as_ref() { - push_json_entry_path( - &mut entries, - &PathBuf::from(metadata_node_file_path(node_id, visit, "status.json")), - status, - ); - } - if let Some(provider_used) = node.provider_used.as_ref() { - entries.push(RunDumpEntry::json( - metadata_node_file_path(node_id, visit, "provider_used.json"), - provider_used.clone(), - )); - } - if let Some(diff) = node.diff.as_ref() { - entries.push(RunDumpEntry::text( - metadata_node_file_path(node_id, visit, "diff.patch"), - diff.clone(), - )); - } - if let Some(script_invocation) = node.script_invocation.as_ref() { - entries.push(RunDumpEntry::json( - metadata_node_file_path(node_id, visit, "script_invocation.json"), - script_invocation.clone(), - )); - } - if let Some(script_timing) = node.script_timing.as_ref() { - entries.push(RunDumpEntry::json( - metadata_node_file_path(node_id, visit, "script_timing.json"), - script_timing.clone(), - )); - } - if let Some(parallel_results) = node.parallel_results.as_ref() { - entries.push(RunDumpEntry::json( - metadata_node_file_path(node_id, visit, "parallel_results.json"), - parallel_results.clone(), - )); - } - } - - Self { entries } - } - - #[must_use] - pub fn metadata_finalize(state: &RunProjection) -> Self { - let mut dump = Self::metadata_checkpoint(state); - if let Some(retro) = state.retro.as_ref() { - push_json_entry(&mut dump.entries, "retro.json", retro); - } - dump - } - - pub fn from_store_state_and_events( - state: &RunProjection, - events: &[EventEnvelope], - ) -> Result { + pub fn from_projection(state: &RunProjection) -> Self { let mut entries = Vec::new(); - if let Some(record) = state.run.as_ref() { - push_json_entry(&mut entries, "run.json", record); - } - if let Some(record) = state.start.as_ref() { - push_json_entry(&mut entries, "start.json", record); - } - if let Some(record) = state.status.as_ref() { - push_json_entry(&mut entries, "status.json", record); - } - if let Some(record) = state.checkpoint.as_ref() { - push_json_entry(&mut entries, "checkpoint.json", record); - } - if let Some(record) = state.conclusion.as_ref() { - push_json_entry(&mut entries, "conclusion.json", record); - } - if let Some(record) = state.retro.as_ref() { - push_json_entry(&mut entries, "retro.json", record); - } + push_json_entry(&mut entries, "run.json", &SerializableProjection(state)); + if let Some(graph_source) = state.graph_source.as_ref() { entries.push(RunDumpEntry::text("graph.fabro", graph_source.clone())); } - if let Some(record) = state.sandbox.as_ref() { - push_json_entry(&mut entries, "sandbox.json", record); - } - let mut node_keys: Vec<_> = state.iter_nodes().map(|(node, _)| node.clone()).collect(); - node_keys.sort(); - for node_key in &node_keys { - let node = state - .node(node_key) - .with_context(|| format!("missing node {node_key:?} in projection"))?; - let node_id_segment = validate_single_path_segment("node id", node_key.node_id())?; - let base = PathBuf::from("nodes") - .join(node_id_segment) - .join(format!("visit-{}", node_key.visit())); + let mut stage_ids: Vec<_> = state + .iter_nodes() + .map(|(stage_id, _)| stage_id.clone()) + .collect(); + stage_ids.sort(); + + for stage_id in stage_ids { + let Some(node) = state.node(&stage_id) else { + continue; + }; + let base = PathBuf::from("stages").join(stage_id.to_string()); if let Some(prompt) = node.prompt.as_ref() { entries.push(RunDumpEntry::text_path( @@ -186,6 +75,36 @@ impl RunDump { if let Some(status) = node.status.as_ref() { push_json_entry_path(&mut entries, &base.join("status.json"), status); } + if let Some(provider_used) = node.provider_used.as_ref() { + entries.push(RunDumpEntry::json_path( + &base.join("provider_used.json"), + provider_used.clone(), + )); + } + if let Some(diff) = node.diff.as_ref() { + entries.push(RunDumpEntry::text_path( + &base.join("diff.patch"), + diff.clone(), + )); + } + if let Some(script_invocation) = node.script_invocation.as_ref() { + entries.push(RunDumpEntry::json_path( + &base.join("script_invocation.json"), + script_invocation.clone(), + )); + } + if let Some(script_timing) = node.script_timing.as_ref() { + entries.push(RunDumpEntry::json_path( + &base.join("script_timing.json"), + script_timing.clone(), + )); + } + if let Some(parallel_results) = node.parallel_results.as_ref() { + entries.push(RunDumpEntry::json_path( + &base.join("parallel_results.json"), + parallel_results.clone(), + )); + } if let Some(stdout) = node.stdout.as_ref() { entries.push(RunDumpEntry::text_path( &base.join("stdout.log"), @@ -207,22 +126,32 @@ impl RunDump { entries.push(RunDumpEntry::text("retro/response.md", response.clone())); } + Self { entries } + } + + pub fn from_store_state_and_events( + state: &RunProjection, + events: &[EventEnvelope], + ) -> Result { + let mut dump = Self::from_projection(state); + let mut events_jsonl = Vec::new(); for event in events { serde_json::to_writer(&mut events_jsonl, event)?; events_jsonl.write_all(b"\n")?; } - entries.push(RunDumpEntry::bytes("events.jsonl", events_jsonl)); + dump.entries + .push(RunDumpEntry::bytes("events.jsonl", events_jsonl)); for (seq, checkpoint) in &state.checkpoints { push_json_entry_path( - &mut entries, + &mut dump.entries, &PathBuf::from("checkpoints").join(format!("{seq:04}.json")), checkpoint, ); } - Ok(Self { entries }) + Ok(dump) } pub fn add_artifact_bytes( @@ -385,14 +314,6 @@ where } } -fn metadata_node_file_path(node_id: &str, visit: u32, filename: &str) -> String { - if visit <= 1 { - format!("nodes/{node_id}/{filename}") - } else { - format!("nodes/{node_id}-visit_{visit}/{filename}") - } -} - fn path_to_string(path: &Path) -> String { path.to_string_lossy().into_owned() } @@ -495,3 +416,174 @@ fn ensure_parent_dir(path: &Path) -> Result<()> { .with_context(|| format!("failed to create {}", parent.display()))?; Ok(()) } + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::path::PathBuf; + + use chrono::{TimeZone, Utc}; + use fabro_store::{NodeState, RunProjection, StageId}; + use fabro_types::graph::Graph; + use fabro_types::run::RunSpec; + use fabro_types::settings::SettingsLayer; + use fabro_types::{ + Checkpoint, Conclusion, NodeStatusRecord, RunStatus, RunStatusRecord, SandboxRecord, + StageStatus, StartRecord, fixtures, + }; + + use super::RunDump; + use crate::run_dump::RunDumpContents; + + fn sample_run_spec() -> RunSpec { + RunSpec { + run_id: fixtures::RUN_1, + settings: SettingsLayer::default(), + graph: Graph::new("ship"), + workflow_slug: Some("demo".to_string()), + working_directory: PathBuf::from("/tmp/project"), + host_repo_path: Some("/tmp/project".to_string()), + repo_origin_url: Some("https://github.com/fabro-sh/fabro.git".to_string()), + base_branch: Some("main".to_string()), + labels: HashMap::from([("team".to_string(), "platform".to_string())]), + provenance: None, + manifest_blob: None, + definition_blob: None, + } + } + + fn sample_checkpoint() -> Checkpoint { + Checkpoint { + timestamp: Utc + .with_ymd_and_hms(2026, 4, 20, 12, 0, 0) + .single() + .unwrap(), + current_node: "build".to_string(), + completed_nodes: vec!["build".to_string()], + node_retries: HashMap::new(), + context_values: HashMap::new(), + node_outcomes: HashMap::new(), + next_node_id: Some("ship".to_string()), + git_commit_sha: Some("abc123".to_string()), + loop_failure_signatures: HashMap::new(), + restart_failure_signatures: HashMap::new(), + node_visits: HashMap::from([("build".to_string(), 2usize)]), + } + } + + #[test] + fn from_projection_uses_stages_layout_and_collapses_top_level_metadata_files() { + let stage_id = StageId::new("build", 2); + let mut projection = RunProjection::default(); + projection.spec = Some(sample_run_spec()); + projection.graph_source = Some("digraph Ship {}".to_string()); + projection.start = Some(StartRecord { + run_id: fixtures::RUN_1, + start_time: Utc + .with_ymd_and_hms(2026, 4, 20, 12, 0, 0) + .single() + .unwrap(), + run_branch: Some("fabro/run/demo".to_string()), + base_sha: Some("deadbeef".to_string()), + }); + projection.status = Some(RunStatusRecord::new(RunStatus::Succeeded, None)); + projection.checkpoint = Some(sample_checkpoint()); + projection.conclusion = Some(Conclusion { + timestamp: Utc + .with_ymd_and_hms(2026, 4, 20, 12, 5, 0) + .single() + .unwrap(), + status: StageStatus::Success, + duration_ms: 5, + failure_reason: None, + final_git_commit_sha: Some("abc123".to_string()), + stages: Vec::new(), + billing: None, + total_retries: 0, + }); + projection.sandbox = Some(SandboxRecord { + provider: "local".to_string(), + working_directory: "/tmp/project".to_string(), + identifier: Some("sandbox-1".to_string()), + host_working_directory: None, + container_mount_point: None, + }); + projection.retro_prompt = Some("retro prompt".to_string()); + projection.retro_response = Some("retro response".to_string()); + projection.set_node(stage_id.clone(), NodeState { + prompt: Some("plan".to_string()), + response: Some("done".to_string()), + status: Some(NodeStatusRecord { + status: StageStatus::Success, + notes: Some("ok".to_string()), + failure_reason: None, + timestamp: Utc + .with_ymd_and_hms(2026, 4, 20, 12, 1, 0) + .single() + .unwrap(), + }), + provider_used: Some(serde_json::json!({ "provider": "openai" })), + diff: Some("diff --git a/a b/a".to_string()), + script_invocation: Some(serde_json::json!({ "command": "cargo test" })), + script_timing: Some(serde_json::json!({ "duration_ms": 10 })), + parallel_results: Some(serde_json::json!([{ "stage": "fanout@1" }])), + stdout: Some("stdout".to_string()), + stderr: Some("stderr".to_string()), + }); + + let dump = RunDump::from_projection(&projection); + let paths: Vec<&str> = dump + .entries() + .iter() + .map(|entry| entry.path.as_str()) + .collect(); + + assert!(paths.contains(&"run.json")); + assert!(paths.contains(&"graph.fabro")); + assert!(paths.contains(&"retro/prompt.md")); + assert!(paths.contains(&"retro/response.md")); + assert!(paths.contains(&"stages/build@2/prompt.md")); + assert!(paths.contains(&"stages/build@2/response.md")); + assert!(paths.contains(&"stages/build@2/status.json")); + assert!(paths.contains(&"stages/build@2/provider_used.json")); + assert!(paths.contains(&"stages/build@2/diff.patch")); + assert!(paths.contains(&"stages/build@2/script_invocation.json")); + assert!(paths.contains(&"stages/build@2/script_timing.json")); + assert!(paths.contains(&"stages/build@2/parallel_results.json")); + assert!(paths.contains(&"stages/build@2/stdout.log")); + assert!(paths.contains(&"stages/build@2/stderr.log")); + assert!(!paths.contains(&"start.json")); + assert!(!paths.contains(&"status.json")); + assert!(!paths.contains(&"checkpoint.json")); + assert!(!paths.contains(&"sandbox.json")); + assert!(!paths.contains(&"retro.json")); + assert!(!paths.contains(&"conclusion.json")); + + let run_json = dump + .entries() + .iter() + .find(|entry| entry.path == "run.json") + .expect("run.json should be emitted"); + let RunDumpContents::Json(value) = &run_json.contents else { + panic!("run.json should be json"); + }; + let round_tripped: RunProjection = serde_json::from_value(value.clone()).unwrap(); + let node = round_tripped.node(&stage_id).expect("node should exist"); + + assert!(round_tripped.spec.is_some()); + assert!(round_tripped.start.is_some()); + assert!(round_tripped.status.is_some()); + assert!(round_tripped.checkpoint.is_some()); + assert!(round_tripped.conclusion.is_some()); + assert!(round_tripped.sandbox.is_some()); + assert_eq!(node.prompt, None); + assert_eq!(node.response, None); + assert_eq!(node.diff, None); + assert_eq!(node.stdout, None); + assert_eq!(node.stderr, None); + assert_eq!( + node.provider_used, + Some(serde_json::json!({ "provider": "openai" })) + ); + } +} diff --git a/lib/crates/fabro-workflow/src/run_lookup.rs b/lib/crates/fabro-workflow/src/run_lookup.rs index 86bc556b6..7f291d86d 100644 --- a/lib/crates/fabro-workflow/src/run_lookup.rs +++ b/lib/crates/fabro-workflow/src/run_lookup.rs @@ -66,7 +66,7 @@ impl RunInfo { self.summary .as_ref() .and_then(|summary| summary.workflow_name.clone()) - .unwrap_or_else(|| "[no run record]".to_string()) + .unwrap_or_else(|| "[no run spec]".to_string()) } pub fn workflow_slug(&self) -> Option<&str> { @@ -407,7 +407,7 @@ mod tests { use super::scan_runs_combined; use crate::event::{Event, append_event}; use crate::operations::make_run_dir; - use crate::records::RunRecord; + use crate::records::RunSpec; fn memory_store() -> Arc { Arc::new(Database::new( @@ -418,8 +418,8 @@ mod tests { )) } - fn sample_run_record() -> RunRecord { - RunRecord { + fn sample_run_spec() -> RunSpec { + RunSpec { run_id: fixtures::RUN_1, settings: SettingsLayer::default(), graph: Graph::new("test"), @@ -442,23 +442,23 @@ mod tests { std::fs::create_dir_all(&run_dir).unwrap(); let store = memory_store(); - let run_record = sample_run_record(); + let run_spec = sample_run_spec(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); append_event(&run_store, &fixtures::RUN_1, &Event::RunCreated { run_id: fixtures::RUN_1, - settings: serde_json::to_value(&run_record.settings).unwrap(), - graph: serde_json::to_value(&run_record.graph).unwrap(), + settings: serde_json::to_value(&run_spec.settings).unwrap(), + graph: serde_json::to_value(&run_spec.graph).unwrap(), workflow_source: None, workflow_config: None, - labels: run_record.labels.clone().into_iter().collect(), + labels: run_spec.labels.clone().into_iter().collect(), run_dir: run_dir.display().to_string(), - working_directory: run_record.working_directory.display().to_string(), - host_repo_path: run_record.host_repo_path.clone(), - repo_origin_url: run_record.repo_origin_url.clone(), - base_branch: run_record.base_branch.clone(), - workflow_slug: run_record.workflow_slug.clone(), + working_directory: run_spec.working_directory.display().to_string(), + host_repo_path: run_spec.host_repo_path.clone(), + repo_origin_url: run_spec.repo_origin_url.clone(), + base_branch: run_spec.base_branch.clone(), + workflow_slug: run_spec.workflow_slug.clone(), db_prefix: None, - provenance: run_record.provenance.clone(), + provenance: run_spec.provenance.clone(), manifest_blob: None, }) .await diff --git a/lib/crates/fabro-workflow/src/runtime_store.rs b/lib/crates/fabro-workflow/src/runtime_store.rs index d2523be14..ac825f176 100644 --- a/lib/crates/fabro-workflow/src/runtime_store.rs +++ b/lib/crates/fabro-workflow/src/runtime_store.rs @@ -118,7 +118,7 @@ mod tests { use super::RunStoreHandle; use crate::event::{Event, append_event}; - use crate::records::RunRecord; + use crate::records::RunSpec; async fn test_run_store() -> fabro_store::RunDatabase { let store = Arc::new(Database::new( @@ -130,8 +130,8 @@ mod tests { store.create_run(&fixtures::RUN_1).await.unwrap() } - fn test_run_record() -> RunRecord { - RunRecord { + fn test_run_spec() -> RunSpec { + RunSpec { run_id: fixtures::RUN_1, settings: SettingsLayer::default(), graph: Graph::new("test"), @@ -150,7 +150,7 @@ mod tests { #[tokio::test] async fn local_handle_loads_state_and_events() { let run_store = test_run_store().await; - let record = test_run_record(); + let record = test_run_spec(); append_event(&run_store, &fixtures::RUN_1, &Event::RunCreated { run_id: fixtures::RUN_1, settings: serde_json::to_value(&record.settings).unwrap(), @@ -175,7 +175,7 @@ mod tests { let state = handle.state().await.unwrap(); let events = handle.list_events().await.unwrap(); - assert_eq!(state.run.unwrap().workflow_slug.as_deref(), Some("test")); + assert_eq!(state.spec.unwrap().workflow_slug.as_deref(), Some("test")); assert_eq!(events.len(), 1); } diff --git a/lib/crates/fabro-workflow/tests/it/daytona_integration.rs b/lib/crates/fabro-workflow/tests/it/daytona_integration.rs index 25db32e37..595b46ca4 100644 --- a/lib/crates/fabro-workflow/tests/it/daytona_integration.rs +++ b/lib/crates/fabro-workflow/tests/it/daytona_integration.rs @@ -743,7 +743,7 @@ async fn daytona_git_checkpoint_remote_emits_events() { ); } - // Verify checkpoint.json has git_commit_sha + // Verify the persisted checkpoint snapshot has git_commit_sha let checkpoint = load_run_checkpoint(dir.path()).expect("checkpoint should load"); assert!( checkpoint.git_commit_sha.is_some(), diff --git a/lib/crates/fabro-workflow/tests/it/integration.rs b/lib/crates/fabro-workflow/tests/it/integration.rs index b2fb8d21b..ec8c8b7f5 100644 --- a/lib/crates/fabro-workflow/tests/it/integration.rs +++ b/lib/crates/fabro-workflow/tests/it/integration.rs @@ -9024,8 +9024,8 @@ async fn downstream_remote_execution_materializes_blob_refs_to_sandbox_files() { // --------------------------------------------------------------------------- /// Verify that revisited nodes get distinct stage directories: -/// visit 1 → `nodes/{id}/` -/// visit 2 → `nodes/{id}-attempt_2/` +/// visit 1 → `stages/{id}@1/` +/// visit 2 → `stages/{id}@2/` #[tokio::test] async fn node_dir_uses_visit_count_on_revisit() { // Handler that fails on first call, succeeds on second. @@ -10515,10 +10515,10 @@ async fn git_checkpoint_host_writes_shadow_branch() { ); // 8. Verify round-trip: shadow checkpoint's completed_nodes matches expected - let run_record = MetadataStore::read_run_record(repo.path(), &run_id.to_string()) - .expect("read_run_record should not error") - .expect("shadow branch should contain run record"); - assert_eq!(run_record.run_id, run_id); + let run_spec = MetadataStore::read_run_spec(repo.path(), &run_id.to_string()) + .expect("read_run_spec should not error") + .expect("shadow branch should contain run spec"); + assert_eq!(run_spec.run_id, run_id); // Cleanup worktree let _ = std::process::Command::new("git") diff --git a/lib/packages/fabro-api-client/src/models/run-projection.ts b/lib/packages/fabro-api-client/src/models/run-projection.ts index ccbfca11c..fe3b3fcac 100644 --- a/lib/packages/fabro-api-client/src/models/run-projection.ts +++ b/lib/packages/fabro-api-client/src/models/run-projection.ts @@ -33,7 +33,7 @@ import type { RunStatusRecord } from './run-status-record'; * Raw internal run projection derived from the event log. */ export interface RunProjection { - 'run'?: { [key: string]: any; } | null; + 'spec'?: { [key: string]: any; } | null; 'graph_source'?: string | null; 'start'?: { [key: string]: any; } | null; 'status'?: RunStatusRecord | null; From 125a73aae3ec0dfb389285ccb24eaaa585292918 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Mon, 20 Apr 2026 22:18:44 -0400 Subject: [PATCH 06/12] fix(client): reject all obfuscated IPv4 host forms at parse time Replace the narrow decimal/hex obfuscation check with a general comparison: if the parsed host is an IPv4 literal and the raw input host differs from the canonical dotted-quad form, the user supplied an obfuscated variant (octal, short-form, mixed radix, leading zeros, decimal integer, hex integer) that url::Url has already normalized to 127.0.0.1. All such variants are rejected. Test now covers decimal, hex, octal, two-/three-part short, mixed hex/ decimal, and leading-zero octets. Co-Authored-By: Claude Opus 4.7 (1M context) --- lib/crates/fabro-client/src/loopback.rs | 12 +++++++++++- lib/crates/fabro-client/src/target.rs | 23 ++++++++++------------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/lib/crates/fabro-client/src/loopback.rs b/lib/crates/fabro-client/src/loopback.rs index bda6f0b25..cea27093f 100644 --- a/lib/crates/fabro-client/src/loopback.rs +++ b/lib/crates/fabro-client/src/loopback.rs @@ -176,7 +176,17 @@ mod tests { #[test] fn rejects_obfuscated_ipv4_literals_at_parse_time() { - for api_url in ["http://2130706433", "http://0x7f000001"] { + let cases = [ + "http://2130706433", // decimal integer + "http://0x7f000001", // hex integer + "http://0177.0.0.1", // octal dotted + "http://127.1", // two-part short + "http://127.0.1", // three-part short + "http://0x7f.0.0.1", // mixed hex/decimal + "http://127.00.0.1", // leading-zero octet + "http://127.0.0.001", // leading-zero octet + ]; + for api_url in cases { assert!( ServerTarget::http_url(api_url).is_err(), "{api_url} should not parse as a server target" diff --git a/lib/crates/fabro-client/src/target.rs b/lib/crates/fabro-client/src/target.rs index 4ad663ac5..b590bcb0a 100644 --- a/lib/crates/fabro-client/src/target.rs +++ b/lib/crates/fabro-client/src/target.rs @@ -146,13 +146,16 @@ fn canonical_http_url(value: &str) -> Result { _ => bail!("server target must be an http(s) URL or absolute Unix socket path"), }; - if raw_url_host(normalized).is_some_and(is_obfuscated_ipv4_literal) { - bail!("server target must be an http(s) URL or absolute Unix socket path"); - } - let Some(host) = url.host_str() else { bail!("server target must be an http(s) URL or absolute Unix socket path"); }; + + if host.parse::().is_ok() + && raw_url_host(normalized).is_some_and(|raw| raw != host) + { + bail!("server target must be an http(s) URL or absolute Unix socket path"); + } + let host = host.to_ascii_lowercase(); let Some(port) = url.port_or_known_default() else { bail!("server target must be an http(s) URL or absolute Unix socket path"); @@ -171,8 +174,9 @@ fn trim_api_path_suffix(value: &str) -> &str { } /// Extract the host substring from `value` without going through -/// [`fabro_http::Url`]. `Url` normalizes decimal/hex IPv4 literals into dotted -/// form, which hides the original input from later inspection. +/// [`fabro_http::Url`]. `Url` normalizes IPv4 literals (decimal/hex/octal/short +/// form) into dotted quads, which hides the original input from later +/// inspection. fn raw_url_host(value: &str) -> Option<&str> { let (_, remainder) = value.split_once("://")?; let authority_end = remainder.find(['/', '?', '#']).unwrap_or(remainder.len()); @@ -189,13 +193,6 @@ fn raw_url_host(value: &str) -> Option<&str> { (!host.is_empty()).then_some(host) } -fn is_obfuscated_ipv4_literal(host: &str) -> bool { - if host.starts_with("0x") || host.starts_with("0X") { - return true; - } - !host.contains('.') && !host.is_empty() && host.bytes().all(|b| b.is_ascii_digit()) -} - fn lexical_normalize_absolute_path(path: &Path) -> Result { if !path.is_absolute() { bail!("server target must be an http(s) URL or absolute Unix socket path"); From 5e482486e71cb90905e2425b4d8e4b3ab324f793 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Tue, 21 Apr 2026 07:44:15 -0400 Subject: [PATCH 07/12] test: finish metadata cleanup sweep Rename the last stale workflow test helpers and assertions that still used pre-refactor checkpoint/retro file terminology, and update the retro docs to describe the exported layout that now exists. --- docs/execution/retros.mdx | 2 +- lib/crates/fabro-workflow/src/operations/fork.rs | 8 ++++---- .../fabro-workflow/src/operations/rewind.rs | 2 +- .../src/operations/test_support.rs | 2 +- .../fabro-workflow/src/pipeline/execute/tests.rs | 2 +- .../fabro-workflow/src/pipeline/finalize.rs | 2 +- lib/crates/fabro-workflow/src/pipeline/retro.rs | 2 +- .../fabro-workflow/tests/it/integration.rs | 16 +++------------- 8 files changed, 13 insertions(+), 23 deletions(-) diff --git a/docs/execution/retros.mdx b/docs/execution/retros.mdx index 8aae4be1e..d0b5f6b63 100644 --- a/docs/execution/retros.mdx +++ b/docs/execution/retros.mdx @@ -143,4 +143,4 @@ Retros are also available via the REST API. See the [list retros](/api-reference ## Storage -Retros are stored in durable run state. If you need files on disk, `fabro store dump` materializes the retro as `retro.json` alongside other exported run data. +Retros are stored in durable run state. If you need files on disk, `fabro store dump` materializes retro text under `retro/` alongside `run.json`, stage files, and the rest of the exported run data. diff --git a/lib/crates/fabro-workflow/src/operations/fork.rs b/lib/crates/fabro-workflow/src/operations/fork.rs index 32037a39c..f0b43153a 100644 --- a/lib/crates/fabro-workflow/src/operations/fork.rs +++ b/lib/crates/fabro-workflow/src/operations/fork.rs @@ -252,15 +252,15 @@ mod tests { for (i, node) in nodes.iter().enumerate() { let mut projection = init_projection.clone(); projection.checkpoint = Some( - serde_json::from_slice(&make_checkpoint_json( + serde_json::from_slice(&make_checkpoint_bytes( node, 1, Some(&run_oids[i].to_string()), )) .unwrap(), ); - let checkpoint_json = serde_json::to_vec_pretty(&projection).unwrap(); - bs.write_entry("run.json", &checkpoint_json, "checkpoint") + let projection_json = serde_json::to_vec_pretty(&projection).unwrap(); + bs.write_entry("run.json", &projection_json, "checkpoint") .unwrap(); } @@ -318,7 +318,7 @@ mod tests { let mut checkpoint_projection = make_run_projection(&run_id); checkpoint_projection.checkpoint = - Some(serde_json::from_slice(&make_checkpoint_json("start", 1, None)).unwrap()); + Some(serde_json::from_slice(&make_checkpoint_bytes("start", 1, None)).unwrap()); let cp = serde_json::to_vec_pretty(&checkpoint_projection).unwrap(); let oid = bs.write_entry("run.json", &cp, "checkpoint").unwrap(); let entry = TimelineEntry { diff --git a/lib/crates/fabro-workflow/src/operations/rewind.rs b/lib/crates/fabro-workflow/src/operations/rewind.rs index 17deb07f4..23603293f 100644 --- a/lib/crates/fabro-workflow/src/operations/rewind.rs +++ b/lib/crates/fabro-workflow/src/operations/rewind.rs @@ -405,7 +405,7 @@ mod tests { ) -> Vec { let mut projection = RunProjection::default(); projection.checkpoint = Some( - serde_json::from_slice(&make_checkpoint_json(current_node, visit, git_commit_sha)) + serde_json::from_slice(&make_checkpoint_bytes(current_node, visit, git_commit_sha)) .unwrap(), ); serde_json::to_vec_pretty(&projection).unwrap() diff --git a/lib/crates/fabro-workflow/src/operations/test_support.rs b/lib/crates/fabro-workflow/src/operations/test_support.rs index 79319fa7b..1065d20b8 100644 --- a/lib/crates/fabro-workflow/src/operations/test_support.rs +++ b/lib/crates/fabro-workflow/src/operations/test_support.rs @@ -13,7 +13,7 @@ pub(super) fn test_sig() -> Signature<'static> { Signature::now("Test", "test@example.com").unwrap() } -pub(super) fn make_checkpoint_json( +pub(super) fn make_checkpoint_bytes( current_node: &str, visit: usize, git_sha: Option<&str>, diff --git a/lib/crates/fabro-workflow/src/pipeline/execute/tests.rs b/lib/crates/fabro-workflow/src/pipeline/execute/tests.rs index 16c824d01..61a8b886f 100644 --- a/lib/crates/fabro-workflow/src/pipeline/execute/tests.rs +++ b/lib/crates/fabro-workflow/src/pipeline/execute/tests.rs @@ -645,7 +645,7 @@ async fn execute_conditional_routing_uses_unconditional_success_path() { } #[tokio::test] -async fn execute_writes_start_json_and_node_status() { +async fn execute_persists_start_record_and_node_status() { let dir = tempfile::tempdir().unwrap(); let mut run_options = test_run_options(dir.path(), "test-run"); run_options.git = Some(GitCheckpointOptions { diff --git a/lib/crates/fabro-workflow/src/pipeline/finalize.rs b/lib/crates/fabro-workflow/src/pipeline/finalize.rs index 188b3fd02..924d43bcf 100644 --- a/lib/crates/fabro-workflow/src/pipeline/finalize.rs +++ b/lib/crates/fabro-workflow/src/pipeline/finalize.rs @@ -342,7 +342,7 @@ mod tests { } #[tokio::test] - async fn finalize_writes_conclusion_json() { + async fn finalize_persists_conclusion_in_projection() { let temp = tempfile::tempdir().unwrap(); let run_dir = temp.path().join("run"); std::fs::create_dir_all(&run_dir).unwrap(); diff --git a/lib/crates/fabro-workflow/src/pipeline/retro.rs b/lib/crates/fabro-workflow/src/pipeline/retro.rs index 300f53afd..74dd181a2 100644 --- a/lib/crates/fabro-workflow/src/pipeline/retro.rs +++ b/lib/crates/fabro-workflow/src/pipeline/retro.rs @@ -312,7 +312,7 @@ mod tests { } #[tokio::test] - async fn retro_phase_writes_retro_json() { + async fn retro_phase_persists_retro_in_projection() { let temp = tempfile::tempdir().unwrap(); let run_dir = temp.path().join("run"); std::fs::create_dir_all(&run_dir).unwrap(); diff --git a/lib/crates/fabro-workflow/tests/it/integration.rs b/lib/crates/fabro-workflow/tests/it/integration.rs index ec8c8b7f5..9c174ea7a 100644 --- a/lib/crates/fabro-workflow/tests/it/integration.rs +++ b/lib/crates/fabro-workflow/tests/it/integration.rs @@ -71,16 +71,6 @@ fn test_run_id(label: &str) -> RunId { } fn load_checkpoint(path: &Path) -> Result> { - if !path.exists() - && path - .file_name() - .is_some_and(|name| name == "checkpoint.json") - { - let run_dir = path - .parent() - .ok_or("checkpoint path should have a parent")?; - return load_run_checkpoint(run_dir); - } let data = std::fs::read_to_string(path)?; Ok(serde_json::from_str(&data)?) } @@ -188,9 +178,9 @@ fn load_run_checkpoint(run_dir: &Path) -> Result ArtifactStore { @@ -1462,7 +1452,7 @@ async fn pipeline_with_many_nodes() { #[test] fn checkpoint_save_and_resume_roundtrip() { let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("checkpoint.json"); + let path = dir.path().join("checkpoint_state.json"); let ctx = Context::new(); ctx.set("goal", serde_json::json!("Test checkpoint")); From 2a3f550e2eb6ee7b8f3292c98860a4c900e3ce22 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Tue, 21 Apr 2026 08:27:59 -0400 Subject: [PATCH 08/12] refactor: simplify metadata snapshot + retro upload paths - MetadataStore: drop redundant write_files (identical to write_snapshot) and the brittle has_projection_data OR-chain in read_run_projection. - operations::rewind: introduce find_run_id_by_prefix_opt using META_BRANCH_PREFIX; delete the duplicate find_run_id_by_prefix_in_refs helper in rebuild_meta and have it call the shared function. - rebuild_meta: stop cloning latest_init_snapshot once the init snapshot has been written; take() the stored snapshot instead of cloning again. - retro::upload_data_files: collapse the ten eager *_path variables into inline base.join(...) args by making upload_file take &Path; replace Vec.join("\n") + "\n" with a streaming String loop. - Migrate retro_agent test std::fs::read_to_string to tokio::fs to satisfy disallowed_methods clippy lint under tokio tests. - Minor: fork.rs drop misnamed `now` var; metadata.rs doc comment describes the unified RunProjection snapshot. Co-Authored-By: Claude Opus 4.7 (1M context) --- lib/crates/fabro-checkpoint/src/metadata.rs | 70 ++------ lib/crates/fabro-retro/src/retro_agent.rs | 163 +++++++++++------- .../fabro-workflow/src/operations/fork.rs | 3 +- .../src/operations/rebuild_meta.rs | 37 +--- .../fabro-workflow/src/operations/rewind.rs | 42 +++-- lib/crates/fabro-workflow/src/run_dump.rs | 2 +- 6 files changed, 142 insertions(+), 175 deletions(-) diff --git a/lib/crates/fabro-checkpoint/src/metadata.rs b/lib/crates/fabro-checkpoint/src/metadata.rs index 8347f7b35..c75f6a0ae 100644 --- a/lib/crates/fabro-checkpoint/src/metadata.rs +++ b/lib/crates/fabro-checkpoint/src/metadata.rs @@ -12,8 +12,8 @@ use crate::git::Store; /// Git-native metadata storage for pipeline runs. /// -/// Stores checkpoint data, run specs, and metadata on an orphan branch -/// (`fabro/meta/{run_id}`) so that runs can be resumed from git alone. +/// Stores a unified `RunProjection` snapshot (plus artifacts) on an orphan +/// branch (`fabro/meta/{run_id}`) so that runs can be resumed from git alone. pub struct MetadataStore { repo_path: PathBuf, author: GitAuthor, @@ -57,23 +57,8 @@ impl MetadataStore { Ok(()) } - /// Write arbitrary files to the metadata branch. - pub fn write_files( - &self, - run_id: &str, - entries: &[(&str, &[u8])], - message: &str, - ) -> Result<(), MetadataError> { - let (store, sig) = self.open_store()?; - let branch = Self::branch_name(run_id); - let branch_store = BranchStore::new(&store, &branch, &sig); - let message = self.commit_message(message); - branch_store.write_entries(entries, &message)?; - Ok(()) - } - - /// Write a projection snapshot commit to the metadata branch and return - /// the new commit SHA. + /// Write a snapshot commit to the metadata branch and return the new + /// commit SHA. pub fn write_snapshot( &self, run_id: &str, @@ -112,39 +97,16 @@ impl MetadataStore { run_id: &str, ) -> Result, MetadataError> { let branch = Self::branch_name(run_id); - match Self::read_file(repo_path, run_id, "run.json")? { - Some(bytes) => { - let projection: RunProjection = - serde_json::from_slice(&bytes).map_err(|source| { - MetadataError::Deserialize { - entity: "run projection", - branch: branch.clone(), - source, - } - })?; - let has_projection_data = projection.spec.is_some() - || projection.start.is_some() - || projection.status.is_some() - || projection.checkpoint.is_some() - || projection.conclusion.is_some() - || projection.sandbox.is_some() - || projection.retro.is_some() - || projection.graph_source.is_some() - || projection.iter_nodes().next().is_some(); - if !has_projection_data { - return Err(MetadataError::Deserialize { - entity: "run projection", - branch, - source: serde_json::Error::io(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "run.json does not contain a serialized projection snapshot", - )), - }); - } - Ok(Some(projection)) - } - None => Ok(None), - } + let Some(bytes) = Self::read_file(repo_path, run_id, "run.json")? else { + return Ok(None); + }; + let projection: RunProjection = + serde_json::from_slice(&bytes).map_err(|source| MetadataError::Deserialize { + entity: "run projection", + branch, + source, + })?; + Ok(Some(projection)) } /// Read a checkpoint from the metadata branch. Returns `None` if branch or @@ -414,7 +376,7 @@ mod tests { } #[test] - fn metadata_store_write_files() { + fn metadata_store_write_snapshot_preserves_prior_files() { let dir = tempfile::tempdir().unwrap(); init_repo(dir.path()); @@ -426,7 +388,7 @@ mod tests { .unwrap(); store - .write_files( + .write_snapshot( &run_id, &[("retro/prompt.md", b"how did it go?")], "finalize run", diff --git a/lib/crates/fabro-retro/src/retro_agent.rs b/lib/crates/fabro-retro/src/retro_agent.rs index 978110032..4792a2f75 100644 --- a/lib/crates/fabro-retro/src/retro_agent.rs +++ b/lib/crates/fabro-retro/src/retro_agent.rs @@ -301,27 +301,36 @@ async fn upload_data_files( _run_dir: &Path, target_dir: &str, ) -> anyhow::Result<()> { - let progress_content = { - let lines: Vec = events - .iter() - .filter_map(|env| serde_json::to_string(&env.event).ok()) - .collect(); - if lines.is_empty() { - None - } else { - Some(lines.join("\n") + "\n") + let progress_content = (!events.is_empty()).then(|| { + let mut buf = String::new(); + for env in events { + if let Ok(line) = serde_json::to_string(&env.event) { + buf.push_str(&line); + buf.push('\n'); + } } - }; - upload_file(sandbox, target_dir, "progress.jsonl", progress_content).await?; - - let run_content = Some(serde_json::to_string_pretty(&SerializableProjection( - state, - ))?); - upload_file(sandbox, target_dir, "run.json", run_content).await?; + buf + }); upload_file( sandbox, target_dir, - "graph.fabro", + Path::new("progress.jsonl"), + progress_content, + ) + .await?; + + let run_content = serde_json::to_string_pretty(&SerializableProjection(state))?; + upload_file( + sandbox, + target_dir, + Path::new("run.json"), + Some(run_content), + ) + .await?; + upload_file( + sandbox, + target_dir, + Path::new("graph.fabro"), state.graph_source.clone(), ) .await?; @@ -337,62 +346,76 @@ async fn upload_data_files( continue; }; let base = PathBuf::from("stages").join(stage_id.to_string()); - let prompt_path = base.join("prompt.md").to_string_lossy().into_owned(); - let response_path = base.join("response.md").to_string_lossy().into_owned(); - let status_path = base.join("status.json").to_string_lossy().into_owned(); - let provider_used_path = base - .join("provider_used.json") - .to_string_lossy() - .into_owned(); - let diff_path = base.join("diff.patch").to_string_lossy().into_owned(); - let script_invocation_path = base - .join("script_invocation.json") - .to_string_lossy() - .into_owned(); - let script_timing_path = base - .join("script_timing.json") - .to_string_lossy() - .into_owned(); - let parallel_results_path = base - .join("parallel_results.json") - .to_string_lossy() - .into_owned(); - let stdout_path = base.join("stdout.log").to_string_lossy().into_owned(); - let stderr_path = base.join("stderr.log").to_string_lossy().into_owned(); - upload_file(sandbox, target_dir, &prompt_path, node.prompt.clone()).await?; - upload_file(sandbox, target_dir, &response_path, node.response.clone()).await?; - upload_json_file(sandbox, target_dir, &status_path, node.status.as_ref()).await?; + upload_file( + sandbox, + target_dir, + &base.join("prompt.md"), + node.prompt.clone(), + ) + .await?; + upload_file( + sandbox, + target_dir, + &base.join("response.md"), + node.response.clone(), + ) + .await?; upload_json_file( sandbox, target_dir, - &provider_used_path, + &base.join("status.json"), + node.status.as_ref(), + ) + .await?; + upload_json_file( + sandbox, + target_dir, + &base.join("provider_used.json"), node.provider_used.as_ref(), ) .await?; - upload_file(sandbox, target_dir, &diff_path, node.diff.clone()).await?; + upload_file( + sandbox, + target_dir, + &base.join("diff.patch"), + node.diff.clone(), + ) + .await?; upload_json_file( sandbox, target_dir, - &script_invocation_path, + &base.join("script_invocation.json"), node.script_invocation.as_ref(), ) .await?; upload_json_file( sandbox, target_dir, - &script_timing_path, + &base.join("script_timing.json"), node.script_timing.as_ref(), ) .await?; upload_json_file( sandbox, target_dir, - ¶llel_results_path, + &base.join("parallel_results.json"), node.parallel_results.as_ref(), ) .await?; - upload_file(sandbox, target_dir, &stdout_path, node.stdout.clone()).await?; - upload_file(sandbox, target_dir, &stderr_path, node.stderr.clone()).await?; + upload_file( + sandbox, + target_dir, + &base.join("stdout.log"), + node.stdout.clone(), + ) + .await?; + upload_file( + sandbox, + target_dir, + &base.join("stderr.log"), + node.stderr.clone(), + ) + .await?; } Ok(()) @@ -401,32 +424,33 @@ async fn upload_data_files( async fn upload_file( sandbox: &Arc, target_dir: &str, - filename: &str, + relative: &Path, content: Option, ) -> anyhow::Result<()> { - if let Some(content) = content { - let path = Path::new(target_dir).join(filename); - let remote_path = path.to_string_lossy().into_owned(); - ensure_remote_dir(sandbox, &path).await?; - sandbox - .write_file(&remote_path, &content) - .await - .map_err(|e| anyhow::anyhow!("Failed to upload {filename}: {e}"))?; - } + let Some(content) = content else { + return Ok(()); + }; + let path = Path::new(target_dir).join(relative); + let remote_path = path.to_string_lossy().into_owned(); + ensure_remote_dir(sandbox, &path).await?; + sandbox + .write_file(&remote_path, &content) + .await + .map_err(|e| anyhow::anyhow!("Failed to upload {}: {e}", relative.display()))?; Ok(()) } async fn upload_json_file( sandbox: &Arc, target_dir: &str, - filename: &str, + relative: &Path, value: Option<&T>, ) -> anyhow::Result<()> where T: serde::Serialize, { let content = value.map(serde_json::to_string_pretty).transpose()?; - upload_file(sandbox, target_dir, filename, content).await + upload_file(sandbox, target_dir, relative, content).await } async fn ensure_remote_dir(sandbox: &Arc, path: &Path) -> anyhow::Result<()> { @@ -456,6 +480,7 @@ mod tests { use fabro_agent::LocalSandbox; use fabro_store::{NodeState, StageId}; use fabro_types::{NodeStatusRecord, StageStatus}; + use tokio::fs; use super::*; @@ -555,7 +580,9 @@ mod tests { .expect("retro files should upload"); let run_json: serde_json::Value = serde_json::from_str( - &std::fs::read_to_string(target_dir.join("run.json")).expect("run.json should exist"), + &fs::read_to_string(target_dir.join("run.json")) + .await + .expect("run.json should exist"), ) .expect("run.json should parse"); assert!(run_json.get("spec").is_some()); @@ -563,22 +590,26 @@ mod tests { assert!(run_json["nodes"]["build@2"]["prompt"].is_null()); assert!(run_json["nodes"]["build@2"]["diff"].is_null()); assert_eq!( - std::fs::read_to_string(target_dir.join("graph.fabro")) + fs::read_to_string(target_dir.join("graph.fabro")) + .await .expect("graph.fabro should exist"), "digraph Ship {}" ); assert_eq!( - std::fs::read_to_string(target_dir.join("stages/build@2/prompt.md")) + fs::read_to_string(target_dir.join("stages/build@2/prompt.md")) + .await .expect("prompt file should exist"), "plan" ); assert_eq!( - std::fs::read_to_string(target_dir.join("stages/build@2/response.md")) + fs::read_to_string(target_dir.join("stages/build@2/response.md")) + .await .expect("response file should exist"), "done" ); assert_eq!( - std::fs::read_to_string(target_dir.join("stages/build@2/stdout.log")) + fs::read_to_string(target_dir.join("stages/build@2/stdout.log")) + .await .expect("stdout file should exist"), "stdout" ); diff --git a/lib/crates/fabro-workflow/src/operations/fork.rs b/lib/crates/fabro-workflow/src/operations/fork.rs index f0b43153a..3b6116ed3 100644 --- a/lib/crates/fabro-workflow/src/operations/fork.rs +++ b/lib/crates/fabro-workflow/src/operations/fork.rs @@ -82,10 +82,9 @@ fn fork_from_entry( .context("source run projection has no spec")?; run_spec.run_id = new_run_id; - let now = new_run_id.created_at(); let start_record = StartRecord { run_id: new_run_id, - start_time: now, + start_time: new_run_id.created_at(), run_branch: Some(new_run_branch.clone()), base_sha: None, }; diff --git a/lib/crates/fabro-workflow/src/operations/rebuild_meta.rs b/lib/crates/fabro-workflow/src/operations/rebuild_meta.rs index 7c64250be..29970863b 100644 --- a/lib/crates/fabro-workflow/src/operations/rebuild_meta.rs +++ b/lib/crates/fabro-workflow/src/operations/rebuild_meta.rs @@ -48,7 +48,7 @@ pub async fn rebuild_metadata_branch( let stored = &event.event; let is_checkpoint = matches!(stored.body, EventBody::CheckpointCompleted(_)); - if !is_checkpoint && projection.spec.is_some() { + if !init_written && !is_checkpoint && projection.spec.is_some() { latest_init_snapshot = Some(projection.clone()); } @@ -56,7 +56,7 @@ pub async fn rebuild_metadata_branch( if is_checkpoint { if !init_written { - let init_snapshot = latest_init_snapshot.clone().unwrap_or_else(|| { + let init_snapshot = latest_init_snapshot.take().unwrap_or_else(|| { let mut snapshot = projection.clone(); snapshot.checkpoint = None; snapshot.checkpoints.clear(); @@ -147,7 +147,7 @@ pub async fn find_run_id_by_prefix_or_store( fabro_store: &DurableStore, prefix: &str, ) -> Result { - if let Some(run_id) = find_run_id_by_prefix_in_refs(repo, prefix)? { + if let Some(run_id) = rewind::find_run_id_by_prefix_opt(repo, prefix)? { return Ok(run_id); } @@ -261,37 +261,6 @@ fn backfill_missing_checkpoint_shas( } } -fn find_run_id_by_prefix_in_refs(repo: &Repository, prefix: &str) -> Result> { - let refs = repo.references()?; - let pattern = "refs/heads/fabro/meta/"; - let mut matches = Vec::new(); - - for reference in refs.flatten() { - let Some(name) = reference.name() else { - continue; - }; - let Some(run_id) = name.strip_prefix(pattern) else { - continue; - }; - let Ok(run_id) = run_id.parse::() else { - continue; - }; - - if run_id.to_string() == prefix { - return Ok(Some(run_id)); - } - if run_id.to_string().starts_with(prefix) { - matches.push(run_id); - } - } - - if matches.is_empty() { - return Ok(None); - } - - resolve_prefix_matches(prefix, matches).map(Some) -} - fn repo_root_path(repo: &Repository) -> PathBuf { repo.workdir() .or_else(|| repo.path().parent()) diff --git a/lib/crates/fabro-workflow/src/operations/rewind.rs b/lib/crates/fabro-workflow/src/operations/rewind.rs index 23603293f..e2e643d7c 100644 --- a/lib/crates/fabro-workflow/src/operations/rewind.rs +++ b/lib/crates/fabro-workflow/src/operations/rewind.rs @@ -3,6 +3,7 @@ use std::fmt::Write; use std::str::FromStr; use anyhow::{Context, Result, bail}; +use fabro_checkpoint::META_BRANCH_PREFIX; use fabro_checkpoint::branch::{BranchStore, CommitInfo}; use fabro_checkpoint::git::Store; use fabro_graphviz::graph::Graph; @@ -319,37 +320,42 @@ fn rewind_to_entry(store: &Store, run_id: &RunId, entry: &TimelineEntry, push: b } pub fn find_run_id_by_prefix(repo: &Repository, prefix: &str) -> Result { + find_run_id_by_prefix_opt(repo, prefix)? + .ok_or_else(|| anyhow::anyhow!("no run found matching '{prefix}'")) +} + +/// Resolve a run id from the metadata branch refs. `Ok(None)` when no run +/// matches; `Err` when the prefix matches more than one. +pub(super) fn find_run_id_by_prefix_opt(repo: &Repository, prefix: &str) -> Result> { let refs = repo.references()?; - let pattern = "refs/heads/fabro/meta/"; + let pattern = format!("refs/heads/{META_BRANCH_PREFIX}"); let mut matches = Vec::new(); for reference in refs.flatten() { let Some(name) = reference.name() else { continue; }; - if let Some(run_id) = name.strip_prefix(pattern) { - let Ok(run_id) = run_id.parse::() else { - continue; - }; - if run_id.to_string() == prefix { - return Ok(run_id); - } - if run_id.to_string().starts_with(prefix) { - matches.push(run_id); - } + let Some(run_id) = name.strip_prefix(&pattern) else { + continue; + }; + let Ok(run_id) = run_id.parse::() else { + continue; + }; + if run_id.to_string() == prefix { + return Ok(Some(run_id)); + } + if run_id.to_string().starts_with(prefix) { + matches.push(run_id); } } match matches.len() { - 0 => bail!("no run found matching '{prefix}'"), - 1 => Ok(matches - .into_iter() - .next() - .expect("exactly one run should match when len is 1")), + 0 => Ok(None), + 1 => Ok(matches.into_iter().next()), _ => { let mut msg = format!("ambiguous run ID prefix '{prefix}', matches:\n"); - for m in &matches { - let _ = writeln!(msg, " {m}"); + for run_id in &matches { + let _ = writeln!(msg, " {run_id}"); } bail!("{msg}") } diff --git a/lib/crates/fabro-workflow/src/run_dump.rs b/lib/crates/fabro-workflow/src/run_dump.rs index 68773e2e5..c9013bda7 100644 --- a/lib/crates/fabro-workflow/src/run_dump.rs +++ b/lib/crates/fabro-workflow/src/run_dump.rs @@ -221,7 +221,7 @@ impl RunDump { .iter() .map(|(path, bytes)| (path.as_str(), bytes.as_slice())) .collect(); - store.write_files(run_id, &refs, message)?; + store.write_snapshot(run_id, &refs, message)?; Ok(()) } From f75e5c2ef6951b74642e1c1e8564d8e772cb1507 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Tue, 21 Apr 2026 08:29:34 -0400 Subject: [PATCH 09/12] refactor(store): extract Record/Repository abstractions Replaces hand-written K/V stores in fabro-store with a shared Record trait plus Repository typed K/V layer. Adds KeyedMutex for per-key serialization and transaction() for all-or-nothing WriteBatch commits. Renames SlateAuthCodeStore/SlateAuthTokenStore to AuthCodeStore/RefreshTokenStore and adds BlobStore and RunCatalogIndex wrappers on top of Repository. Deletes catalog.rs in favor of RunCatalogIndex. Database gains blobs() and catalog_index() accessors; auth_tokens() is renamed refresh_tokens(). Plan: docs/plans/2026-04-20-003-refactor-fabro-store-record-abstractions-plan.md Co-Authored-By: Claude Opus 4.7 (1M context) --- Cargo.lock | 1 + ...-store-record-abstractions-requirements.md | 146 ++++ ...or-fabro-store-record-abstractions-plan.md | 725 ++++++++++++++++++ lib/crates/fabro-server/src/auth/cli_flow.rs | 20 +- lib/crates/fabro-server/src/serve.rs | 10 +- .../tests/it/api/cli_auth_token.rs | 5 +- lib/crates/fabro-store/Cargo.toml | 1 + lib/crates/fabro-store/src/error.rs | 4 + lib/crates/fabro-store/src/keyed_mutex.rs | 180 +++++ lib/crates/fabro-store/src/keys.rs | 80 +- lib/crates/fabro-store/src/lib.rs | 7 +- lib/crates/fabro-store/src/record/codec.rs | 95 +++ lib/crates/fabro-store/src/record/mod.rs | 25 + .../fabro-store/src/record/record_id.rs | 97 +++ .../fabro-store/src/record/repository.rs | 504 ++++++++++++ .../fabro-store/src/record/transaction.rs | 211 +++++ .../fabro-store/src/slate/auth_codes.rs | 120 ++- .../fabro-store/src/slate/auth_tokens.rs | 120 +-- .../fabro-store/src/slate/blob_store.rs | 132 ++++ lib/crates/fabro-store/src/slate/catalog.rs | 51 -- lib/crates/fabro-store/src/slate/mod.rs | 59 +- .../src/slate/run_catalog_index.rs | 150 ++++ lib/crates/fabro-store/src/slate/run_store.rs | 15 +- 23 files changed, 2442 insertions(+), 316 deletions(-) create mode 100644 docs/brainstorms/2026-04-20-fabro-store-record-abstractions-requirements.md create mode 100644 docs/plans/2026-04-20-003-refactor-fabro-store-record-abstractions-plan.md create mode 100644 lib/crates/fabro-store/src/keyed_mutex.rs create mode 100644 lib/crates/fabro-store/src/record/codec.rs create mode 100644 lib/crates/fabro-store/src/record/mod.rs create mode 100644 lib/crates/fabro-store/src/record/record_id.rs create mode 100644 lib/crates/fabro-store/src/record/repository.rs create mode 100644 lib/crates/fabro-store/src/record/transaction.rs create mode 100644 lib/crates/fabro-store/src/slate/blob_store.rs delete mode 100644 lib/crates/fabro-store/src/slate/catalog.rs create mode 100644 lib/crates/fabro-store/src/slate/run_catalog_index.rs diff --git a/Cargo.lock b/Cargo.lock index c779b34bf..e6d735c00 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2074,6 +2074,7 @@ dependencies = [ "dashmap", "fabro-types", "futures", + "insta", "object_store", "percent-encoding", "serde", diff --git a/docs/brainstorms/2026-04-20-fabro-store-record-abstractions-requirements.md b/docs/brainstorms/2026-04-20-fabro-store-record-abstractions-requirements.md new file mode 100644 index 000000000..90848007f --- /dev/null +++ b/docs/brainstorms/2026-04-20-fabro-store-record-abstractions-requirements.md @@ -0,0 +1,146 @@ +--- +date: 2026-04-20 +topic: fabro-store-record-abstractions +--- + +# Fabro-store Record Abstractions + +## Problem Frame + +Adding a new persisted record type to `fabro-store` today means writing ~200–400 LOC of repetitive plumbing in a new `Slate*Store`: key construction, JSON serialization, `get`/`put`/`delete`/`scan_prefix` wrappers, optional per-key consume mutex, optional GC-by-prefix-scan, optional secondary index. The existing four record families — `RefreshToken`, `AuthCode`, `Blob`, and the Run catalog index — share most of that plumbing mechanically, but each duplicates it inside `lib/crates/fabro-store/src/slate/`. As more record types land (new auth flows, sessions, vault entries, agent state), the duplication compounds and each copy is one more place where serialization, locking, or key encoding can drift. + +Greenfield, no production deployments — backwards compat can be broken freely. + +## Architecture + +``` +lib/crates/fabro-store/ + Database + .refresh_tokens() -> Arc + .auth_codes() -> Arc + .blobs() -> Arc + .catalog_index() -> Arc + .runs() -> Runs (unchanged) + + *Store wrappers (one per Record) + own: Repository + domain helpers (KeyedMutex, ReplayCache, …) + expose: domain-named methods (consume_and_rotate, delete_chain, …) + + Repository ← reusable typed K/V layer + get / put / delete / scan_stream / scan_prefix_stream / gc + serializes via R::Codec + keys built from R::PREFIX + R::Id::key_segments + pub(crate) — wrapper Stores own it; never exposed on Database + + trait Record / trait RecordId / trait Codec + Record::PREFIX (const) + Record::Codec (associated type; each impl writes `type Codec = JsonCodec;`) + Record::id(&self) (-> Self::Id) + RecordId::key_segments(&self) -> Vec (Repository assembles SlateKey) + + transaction(&db, |tx| { tx.put(&r1)?; tx.put(&r2)?; }) + ← cross-record atomic batch +``` + +Run's event log, projection cache, broadcast channel, and per-run mutex stay in `RunDatabase` and are deliberately out of scope. The Run catalog *index* (today's `slate/catalog.rs`) is in scope and becomes `RunCatalogIndex` on top of `Repository`. + +## Records in Scope + +| Record | PREFIX | Id type | Codec | Special semantics | +|------------------|-------------------------|---------------------|-----------|---------------------------------------------------------| +| RefreshToken | `auth/refresh` | `[u8; 32]` (hex) | JsonCodec | KeyedMutex consume lock, in-memory replay revocation | +| AuthCode | `auth/code` | `String` (opaque) | JsonCodec | KeyedMutex consume lock, single-use | +| Blob | `blobs/sha256` | `RunBlobId` | RawBytesCodec | Immutable, content-addressed, never deleted | +| RunCatalogEntry | `runs/_index/by-start` | `RunId` | EmptyCodec | Empty value; date segment derived from `RunId.created_at()` | + +## Requirements + +**Core abstraction** +- R1. Define `trait Record: Sized + Send + Sync + 'static` with associated `type Id: RecordId`, `type Codec: Codec`, `const PREFIX: &'static str`, and `fn id(&self) -> Self::Id`. Each `impl Record` writes `type Codec = JsonCodec;` (or its override) explicitly — no associated-type default, since `associated_type_defaults` is unstable on stable Rust as of 2026. One extra line per impl is the lowest-magic alternative; the workspace doesn't have a proc-macro sub-crate today and adding one is more weight than the savings justifies. `PREFIX` is a `/`-separated path of segments (e.g. `"auth/refresh"`, `"runs/_index/by-start"`) that `Repository` splits before assembling the `\0`-separated `SlateKey`; `/` is therefore reserved inside any single segment. +- R2. Define `trait RecordId { fn key_segments(&self) -> Vec; }`. Implementations return segment data; `Repository` assembles the SlateKey internally so `SlateKey` (and its `\0`-segment-boundary invariant) stay `pub(crate)`. Built-in impls: `[u8; 32]` writes one hex segment; `String` writes itself; `RunBlobId` writes its sha256 hex segment; `RunId` writes two segments — `` derived from `RunId.created_at()` followed by the RunId string. (`RunId`-as-Record-Id is used only by `RunCatalogEntry` today; if a future record needs a single-segment RunId encoding it uses a newtype wrapper.) +- R3. Define `trait Codec { fn encode(r: &R) -> Result>; fn decode(bytes: &[u8]) -> Result; }` with built-in implementations: `JsonCodec` (used by most records), `RawBytesCodec` (for `Blob`), `EmptyCodec` (for index entries with no value). `JsonCodec::encode` is implemented as a one-line literal forwarding to `serde_json::to_vec(&value)` (and `decode` to `serde_json::from_slice`); the implementation IS the proof of byte-identity. A snapshot test on a representative `RefreshToken` and `AuthCode` is kept as a regression net so any accidental change to the forwarding impl (e.g. someone adds an envelope) fails CI loudly. The snapshot files commit the on-disk wire format to the repo and are reviewed as wire-format changes when they change. +- R4. Provide `Repository` exposing the typed K/V primitives: `get(&R::Id)`, `put(&R)`, `delete(&R::Id)`, `scan_stream() -> impl Stream>` (full prefix), `scan_prefix_stream(extra_segments)`, and `gc(predicate: impl Fn(&R) -> bool) -> Result` (scans, filters, deletes — replaces the three hand-rolled GC-by-prefix-scan loops in today's stores). `Repository` is `pub(crate)`. The `gc` predicate is sync `Fn` (so it cannot mutate caller state across the scan), MUST NOT perform I/O or block (it runs once per scanned record), and MUST be free of side effects (gc may be called more than once on the same record set during retries or future caller logic). `gc` issues all deletes in a single `slatedb::WriteBatch` — atomic vs today's per-key delete loop, deliberate behaviour change. No capability traits, no marker subtraits — domain logic lives in the wrapper Store. + +**Wrapper stores** +- R5. Each record family has a named domain `*Store` type that owns a `Repository` and any domain-specific helpers (per-key mutex, in-memory caches, replay revocation set). Trivial stores (`BlobStore`, `RunCatalogIndex`) are still concrete named types even when they are thin pass-throughs. **Security boundary**: `Repository` is `pub(crate)`; construction is gated to the wrapper Store and to test fixtures. The `Repository` field on each wrapper Store is private (`pub(super)` at most); wrapper Stores never expose `.scan_stream()` / `.gc()` / `.repository()` accessors. `Database` exposes only the wrapper Stores, never `Repository` directly. Domain methods that need a scan (e.g. `gc_expired`, `delete_chain`) perform it internally and return aggregate results, never raw records. The boundary is enforced for crate-external callers by the `pub(crate)` visibility; for crate-internal contributors it is convention + code review (a future fabro-store author who constructs `Repository` directly is bypassing the design and the PR review should catch it). +- R6. `RefreshTokenStore` keeps its existing API (`insert_refresh_token`, `find_refresh_token`, `consume_and_rotate`, `delete_chain`, `gc_expired`, `mark_refresh_token_replay`, `was_recently_replay_revoked`). `consume_and_rotate` uses the new `transaction` helper while still holding its `KeyedMutex` guard around the call. The replay revocation set (`mark_refresh_token_replay` / `was_recently_replay_revoked`) is in-memory only with a 60s TTL — it MUST NOT be moved into `Repository` or otherwise persisted. Rationale: it's a transient signal so the current process can recognize replays within the rotation window; persisting attacker-supplied token hashes adds disk I/O and a new GC surface with no security benefit and an unbounded-growth risk under token-stuffing attack. +- R7. `AuthCodeStore` keeps its existing API (`insert`, `consume`, `gc_expired`). The `AuthCode` struct gains a `code: String` first field carrying its own key (today's struct receives the code as a separate parameter to `insert`); this lets `Record::id(&self)` return the key from the value, which `Repository::scan_stream` requires. The on-disk JSON shape gains one field — acceptable under greenfield. +- R8. `BlobStore` exposes `read(&RunBlobId)`, `write(bytes) -> RunBlobId`, `exists(&RunBlobId)`. `delete` is intentionally not exposed — current behaviour is that blobs survive run deletion (enforced by `delete_run_keeps_global_cas_blobs` test). `RunDatabase::write_blob` / `read_blob` / `list_blobs` keep their signatures and delegate to `BlobStore` internally so existing callers (and tests) require no changes. +- R9. `RunCatalogIndex` exposes `add(&RunId)`, `remove(&RunId)`, `list(query: &ListRunsQuery) -> Vec`, replacing the free functions in `lib/crates/fabro-store/src/slate/catalog.rs`. `list` scans the prefix, applies `query.start`/`query.end` filtering against `run_id.created_at()`, and sorts by the same key as today's `catalog.rs:39-49` — `(year, month, day, hour, minute, run_id)` ascending — derived inside `RunCatalogIndex::list` from each `RunId`'s embedded ULID timestamp. The post-list summary-building loop in `Database::list_runs` (`slate/mod.rs:170-183`) is **not** absorbed into `RunCatalogIndex`; only the catalog scan + filter + sort move. + +**Cross-record atomic writes** +- R10. Provide a `transaction(&db, |tx| { … })` helper producing a single `slatedb::WriteBatch`. `Tx` exposes `put(&R)` and `delete(&R::Id)` for any `R: Record`, so a single transaction can span record types. Replaces the hand-built `slatedb::WriteBatch` in `RefreshTokenStore::consume_and_rotate`. **Atomicity invariants** (security-critical for token rotation): the closure is `FnOnce(&mut Tx) -> Result` (synchronous); on closure `Err`, no slatedb write occurs (codec errors short-circuit via `?`); on closure `Ok`, exactly one `slatedb::WriteBatch::write` commit is issued; no retry, no partial flush, no commit-on-drop. A fault-injection test asserts that an encode failure on the Nth `put` leaves the database unchanged. To restrict the cross-type capability to authorized callers, `transaction` is `pub(crate)` — wrapper Stores expose domain-named atomic methods (e.g. `RefreshTokenStore::consume_and_rotate`) that compose `transaction` internally. + +**Database surface** +- R11. `Database` exposes `refresh_tokens()`, `auth_codes()`, `blobs()`, `catalog_index()`, and `runs()`. Each lazily initializes its store via `OnceCell` and returns `Arc<*Store>` (mirrors today's pattern in `lib/crates/fabro-store/src/slate/mod.rs:208-228`). +- R12. `RunDatabase`'s event-sourcing machinery (event log, projection cache, broadcast channel, atomic seq counter, per-run state mutex) and public method signatures are unchanged. Two internal touch-points exist: (a) the catalog call sites all live in `Database` itself — `Database::create_run` (`slate/mod.rs:119,127`), `Database::list_runs` (`slate/mod.rs:169`), and `Database::delete_run` (`slate/mod.rs:204`) — and migrate from `catalog::write_index` / `catalog::delete_index` / `catalog::list_run_ids` to the new `RunCatalogIndex` (lazily initialized on `Database` via `OnceCell`, like the other stores); (b) `RunDatabase::write_blob` / `read_blob` / `list_blobs` (`run_store.rs:283-302`) keep their public signatures and behaviour but delegate internally to `BlobStore` per R8. No callers change. + +**Reusable helpers** +- R13. Extract `KeyedMutex` (per-key async mutex on top of `DashMap>>` with auto-cleanup when strong count drops to 2) as a shared helper inside `fabro-store`, used by `RefreshTokenStore` and `AuthCodeStore`. **Security purpose**: `KeyedMutex` serializes concurrent `consume` operations on the same key. Required by single-use semantics for `AuthCode` and replay detection for `RefreshToken` — without it, a TOCTOU race between the find/get and the delete/mark-used write permits double-consumption. Auto-cleanup at `strong_count == 2` bounds the map size so a high volume of distinct codes/hashes cannot grow it without limit. The cleanup check and the entry insertion happen under the same `DashMap` shard lock so a concurrent `lock(&key)` cannot observe a soon-to-be-dropped Arc and acquire a different `Mutex` instance for the same key. `KeyedMutex` exposes only `lock(&self, key: K) -> Guard<'_>` — never returns or clones the inner `Arc`; `Guard` holds the Arc internally so the strong-count==2 check happens after `Guard` drops. The lock MUST NOT be made optional, replaced with a TTL/janitor, or relocated to a separate process without explicit security review. + +## Success Criteria + +- Adding a 5th simple K/V record type requires writing only: a struct + `impl Record` + an `impl RecordId` (if its ID type isn't already covered) + a thin `*Store` wrapper. No new key-construction code, no `serde_json::to_vec`/`from_slice` per store, no copy of get/put/delete/scan plumbing. +- All existing public store APIs (`RefreshTokenStore::consume_and_rotate`, `AuthCodeStore::consume`, etc.) keep their signatures and observable behaviour. Existing unit and integration tests pass without modification. +- `lib/crates/fabro-store/src/slate/auth_tokens.rs` and `auth_codes.rs` shrink to roughly their domain logic (consume lock, GC traversal, replay cache) — kv plumbing moves into `Repository`. +- `lib/crates/fabro-store/src/slate/catalog.rs` is deleted; `RunCatalogIndex` exposes the same operations through `Repository`. +- A single `transaction(…)` call replaces the hand-built `slatedb::WriteBatch` in `RefreshTokenStore::consume_and_rotate`. + +## Scope Boundaries + +- Run aggregate's event-sourcing machinery (`RunDatabase` event log, projection cache, broadcast channel, `recover_next_seq`, `EventProjectionCache`, atomic seq counter) is not refactored. `RunDatabase`'s blob methods are touched only to delegate to `BlobStore` per R8 — no signature or behaviour change. +- No marker capability traits (`ExpiringRecord`, `ConsumableRecord`, `IndexedRecord`, `ContentAddressed`). Capabilities are hand-written per `*Store`. +- No record schema versioning or migration framework. Greenfield, can break compat. +- No swap-out of the storage backend (still SlateDB on `Arc`). Repository is parameterized by `R`, not by storage. +- No changes to `ArtifactStore` or other non-SlateDB storage. +- No changes to the public `Database::create_run` / `open_run` / `delete_run` / `list_runs` API signatures or observable behaviour. Internal implementations of `create_run`, `list_runs`, and `delete_run` consume the new `RunCatalogIndex` per R12. + +## Key Decisions + +- **Two worlds** (Run separate from simple K/V records): unifying an event-sourced aggregate (atomic seq counter, projection cache, live broadcast, per-run state mutex) with single-key records would either flatten to a lowest-common-denominator API or force Run's complexity onto records that don't need it. +- **Hand-written `*Store` wrappers on a thin `Repository`** (not marker capability traits): chose lowest magic. Capability variations are small in number, infrequently added, and easy to read inline. A reader of `RefreshTokenStore` should see exactly what `consume_and_rotate` does without crossing trait-bound boundaries. +- **Multi-segment IDs via data-only `key_segments() -> Vec`**: `SlateKey` stays `pub(crate)`. `Repository` assembles the key internally — `RecordId` impls cannot violate the `\0`-segment-boundary invariant by accident. Small allocation per key derivation is acceptable for the encapsulation gain. +- **Codec specified explicitly per impl** (no associated-type default): `associated_type_defaults` is unstable on stable Rust as of 2026. Rather than wait, pull in a derive-macro sub-crate, or split into sub-traits, every `impl Record` writes `type Codec = JsonCodec;` (or its override). One extra line per impl, no nightly, no proc-macro infra. +- **`transaction(...)` as a single-batch atomic primitive, `pub(crate)`**: `consume_and_rotate` already needs an atomic two-key write; one helper covers it and any future cross-record flows. Closure is `FnOnce`, all-or-nothing, no commit-on-drop. Crate-private visibility prevents callers from bypassing per-store invariants — wrapper Stores expose domain-named atomic methods that compose `transaction` internally. +- **Repository for security-sensitive records is store-private**: `Database` exposes only wrapper Stores. `RefreshTokenStore` and `AuthCodeStore` never hand out a `Repository` accessor or generic `scan`. Prevents a new caller in `fabro-server` (or anywhere else) from walking the entire token table. +- **Named `*Store` for every record**: discoverability matters more than line count. `db.blobs()` and `db.catalog_index()` read more clearly than `db.repository::()`. The trivial wrappers (`BlobStore`, `RunCatalogIndex`) are accepted overhead — see Alternatives Considered for the leaner shape we rejected. + +## Alternatives Considered + +Three lighter shapes were considered. The full scaffold (R1–R13) was chosen anyway. Comparison: + +| Option | New types | Approx. scaffold | Per-record cost (5th) | Trade | +|---|---|---|---|---| +| (1) Copy-paste-and-rename | 0 | 0 LOC | ~200 LOC duplicated | Zero abstraction tax now; drift risk grows linearly. | +| (2) `JsonStore` helper only | 1 (~30 LOC) | ~30 LOC | ~80 LOC | Removes JSON serde duplication only. No typed key, no codec abstraction, no transaction helper, no security boundary. Each store keeps bespoke key construction and consume locks. | +| (3) Minimum-viable subset (R1+R2+R3 JsonCodec only+R4+R6+R7+R13) | ~5 traits + Repository + KeyedMutex | ~120 LOC | ~30 LOC | Touches only `auth_tokens.rs` and `auth_codes.rs`. Drops `BlobStore`, `RunCatalogIndex`, cross-record `transaction`, `RawBytesCodec`, `EmptyCodec`. Loses the boundary requirement (no Blob/catalog pass-through to demonstrate the pattern). | +| **(chosen) Full scaffold (R1–R13)** | ~6 traits + Repository + Tx + 4 wrapper stores + KeyedMutex | ~250 LOC | ~30 LOC | All of (3) plus Blob CAS and catalog refactor onto the same shape. R8 (`BlobStore` as a public Database surface) and R9 (catalog migration) are forward-looking in their *value* — they don't have a new today-consumer beyond what already works through `RunDatabase::write_blob` and `catalog::*` — but they are concrete refactors of existing code, not speculative new code. R10's cross-type Tx is similarly forward-looking. | + +**Why (chosen) over (3)**: standardizes the shape across every K/V record the crate has, so new entrants (sessions, vault, agent state) hit one well-understood pattern instead of choosing between "use `Repository`" and "use the older bespoke style." Designs the security boundary (R5) once for the whole crate so a future security-sensitive record inherits the discipline by default rather than reinventing it. (Note: only `RefreshToken` and `AuthCode` actually need the boundary today; for `Blob` and `RunCatalogEntry` the boundary is "free" — there's nothing sensitive to protect — so the security argument is forward-leaning, not load-bearing right now.) + +**Why not (2) `JsonStore`**: removes the JSON duplication but nothing else. Each store still hand-writes its own key construction (so the scan/key-parsing invariants stay scattered), its own consume mutex (so KeyedMutex can't naturally land), and its own atomic-write batch (so the security invariants in R10 stay implicit). Reduction in scaffolding is real, reduction in surface-to-think-about is much smaller than it looks. + +**Why not (1) copy-paste**: works fine until two records drift. The first such drift (e.g., one store fixes a key-encoding bug another doesn't) is a class of bug the abstraction prevents structurally. + +**Acknowledged speculative scope**: R8 (`BlobStore` public surface), R9 (catalog sweep), and R10's cross-type Tx are all forward-looking. The bet is that paying the setup cost once now is cheaper than retrofitting later. If that bet is wrong, the cost is the LOC delta between (3) and (chosen) plus the ongoing readability cost: a new contributor must learn `Record`/`RecordId`/`Codec`/`Repository`/`Tx`/`KeyedMutex` to add a record, vs. learning one `JsonStore` API; trait-resolution makes "where is this key constructed?" harder to grep; generic error messages reference `::Codec` instead of `JsonCodec`. The per-record LOC numbers in the table above are rough estimates — the actual delta between (2)/(3) and (chosen) for a 5th record is small (single-digit LOC). The primary argument for (chosen) over (3) is consistency-across-records and one-time security-boundary design, not per-record LOC. + +## Dependencies / Assumptions + +- SlateDB's `WriteBatch` is suitable as the underlying primitive for the `transaction` helper. Already used by `RefreshTokenStore::consume_and_rotate`. +- The current public consumers of `Database::auth_codes()` / `auth_tokens()` are limited to `fabro-server` and crate-internal tests. The `Slate*Store` type names are referenced in `fabro-server` (`serve.rs:799-829` types `Arc` etc.); renaming to `*Store` is a mechanical import update on the order of one PR for `fabro-server`, not a deep API change. + +## Outstanding Questions + +### Resolve Before Planning + +(none) + +### Deferred to Planning + +- [Affects R13] [Technical] Whether `KeyedMutex` lives at `lib/crates/fabro-store/src/keyed_mutex.rs` or in a small shared util module. Either is fine. +- [Affects R5–R9] [Technical] Migration order — which store moves to `Repository` first and in what PR sequence. Suggested order: R13 (`KeyedMutex` extraction, can land first as a precursor PR) → `BlobStore` (greenfield, simplest) → `RunCatalogIndex` (replaces `catalog.rs`) → `AuthCodeStore` → `RefreshTokenStore` (most complex due to consume-and-rotate + replay cache). Planning concern. +- [Affects R10] [Technical] Whether to add a separate `per_repository_batch` helper for the common single-type case so callers don't pay the cross-record machinery cost when they don't need it. Performance/ergonomics tweak, not a correctness question. + +## Next Steps + +→ `/ce:plan` for structured implementation planning diff --git a/docs/plans/2026-04-20-003-refactor-fabro-store-record-abstractions-plan.md b/docs/plans/2026-04-20-003-refactor-fabro-store-record-abstractions-plan.md new file mode 100644 index 000000000..335912d1b --- /dev/null +++ b/docs/plans/2026-04-20-003-refactor-fabro-store-record-abstractions-plan.md @@ -0,0 +1,725 @@ +--- +title: "refactor: Extract Record/Repository abstractions in fabro-store" +type: refactor +status: completed +date: 2026-04-20 +origin: docs/brainstorms/2026-04-20-fabro-store-record-abstractions-requirements.md +--- + +# refactor: Extract Record/Repository abstractions in fabro-store + +## Overview + +`fabro-store` currently has four hand-written `Slate*Store` types (`SlateAuthCodeStore`, `SlateAuthTokenStore`, blob plumbing on `RunDatabase`, free functions in `catalog.rs`) that each duplicate the same K/V plumbing on top of SlateDB: key construction, JSON serialization, get/put/delete/scan loops, optional per-key consume mutex, optional GC-by-prefix-scan. This plan replaces that duplication with a thin reusable typed K/V layer (`Repository`) plus per-record wrapper Stores that hold any domain-specific helpers (consume locks, replay caches, etc.). + +The Run aggregate (event log, projection cache, broadcast channel) stays as it is. Only the simple K/V records (RefreshToken, AuthCode, Blob, RunCatalogEntry) move to the new shape. + +This is a greenfield refactor — no production deployments, backwards-compat can be broken freely (see origin: `docs/brainstorms/2026-04-20-fabro-store-record-abstractions-requirements.md`). + +## Problem Frame + +Adding a new persisted record type today requires copying ~200 LOC of plumbing into a new `Slate*Store` and editing names. Each copy is one more place where serialization, locking, or key encoding can drift. As more record families land (sessions, vault entries, agent state), the duplication compounds. + +The brainstorm's chosen response: a `Record` trait + `Repository` typed K/V layer + named domain `*Store` wrappers + a security boundary that's designed once for the whole crate (see origin Key Decisions). + +## Requirements Trace + +All requirements traced from the origin document. + +- **R1.** `trait Record` with associated `type Id: RecordId`, `type Codec: Codec` (no default — explicit per impl), `const PREFIX: &'static str`, and `fn id(&self) -> Self::Id`. `PREFIX` MUST be a non-empty `/`-separated path of non-empty segments — no leading or trailing `/`, no empty segments, `/` is reserved within a segment. `Repository::new` `debug_assert!`s the constraint. (origin R1) +- **R2.** `trait RecordId: Sized { fn key_segments(&self) -> Vec; fn from_key_segments(segs: &[&str]) -> Result; }` with built-in impls for `[u8; 32]` (hex), `String`, `RunBlobId`, and `RunId` (date + ulid). `from_key_segments` is the parse counterpart used by `Repository::scan_stream` and `scan_ids_stream` — the only path to reconstruct `R::Id` from the SlateDB key for both value-carrying records and ZST marker records (e.g. `RunCatalogEntry`). Implementations of `key_segments` MUST NOT include the `\0` byte in any returned segment. `Repository` validates this **at runtime** before assembling any SlateKey (in `put`/`put_at`/`get`/`delete`/`exists`/scan-prefix construction): a segment containing `\0` returns `Err(Error::InvalidKeySegment)` rather than silently corrupting the keyspace. Validation is one byte scan per segment per call — negligible vs the SlateDB op. (origin R2 — extended at plan time) +- **R3.** `trait Codec` with three built-in implementations: `JsonCodec` (literal `serde_json` forwarding for value-carrying records), `RawBytesCodec` (byte-identity for `Blob`), `MarkerCodec` (for ZST marker records — `encode(_) -> Vec::new()`, `decode(&[]) -> R::default()`; bound `R: Default + Sized`). Snapshot test on a stable synthetic struct locks the `JsonCodec` wire format. **Wire-format change tracking:** `RefreshToken` JSON wire format is byte-identical to today (literal serde_json forward). `AuthCode` JSON wire format gains a new `code` field per R7 — acceptable under greenfield. (origin R3) +- **R4.** `Repository` (`pub(crate)`) exposes: + - For value-carrying records: `get(&id)`, `put(&r)`, `delete(&id)`, `scan_stream() -> Stream<(R::Id, R)>`, `scan_prefix_stream(extra_segments)`, `gc(predicate)`. + - For all records (including markers): `put_at(&id, &r)` (writes a key derived from the explicit id rather than from `r.id()`; `put` becomes sugar for `put_at(&r.id(), r)`), `exists(&id) -> bool`, `scan_ids_stream() -> Stream` (keys-only enumeration; doesn't touch values). Marker records (`R::Codec = MarkerCodec`) interact with `Repository` exclusively through the id-only API: `put_at`, `delete`, `exists`, `scan_ids_stream`. (origin R4 — extended at plan time) +- **R5.** Named domain `*Store` wrappers; `Repository` field is private; security boundary against external callers walking sensitive tables. (origin R5) +- **R6.** `RefreshTokenStore` keeps existing API; `consume_and_rotate` uses `transaction(...)` under `KeyedMutex` guard; replay cache stays in-memory. (origin R6) +- **R7.** `AuthCodeStore` keeps observable behaviour but `insert` changes signature from `insert(code: &str, entry: AuthCode)` to `insert(entry: AuthCode)` since `AuthCode` now carries its own `code` field (per the brainstorm decision). All existing callers update their construction pattern: compute the code, then construct `AuthCode { code, identity, ... }` in one literal. This is a public API change accepted under greenfield. (origin R7 — clarified at plan time) +- **R8.** `BlobStore` exposes `read`/`write`/`exists` (no `delete`, no `list`/`scan` — see R5 boundary). `RunDatabase::write_blob` and `RunDatabase::read_blob` keep signatures and delegate to `BlobStore`. `RunDatabase::list_blobs` keeps signature and continues using its existing free-function key scan path until/unless a future record needs blob enumeration through `Repository`. (origin R8 — clarified at plan time) +- **R9.** `RunCatalogIndex` exposes `add(&run_id)` (writes a `RunCatalogEntry` marker key), `remove(&run_id)`, `list(query)` (uses `Repository::scan_ids_stream` to enumerate marker keys, applies query filter, sorts by `(year, month, day, hour, minute, run_id)` per `catalog.rs:39-49`). Replaces `catalog.rs` free functions. `RunCatalogEntry` is a ZST marker (no fields); the run id lives only in the key. (origin R9 — extended at plan time) +- **R10.** `transaction(&db, |tx| ...)` helper, `pub(crate)`, `FnOnce`, single-batch atomic write, encode errors short-circuit; fault-injection test required. (origin R10) +- **R11.** `Database` exposes five accessors (lazy `OnceCell` init): `blobs()` lands in U2b, `catalog_index()` in U4, `auth_codes()` is preserved in U5, `refresh_tokens()` (renamed from `auth_tokens()`) lands in U6, `runs()` is unchanged. (origin R11) +- **R12.** `RunDatabase`'s event-sourcing machinery + public method signatures unchanged. `Database` consumes `RunCatalogIndex`. `RunDatabase::write_blob`/`read_blob` delegate to `BlobStore` (per R8); `RunDatabase::list_blobs` keeps its current free-function path. (origin R12 — clarified at plan time) +- **R13.** `KeyedMutex` extracted as shared helper; cleanup-under-shard-lock invariant; never exposes the inner Arc. (origin R13) + +## Scope Boundaries + +- Run aggregate event-sourcing machinery (`RunDatabase` event log, projection cache, broadcast channel, `recover_next_seq`, `EventProjectionCache`, atomic seq counter) is not refactored. +- No marker capability traits (`ExpiringRecord`, `ConsumableRecord`, `IndexedRecord`, `ContentAddressed`). +- No record schema versioning or migration framework. +- No swap-out of the storage backend (still SlateDB on `Arc`). +- No changes to `ArtifactStore` or other non-SlateDB storage. +- No changes to public `Database::create_run` / `open_run` / `delete_run` / `list_runs` API signatures or observable behaviour. Internals consume new `RunCatalogIndex` per R12. + +## Context & Research + +### Relevant Code and Patterns + +- `lib/crates/fabro-store/src/slate/auth_tokens.rs` — `SlateAuthTokenStore`, the most complex existing store (consume_and_rotate with WriteBatch, KeyedMutex, replay cache). +- `lib/crates/fabro-store/src/slate/auth_codes.rs` — `SlateAuthCodeStore`, mirror of the same pattern minus replay cache. +- `lib/crates/fabro-store/src/slate/run_store.rs:283-302` — `RunDatabase::write_blob`/`read_blob`/`list_blobs`, today's blob plumbing. +- `lib/crates/fabro-store/src/slate/catalog.rs` — three free functions (`write_index`, `delete_index`, `list_run_ids`) called from `Database::create_run`/`list_runs`/`delete_run`. +- `lib/crates/fabro-store/src/slate/mod.rs` — `Database` (lazy `OnceCell` per store), `Runs` accessor, run lifecycle methods that thread catalog calls. +- `lib/crates/fabro-store/src/keys.rs` — `SlateKey` builder (`pub(crate)`, `\0`-separated segments), per-record key constructors, parse helpers. **Stays `pub(crate)`** under R2's data-only `key_segments()` design. +- `lib/crates/fabro-store/src/lib.rs` — current `pub use` re-exports of `Slate*Store` types. + +### Pattern: lazy store init via `OnceCell` + +Today (`slate/mod.rs:208-228`): + +```text +auth_codes: Arc>> +auth_tokens: Arc>> +``` + +`Database::auth_codes()` / `auth_tokens()` use `get_or_try_init` to construct the store with `Arc::new(self.open_db().await?)`. New `BlobStore`, `RunCatalogIndex` follow the same shape. No new pattern needed. + +### Pattern: KeyedMutex (today inlined per store) + +Both `auth_tokens.rs:85-118` and `auth_codes.rs:48-77` carry the same `DashMap>>` + clone Arc + lock + decide-on-drop-via-strong-count==2 pattern. R13 extracts it. + +### Pattern: today's WriteBatch usage + +`auth_tokens.rs:101-110` builds one `slatedb::WriteBatch`, puts both keys, calls `db.write(batch).await?`. The `transaction(...)` helper preserves this exact shape — single batch, single commit. + +### External References + +None — this is an internal refactor. SlateDB API is already understood from existing usage. No new dependencies. + +### Institutional Learnings + +No prior `docs/solutions/` entries on this topic — first refactor of this kind in `fabro-store`. + +## Key Technical Decisions + +Most decisions carry forward from the origin Key Decisions section. + +- **Two worlds**: Run aggregate stays as-is; only K/V records move to `Repository`. (origin) +- **Hand-written `*Store` wrappers on a thin `Repository`**, no marker capability traits. (origin) +- **Multi-segment IDs via data-only `key_segments() -> Vec`**: keeps `SlateKey` `pub(crate)`. (origin) +- **Codec specified explicitly per impl** (no associated-type default): avoids unstable `associated_type_defaults`. (origin) +- **`transaction(...)` is `pub(crate)`**, single-batch atomic, `FnOnce`. Wrapper Stores compose it via domain-named methods. (origin) +- **Repository for security-sensitive records is store-private** (`pub(crate)` Repository, `pub(super)` field on the wrapper Store). External boundary enforced by visibility; internal discipline by code review. (origin + pass-2 review) + +Plan-time additions: + +- **KeyedMutex location**: `lib/crates/fabro-store/src/keyed_mutex.rs` (top-level module, not under `slate/`, since it's not slatedb-specific). Resolves origin deferred Q1. +- **Migration order** (resolves origin deferred Q2): U1 (KeyedMutex precursor) and U2a (trait scaffolding + Repository + transaction) are independent and can land in either order. U2b (Blob + BlobStore + Database::blobs) follows U2a. Then U3, U4, U5, U6 can be parallelized. U7 closes with re-exports + fabro-server cleanup. Each unit lands as one commit; each compiles and tests pass independently. See dependency graph in Implementation Units. +- **Per-Repository batch helper deferred** (resolves origin deferred Q3): not added in this refactor. Only `transaction(...)` ships. If profiling shows the cross-record machinery cost is meaningful for the single-type case, add later. +- **Module organization**: traits + Repository + Tx live in a new top-level `record/` module (`lib/crates/fabro-store/src/record/{mod,codec,repository,transaction}.rs`); per-record wrapper Stores stay under `slate/` (since they're slatedb-bound). `KeyedMutex` is its own top-level module. +- **`AuthCode` schema change**: `code` becomes the first field. JSON shape changes by one field. Acceptable under greenfield; test fixtures update mechanically. +- **Test approach**: continue the existing pattern — real SlateDB on `object_store::memory::InMemory`, no mocks (per repo `CLAUDE.md` testing posture and existing `auth_tokens.rs:194-201` style). + +## Open Questions + +### Resolved During Planning + +- **Where does `KeyedMutex` live?** → `lib/crates/fabro-store/src/keyed_mutex.rs` (top-level module). +- **Migration order?** → U1 + U2a parallelizable → U2b → U3, U4, U5, U6 parallelizable → U7. See Implementation Units dependency graph. +- **Per-Repository batch helper?** → Deferred, not in this refactor. +- **Module organization for traits?** → New `record/` module at `lib/crates/fabro-store/src/record/`. + +### Deferred to Implementation + +- **Exact internal types for `Tx`** — whether it owns or borrows the `WriteBatch`, whether `put` returns `&mut Self` for chaining. Mechanical; affects no caller. The contract (`FnOnce`, single commit, encode short-circuit) is fixed in R10. +- **Stream adapter for `scan_stream`** — `slatedb::DbIterator` exposes `async fn next(&mut self)` but does not impl `Stream`. Implementer chooses between `futures::stream::unfold`, a hand-rolled `poll_next`, or returning the iterator directly behind a thin newtype. None affect callers; pick the simplest that types cleanly. +- **`gc(predicate)` exact signature for the closure capture lifetime** — `impl Fn(&R) -> bool` may need a `+ Send` bound depending on how the stream-and-batch implementation interleaves with `Send` futures. Implementer adjusts at the type-error site. +- **Whether `BlobStore` lives at `lib/crates/fabro-store/src/slate/blob_store.rs` or a deeper path** — file location is mechanical. + +## High-Level Technical Design + +> *This illustrates the intended approach and is directional guidance for review, not implementation specification. The implementing agent should treat it as context, not code to reproduce.* + +### Trait scaffolding shape + +```text +trait Record: Sized + Send + Sync + 'static { + type Id: RecordId; + type Codec: Codec; + const PREFIX: &'static str; // "/"-separated, split by Repository + fn id(&self) -> Self::Id; +} + +trait RecordId: Sized { + fn key_segments(&self) -> Vec; + fn from_key_segments(segs: &[&str]) -> Result; +} + +trait Codec { + fn encode(r: &R) -> Result>; + fn decode(bytes: &[u8]) -> Result; +} + +// Built-in codecs +struct JsonCodec; // literal forward to serde_json::to_vec / from_slice +struct RawBytesCodec; // identity on bytes — encode(&Blob(bytes)) returns bytes; decode(bytes) returns Blob(bytes) +struct MarkerCodec; // for ZST marker records (R: Default + Sized) — encode(_) -> Vec::new(); decode(&[]) -> R::default() + +// Built-in RecordId impls (each implements both directions) +impl RecordId for [u8; 32] { /* key_segments: vec![hex(self)]; from_key_segments: parse hex */ } +impl RecordId for String { /* key_segments: vec![self.clone()]; from_key_segments: clone segs[0] */ } +impl RecordId for RunBlobId { /* one hex segment, both directions */ } +impl RecordId for RunId { /* key_segments: [, ]; from_key_segments: parse segs[1] */ } +``` + +### Repository shape + +```text +pub(crate) struct Repository { + db: Arc, + _r: PhantomData, +} + +impl Repository { + // Value-carrying API + pub(crate) async fn get(&self, id: &R::Id) -> Result>; + pub(crate) async fn put(&self, r: &R) -> Result<()>; // sugar for put_at(&r.id(), r) + pub(crate) async fn delete(&self, id: &R::Id) -> Result<()>; + pub(crate) fn scan_stream(&self) -> impl Stream>; + pub(crate) fn scan_prefix_stream(&self, extra: &[&str]) -> impl Stream>; + pub(crate) async fn gc(&self, predicate: impl Fn(&R) -> bool) -> Result; + + // Id-only API — works for any record, required for ZST marker records + pub(crate) async fn put_at(&self, id: &R::Id, r: &R) -> Result<()>; + pub(crate) async fn exists(&self, id: &R::Id) -> Result; + pub(crate) fn scan_ids_stream(&self) -> impl Stream>; + pub(crate) fn scan_prefix_ids_stream(&self, extra: &[&str]) -> impl Stream>; +} + +// Marker (index) record example +#[derive(Default)] +pub(crate) struct RunCatalogEntry; // ZST — id lives only in the key + +impl Record for RunCatalogEntry { + type Id = RunId; + type Codec = MarkerCodec; + const PREFIX: &'static str = "runs/_index/by-start"; + fn id(&self) -> RunId { unreachable!("marker — use put_at(&id, &RunCatalogEntry)") } +} + +// RunCatalogIndex usage: +// add(&run_id) -> repo.put_at(&run_id, &RunCatalogEntry).await +// remove(&run_id) -> repo.delete(&run_id).await +// list(query) -> repo.scan_ids_stream().filter(query).sort() +``` + +### transaction shape + +```text +pub(crate) async fn transaction(db: &slatedb::Db, f: F) -> Result +where F: FnOnce(&mut Tx) -> Result +{ + let mut tx = Tx::new(); + let value = f(&mut tx)?; // closure Err → return early, no DB write + db.write(tx.into_batch()).await?; // exactly one commit + Ok(value) +} + +pub(crate) struct Tx { batch: WriteBatch } +impl Tx { + pub(crate) fn put(&mut self, r: &R) -> Result<&mut Self> { ... } + pub(crate) fn delete(&mut self, id: &R::Id) -> Result<&mut Self> { ... } +} +``` + +### Wrapper Store shape (RefreshTokenStore example) + +```text +pub struct RefreshTokenStore { + db: Arc, // for transaction(...) + repo: Repository, // pub(super) at most + consume_locks: KeyedMutex<[u8; 32]>, + replay_revocations: DashMap<[u8; 32], DateTime>, // in-memory only +} + +impl RefreshTokenStore { + pub async fn consume_and_rotate(...) -> Result { + let _guard = self.consume_locks.lock(presented_hash).await; + let existing = self.repo.get(&presented_hash).await?; + match existing { + None => Ok(ConsumeOutcome::NotFound), + Some(token) if now >= token.expires_at => Ok(ConsumeOutcome::Expired), + Some(token) if token.used => Ok(ConsumeOutcome::Reused(token)), + Some(token) => { + let mut old = token.clone(); + old.used = true; + old.last_used_at = now; + transaction(&self.db, |tx| { + tx.put(&old)?; + tx.put(&new_token)?; + Ok(()) + }).await?; + Ok(ConsumeOutcome::Rotated(old, Box::new(new_token))) + } + } + } +} +``` + +## Implementation Units + +```mermaid +graph TB + U1[U1: KeyedMutex extraction] + U2a[U2a: Traits + Repository + transaction] + U2b[U2b: Blob + BlobStore + Database::blobs] + U3[U3: RunDatabase blob delegation] + U4[U4: RunCatalogIndex; delete catalog.rs] + U5[U5: AuthCodeStore migration] + U6[U6: RefreshTokenStore migration; transaction wired in] + U7[U7: Re-exports + fabro-server import cleanup] + + U2a --> U2b + U2b --> U3 + U2a --> U4 + U2a --> U5 + U2a --> U6 + U1 --> U5 + U1 --> U6 + U2b --> U7 + U3 --> U7 + U4 --> U7 + U5 --> U7 + U6 --> U7 +``` + +U1 and U2a are independent of each other and can land in either order. U2b depends on U2a. U3 depends on U2b (it needs `BlobStore`). U4, U5, U6 each depend on U2a (they need `Repository` and `transaction`); U5 and U6 also depend on U1 (`KeyedMutex`). U3, U4, U5, U6 can be parallelized after their dependencies land. U7 is the final cleanup pass. + +--- + +- [x] **Unit 1: Extract `KeyedMutex` shared helper** + +**Goal:** Move the per-key `DashMap>>` + auto-cleanup-at-strong-count-2 pattern out of `auth_tokens.rs` and `auth_codes.rs` into a single shared helper. No behaviour change. + +**Requirements:** R13. + +**Dependencies:** None. Independent precursor PR. + +**Files:** +- Create: `lib/crates/fabro-store/src/keyed_mutex.rs` +- Modify: `lib/crates/fabro-store/src/lib.rs` (add `mod keyed_mutex;` + `pub(crate) use`) +- Modify: `lib/crates/fabro-store/src/slate/auth_tokens.rs` (replace `refresh_locks: DashMap<...>` field + inline lock pattern with `consume_locks: KeyedMutex<[u8; 32]>`) +- Modify: `lib/crates/fabro-store/src/slate/auth_codes.rs` (replace `code_locks: DashMap<...>` similarly) +- Test: `lib/crates/fabro-store/src/keyed_mutex.rs` (unit tests inline) + +**Approach:** +- `KeyedMutex` exposes only `pub async fn lock(&self, key: K) -> Guard<'_>`. +- `Guard<'_>` holds the `Arc>` internally, releases the inner `Mutex` on drop, then performs the cleanup check (strong_count == 2 → remove from `DashMap`). +- The cleanup check and entry insertion happen under the same `DashMap` shard lock — use `DashMap::entry().and_modify().or_insert_with()` or equivalent so a concurrent `lock(&key)` cannot observe a stale Arc. +- No public access to the inner `Arc`. + +**Patterns to follow:** +- `lib/crates/fabro-store/src/slate/auth_tokens.rs:85-118` (today's inline pattern — preserve semantics). +- `lib/crates/fabro-store/src/slate/auth_codes.rs:48-77` (mirror pattern). + +**Test scenarios:** +- *Happy path:* `lock(K1)` then `lock(K2)` proceed independently (no contention between distinct keys). +- *Edge case:* `lock(K1)` from two tasks serializes — second waits until first drops Guard. +- *Edge case:* Auto-cleanup — after the only outstanding Guard drops, the `DashMap` no longer contains the entry for that key (verified via internal accessor or by counting `len()`). +- *Integration:* Stress test — 16 concurrent tasks `lock(K)` on the same key; verify exactly one progresses at a time and the map is empty after all complete. +- *Integration race regression:* Stress test that hammers the same key from N tasks across many drop cycles — verify two threads cannot acquire `lock(&key)` and end up with different `Mutex` instances (cleanup-under-shard-lock invariant). + +**Verification:** +- `cargo nextest run -p fabro-store` passes. +- `auth_tokens.rs` and `auth_codes.rs` no longer carry their inline `refresh_locks` / `code_locks` fields and inline lock plumbing. +- All existing concurrent_consume_has_one_winner / concurrent_rotation_has_one_winner tests pass unmodified. + +--- + +- [x] **Unit 2a: Trait scaffolding (`Record`, `RecordId`, `Codec`, `Repository`, `transaction`)** + +**Goal:** Land the new abstraction with a synthetic test-only Record exercising the full `Repository` API and the `transaction(...)` atomicity contract. No production consumer yet. + +**Requirements:** R1, R2, R3, R4, R5 (boundary spec), R10 (transaction). + +**Dependencies:** None. Independent of U1; either order fine. + +**Files:** +- Create: `lib/crates/fabro-store/src/record/mod.rs` (re-exports `Record`, `RecordId`) +- Create: `lib/crates/fabro-store/src/record/codec.rs` (`trait Codec`, `JsonCodec`, `RawBytesCodec`, `MarkerCodec`) +- Create: `lib/crates/fabro-store/src/record/repository.rs` (`Repository` and methods) +- Create: `lib/crates/fabro-store/src/record/transaction.rs` (`Tx`, `transaction()`) +- Create: `lib/crates/fabro-store/src/record/record_id.rs` (built-in `impl RecordId` for `[u8; 32]`, `String`, `RunId`, `RunBlobId` — both `key_segments` and `from_key_segments` directions) +- Modify: `lib/crates/fabro-store/src/error.rs` — add a new public variant `Error::InvalidKeySegment { segment: String }` returned by `Repository` key-assembly when a `RecordId::key_segments` impl produces a segment containing the `\0` separator byte. Also add `Error::KeyParse(String)` returned by `Repository::scan_stream` / `scan_ids_stream` when `RecordId::from_key_segments` rejects the parsed segments. +- Modify: `lib/crates/fabro-store/src/lib.rs` (add `mod record;`, `pub(crate) use record::...;`) +- Test: `lib/crates/fabro-store/src/record/repository.rs` (unit tests with a synthetic test-only Record to exercise get/put/delete/scan_stream/scan_prefix_stream/gc) +- Test: `lib/crates/fabro-store/src/record/transaction.rs` (unit tests for `transaction(...)` including encode-failure fault injection) +- Test: `lib/crates/fabro-store/src/record/codec.rs` (snapshot test for `JsonCodec` byte-identity using a stable synthetic struct via `insta::assert_snapshot!`) + +**Approach:** +- `Record::PREFIX` is a `&'static str` containing `/`-separated segments (`"auth/refresh"`, `"blobs/sha256"`, etc.). `Repository` splits on `/` once and writes `\0`-separated `SlateKey` segments. `/` is reserved inside any single segment. **Key-assembly invariant (runtime-enforced):** `Repository` checks every segment (PREFIX-derived OR id-derived) for `\0` before assembling the SlateKey, in every operation that constructs a key (`put`/`put_at`/`get`/`delete`/`exists`/scan-prefix). A segment containing `\0` returns `Err(Error::InvalidKeySegment { segment: String })`. This is a runtime error, not a `debug_assert!` — release builds get the same protection. Adds one byte-scan per segment per call (negligible). +- `Repository::scan_stream` adapts `slatedb::DbIterator::next` into `impl Stream>` via `futures::stream::unfold` or hand-rolled `poll_next` (implementer's choice). For each scanned entry: parse the SlateDB key into segments (split on `\0`), strip the `R::PREFIX` segments, pass the remaining segments to `R::Id::from_key_segments` to reconstruct the typed Id; decode the value via `R::Codec::decode`; yield `(id, value)`. For ZST marker records using `MarkerCodec`, decode is trivially `R::default()` — yielding `(id, marker)` where the marker carries no data and the id from the key is the actual information. +- `Repository::scan_ids_stream` is a keys-only variant: same key-parsing as `scan_stream`, but skips value bytes entirely (yields `Stream>`). Required for ZST marker records (where `decode` would be wasted), useful as a perf optimization for any record where the caller only needs ids. +- `Repository::put_at(id, &r)` is the explicit-id put: writes the key derived from `id` (not from `r.id()`). `Repository::put(&r)` is sugar for `put_at(&r.id(), r)`. ZST markers use `put_at` exclusively because they don't carry an id (their `Record::id` is `unreachable!`). +- `Repository::gc(predicate)` scans the prefix, decodes each value, evaluates the (sync, no-I/O) predicate, collects matching keys into a `Vec`, then issues a single `slatedb::WriteBatch` containing all deletes. Returns the count. +- `JsonCodec::encode` is literally `serde_json::to_vec(value)`; `decode` is `serde_json::from_slice(bytes)`. The implementation IS the byte-identity proof. Snapshot test guards against accidental future changes. +- `MarkerCodec::encode(_: &R) -> Ok(Vec::new())`; `decode(bytes) -> Ok(R::default())` (asserts `bytes.is_empty()`). +- `transaction(&db, f)` runs `f(&mut tx)`. On `Err`, returns the error without writing anything. On `Ok`, calls `db.write(tx.into_batch()).await`. `Tx::put` calls `R::Codec::encode(value)` and pushes onto the batch — encode errors propagate via `?` from inside the closure. `Tx::put_at(&id, &r)` is the explicit-id variant for ZST markers. + +**Patterns to follow:** +- `lib/crates/fabro-store/src/slate/mod.rs:208-228` (`OnceCell`-based store accessor on `Database`). +- `lib/crates/fabro-store/src/slate/auth_tokens.rs:79-121` (single `WriteBatch` + single `db.write` — `transaction(...)` mirrors this). +- `lib/crates/fabro-store/src/keys.rs:11-23` (`SlateKey` builder usage — `Repository` calls these `pub(crate)` methods). +- `lib/crates/fabro-store/src/slate/auth_tokens.rs:194-201` (test fixture pattern: `Database::new(Arc::new(InMemory::new()), "", Duration::from_millis(1), None)` then `db.().await.unwrap()`). + +**Test scenarios:** +- *Happy path (Repository):* `put` then `get` returns the same value; `delete` then `get` returns `None`. +- *Happy path (scan_stream):* `put` 5 records, `scan_stream().collect()` yields all 5 with correct `(id, value)` pairs. +- *Happy path (scan_prefix_stream):* records under different sub-prefixes filter correctly. +- *Happy path (gc):* `put` 10 records, `gc(|r| predicate true for half)` deletes 5, returns count 5; `scan_stream` confirms 5 remain. +- *Edge case (Repository):* `get` on missing id returns `None`; `delete` on missing id is a no-op. +- *Edge case (Repository):* empty prefix scan returns empty stream. +- *Error path (Codec):* malformed bytes from `decode` propagate as `Error::Other` (or whatever the existing error type is). +- *Error path (transaction):* closure returns `Err` → `db.write` is NOT called (verify via store state unchanged after). +- *Error path (transaction fault injection):* in the test module, define `TestRecord` with `type Codec = TestFailCodec;` where `TestFailCodec::encode` returns `Err` on a poisoned input. Run `transaction(&db, |tx| { tx.put(&good)?; tx.put(&poisoned)?; Ok(()) })` and assert: (1) the helper returns `Err`, (2) `db.get(key_for(&good))` returns `None`, (3) `db.get(key_for(&poisoned))` returns `None`. This proves the all-or-nothing invariant against the encode-error-mid-batch path (not just the closure-Err short-circuit). **Required by R10.** +- *Edge case (transaction):* empty closure (no puts/deletes) → no batch commit overhead, returns `Ok`. +- *Happy path (MarkerCodec / ZST marker):* define a synthetic `TestMarker` ZST + `impl Record for TestMarker { type Codec = MarkerCodec; ... }`. `repo.put_at(&id, &TestMarker)` then `repo.exists(&id) == true`; `repo.scan_ids_stream().collect()` yields the id; `repo.delete(&id)` then `exists == false`. +- *Edge case (key-assembly invariant):* in release builds, `RecordId::key_segments` returning a string containing `\0` causes the next `Repository` op to return `Err(Error::InvalidKeySegment)` rather than silently corrupting the keyspace. Test asserts the error is returned and the DB is unchanged. +- *Snapshot (JsonCodec byte-identity):* `insta::assert_snapshot!(std::str::from_utf8(&JsonCodec::encode(&sample)?).unwrap())` against a fixed synthetic struct with deterministic timestamps and IDs. Locks the JSON wire format string itself; any future change to `JsonCodec` (envelope, version tag, field reordering) fails the snapshot. + +**Verification:** +- `cargo nextest run -p fabro-store` passes including the new synthetic-record tests. +- `cargo +nightly-2026-04-14 clippy --workspace --all-targets -- -D warnings` passes. +- `Repository` and `transaction` are `pub(crate)`; the `record/` module is internal to `fabro-store`. + +--- + +- [x] **Unit 2b: `Blob` record + `BlobStore` wrapper + `Database::blobs()` accessor** + +**Goal:** First production consumer of the new abstraction. Land `BlobStore` and the `Database::blobs()` accessor. `RunDatabase::write_blob` continues to use the raw DB this unit (delegation lands in U3). + +**Requirements:** R5 (boundary applied to first wrapper), R8 (`BlobStore` API), R11 (`Database::blobs()`). + +**Dependencies:** U2a. + +**Files:** +- Create: `lib/crates/fabro-store/src/slate/blob_store.rs` (`Blob` record + `BlobStore` wrapper) +- Modify: `lib/crates/fabro-store/src/lib.rs` (public re-export of `BlobStore` and `Blob`) +- Modify: `lib/crates/fabro-store/src/slate/mod.rs` (`Database` gets `blobs: Arc>>` field + `Database::blobs() -> Result>` accessor) +- Test: `lib/crates/fabro-store/src/slate/blob_store.rs` (unit tests inline) + +**Approach:** +- `Blob(pub Bytes)` newtype. `impl Record for Blob { type Id = RunBlobId; type Codec = RawBytesCodec; const PREFIX = "blobs/sha256"; fn id(&self) -> RunBlobId { RunBlobId::new(&self.0) } }`. (`RunBlobId::new(content)` is the actual constructor in `lib/crates/fabro-types/src/run_blob_id.rs`; it computes the sha256 internally.) +- `BlobStore` wraps `Repository`. `Repository` field is private. No `.scan_stream()` or `.gc()` exposed. +- `Database::blobs()` follows the existing `auth_codes()`/`auth_tokens()` pattern at `slate/mod.rs:208-228`. + +**Patterns to follow:** +- `lib/crates/fabro-store/src/slate/mod.rs:208-228` (`OnceCell`-based store accessor on `Database`). +- `lib/crates/fabro-store/src/slate/auth_tokens.rs:194-201` (test fixture pattern: `Database::new(Arc::new(InMemory::new()), "", Duration::from_millis(1), None)`). + +**Test scenarios:** +- *Happy path:* `write(bytes_a)` returns `id_a`; `read(&id_a)` returns `Some(bytes_a)`; `write(bytes_a)` again returns the same `id_a` (content-addressed); `exists(&id_a) == true`, `exists(&unknown) == false`. +- *Edge case:* `write(empty bytes)` succeeds; reading back returns `Some(empty)`. +- *Integration (no delete):* `BlobStore` exposes no `delete` — verified at compile time by the absence of the method. +- *Integration (cross-path equivalence with raw DB):* a blob written via `BlobStore::write(bytes)` is byte-identical when read directly from SlateDB via `db.get(blob_key(&id))`. Locks the invariant that `RawBytesCodec::encode` is byte-identity (no envelope, no length prefix). The same equivalence is checked end-to-end in U3 once `RunDatabase::write_blob` delegates. + +**Verification:** +- `cargo nextest run -p fabro-store` passes including new tests. +- `Database::blobs()` returns an `Arc` and supports the round-trip tests. +- Existing tests (including `delete_run_keeps_global_cas_blobs` at `slate/mod.rs:451-466`) still pass — `RunDatabase::write_blob` continues to use the raw DB this unit, so no existing observable behaviour changes. + +--- + +- [x] **Unit 3: `RunDatabase` blob methods delegate to `BlobStore`** + +**Goal:** Internal refactor only — `RunDatabase::write_blob` and `RunDatabase::read_blob` keep their public signatures and delegate to `BlobStore`. `RunDatabase::list_blobs` is **not touched**; it keeps its existing free-function key-scan path at `run_store.rs:365-380` since `BlobStore` intentionally has no `list` method per R8's boundary. No callers change. No tests change. (If a future record needs blob enumeration through `Repository`, `BlobStore::list_ids` could be added later via `Repository::scan_ids_stream`; out of scope for this refactor.) + +**Requirements:** R8 (delegation), R12 (`RunDatabase` public surface unchanged). + +**Dependencies:** U2b. + +**Files:** +- Modify: `lib/crates/fabro-store/src/slate/run_store.rs` — `write_blob` and `read_blob` only. Each constructs a stack-local `BlobStore::new(self.inner.db.clone())` and delegates. `list_blobs` is left exactly as-is. No `RunDatabaseInner` field changes. No constructor signature changes. No `Database::create_run`/`open_run`/`open_run_reader` changes. + +**Approach:** +- `RunDatabase` keeps the raw `Db` field as today. `write_blob` and `read_blob` construct a stack-local `BlobStore` from the existing `Arc` and delegate (~24-byte stack alloc per call; `BlobStore` is stateless). `list_blobs` is unchanged — it continues calling the existing free function at `run_store.rs:365-380`. +- Existing `delete_run_keeps_global_cas_blobs` test (`slate/mod.rs:451-466`) MUST continue to pass — it accesses blobs through `RunDatabase::write_blob`/`read_blob` and the test is the canonical contract that blobs survive run deletion. + +**Patterns to follow:** +- `lib/crates/fabro-store/src/slate/run_store.rs:283-302` (today's `write_blob`/`read_blob` bodies — preserve return types and error behavior; only the implementation changes). +- `lib/crates/fabro-store/src/slate/run_store.rs:365-380` (today's `list_blobs` free function — left untouched). + +**Test scenarios:** +- *Integration:* `RunDatabase::write_blob(bytes)` followed by `RunDatabase::read_blob(&id)` returns `Some(bytes)` (existing behaviour). +- *Integration:* `Database::create_run` → `run.write_blob` → `Database::delete_run` → blob still readable via a different run's `read_blob` (the `delete_run_keeps_global_cas_blobs` invariant). +- *Integration:* `RunDatabase::list_blobs` returns the same set as before for a run that wrote N blobs. + +**Verification:** +- All existing tests in `lib/crates/fabro-store/src/slate/mod.rs` pass unmodified. +- `RunDatabase::write_blob`/`read_blob`/`list_blobs` signatures and behaviour are observably unchanged (no caller in `fabro-workflow`, `fabro-server`, etc. needs updating). +- `cargo nextest run --workspace` passes. + +--- + +- [x] **Unit 4: `RunCatalogIndex` replaces `catalog.rs`** + +**Goal:** Migrate the three free functions in `catalog.rs` to a `RunCatalogIndex` wrapper Store on top of `Repository`. Wire `Database::create_run`/`list_runs`/`delete_run` to use it. Delete `catalog.rs`. + +**Requirements:** R9 (`RunCatalogIndex` API + sort matches today), R12 (catalog calls live on `Database`), R11 (`Database::catalog_index()`). + +**Dependencies:** U2a. + +**Files:** +- Create: `lib/crates/fabro-store/src/slate/run_catalog_index.rs` (`RunCatalogEntry` record + `RunCatalogIndex` wrapper) +- Modify: `lib/crates/fabro-store/src/slate/mod.rs`: + - Add `catalog_index: Arc>>` field on `Database` + - Add `Database::catalog_index()` accessor + - `Database::create_run` (lines 119, 127) replaces `catalog::write_index(&db, run_id).await?` with `self.catalog_index().await?.add(run_id).await?` + - `Database::list_runs` (line 169) replaces `catalog::list_run_ids(&db, query).await?` with `self.catalog_index().await?.list(query).await?` + - `Database::delete_run` (line 204) replaces `catalog::delete_index(&db, run_id).await?` with `self.catalog_index().await?.remove(run_id).await?` +- Delete: `lib/crates/fabro-store/src/slate/catalog.rs` +- Modify: `lib/crates/fabro-store/src/slate/mod.rs` (remove `mod catalog;` import; remove the test-side `use catalog::*` if any) +- Test: `lib/crates/fabro-store/src/slate/run_catalog_index.rs` (unit tests inline) + +**Approach:** +- `RunCatalogEntry` is a ZST marker: `#[derive(Default)] pub(crate) struct RunCatalogEntry;`. `Record::Id = RunId`, `Codec = MarkerCodec`, `PREFIX = "runs/_index/by-start"`. `id(&self)` is `unreachable!("marker — use put_at(&id, &RunCatalogEntry)")` since the marker carries no id. +- `impl RecordId for RunId` writes two segments: `` (from `self.created_at().format("%Y-%m-%d")`) followed by `self.to_string()`. `from_key_segments` parses the ULID from `segs[1]` (the date segment is redundant — derivable from the ULID timestamp). +- `RunCatalogIndex::add(&RunId)` calls `repo.put_at(&run_id, &RunCatalogEntry).await`. Value bytes are empty per `MarkerCodec`. +- `RunCatalogIndex::remove(&RunId)` calls `repo.delete(&run_id).await`. +- `RunCatalogIndex::list(query)` uses `repo.scan_ids_stream()` to enumerate marker keys (no value decoding — cheap), collects into a `Vec`, filters by `query.start`/`query.end` against `run_id.created_at()` (today's logic at `catalog.rs:27-37`), and sorts by `(year, month, day, hour, minute, run_id)` — the same key as `catalog.rs:39-49`. +- `Database::list_runs`'s post-list summary-building loop (lines 170-183) is **NOT** absorbed into `RunCatalogIndex` — only the catalog scan + filter + sort move. The summary loop stays in `Database` because it needs `RunDatabase::has_any_events` and `RunDatabase::build_summary`. + +**Patterns to follow:** +- `lib/crates/fabro-store/src/slate/catalog.rs:7-50` (today's three functions — preserve filter and sort semantics exactly). +- `lib/crates/fabro-store/src/slate/auth_codes.rs::SlateAuthCodeStore::new` (wrapper Store construction signature). +- `lib/crates/fabro-store/src/slate/mod.rs:208-228` (`OnceCell` accessor pattern). + +**Test scenarios:** +- *Happy path:* `add(run_id_1); add(run_id_2); list(default query)` returns both, sorted by `(year, month, day, hour, minute, run_id)` ascending exactly per `catalog.rs:39-49`. Existing `create_open_list_and_delete_full_lifecycle_in_shared_db` test (`slate/mod.rs:421-447`) is the canonical regression — it MUST pass with identical output before and after migration. +- *Edge case:* `list` with `query.start` set later than all `run_id.created_at()` returns empty. +- *Edge case:* `list` with `query.end` set earlier than all `run_id.created_at()` returns empty. +- *Edge case:* `add(run_id); remove(run_id); list()` returns empty. +- *Integration:* `Database::create_run(rid_1); Database::list_runs(default)` returns the run summary (full integration through `Database` — verifies the migration didn't break the surface). +- *Integration:* `Database::delete_run(rid_1); Database::list_runs(default)` no longer returns it. +- *Integration:* All existing tests in `slate/mod.rs::tests` (notably `create_open_list_and_delete_full_lifecycle_in_shared_db`, `delete_run_keeps_global_cas_blobs`, `reopening_store_rebuilds_from_shared_db`) pass unmodified. + +**Verification:** +- `lib/crates/fabro-store/src/slate/catalog.rs` is deleted. +- `cargo nextest run -p fabro-store` passes. +- `Database::list_runs` produces identical results to the pre-refactor state for the same input. + +--- + +- [x] **Unit 5: Migrate `AuthCodeStore` to `Repository`; add `code` field to `AuthCode`** + +**Goal:** Refactor `SlateAuthCodeStore` → `AuthCodeStore` on top of `Repository` and `KeyedMutex`. Add `code: String` as the first field of `AuthCode` so `Record::id` can return it. Update insert/consume signatures and call sites. + +**Requirements:** R7 (API + struct change), R5 (security boundary), R13 (uses KeyedMutex from U1). + +**Dependencies:** U1, U2a. + +**Files:** +- Modify: `lib/crates/fabro-store/src/slate/auth_codes.rs`: + - Rename `SlateAuthCodeStore` → `AuthCodeStore` + - Add `code: String` as first field of `AuthCode` + - `impl Record for AuthCode { type Id = String; type Codec = JsonCodec; const PREFIX = "auth/code"; fn id(&self) -> String { self.code.clone() } }` + - Refactor `insert(code: &str, entry: AuthCode)` → `insert(&self, entry: AuthCode)` per R7 (the `entry` now carries its own `code`; one-arg form is the chosen signature) + - Refactor `consume(&self, code: &str)` to use `self.repo.get(&code.to_string())` then `self.repo.delete(...)` under `KeyedMutex` guard + - Refactor `gc_expired(&self, cutoff)` to use `self.repo.gc(|entry| entry.expires_at <= cutoff)` + - Field changes: drop `db: Arc` (now via `repo`); drop `code_locks: DashMap<...>` (now via `consume_locks: KeyedMutex`) +- Modify: `lib/crates/fabro-store/src/slate/mod.rs`: + - Rename the field `auth_codes: Arc>>` → `Arc>>` + - Update `Database::auth_codes()` return type +- Modify: `lib/crates/fabro-store/src/lib.rs`: drop `SlateAuthCodeStore` from `pub use`; add `AuthCodeStore` and `AuthCode`; **add temporary alias** `pub type SlateAuthCodeStore = AuthCodeStore;` so `fabro-server` keeps compiling until U7 lands. (Without this alias the workspace breaks between U5 and U7.) +- Modify: `lib/crates/fabro-server/src/auth/cli_flow.rs:1226` (OAuth code-mint AuthCode literal — add `code: code.clone()` as first field; reorder so `code` is computed before the literal). After this change, the call switches from `store.insert(&code, entry)` (today's `auth_codes.rs:41`) to `store.insert(entry)` per R7. +- Modify: `lib/crates/fabro-server/src/auth/cli_flow.rs:1390` (test fixture AuthCode literal) +- Modify: `lib/crates/fabro-server/tests/it/api/cli_auth_token.rs:82` (test fixture AuthCode literal) +- Test: `lib/crates/fabro-store/src/slate/auth_codes.rs` (existing test module updates to construct `AuthCode { code: "...", ... }` — `auth_codes.rs:126`) + +**Approach:** +- The `code` field becomes the first field of `AuthCode` so the JSON shape is `{"code":"...","identity":...,...}` — adds one field to the wire format. Greenfield → acceptable. +- `AuthCodeStore::insert(entry)` calls `self.repo.put(&entry)` (no extra arg needed since `entry.code` carries the key). +- `AuthCodeStore::consume(code)` takes `&str`, holds `consume_locks.lock(code.to_string())` guard, then `repo.get(&code.to_string())`, branches on expiry/found, deletes via `repo.delete(&code.to_string())`. Single-use semantics preserved. +- `gc_expired` is one line via `repo.gc(|c| c.expires_at <= cutoff)`. +- Repository field is `pub(super)` per R5 boundary. + +**Patterns to follow:** +- `lib/crates/fabro-store/src/slate/auth_codes.rs:33-100` (preserve API behavior). +- `lib/crates/fabro-store/src/slate/auth_codes.rs:103-200` (existing test patterns — update for new struct shape, otherwise preserve). + +**Test scenarios:** +- *Happy path:* `insert(AuthCode { code, ... })` then `consume(&code)` returns the entry and a second `consume` returns `None` (single-use). +- *Edge case:* `consume` on a non-existent code returns `None`. +- *Edge case:* `consume` on an expired code deletes the row and returns `None` (today's behaviour at `auth_codes.rs:67-71`). +- *Error path:* none surfaced by today's code beyond serde errors — preserve. +- *Integration:* All existing `auth_codes.rs` tests pass with updated struct construction (e.g., `concurrent_consume_has_one_winner` at `auth_codes.rs:152` MUST still pass — KeyedMutex preserves single-winner behaviour). +- *Integration (boundary):* `AuthCodeStore`'s `repo` field is not accessible from outside the wrapper (compile-time check via test that the field is private; or simply: no public method on `AuthCodeStore` returns `&Repository`). + +**Verification:** +- `cargo nextest run -p fabro-store` passes. +- `cargo nextest run -p fabro-server` passes (if server uses `AuthCode` struct literally; update imports per U7 if needed). +- `lib/crates/fabro-store/src/slate/auth_codes.rs` no longer imports `slatedb::Db` directly. + +--- + +- [x] **Unit 6: Migrate `RefreshTokenStore` to `Repository` + `transaction(...)` for `consume_and_rotate`** + +**Goal:** The most complex migration. Refactor `SlateAuthTokenStore` → `RefreshTokenStore` on top of `Repository` and `KeyedMutex<[u8; 32]>`. Replace the hand-built `slatedb::WriteBatch` in `consume_and_rotate` with `transaction(&db, ...)`. Preserve replay revocation cache as in-memory only. R10 atomicity is verified by U2a's transaction-layer fault-injection test (no separate U6 fault-injection test — see Execution note). + +**Requirements:** R6 (API + transaction usage + replay cache invariant), R5 (security boundary), R10 (transaction atomicity), R13 (KeyedMutex). + +**Dependencies:** U1, U2a. + +**Execution note:** `consume_and_rotate` is security-sensitive. The R10 atomicity contract is exercised by U2a's transaction-layer fault-injection test using a synthetic test-only `Record` + `Codec` whose `encode` fails on the second `put` (`RefreshToken` cannot itself trigger a JSON encode failure). U6 keeps the existing `concurrent_rotation_has_one_winner` (`auth_tokens.rs:311`) test as the consume-layer atomicity regression net. + +**Files:** +- Modify: `lib/crates/fabro-store/src/slate/auth_tokens.rs`: + - Rename `SlateAuthTokenStore` → `RefreshTokenStore` + - `impl Record for RefreshToken { type Id = [u8; 32]; type Codec = JsonCodec; const PREFIX = "auth/refresh"; fn id(&self) -> [u8; 32] { self.token_hash } }` + - Refactor `insert_refresh_token(token)` → uses `self.repo.put(&token)` + - Refactor `find_refresh_token(&hash)` → uses `self.repo.get(&hash)` + - Refactor `consume_and_rotate(presented_hash, new_token, now)` → uses `transaction(&self.db, |tx| { tx.put(&old_token)?; tx.put(&new_token)?; Ok(()) })` while holding `consume_locks.lock(presented_hash)` guard. Outcomes (`Rotated`, `Reused`, `Expired`, `NotFound`) preserved exactly. + - Refactor `delete_chain(chain_id)` → uses `self.repo.gc(|t| t.chain_id == chain_id)` + - Refactor `gc_expired(cutoff)` → uses `self.repo.gc(|t| t.expires_at <= cutoff)` + - Field changes: drop `refresh_locks: DashMap<...>` (now `consume_locks: KeyedMutex<[u8; 32]>`); keep `replay_revocations: DashMap<[u8; 32], DateTime>` UNCHANGED (in-memory only per R6). Add a doc-comment on the `replay_revocations` field: `/// In-memory only by design (origin R6) — persisting attacker-supplied hashes adds an unbounded-growth surface under token-stuffing attack with no security benefit. Do NOT migrate to Repository.` so future contributors don't "complete the abstraction" by routing this through a persisted Repository. + - `mark_refresh_token_replay` and `was_recently_replay_revoked` keep their existing impls — they touch only the in-memory `DashMap`. +- Modify: `lib/crates/fabro-store/src/slate/mod.rs` (rename field type; update `Database::auth_tokens()` → `Database::refresh_tokens()` per R11; **add temporary inherent shim** `pub async fn auth_tokens(&self) -> Result> { self.refresh_tokens().await }` so existing `fabro-server` callers compile until U7 removes them.) +- Modify: `lib/crates/fabro-store/src/lib.rs`: drop `SlateAuthTokenStore` from `pub use`; add `RefreshTokenStore` and `RefreshToken`; **add temporary alias** `pub type SlateAuthTokenStore = RefreshTokenStore;` so `fabro-server` keeps compiling until U7 lands. +- Modify: `lib/crates/fabro-store/src/slate/auth_tokens.rs:201` (the in-crate test fixture call `db.auth_tokens().await.unwrap()` → `db.refresh_tokens().await.unwrap()`) +- Test: `lib/crates/fabro-store/src/slate/auth_tokens.rs` (preserve all existing tests; the U2a transaction-layer fault-injection test covers R10) + +**Approach:** +- `RefreshTokenStore` holds `db: Arc` (for `transaction`), `repo: Repository` (`pub(super)`), `consume_locks: KeyedMutex<[u8; 32]>`, `replay_revocations: DashMap<[u8; 32], DateTime>`. +- `consume_and_rotate` flow: + 1. `let _guard = self.consume_locks.lock(presented_hash).await;` + 2. `let existing = self.repo.get(&presented_hash).await?;` + 3. Branch on `None` / expired / used / proceed exactly as today. + 4. For the "proceed" branch: clone `existing` as `old_token`, set `used = true`, `last_used_at = now`. Then `transaction(&self.db, |tx| { tx.put(&old_token)?; tx.put(&new_token)?; Ok(()) }).await?`. + 5. Return `ConsumeOutcome::Rotated(old_token, Box::new(new_token))`. + 6. Mutex strong-count cleanup happens via `KeyedMutex` per U1 — no inline check here. +- Replay cache: `mark_refresh_token_replay` and `was_recently_replay_revoked` stay as-is. R6 explicitly forbids moving them into `Repository`. + +**Patterns to follow:** +- `lib/crates/fabro-store/src/slate/auth_tokens.rs:79-121` (today's `consume_and_rotate` — preserve outcomes and ordering exactly; only the WriteBatch construction moves into `transaction(...)`). +- `lib/crates/fabro-store/src/slate/auth_tokens.rs:123-161` (today's `delete_chain` and `gc_expired` — both collapse to `repo.gc(predicate)` calls). +- `lib/crates/fabro-store/src/slate/auth_tokens.rs:163-178` (replay revocation methods — preserve unchanged). + +**Test scenarios:** +- *Happy path:* `insert_refresh_token(t1); find_refresh_token(&hash) == Some(t1)`. +- *Happy path (rotation):* `insert(old); consume_and_rotate(old_hash, new, now)` returns `Rotated(old_used, new)`; both `find(&old_hash)` and `find(&new_hash)` succeed; `old.used == true`. +- *Edge case (NotFound):* `consume_and_rotate(unknown_hash, ...)` returns `NotFound`. +- *Edge case (Expired):* `consume_and_rotate(expired_hash, ...)` returns `Expired`; old token is NOT marked `used`. +- *Edge case (Reused):* `consume_and_rotate(used_hash, ...)` returns `Reused(existing)`. +- *Integration (concurrency):* 16 concurrent `consume_and_rotate` on the same hash — exactly one returns `Rotated`, 15 return `Reused`. (Existing `concurrent_rotation_has_one_winner` test, MUST pass unmodified.) +- *Integration (delete_chain):* `insert` two tokens with the same `chain_id`, `delete_chain(chain_id)` returns 2, both vanish. +- *Integration (gc_expired):* `insert` an expired token + a live token, `gc_expired(now)` returns 1, expired vanishes, live remains. +- *Integration (replay revocation):* `mark_refresh_token_replay(h)` then `was_recently_replay_revoked(&h)` is true; after 60s TTL, false. +- *Integration (atomicity):* the R10 transaction-layer atomicity test lives in U2a (synthetic Record + Codec). U6 does not duplicate it — `RefreshToken` cannot itself trigger a JSON encode failure without unrealistic fixtures, so the consume-layer test would either reduce to a closure-Err short-circuit (strictly weaker) or require a fragile mock. The U2a test plus `concurrent_rotation_has_one_winner` are the regression net. + +**Verification:** +- All existing tests in `auth_tokens.rs:181-403` pass without behavioural change (the test fixtures may need import-rename updates only). +- `cargo nextest run -p fabro-store` passes. +- `cargo nextest run --workspace` passes. +- `lib/crates/fabro-store/src/slate/auth_tokens.rs` no longer imports `slatedb::WriteBatch` directly. +- `RefreshTokenStore`'s `repo` field is private (compile-time check). + +--- + +- [x] **Unit 7: Update public re-exports + `fabro-server` imports** + +**Goal:** Update `fabro-store::lib.rs` re-exports to the new names; update `fabro-server` to use the new type names. Mechanical cleanup. + +**Requirements:** R11 (Database surface). + +**Dependencies:** U2b, U3, U4, U5, U6. + +**Files:** +- Modify: `lib/crates/fabro-store/src/lib.rs`: + - Drop temporary `pub type SlateAuthCodeStore = AuthCodeStore;` and `pub type SlateAuthTokenStore = RefreshTokenStore;` aliases that U5/U6 introduced + - Add `BlobStore`, `RunCatalogIndex` to `pub use slate::{...}` (the new accessor types from U2b/U4) + - Verify other re-exports (`AuthCode`, `RefreshToken`, `ConsumeOutcome`, `Database`, `RunDatabase`, `Runs`, `AuthCodeStore`, `RefreshTokenStore`) are still correct +- Modify: `lib/crates/fabro-store/src/slate/mod.rs`: drop the temporary `Database::auth_tokens()` shim that U6 introduced (callers now use `refresh_tokens()` directly) +- Modify: `lib/crates/fabro-server/src/serve.rs:799-829` (4 type-name sites: `Arc` → `Arc`, `Arc` → `Arc`) +- Modify: `lib/crates/fabro-server/src/serve.rs:507,508` (2 accessor sites: `db.auth_tokens()` → `db.refresh_tokens()`) +- Modify: `lib/crates/fabro-server/src/auth/cli_flow.rs:380, 462, 541, 660, 1236, 1388, 1660, 1898, 1987, 2042, 2126` (12 accessor sites: `db.auth_tokens()` → `db.refresh_tokens()`; verify each by grep before edit) +- Modify: `lib/crates/fabro-server/src/auth/cli_flow.rs:449, 1039, 1410, 2129` (4 `RefreshToken { ... }` literal sites — should be unchanged in shape since `RefreshToken` struct itself doesn't change, but verify imports) +- Modify: `lib/crates/fabro-server/tests/it/api/cli_auth_token.rs:80, 82, 146, 149` (4 sites: accessor calls + `AuthCode { ... }` and `RefreshToken { ... }` literals — `AuthCode` literal needs `code` field per U5) +- Note: `server.rs`, `jwt_auth.rs`, `auth/mod.rs`, `web_auth.rs` were spot-checked and contain no `SlateAuth*Store` or accessor references — no edits needed there. Final verification grep below catches drift. + +**Approach:** +- Rename via grep-replace, since the types are referenced by qualified name only. +- `Database::auth_tokens()` → `Database::refresh_tokens()` per R11. Update all callers accordingly. +- If any `fabro-server` code constructs `AuthCode` literals (e.g., during the OAuth code flow), add the `code` field to those literals. + +**Patterns to follow:** +- N/A — pure rename + import update. + +**Test scenarios:** +- *Test expectation: none — pure rename / re-export update.* No new behavioural test required. +- *Verification:* `cargo build --workspace` passes; `cargo nextest run --workspace` passes. + +**Verification:** +- `cargo build --workspace` succeeds. +- `cargo nextest run --workspace` passes (including `fabro-server` and `fabro-cli` integration tests). +- `cargo +nightly-2026-04-14 clippy --workspace --all-targets -- -D warnings` passes. +- `cargo +nightly-2026-04-14 fmt --check --all` passes. +- `grep -r "Slate\(Auth\|RefreshToken\)" lib/ apps/` returns no production references (test-fixture references are also updated). +- `grep -rn '\.auth_tokens()' lib/ apps/` returns zero matches (the `auth_tokens()` accessor was renamed to `refresh_tokens()`; this catches any caller the import-update missed, since a stale `.auth_tokens()` would only fail at compile time if no other type in scope happens to expose a method by that name). + +## System-Wide Impact + +- **Interaction graph:** `fabro-server` (`serve.rs`, `auth/mod.rs`, `auth/cli_flow.rs`, `jwt_auth.rs`, `web_auth.rs`) imports `Slate*Store` types from `fabro-store` and calls `Database::auth_codes()` / `auth_tokens()`. All of these need import + accessor-name updates in U7. No `fabro-server` logic changes — purely mechanical. +- **Error propagation:** `fabro_store::Error` gains two new public variants in U2a: `InvalidKeySegment { segment: String }` (returned by `Repository` key-assembly when a `RecordId::key_segments` impl produces a segment containing the `\0` separator) and `KeyParse(String)` (returned by `scan_stream`/`scan_ids_stream` when `RecordId::from_key_segments` rejects the parsed segments). Existing variants are unchanged. New error paths from `JsonCodec::encode` (`serde_json::Error`) and `MarkerCodec` propagate through the same `Result` shape via the existing variants. +- **State lifecycle risks:** + - U2a's `transaction(...)` MUST commit all-or-nothing — covered by R10's fault-injection test in U2a (synthetic Record + TestFailCodec). + - U5's `AuthCode` schema change (adding `code` field) means any in-flight or persisted code from before the migration would not deserialize. Greenfield → no concern in practice; flag in PR description anyway. + - U1's `KeyedMutex` cleanup race (cleanup-under-shard-lock invariant) is the load-bearing concurrency guarantee — covered by U1 stress test. +- **API surface parity:** `Database::auth_tokens()` rename → `refresh_tokens()` is observable. Callers in `fabro-server` are the only consumers; updating them in U7. No external repos consume `fabro-store`. +- **Integration coverage:** Cross-layer scenarios (Database → wrapper Store → Repository → SlateDB) are covered by: + - U2a's transaction fault-injection test (proves R10 atomicity at the helper layer) + - U2b's `BlobStore` round-trip + cross-path equivalence test (proves the full stack works for the simplest record and that `RawBytesCodec` is byte-identity) + - U4's `Database::list_runs` integration test (proves the catalog migration preserves observable behaviour) + - U6's `concurrent_rotation_has_one_winner` (proves KeyedMutex + transaction integration at the consume layer) + - The `delete_run_keeps_global_cas_blobs` test in `slate/mod.rs:451` continues to enforce the cross-store blob invariant. +- **Unchanged invariants:** + - `RunDatabase` event log, projection cache, broadcast channel, `recover_next_seq`, `EventProjectionCache` — all untouched. + - `Database::create_run` / `open_run` / `delete_run` / `list_runs` public signatures and observable behaviour — preserved. + - `RunDatabase::write_blob` / `read_blob` / `list_blobs` public signatures — preserved (delegation is internal). + - All existing `*_db.rs` tests pass without modification (tests touching renamed types update imports; tests touching `AuthCode` literals add the `code` field). + - Wire format for `RefreshToken` JSON is byte-identical to today (`JsonCodec` is literal `serde_json::to_vec` forwarding; U2a's snapshot test on a synthetic struct catches accidental future envelope/version/reordering changes). + - Wire format for `AuthCode` JSON gains a new `code` field per R7/U5 — the value goes from `{"identity":...,"login":...,...}` to `{"code":"...","identity":...,...}`. Acceptable under greenfield (no production deployments). All existing AuthCode rows in any non-empty test/dev SlateDB become un-decodable after upgrade and must be discarded. + +## Risks & Dependencies + +| Risk | Mitigation | +|------|------------| +| `transaction(...)` is implemented with non-atomic semantics (e.g., commits per-put on closure error), silently weakening refresh-token rotation atomicity. | R10 fault-injection test in U2a explicitly asserts an encode error on the Nth `put` leaves the DB unchanged. Implementer cannot ship a non-atomic helper without breaking the test. `concurrent_rotation_has_one_winner` in U6 is the consume-layer regression net. | +| `AuthCode` struct change in U5 breaks `fabro-server` OAuth code flow. | U5 includes a grep for `AuthCode { ` literals; all are updated in the same commit. `cargo build --workspace` failure is the canary. | +| `KeyedMutex` cleanup race: a concurrent `lock(K)` observes a stale Arc after `strong_count == 2` cleanup, leading to two threads holding "the lock for K" against different Mutex instances. | U1 implementation MUST hold the `DashMap` shard lock for both the cleanup check and the entry insertion. U1 includes a stress test that hammers the same key across many drop cycles. | +| `Repository::scan_stream` cannot reconstruct `R::Id` from values for records whose Id isn't a field (e.g. ZST markers, content-addressed records). | Resolved at plan time: `RecordId::from_key_segments` parses Id from the key, and `Repository::scan_ids_stream` enumerates ids without touching values. ZST markers (`MarkerCodec`) use the id-only API exclusively. | +| `RunDatabase` blob delegation in U3 changes construction of `RunDatabase` inside `Database::create_run`/`open_run`/`open_run_reader`, breaking subtle ordering assumptions. | Resolved at plan time: U3 uses stack-local `BlobStore` construction inside each blob method — no `RunDatabaseInner` field changes, no constructor signature changes. Existing `delete_run_keeps_global_cas_blobs` and full-lifecycle tests are the regression net. | +| `gc(predicate)` predicate is `impl Fn(&R) -> bool` — a future caller passes an `async` closure or one that does I/O, blocking the scan. | R4 explicitly states the predicate is sync `Fn` and MUST NOT perform I/O or block. Documented; relies on convention + code review (no compile-time enforcement). | +| `JsonCodec::encode` is "literally `serde_json::to_vec`" but a future maintainer adds an envelope/version tag for "v2" without realizing it breaks every existing on-disk record. | U2a's snapshot test on a stable synthetic struct fails loudly on any byte change to JsonCodec output (envelope, version tag, field reordering). PR review should catch the snapshot acceptance. | +| Migration order assumption: U2b lands and exposes `BlobStore` publicly before U3 wires `RunDatabase` to it; an external caller could start using `Database::blobs()` directly between U2b and U3. | Acceptable — both paths produce identical results; cross-path equivalence is locked by U2b's test (RawBytesCodec is byte-identity). After U3, `RunDatabase::write_blob` and `Database::blobs().write` are equivalent surfaces. | +| Pinning `Repository` to `pub(crate)` is by convention only against future in-crate contributors — nothing prevents a new module under `fabro-store/src/` from constructing a second `Repository` and bypassing `RefreshTokenStore`. | Documented in R5: "for crate-internal contributors it is convention + code review." Considered alternative (sealed-per-record module with `pub(super)`) was rejected in pass-2 review for being heavier than the threat warrants. | + +## Documentation / Operational Notes + +- No external docs to update — `fabro-store` has no public-facing documentation in `docs/`. +- No CLI changes; no rollout/monitoring impact. +- No migration script needed — greenfield, on-disk format for `RefreshToken` is byte-identical (locked by snapshot); `AuthCode` schema change in U5 is an additive field, but greenfield means existing dev databases are throwaway. +- After U7 lands, a follow-up issue may be filed for: (a) per-Repository batch helper (deferred from origin Q3); (b) `BlobStore` lifecycle question if the simpler stack-local construction in U3 ages poorly. + +## Verification Notes (2026-04-21) + +Post-implementation audit against the plan. All 7 units are implemented; build, nextest (`fabro-store` 68 / `fabro-server` 359), clippy (nightly-2026-04-14 `-D warnings`), and fmt pass. U7 grep checks (`Slate(Auth|RefreshToken)`, `.auth_tokens()`) return zero hits. `slate/catalog.rs` is deleted. + +Minor gaps that do not affect correctness or production surface: + +- **U2a — `scan_prefix_ids_stream` not implemented.** Appears in the Repository *design* sketch (line 186) but is not listed in R4. No current caller. Add lazily if a future consumer needs prefix-scoped id enumeration. +- **U4 — two edge-case tests from plan not individually present.** `run_catalog_index.rs` tests cover add/list/remove round-trip and start/end filter, but not (a) `add(rid); remove(rid); list()` returns empty, nor (b) start-later-than-all / end-earlier-than-all returning empty. The integration paths in `slate::tests` and `list_applies_start_and_end_filters` exercise the same code paths, so no functional regression risk. +- **U3 — `BlobStore::new(Arc::new(self.inner.db.clone()))` shape.** Plan suggested stack-local `BlobStore::new(self.inner.db.clone())`, but `BlobStore::new` requires `Arc` so the call wraps in `Arc::new`. Functionally equivalent; cosmetic. + +## Sources & References + +- **Origin document:** `docs/brainstorms/2026-04-20-fabro-store-record-abstractions-requirements.md` +- Today's K/V stores: + - `lib/crates/fabro-store/src/slate/auth_tokens.rs` + - `lib/crates/fabro-store/src/slate/auth_codes.rs` + - `lib/crates/fabro-store/src/slate/catalog.rs` + - `lib/crates/fabro-store/src/slate/run_store.rs` +- Database accessor pattern: `lib/crates/fabro-store/src/slate/mod.rs:208-228` +- Key construction: `lib/crates/fabro-store/src/keys.rs` +- Public re-exports today: `lib/crates/fabro-store/src/lib.rs` +- Server callers: `lib/crates/fabro-server/src/serve.rs:799-829`, `lib/crates/fabro-server/src/server.rs`, `lib/crates/fabro-server/src/auth/mod.rs` +- Workspace build commands and test posture: `CLAUDE.md` (`cargo nextest`, `cargo +nightly-2026-04-14 clippy`, `cargo +nightly-2026-04-14 fmt`) diff --git a/lib/crates/fabro-server/src/auth/cli_flow.rs b/lib/crates/fabro-server/src/auth/cli_flow.rs index 6d6df73ac..0fa1d163a 100644 --- a/lib/crates/fabro-server/src/auth/cli_flow.rs +++ b/lib/crates/fabro-server/src/auth/cli_flow.rs @@ -459,7 +459,7 @@ async fn token( used: false, user_agent: sanitize_user_agent(request_user_agent(&headers)), }; - let auth_tokens = match state.store_ref().auth_tokens().await { + let auth_tokens = match state.store_ref().refresh_tokens().await { Ok(store) => store, Err(err) => { warn!(error = %err, "Failed to open refresh token store"); @@ -538,7 +538,7 @@ async fn refresh( "Could not refresh authentication", ); }; - let auth_tokens = match state.store_ref().auth_tokens().await { + let auth_tokens = match state.store_ref().refresh_tokens().await { Ok(store) => store, Err(err) => { warn!(error = %err, "Failed to open refresh token store"); @@ -657,7 +657,7 @@ async fn logout( let Some(secret) = refresh_secret_from_headers(&headers) else { return StatusCode::NO_CONTENT.into_response(); }; - let auth_tokens = match state.store_ref().auth_tokens().await { + let auth_tokens = match state.store_ref().refresh_tokens().await { Ok(store) => store, Err(err) => { warn!(error = %err, "Failed to open refresh token store"); @@ -1224,6 +1224,7 @@ async fn issue_auth_code_response( return static_error_page(INVALID_REDIRECT_URI); }; let entry = AuthCode { + code: code.clone(), identity, login: session.login.clone(), name: session.name.clone(), @@ -1246,7 +1247,7 @@ async fn issue_auth_code_response( } }; - if let Err(err) = store.insert(&code, entry).await { + if let Err(err) = store.insert(entry).await { warn!(error = %err, "Failed to persist auth code"); return redirect_with_error( &redirect_uri, @@ -1387,7 +1388,8 @@ mod tests { async fn insert_auth_code(state: &crate::server::AppState, code: &str, verifier: &str) { let auth_codes = state.store_ref().auth_codes().await.unwrap(); auth_codes - .insert(code, AuthCode { + .insert(AuthCode { + code: code.to_string(), identity: fabro_types::IdpIdentity::new("https://github.com", "12345") .expect("identity should be valid"), login: "octocat".to_string(), @@ -1895,7 +1897,7 @@ mod tests { .unwrap() .strip_prefix("fabro_refresh_") .unwrap(); - let auth_tokens = state.store_ref().auth_tokens().await.unwrap(); + let auth_tokens = state.store_ref().refresh_tokens().await.unwrap(); let refresh = auth_tokens .find_refresh_token(&hash_refresh_secret(refresh_secret)) .await @@ -1984,7 +1986,7 @@ mod tests { async fn refresh_rotates_tokens_and_replay_revokes_chain() { let (app, state) = test_router(github_settings("https://fabro.example")); let initial_secret = "refresh-secret-1"; - let auth_tokens = state.store_ref().auth_tokens().await.unwrap(); + let auth_tokens = state.store_ref().refresh_tokens().await.unwrap(); auth_tokens .insert_refresh_token(refresh_row(initial_secret)) .await @@ -2039,7 +2041,7 @@ mod tests { async fn concurrent_refresh_has_one_winner_and_revokes_chain() { let (app, state) = test_router(github_settings("https://fabro.example")); let initial_secret = "refresh-secret-concurrent"; - let auth_tokens = state.store_ref().auth_tokens().await.unwrap(); + let auth_tokens = state.store_ref().refresh_tokens().await.unwrap(); auth_tokens .insert_refresh_token(refresh_row(initial_secret)) .await @@ -2123,7 +2125,7 @@ mod tests { let secret = "refresh-secret-logout"; let token = refresh_row(secret); let chain_id = token.chain_id; - let auth_tokens = state.store_ref().auth_tokens().await.unwrap(); + let auth_tokens = state.store_ref().refresh_tokens().await.unwrap(); auth_tokens.insert_refresh_token(token).await.unwrap(); let sibling = RefreshToken { diff --git a/lib/crates/fabro-server/src/serve.rs b/lib/crates/fabro-server/src/serve.rs index cbb35d066..7b4b2ba96 100644 --- a/lib/crates/fabro-server/src/serve.rs +++ b/lib/crates/fabro-server/src/serve.rs @@ -505,7 +505,7 @@ where cache_path, )); let auth_code_store = store.auth_codes().await?; - let auth_token_store = store.auth_tokens().await?; + let auth_token_store = store.refresh_tokens().await?; let (artifact_object_store, artifact_prefix) = build_artifact_object_store(&resolved_server_settings)?; let artifact_store = fabro_store::ArtifactStore::new(artifact_object_store, artifact_prefix); @@ -796,8 +796,8 @@ async fn wait_for_shutdown(mut shutdown_rx: watch::Receiver) { } fn spawn_auth_store_reapers( - auth_codes: Arc, - auth_tokens: Arc, + auth_codes: Arc, + auth_tokens: Arc, shutdown_rx: watch::Receiver, ) { spawn_auth_code_reaper(auth_codes, shutdown_rx.clone()); @@ -805,7 +805,7 @@ fn spawn_auth_store_reapers( } fn spawn_auth_code_reaper( - auth_codes: Arc, + auth_codes: Arc, mut shutdown_rx: watch::Receiver, ) { tokio::spawn(async move { @@ -826,7 +826,7 @@ fn spawn_auth_code_reaper( } fn spawn_refresh_token_reaper( - auth_tokens: Arc, + auth_tokens: Arc, mut shutdown_rx: watch::Receiver, ) { tokio::spawn(async move { diff --git a/lib/crates/fabro-server/tests/it/api/cli_auth_token.rs b/lib/crates/fabro-server/tests/it/api/cli_auth_token.rs index 818cbd928..1fb8ab3ea 100644 --- a/lib/crates/fabro-server/tests/it/api/cli_auth_token.rs +++ b/lib/crates/fabro-server/tests/it/api/cli_auth_token.rs @@ -79,7 +79,8 @@ client_id = "Iv1.test" )); let auth_codes = store.auth_codes().await.unwrap(); auth_codes - .insert("integration-code", AuthCode { + .insert(AuthCode { + code: "integration-code".to_string(), identity: fabro_types::IdpIdentity::new("https://github.com", "12345").unwrap(), login: "octocat".to_string(), name: "The Octocat".to_string(), @@ -143,7 +144,7 @@ url = "https://fabro.example" client_id = "Iv1.test" "#, )); - let auth_tokens = store.auth_tokens().await.unwrap(); + let auth_tokens = store.refresh_tokens().await.unwrap(); let now = chrono::Utc::now(); auth_tokens .insert_refresh_token(RefreshToken { diff --git a/lib/crates/fabro-store/Cargo.toml b/lib/crates/fabro-store/Cargo.toml index ce458b884..6b3fe893b 100644 --- a/lib/crates/fabro-store/Cargo.toml +++ b/lib/crates/fabro-store/Cargo.toml @@ -33,3 +33,4 @@ uuid.workspace = true tokio = { workspace = true, features = ["test-util", "macros"] } tempfile = "3" ulid.workspace = true +insta = { workspace = true } diff --git a/lib/crates/fabro-store/src/error.rs b/lib/crates/fabro-store/src/error.rs index f73aa472a..e66da4134 100644 --- a/lib/crates/fabro-store/src/error.rs +++ b/lib/crates/fabro-store/src/error.rs @@ -16,6 +16,10 @@ pub enum Error { RunAlreadyExists(String), #[error("run store is read-only")] ReadOnly, + #[error("invalid key segment: {segment:?}")] + InvalidKeySegment { segment: String }, + #[error("failed to parse key: {0}")] + KeyParse(String), #[error("{0}")] Other(String), } diff --git a/lib/crates/fabro-store/src/keyed_mutex.rs b/lib/crates/fabro-store/src/keyed_mutex.rs new file mode 100644 index 000000000..576ce3352 --- /dev/null +++ b/lib/crates/fabro-store/src/keyed_mutex.rs @@ -0,0 +1,180 @@ +use std::hash::Hash; + +use dashmap::DashMap; +use dashmap::mapref::entry::Entry; +use tokio::sync::{Mutex, OwnedMutexGuard}; + +#[derive(Debug)] +pub(crate) struct KeyedMutex +where + K: Eq + Hash + Clone, +{ + mutexes: DashMap>>, +} + +impl Default for KeyedMutex +where + K: Eq + Hash + Clone, +{ + fn default() -> Self { + Self::new() + } +} + +impl KeyedMutex +where + K: Eq + Hash + Clone, +{ + pub(crate) fn new() -> Self { + Self { + mutexes: DashMap::new(), + } + } + + #[cfg(test)] + pub(crate) fn len(&self) -> usize { + self.mutexes.len() + } +} + +impl KeyedMutex +where + K: Eq + Hash + Clone, +{ + pub(crate) async fn lock(&self, key: K) -> Guard<'_, K> { + let mutex = self + .mutexes + .entry(key.clone()) + .or_insert_with(|| std::sync::Arc::new(Mutex::new(()))) + .clone(); + + Guard { + keyed_mutex: self, + key, + held_lock: Some(mutex.lock_owned().await), + } + } +} + +pub(crate) struct Guard<'a, K> +where + K: Eq + Hash + Clone, +{ + keyed_mutex: &'a KeyedMutex, + key: K, + held_lock: Option>, +} + +impl Drop for Guard<'_, K> +where + K: Eq + Hash + Clone, +{ + fn drop(&mut self) { + self.held_lock.take(); + + if let Entry::Occupied(entry) = self.keyed_mutex.mutexes.entry(self.key.clone()) { + if std::sync::Arc::strong_count(entry.get()) == 1 { + entry.remove(); + } + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::time::Duration; + + use tokio::task::{JoinSet, yield_now}; + use tokio::time::{sleep, timeout}; + + use super::KeyedMutex; + + #[tokio::test] + async fn distinct_keys_do_not_contend() { + let keyed_mutex = KeyedMutex::new(); + let _first = keyed_mutex.lock("alpha".to_string()).await; + + timeout( + Duration::from_millis(50), + keyed_mutex.lock("beta".to_string()), + ) + .await + .expect("distinct keys should not block"); + } + + #[tokio::test] + async fn same_key_serializes_access() { + let keyed_mutex = Arc::new(KeyedMutex::new()); + let first = keyed_mutex.lock("alpha".to_string()).await; + let acquired = Arc::new(AtomicBool::new(false)); + + let worker_mutex = Arc::clone(&keyed_mutex); + let worker_acquired = Arc::clone(&acquired); + let waiter = tokio::spawn(async move { + let _second = worker_mutex.lock("alpha".to_string()).await; + worker_acquired.store(true, Ordering::SeqCst); + }); + + sleep(Duration::from_millis(10)).await; + assert!( + !acquired.load(Ordering::SeqCst), + "second waiter should block while first guard is held" + ); + + drop(first); + timeout(Duration::from_millis(100), waiter) + .await + .expect("waiter should proceed after first guard drops") + .unwrap(); + assert!(acquired.load(Ordering::SeqCst)); + } + + #[tokio::test] + async fn drops_unused_entries_after_last_guard_releases() { + let keyed_mutex = KeyedMutex::new(); + let guard = keyed_mutex.lock("alpha".to_string()).await; + + assert_eq!(keyed_mutex.len(), 1); + drop(guard); + + assert_eq!(keyed_mutex.len(), 0); + } + + #[tokio::test] + async fn stress_same_key_keeps_single_mutex_and_cleans_up() { + let keyed_mutex = Arc::new(KeyedMutex::new()); + let active = Arc::new(AtomicUsize::new(0)); + let max_active = Arc::new(AtomicUsize::new(0)); + let violations = Arc::new(AtomicUsize::new(0)); + let mut tasks = JoinSet::new(); + + for _ in 0..16 { + let keyed_mutex = Arc::clone(&keyed_mutex); + let active = Arc::clone(&active); + let max_active = Arc::clone(&max_active); + let violations = Arc::clone(&violations); + tasks.spawn(async move { + for _ in 0..64 { + let _guard = keyed_mutex.lock("alpha".to_string()).await; + let current = active.fetch_add(1, Ordering::SeqCst) + 1; + max_active.fetch_max(current, Ordering::SeqCst); + if current != 1 { + violations.fetch_add(1, Ordering::SeqCst); + } + yield_now().await; + active.fetch_sub(1, Ordering::SeqCst); + } + }); + } + + while let Some(result) = tasks.join_next().await { + result.unwrap(); + } + + assert_eq!(violations.load(Ordering::SeqCst), 0); + assert_eq!(max_active.load(Ordering::SeqCst), 1); + assert_eq!(keyed_mutex.len(), 0); + } +} diff --git a/lib/crates/fabro-store/src/keys.rs b/lib/crates/fabro-store/src/keys.rs index d3bcb7d76..44b1a9753 100644 --- a/lib/crates/fabro-store/src/keys.rs +++ b/lib/crates/fabro-store/src/keys.rs @@ -8,17 +8,17 @@ pub(crate) struct SlateKey(String); impl SlateKey { const SEP: char = '\0'; - fn new(segment: impl fmt::Display) -> Self { + pub(crate) fn new(segment: impl fmt::Display) -> Self { Self(segment.to_string()) } - fn with(mut self, segment: impl fmt::Display) -> Self { + pub(crate) fn with(mut self, segment: impl fmt::Display) -> Self { self.0.push(Self::SEP); write!(&mut self.0, "{segment}").expect("write to String cannot fail"); self } - fn into_prefix(mut self) -> Self { + pub(crate) fn into_prefix(mut self) -> Self { self.0.push(Self::SEP); self } @@ -28,7 +28,7 @@ impl SlateKey { &self.0 } - fn segments(raw: &str) -> impl Iterator { + pub(crate) fn segments(raw: &str) -> impl Iterator { raw.split(Self::SEP) } } @@ -41,21 +41,6 @@ impl AsRef<[u8]> for SlateKey { // --- Construction --- -pub(crate) fn runs_index_by_start_prefix() -> SlateKey { - SlateKey::new("runs") - .with("_index") - .with("by-start") - .into_prefix() -} - -pub(crate) fn runs_index_by_start_key(run_id: &RunId) -> SlateKey { - SlateKey::new("runs") - .with("_index") - .with("by-start") - .with(run_id.created_at().format("%Y-%m-%d")) - .with(run_id) -} - pub(crate) fn run_data_prefix(run_id: &RunId) -> SlateKey { SlateKey::new("runs").with(run_id).into_prefix() } @@ -78,30 +63,6 @@ pub(crate) fn blobs_prefix() -> SlateKey { SlateKey::new("blobs").with("sha256").into_prefix() } -pub(crate) fn blob_key(id: &RunBlobId) -> SlateKey { - SlateKey::new("blobs").with("sha256").with(id) -} - -pub(crate) fn auth_code_prefix() -> SlateKey { - SlateKey::new("auth").with("code").into_prefix() -} - -pub(crate) fn auth_code_key(code: &str) -> SlateKey { - SlateKey::new("auth").with("code").with(code) -} - -pub(crate) fn auth_refresh_prefix() -> SlateKey { - SlateKey::new("auth").with("refresh").into_prefix() -} - -pub(crate) fn auth_refresh_key(token_hash: &[u8; 32]) -> SlateKey { - let mut encoded = String::with_capacity(token_hash.len() * 2); - for byte in token_hash { - write!(&mut encoded, "{byte:02x}").expect("write to String cannot fail"); - } - SlateKey::new("auth").with("refresh").with(encoded) -} - // --- Parsing --- pub(crate) fn parse_event_seq(key: &str) -> Option { @@ -129,15 +90,6 @@ pub(crate) fn parse_blob_id(key: &str) -> Option { id.parse().ok() } -pub(crate) fn parse_run_id_from_index_key(key: &str) -> Option { - let mut segments = SlateKey::segments(key); - let _ = segments.next()?; // "runs" - let _ = segments.next()?; // "_index" - let _ = segments.next()?; // "by-start" - let _ = segments.next()?; // date - segments.next()?.parse().ok() -} - #[cfg(test)] mod tests { use fabro_types::RunId; @@ -169,24 +121,10 @@ mod tests { ]); } - #[test] - fn index_key_segments() { - let run_id: RunId = "01JT56VE4Z5NZ814GZN2JZD65A".parse().unwrap(); - let key = runs_index_by_start_key(&run_id); - let segments: Vec<&str> = SlateKey::segments(key.as_str()).collect(); - assert_eq!(segments, [ - "runs", - "_index", - "by-start", - &run_id.created_at().format("%Y-%m-%d").to_string(), - &run_id.to_string(), - ]); - } - #[test] fn blob_key_segments() { let blob_id = RunBlobId::new(b"summary"); - let key = blob_key(&blob_id); + let key = SlateKey::new("blobs").with("sha256").with(blob_id); let segments: Vec<&str> = SlateKey::segments(key.as_str()).collect(); assert_eq!(segments, ["blobs", "sha256", &blob_id.to_string()]); } @@ -208,12 +146,8 @@ mod tests { ); let blob_id = RunBlobId::new(b"summary"); - assert_eq!(parse_blob_id(blob_key(&blob_id).as_str()), Some(blob_id)); - - assert_eq!( - parse_run_id_from_index_key(runs_index_by_start_key(&run_id).as_str()), - Some(run_id) - ); + let key = SlateKey::new("blobs").with("sha256").with(blob_id); + assert_eq!(parse_blob_id(key.as_str()), Some(blob_id)); } #[test] diff --git a/lib/crates/fabro-store/src/lib.rs b/lib/crates/fabro-store/src/lib.rs index 55ff6a427..1bb69834a 100644 --- a/lib/crates/fabro-store/src/lib.rs +++ b/lib/crates/fabro-store/src/lib.rs @@ -2,7 +2,9 @@ use chrono::{DateTime, Utc}; mod artifact_store; mod error; +mod keyed_mutex; mod keys; +mod record; mod run_state; mod slate; mod types; @@ -10,10 +12,11 @@ mod types; pub use artifact_store::{ArtifactStore, NodeArtifact}; pub use error::{Error, Result}; pub use fabro_types::{RunBlobId, StageId}; +pub(crate) use keyed_mutex::KeyedMutex; pub use run_state::{NodeState, PendingInterviewRecord, RunProjection}; pub use slate::{ - AuthCode, ConsumeOutcome, Database, RefreshToken, RunDatabase, Runs, SlateAuthCodeStore, - SlateAuthTokenStore, + AuthCode, AuthCodeStore, Blob, BlobStore, ConsumeOutcome, Database, RefreshToken, + RefreshTokenStore, RunCatalogIndex, RunDatabase, Runs, }; pub use types::{EventEnvelope, EventPayload, RunSummary}; diff --git a/lib/crates/fabro-store/src/record/codec.rs b/lib/crates/fabro-store/src/record/codec.rs new file mode 100644 index 000000000..2a1eef181 --- /dev/null +++ b/lib/crates/fabro-store/src/record/codec.rs @@ -0,0 +1,95 @@ +use bytes::Bytes; +use serde::Serialize; +use serde::de::DeserializeOwned; + +use crate::{Error, Result}; + +pub(crate) trait Codec: Send + Sync + 'static { + fn encode(value: &R) -> Result>; + + fn decode(bytes: &[u8]) -> Result; +} + +pub(crate) struct JsonCodec; + +impl Codec for JsonCodec +where + R: Serialize + DeserializeOwned, +{ + fn encode(value: &R) -> Result> { + serde_json::to_vec(value).map_err(Into::into) + } + + fn decode(bytes: &[u8]) -> Result { + serde_json::from_slice(bytes).map_err(Into::into) + } +} + +pub(crate) struct RawBytesCodec; + +impl Codec for RawBytesCodec +where + R: AsRef<[u8]> + From, +{ + fn encode(value: &R) -> Result> { + Ok(value.as_ref().to_vec()) + } + + fn decode(bytes: &[u8]) -> Result { + Ok(R::from(Bytes::copy_from_slice(bytes))) + } +} + +pub(crate) struct MarkerCodec; + +impl Codec for MarkerCodec +where + R: Default, +{ + fn encode(_: &R) -> Result> { + Ok(Vec::new()) + } + + fn decode(bytes: &[u8]) -> Result { + if bytes.is_empty() { + return Ok(R::default()); + } + Err(Error::Other( + "marker records must decode from an empty byte slice".to_string(), + )) + } +} + +#[cfg(test)] +mod tests { + use chrono::{TimeZone, Utc}; + use serde::{Deserialize, Serialize}; + + use super::{Codec, JsonCodec}; + + #[derive(Debug, Serialize, Deserialize)] + struct SnapshotRecord { + code: String, + issued_at: chrono::DateTime, + expires_at: chrono::DateTime, + attempts: u32, + } + + #[test] + fn json_codec_matches_snapshot() { + let record = SnapshotRecord { + code: "code-123".to_string(), + issued_at: Utc.with_ymd_and_hms(2026, 4, 20, 12, 34, 56).unwrap(), + expires_at: Utc.with_ymd_and_hms(2026, 4, 20, 12, 39, 56).unwrap(), + attempts: 2, + }; + + let encoded = JsonCodec::encode(&record).unwrap(); + let encoded = std::str::from_utf8(&encoded).unwrap(); + + insta::assert_snapshot!( + encoded, + @"{\"code\":\"code-123\",\"issued_at\":\"2026-04-20T12:34:56Z\",\"expires_at\":\"2026-04-20T12:39:56Z\",\"attempts\":2}" + ); + } +} diff --git a/lib/crates/fabro-store/src/record/mod.rs b/lib/crates/fabro-store/src/record/mod.rs new file mode 100644 index 000000000..9c455c205 --- /dev/null +++ b/lib/crates/fabro-store/src/record/mod.rs @@ -0,0 +1,25 @@ +mod codec; +mod record_id; +mod repository; +mod transaction; + +pub(crate) use codec::{Codec, JsonCodec, MarkerCodec, RawBytesCodec}; +pub(crate) use repository::Repository; +pub(crate) use transaction::transaction; + +use crate::Result; + +pub(crate) trait Record: Sized + Send + Sync + 'static { + type Id: RecordId; + type Codec: Codec; + + const PREFIX: &'static str; + + fn id(&self) -> Self::Id; +} + +pub(crate) trait RecordId: Sized { + fn key_segments(&self) -> Vec; + + fn from_key_segments(segs: &[&str]) -> Result; +} diff --git a/lib/crates/fabro-store/src/record/record_id.rs b/lib/crates/fabro-store/src/record/record_id.rs new file mode 100644 index 000000000..21a483a67 --- /dev/null +++ b/lib/crates/fabro-store/src/record/record_id.rs @@ -0,0 +1,97 @@ +use std::fmt::Write; + +use fabro_types::{RunBlobId, RunId}; + +use super::RecordId; +use crate::{Error, Result}; + +impl RecordId for [u8; 32] { + fn key_segments(&self) -> Vec { + let mut encoded = String::with_capacity(self.len() * 2); + for byte in self { + write!(&mut encoded, "{byte:02x}").expect("write to String cannot fail"); + } + vec![encoded] + } + + fn from_key_segments(segs: &[&str]) -> Result { + let [segment] = segs else { + return Err(Error::KeyParse(format!( + "expected 1 segment for [u8; 32], got {}", + segs.len() + ))); + }; + + if segment.len() != 64 { + return Err(Error::KeyParse(format!( + "expected 64 hex characters for [u8; 32], got {}", + segment.len() + ))); + } + + let mut bytes = [0_u8; 32]; + for (index, chunk) in segment.as_bytes().chunks_exact(2).enumerate() { + let chunk = std::str::from_utf8(chunk).map_err(|err| { + Error::KeyParse(format!("hex segment was not valid UTF-8: {err}")) + })?; + bytes[index] = u8::from_str_radix(chunk, 16) + .map_err(|err| Error::KeyParse(format!("invalid hex byte {chunk:?}: {err}")))?; + } + Ok(bytes) + } +} + +impl RecordId for String { + fn key_segments(&self) -> Vec { + vec![self.clone()] + } + + fn from_key_segments(segs: &[&str]) -> Result { + let [segment] = segs else { + return Err(Error::KeyParse(format!( + "expected 1 segment for String, got {}", + segs.len() + ))); + }; + Ok((*segment).to_string()) + } +} + +impl RecordId for RunBlobId { + fn key_segments(&self) -> Vec { + vec![self.to_string()] + } + + fn from_key_segments(segs: &[&str]) -> Result { + let [segment] = segs else { + return Err(Error::KeyParse(format!( + "expected 1 segment for RunBlobId, got {}", + segs.len() + ))); + }; + segment + .parse() + .map_err(|err| Error::KeyParse(format!("invalid RunBlobId segment {segment:?}: {err}"))) + } +} + +impl RecordId for RunId { + fn key_segments(&self) -> Vec { + vec![ + self.created_at().format("%Y-%m-%d").to_string(), + self.to_string(), + ] + } + + fn from_key_segments(segs: &[&str]) -> Result { + if segs.len() != 2 { + return Err(Error::KeyParse(format!( + "expected 2 segments for RunId, got {}", + segs.len() + ))); + } + segs[1] + .parse() + .map_err(|err| Error::KeyParse(format!("invalid RunId segment {:?}: {err}", segs[1]))) + } +} diff --git a/lib/crates/fabro-store/src/record/repository.rs b/lib/crates/fabro-store/src/record/repository.rs new file mode 100644 index 000000000..ae77e103e --- /dev/null +++ b/lib/crates/fabro-store/src/record/repository.rs @@ -0,0 +1,504 @@ +use std::marker::PhantomData; +use std::pin::Pin; +use std::sync::Arc; + +use futures::stream::{self}; +use futures::{Stream, StreamExt}; +use slatedb::{Db, KeyValue, WriteBatch}; + +use super::{Codec, Record, RecordId}; +use crate::{Error, Result, keys}; + +pub(crate) struct Repository { + db: Arc, + prefix_segments: Vec<&'static str>, + _record: PhantomData, +} + +impl Repository { + pub(crate) fn new(db: Arc) -> Self { + Self { + db, + prefix_segments: prefix_segments::(), + _record: PhantomData, + } + } + + pub(crate) async fn get(&self, id: &R::Id) -> Result> { + self.db + .get(key_for_id::(id)?) + .await? + .map(|bytes| R::Codec::decode(&bytes)) + .transpose() + } + + pub(crate) async fn put(&self, record: &R) -> Result<()> { + let id = record.id(); + self.put_at(&id, record).await + } + + pub(crate) async fn put_at(&self, id: &R::Id, record: &R) -> Result<()> { + self.db + .put(key_for_id::(id)?, R::Codec::encode(record)?) + .await?; + Ok(()) + } + + pub(crate) async fn delete(&self, id: &R::Id) -> Result<()> { + self.db.delete(key_for_id::(id)?).await?; + Ok(()) + } + + pub(crate) async fn exists(&self, id: &R::Id) -> Result { + Ok(self.db.get(key_for_id::(id)?).await?.is_some()) + } + + #[allow( + dead_code, + reason = "Part of the shared Repository surface; current consumers do not need value scans yet" + )] + pub(crate) fn scan_stream(&self) -> RepositoryStream<'_, (R::Id, R)> { + self.scan_prefix_stream(&[]) + } + + #[allow( + dead_code, + reason = "Part of the shared Repository surface; current consumers do not need value scans by sub-prefix yet" + )] + pub(crate) fn scan_prefix_stream<'a>( + &'a self, + extra_segments: &'a [&'a str], + ) -> RepositoryStream<'a, (R::Id, R)> { + match prefix_key::(extra_segments) { + Ok(prefix) => { + let prefix_segments = self.prefix_segments.clone(); + Box::pin( + scan_entries(Arc::clone(&self.db), &prefix).map(move |result| { + result + .map_err(Into::into) + .and_then(|entry| decode_entry::(&entry, &prefix_segments)) + }), + ) + } + Err(err) => Box::pin(stream::once(async move { Err(err) })), + } + } + + pub(crate) fn scan_ids_stream(&self) -> RepositoryStream<'_, R::Id> { + match prefix_key::(&[]) { + Ok(prefix) => { + let prefix_segments = self.prefix_segments.clone(); + Box::pin( + scan_entries(Arc::clone(&self.db), &prefix).map(move |result| { + result + .map_err(Into::into) + .and_then(|entry| parse_entry_id::(&entry, &prefix_segments)) + }), + ) + } + Err(err) => Box::pin(stream::once(async move { Err(err) })), + } + } + + pub(crate) async fn gc(&self, predicate: F) -> Result + where + F: Fn(&R) -> bool + Send + Sync, + { + let mut iter = self.db.scan_prefix(prefix_key::(&[])?).await?; + let mut batch = WriteBatch::new(); + let mut deletes = 0_u64; + + while let Some(entry) = iter.next().await? { + let value = R::Codec::decode(&entry.value)?; + if predicate(&value) { + batch.delete(entry.key); + deletes += 1; + } + } + + if deletes > 0 { + self.db.write(batch).await?; + } + + Ok(deletes) + } +} + +pub(crate) type RepositoryStream<'a, T> = Pin> + Send + 'a>>; + +pub(super) fn key_for_id(id: &R::Id) -> Result { + let prefix_segments = prefix_segments::(); + let id_segments = id.key_segments(); + let id_segments: Vec<&str> = id_segments.iter().map(String::as_str).collect(); + key_from_segments( + prefix_segments + .iter() + .copied() + .chain(id_segments.iter().copied()), + ) +} + +pub(super) fn prefix_key(extra_segments: &[&str]) -> Result { + let prefix_segments = prefix_segments::(); + prefix_from_segments( + prefix_segments + .iter() + .copied() + .chain(extra_segments.iter().copied()), + ) +} + +fn decode_entry(entry: &KeyValue, prefix_segments: &[&str]) -> Result<(R::Id, R)> { + let id = parse_entry_id::(entry, prefix_segments)?; + let value = R::Codec::decode(&entry.value)?; + Ok((id, value)) +} + +fn parse_entry_id(entry: &KeyValue, prefix_segments: &[&str]) -> Result { + let raw_key = String::from_utf8(entry.key.to_vec()) + .map_err(|err| Error::Other(format!("stored key is not valid UTF-8: {err}")))?; + let segments: Vec<&str> = keys::SlateKey::segments(&raw_key).collect(); + if segments.len() < prefix_segments.len() { + return Err(Error::KeyParse(format!( + "key {raw_key:?} had {} segments, expected at least {} for prefix {}", + segments.len(), + prefix_segments.len(), + R::PREFIX + ))); + } + if segments[..prefix_segments.len()] != prefix_segments[..] { + return Err(Error::KeyParse(format!( + "key {raw_key:?} did not match expected prefix {}", + R::PREFIX + ))); + } + R::Id::from_key_segments(&segments[prefix_segments.len()..]) +} + +fn scan_entries( + db: Arc, + prefix: &keys::SlateKey, +) -> impl Stream> + Send { + enum ScanState { + Opening { db: Arc, prefix: Vec }, + Iterating(Box), + } + + stream::try_unfold( + ScanState::Opening { + db, + prefix: prefix.as_ref().to_vec(), + }, + |state| async move { + let mut iter = match state { + ScanState::Opening { db, prefix } => db.scan_prefix(prefix).await?, + ScanState::Iterating(iter) => *iter, + }; + + match iter.next().await? { + Some(entry) => Ok(Some((entry, ScanState::Iterating(Box::new(iter))))), + None => Ok(None), + } + }, + ) +} + +fn prefix_segments() -> Vec<&'static str> { + debug_assert!( + !R::PREFIX.is_empty() + && !R::PREFIX.starts_with('/') + && !R::PREFIX.ends_with('/') + && R::PREFIX.split('/').all(|segment| !segment.is_empty()), + "Record::PREFIX must be a non-empty '/'-separated path with no empty segments: {}", + R::PREFIX + ); + R::PREFIX.split('/').collect() +} + +fn key_from_segments<'a>(segments: impl IntoIterator) -> Result { + let mut segments = segments.into_iter(); + let first = segments.next().ok_or_else(|| { + Error::Other("record key assembly requires at least one segment".to_string()) + })?; + validate_key_segment(first)?; + + let mut key = keys::SlateKey::new(first); + for segment in segments { + validate_key_segment(segment)?; + key = key.with(segment); + } + Ok(key) +} + +fn prefix_from_segments<'a>(segments: impl IntoIterator) -> Result { + Ok(key_from_segments(segments)?.into_prefix()) +} + +fn validate_key_segment(segment: &str) -> Result<()> { + if segment.as_bytes().contains(&b'\0') { + return Err(Error::InvalidKeySegment { + segment: segment.to_string(), + }); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use futures::TryStreamExt; + use object_store::memory::InMemory; + use serde::{Deserialize, Serialize}; + + use super::Repository; + use crate::record::{JsonCodec, MarkerCodec, RawBytesCodec, Record, RecordId}; + use crate::{Error, Result}; + + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] + struct TestRecord { + id: TestId, + payload: String, + delete_me: bool, + } + + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] + struct TestId { + bucket: String, + name: String, + } + + impl RecordId for TestId { + fn key_segments(&self) -> Vec { + vec![self.bucket.clone(), self.name.clone()] + } + + fn from_key_segments(segs: &[&str]) -> Result { + let [bucket, name] = segs else { + return Err(Error::KeyParse(format!( + "expected 2 segments for TestId, got {}", + segs.len() + ))); + }; + Ok(Self { + bucket: (*bucket).to_string(), + name: (*name).to_string(), + }) + } + } + + impl Record for TestRecord { + type Id = TestId; + type Codec = JsonCodec; + + const PREFIX: &'static str = "test/repository"; + + fn id(&self) -> Self::Id { + self.id.clone() + } + } + + #[derive(Debug, Clone, PartialEq, Eq, Default)] + struct TestMarker; + + impl Record for TestMarker { + type Id = String; + type Codec = MarkerCodec; + + const PREFIX: &'static str = "test/marker"; + + fn id(&self) -> Self::Id { + unreachable!("marker records must use put_at") + } + } + + #[derive(Debug, Clone, PartialEq, Eq)] + struct RawBlob(bytes::Bytes); + + impl AsRef<[u8]> for RawBlob { + fn as_ref(&self) -> &[u8] { + self.0.as_ref() + } + } + + impl From for RawBlob { + fn from(value: bytes::Bytes) -> Self { + Self(value) + } + } + + impl Record for RawBlob { + type Id = String; + type Codec = RawBytesCodec; + + const PREFIX: &'static str = "test/raw"; + + fn id(&self) -> Self::Id { + "blob".to_string() + } + } + + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] + struct InvalidSegmentRecord { + id: String, + } + + impl Record for InvalidSegmentRecord { + type Id = String; + type Codec = JsonCodec; + + const PREFIX: &'static str = "test/invalid"; + + fn id(&self) -> Self::Id { + self.id.clone() + } + } + + async fn db() -> Arc { + Arc::new( + slatedb::Db::open("repository-tests", Arc::new(InMemory::new())) + .await + .unwrap(), + ) + } + + fn record(bucket: &str, name: &str, delete_me: bool) -> TestRecord { + TestRecord { + id: TestId { + bucket: bucket.to_string(), + name: name.to_string(), + }, + payload: format!("{bucket}/{name}"), + delete_me, + } + } + + #[tokio::test] + async fn put_get_delete_and_scan_round_trip() { + let repo = Repository::::new(db().await); + let saved = record("bucket-a", "alpha", false); + + assert!(repo.get(&saved.id()).await.unwrap().is_none()); + repo.put(&saved).await.unwrap(); + assert_eq!(repo.get(&saved.id()).await.unwrap(), Some(saved.clone())); + + let records = [ + saved.clone(), + record("bucket-a", "beta", false), + record("bucket-b", "alpha", false), + record("bucket-b", "beta", false), + record("bucket-c", "gamma", false), + ]; + for record in &records[1..] { + repo.put(record).await.unwrap(); + } + + let scanned = repo.scan_stream().try_collect::>().await.unwrap(); + assert_eq!(scanned.len(), records.len()); + assert_eq!(scanned[0], (records[0].id(), records[0].clone())); + + let bucket_a = repo + .scan_prefix_stream(&["bucket-a"]) + .try_collect::>() + .await + .unwrap(); + assert_eq!(bucket_a, vec![ + (records[0].id(), records[0].clone()), + (records[1].id(), records[1].clone()), + ]); + + repo.delete(&saved.id()).await.unwrap(); + assert!(repo.get(&saved.id()).await.unwrap().is_none()); + } + + #[tokio::test] + async fn gc_deletes_matching_records() { + let repo = Repository::::new(db().await); + for record in [ + record("bucket-a", "keep", false), + record("bucket-a", "delete", true), + record("bucket-b", "keep", false), + record("bucket-b", "delete", true), + ] { + repo.put(&record).await.unwrap(); + } + + assert_eq!(repo.gc(|record| record.delete_me).await.unwrap(), 2); + + let remaining = repo.scan_stream().try_collect::>().await.unwrap(); + assert_eq!(remaining.len(), 2); + assert!(remaining.iter().all(|(_, record)| !record.delete_me)); + } + + #[tokio::test] + async fn marker_records_use_put_at_exists_and_scan_ids() { + let repo = Repository::::new(db().await); + let marker = TestMarker; + + repo.put_at(&"marker-a".to_string(), &marker).await.unwrap(); + repo.put_at(&"marker-b".to_string(), &marker).await.unwrap(); + + assert!(repo.exists(&"marker-a".to_string()).await.unwrap()); + + let ids = repo + .scan_ids_stream() + .try_collect::>() + .await + .unwrap(); + assert_eq!(ids, vec!["marker-a".to_string(), "marker-b".to_string()]); + + repo.delete(&"marker-a".to_string()).await.unwrap(); + assert!(!repo.exists(&"marker-a".to_string()).await.unwrap()); + } + + #[tokio::test] + async fn raw_bytes_codec_round_trips_bytes() { + let repo = Repository::::new(db().await); + let blob = RawBlob(bytes::Bytes::from_static(b"hello")); + + repo.put(&blob).await.unwrap(); + + assert_eq!(repo.get(&"blob".to_string()).await.unwrap(), Some(blob)); + } + + #[tokio::test] + async fn malformed_bytes_propagate_decode_errors() { + let db = db().await; + let repo = Repository::::new(Arc::clone(&db)); + db.put( + super::key_for_id::(&TestId { + bucket: "bucket-a".to_string(), + name: "broken".to_string(), + }) + .unwrap(), + b"not-json", + ) + .await + .unwrap(); + + let error = repo + .get(&TestId { + bucket: "bucket-a".to_string(), + name: "broken".to_string(), + }) + .await + .unwrap_err(); + assert!(matches!(error, Error::Serde(_))); + } + + #[tokio::test] + async fn invalid_key_segments_return_runtime_error() { + let repo = Repository::::new(db().await); + let error = repo + .put(&InvalidSegmentRecord { + id: "bad\0segment".to_string(), + }) + .await + .unwrap_err(); + + match error { + Error::InvalidKeySegment { segment } => assert_eq!(segment, "bad\0segment"), + other => panic!("expected invalid key segment error, got {other:?}"), + } + } +} diff --git a/lib/crates/fabro-store/src/record/transaction.rs b/lib/crates/fabro-store/src/record/transaction.rs new file mode 100644 index 000000000..a66e7e80f --- /dev/null +++ b/lib/crates/fabro-store/src/record/transaction.rs @@ -0,0 +1,211 @@ +use slatedb::{Db, WriteBatch}; + +use super::repository::key_for_id; +use super::{Codec, Record}; +use crate::Result; + +pub(crate) async fn transaction(db: &Db, f: F) -> Result +where + F: FnOnce(&mut Tx) -> Result, +{ + let mut tx = Tx::new(); + let value = f(&mut tx)?; + if !tx.has_ops { + return Ok(value); + } + db.write(tx.into_batch()).await?; + Ok(value) +} + +pub(crate) struct Tx { + batch: WriteBatch, + has_ops: bool, +} + +impl Tx { + pub(crate) fn new() -> Self { + Self { + batch: WriteBatch::new(), + has_ops: false, + } + } + + pub(crate) fn put(&mut self, record: &R) -> Result<&mut Self> { + let id = record.id(); + self.put_at(&id, record) + } + + pub(crate) fn put_at(&mut self, id: &R::Id, record: &R) -> Result<&mut Self> { + self.batch + .put(key_for_id::(id)?, R::Codec::encode(record)?); + self.has_ops = true; + Ok(self) + } + + #[allow( + dead_code, + reason = "Shared transaction surface; current production callers only use put paths" + )] + pub(crate) fn delete(&mut self, id: &R::Id) -> Result<&mut Self> { + self.batch.delete(key_for_id::(id)?); + self.has_ops = true; + Ok(self) + } + + fn into_batch(self) -> WriteBatch { + self.batch + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use object_store::memory::InMemory; + use serde::{Deserialize, Serialize}; + + use super::{Record, Tx, transaction}; + use crate::record::{Codec, JsonCodec, Repository}; + use crate::{Error, Result}; + + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] + struct TxRecord { + id: String, + payload: String, + poisoned: bool, + } + + impl Record for TxRecord { + type Id = String; + type Codec = JsonCodec; + + const PREFIX: &'static str = "test/transaction"; + + fn id(&self) -> Self::Id { + self.id.clone() + } + } + + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] + struct FailingRecord { + id: String, + poisoned: bool, + } + + struct FailingCodec; + + impl Codec for FailingCodec { + fn encode(value: &FailingRecord) -> Result> { + if value.poisoned { + return Err(Error::Other( + "poisoned record refused to encode".to_string(), + )); + } + serde_json::to_vec(value).map_err(Into::into) + } + + fn decode(bytes: &[u8]) -> Result { + serde_json::from_slice(bytes).map_err(Into::into) + } + } + + impl Record for FailingRecord { + type Id = String; + type Codec = FailingCodec; + + const PREFIX: &'static str = "test/failing-transaction"; + + fn id(&self) -> Self::Id { + self.id.clone() + } + } + + async fn db() -> Arc { + Arc::new( + slatedb::Db::open("transaction-tests", Arc::new(InMemory::new())) + .await + .unwrap(), + ) + } + + #[tokio::test] + async fn closure_error_short_circuits_without_writing() { + let db = db().await; + let repo = Repository::::new(Arc::clone(&db)); + let record = TxRecord { + id: "record-1".to_string(), + payload: "hello".to_string(), + poisoned: false, + }; + + let error = transaction::<(), _>(&db, |tx| { + tx.put(&record)?; + Err(Error::Other("stop before commit".to_string())) + }) + .await + .unwrap_err(); + + assert_eq!(error.to_string(), "stop before commit"); + assert!(repo.get(&record.id()).await.unwrap().is_none()); + } + + #[tokio::test] + async fn encode_failure_discards_the_entire_batch() { + let db = db().await; + let repo = Repository::::new(Arc::clone(&db)); + let good = FailingRecord { + id: "good".to_string(), + poisoned: false, + }; + let bad = FailingRecord { + id: "bad".to_string(), + poisoned: true, + }; + + let error = transaction::<(), _>(&db, |tx| { + tx.put(&good)?; + tx.put(&bad)?; + Ok(()) + }) + .await + .unwrap_err(); + + assert_eq!(error.to_string(), "poisoned record refused to encode"); + assert!(repo.get(&good.id()).await.unwrap().is_none()); + assert!(repo.get(&bad.id()).await.unwrap().is_none()); + } + + #[tokio::test] + async fn empty_transaction_returns_without_writing() { + let db = db().await; + let repo = Repository::::new(Arc::clone(&db)); + + let value = transaction(&db, |_tx: &mut Tx| Ok::<_, Error>("ok")) + .await + .unwrap(); + + assert_eq!(value, "ok"); + assert!(repo.get(&"missing".to_string()).await.unwrap().is_none()); + } + + #[tokio::test] + async fn delete_operations_are_committed() { + let db = db().await; + let repo = Repository::::new(Arc::clone(&db)); + let record = TxRecord { + id: "delete-me".to_string(), + payload: "hello".to_string(), + poisoned: false, + }; + repo.put(&record).await.unwrap(); + + transaction::<(), _>(&db, |tx| { + tx.delete::(&record.id())?; + Ok(()) + }) + .await + .unwrap(); + + assert!(repo.get(&record.id()).await.unwrap().is_none()); + } +} diff --git a/lib/crates/fabro-store/src/slate/auth_codes.rs b/lib/crates/fabro-store/src/slate/auth_codes.rs index 7171a29cc..f3104c884 100644 --- a/lib/crates/fabro-store/src/slate/auth_codes.rs +++ b/lib/crates/fabro-store/src/slate/auth_codes.rs @@ -1,15 +1,15 @@ use std::sync::Arc; use chrono::{DateTime, Utc}; -use dashmap::DashMap; use fabro_types::IdpIdentity; use serde::{Deserialize, Serialize}; -use tokio::sync::Mutex; -use crate::{Result, keys}; +use crate::record::{JsonCodec, Record, Repository}; +use crate::{KeyedMutex, Result}; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct AuthCode { + pub code: String, pub identity: IdpIdentity, pub login: String, pub name: String, @@ -19,84 +19,63 @@ pub struct AuthCode { pub expires_at: DateTime, } -pub struct SlateAuthCodeStore { - db: Arc, - code_locks: DashMap>>, -} +impl Record for AuthCode { + type Id = String; + type Codec = JsonCodec; -impl std::fmt::Debug for SlateAuthCodeStore { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("SlateAuthCodeStore").finish_non_exhaustive() + const PREFIX: &'static str = "auth/code"; + + fn id(&self) -> Self::Id { + self.code.clone() } } -impl SlateAuthCodeStore { +pub struct AuthCodeStore { + repo: Repository, + consume_locks: KeyedMutex, +} + +impl std::fmt::Debug for AuthCodeStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AuthCodeStore").finish_non_exhaustive() + } +} + +impl AuthCodeStore { pub(crate) fn new(db: Arc) -> Self { Self { - db, - code_locks: DashMap::new(), + repo: Repository::new(db), + consume_locks: KeyedMutex::new(), } } - pub async fn insert(&self, code: &str, entry: AuthCode) -> Result<()> { - self.db - .put(keys::auth_code_key(code), serde_json::to_vec(&entry)?) - .await?; - Ok(()) + pub async fn insert(&self, entry: AuthCode) -> Result<()> { + self.repo.put(&entry).await } pub async fn consume(&self, code: &str) -> Result> { - let mutex = self - .code_locks - .entry(code.to_string()) - .or_insert_with(|| Arc::new(Mutex::new(()))) - .clone(); - let _guard = mutex.lock().await; - - let key = keys::auth_code_key(code); - let entry = self - .db - .get(&key) - .await? - .map(|bytes| serde_json::from_slice::(&bytes)) - .transpose()?; + let code = code.to_string(); + let _guard = self.consume_locks.lock(code.clone()).await; + let entry = self.repo.get(&code).await?; let result = match entry { Some(entry) if entry.expires_at > Utc::now() => { - self.db.delete(&key).await?; + self.repo.delete(&code).await?; Some(entry) } Some(_) => { - self.db.delete(&key).await?; + self.repo.delete(&code).await?; None } None => None, }; - if Arc::strong_count(&mutex) == 2 { - self.code_locks.remove(code); - } - Ok(result) } pub async fn gc_expired(&self, cutoff: DateTime) -> Result { - let mut iter = self.db.scan_prefix(keys::auth_code_prefix()).await?; - let mut keys_to_delete = Vec::new(); - while let Some(entry) = iter.next().await? { - let auth_code: AuthCode = serde_json::from_slice(&entry.value)?; - if auth_code.expires_at <= cutoff { - keys_to_delete.push( - String::from_utf8(entry.key.to_vec()) - .expect("slatedb keys should be valid utf-8"), - ); - } - } - - for key in &keys_to_delete { - self.db.delete(key).await?; - } - - Ok(keys_to_delete.len() as u64) + self.repo + .gc(|auth_code| auth_code.expires_at <= cutoff) + .await } } @@ -109,10 +88,10 @@ mod tests { use object_store::memory::InMemory; use tokio::task::JoinSet; - use super::{AuthCode, SlateAuthCodeStore}; + use super::{AuthCode, AuthCodeStore}; use crate::Database; - async fn store() -> Arc { + async fn store() -> Arc { let db = Database::new( Arc::new(InMemory::new()), "", @@ -122,8 +101,9 @@ mod tests { db.auth_codes().await.unwrap() } - fn auth_code(expires_at: chrono::DateTime) -> AuthCode { + fn auth_code(code: &str, expires_at: chrono::DateTime) -> AuthCode { AuthCode { + code: code.to_string(), identity: fabro_types::IdpIdentity::new("https://github.com", "12345").unwrap(), login: "octocat".to_string(), name: "The Octocat".to_string(), @@ -138,10 +118,10 @@ mod tests { async fn insert_and_consume_is_single_use() { let store = store().await; store - .insert( + .insert(auth_code( "code-1", - auth_code(chrono::Utc::now() + ChronoDuration::seconds(60)), - ) + chrono::Utc::now() + ChronoDuration::seconds(60), + )) .await .unwrap(); @@ -153,10 +133,10 @@ mod tests { async fn concurrent_consume_has_one_winner() { let store = store().await; store - .insert( + .insert(auth_code( "code-2", - auth_code(chrono::Utc::now() + ChronoDuration::seconds(60)), - ) + chrono::Utc::now() + ChronoDuration::seconds(60), + )) .await .unwrap(); @@ -180,17 +160,17 @@ mod tests { async fn gc_expired_removes_only_expired_codes() { let store = store().await; store - .insert( + .insert(auth_code( "expired", - auth_code(chrono::Utc::now() - ChronoDuration::seconds(1)), - ) + chrono::Utc::now() - ChronoDuration::seconds(1), + )) .await .unwrap(); store - .insert( + .insert(auth_code( "live", - auth_code(chrono::Utc::now() + ChronoDuration::seconds(60)), - ) + chrono::Utc::now() + ChronoDuration::seconds(60), + )) .await .unwrap(); diff --git a/lib/crates/fabro-store/src/slate/auth_tokens.rs b/lib/crates/fabro-store/src/slate/auth_tokens.rs index 91b4213a8..9357dd5b8 100644 --- a/lib/crates/fabro-store/src/slate/auth_tokens.rs +++ b/lib/crates/fabro-store/src/slate/auth_tokens.rs @@ -4,11 +4,10 @@ use chrono::{DateTime, Utc}; use dashmap::DashMap; use fabro_types::IdpIdentity; use serde::{Deserialize, Serialize}; -use slatedb::WriteBatch; -use tokio::sync::Mutex; use uuid::Uuid; -use crate::{Result, keys}; +use crate::record::{JsonCodec, Record, Repository, transaction}; +use crate::{KeyedMutex, Result}; const REPLAY_REVOCATION_TTL_SECONDS: i64 = 60; @@ -27,6 +26,17 @@ pub struct RefreshToken { pub user_agent: String, } +impl Record for RefreshToken { + type Id = [u8; 32]; + type Codec = JsonCodec; + + const PREFIX: &'static str = "auth/refresh"; + + fn id(&self) -> Self::Id { + self.token_hash + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub enum ConsumeOutcome { Rotated(RefreshToken, Box), @@ -35,45 +45,38 @@ pub enum ConsumeOutcome { NotFound, } -pub struct SlateAuthTokenStore { +pub struct RefreshTokenStore { db: Arc, - refresh_locks: DashMap<[u8; 32], Arc>>, + repo: Repository, + consume_locks: KeyedMutex<[u8; 32]>, + /// In-memory only by design (origin R6): persisting attacker-supplied + /// hashes adds an unbounded-growth surface under token-stuffing attack with + /// no security benefit. Do not migrate this into Repository. replay_revocations: DashMap<[u8; 32], DateTime>, } -impl std::fmt::Debug for SlateAuthTokenStore { +impl std::fmt::Debug for RefreshTokenStore { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("SlateAuthTokenStore") - .finish_non_exhaustive() + f.debug_struct("RefreshTokenStore").finish_non_exhaustive() } } -impl SlateAuthTokenStore { +impl RefreshTokenStore { pub(crate) fn new(db: Arc) -> Self { Self { + repo: Repository::new(Arc::clone(&db)), db, - refresh_locks: DashMap::new(), + consume_locks: KeyedMutex::new(), replay_revocations: DashMap::new(), } } pub async fn insert_refresh_token(&self, token: RefreshToken) -> Result<()> { - self.db - .put( - keys::auth_refresh_key(&token.token_hash), - serde_json::to_vec(&token)?, - ) - .await?; - Ok(()) + self.repo.put(&token).await } pub async fn find_refresh_token(&self, token_hash: &[u8; 32]) -> Result> { - self.db - .get(keys::auth_refresh_key(token_hash)) - .await? - .map(|bytes| serde_json::from_slice::(&bytes)) - .transpose() - .map_err(Into::into) + self.repo.get(token_hash).await } pub async fn consume_and_rotate( @@ -82,14 +85,9 @@ impl SlateAuthTokenStore { new_token: RefreshToken, now: DateTime, ) -> Result { - let mutex = self - .refresh_locks - .entry(presented_hash) - .or_insert_with(|| Arc::new(Mutex::new(()))) - .clone(); - let _guard = mutex.lock().await; + let _guard = self.consume_locks.lock(presented_hash).await; - let outcome = match self.find_refresh_token(&presented_hash).await? { + let outcome = match self.repo.get(&presented_hash).await? { None => ConsumeOutcome::NotFound, Some(existing) if now >= existing.expires_at => ConsumeOutcome::Expired, Some(existing) if existing.used => ConsumeOutcome::Reused(existing), @@ -98,66 +96,26 @@ impl SlateAuthTokenStore { old_token.used = true; old_token.last_used_at = now; - let mut batch = WriteBatch::new(); - batch.put( - keys::auth_refresh_key(&presented_hash), - serde_json::to_vec(&old_token)?, - ); - batch.put( - keys::auth_refresh_key(&new_token.token_hash), - serde_json::to_vec(&new_token)?, - ); - self.db.write(batch).await?; + transaction(&self.db, |tx| { + tx.put(&old_token)?; + tx.put(&new_token)?; + Ok(()) + }) + .await?; ConsumeOutcome::Rotated(old_token, Box::new(new_token)) } }; - if Arc::strong_count(&mutex) == 2 { - self.refresh_locks.remove(&presented_hash); - } - Ok(outcome) } pub async fn delete_chain(&self, chain_id: Uuid) -> Result { - let mut iter = self.db.scan_prefix(keys::auth_refresh_prefix()).await?; - let mut keys_to_delete = Vec::new(); - while let Some(entry) = iter.next().await? { - let token: RefreshToken = serde_json::from_slice(&entry.value)?; - if token.chain_id == chain_id { - keys_to_delete.push( - String::from_utf8(entry.key.to_vec()) - .expect("slatedb keys should be valid utf-8"), - ); - } - } - - for key in &keys_to_delete { - self.db.delete(key).await?; - } - - Ok(keys_to_delete.len() as u64) + self.repo.gc(|token| token.chain_id == chain_id).await } pub async fn gc_expired(&self, cutoff: DateTime) -> Result { - let mut iter = self.db.scan_prefix(keys::auth_refresh_prefix()).await?; - let mut keys_to_delete = Vec::new(); - while let Some(entry) = iter.next().await? { - let token: RefreshToken = serde_json::from_slice(&entry.value)?; - if token.expires_at <= cutoff { - keys_to_delete.push( - String::from_utf8(entry.key.to_vec()) - .expect("slatedb keys should be valid utf-8"), - ); - } - } - - for key in &keys_to_delete { - self.db.delete(key).await?; - } - - Ok(keys_to_delete.len() as u64) + self.repo.gc(|token| token.expires_at <= cutoff).await } pub fn mark_refresh_token_replay(&self, token_hash: [u8; 32], now: DateTime) { @@ -188,17 +146,17 @@ mod tests { use tokio::task::JoinSet; use uuid::Uuid; - use super::{ConsumeOutcome, RefreshToken, SlateAuthTokenStore}; + use super::{ConsumeOutcome, RefreshToken, RefreshTokenStore}; use crate::Database; - async fn store() -> Arc { + async fn store() -> Arc { let db = Database::new( Arc::new(InMemory::new()), "", Duration::from_millis(1), None, ); - db.auth_tokens().await.unwrap() + db.refresh_tokens().await.unwrap() } fn refresh_token(hash: [u8; 32], chain_id: Uuid, used: bool) -> RefreshToken { diff --git a/lib/crates/fabro-store/src/slate/blob_store.rs b/lib/crates/fabro-store/src/slate/blob_store.rs new file mode 100644 index 000000000..3c058b3c8 --- /dev/null +++ b/lib/crates/fabro-store/src/slate/blob_store.rs @@ -0,0 +1,132 @@ +use std::sync::Arc; + +use bytes::Bytes; +use fabro_types::RunBlobId; + +use crate::Result; +use crate::record::{RawBytesCodec, Record, Repository}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Blob(pub Bytes); + +impl AsRef<[u8]> for Blob { + fn as_ref(&self) -> &[u8] { + self.0.as_ref() + } +} + +impl From for Blob { + fn from(value: Bytes) -> Self { + Self(value) + } +} + +impl Record for Blob { + type Id = RunBlobId; + type Codec = RawBytesCodec; + + const PREFIX: &'static str = "blobs/sha256"; + + fn id(&self) -> Self::Id { + RunBlobId::new(&self.0) + } +} + +pub struct BlobStore { + repo: Repository, +} + +impl std::fmt::Debug for BlobStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("BlobStore").finish_non_exhaustive() + } +} + +impl BlobStore { + pub(crate) fn new(db: Arc) -> Self { + Self { + repo: Repository::new(db), + } + } + + pub async fn write(&self, bytes: &[u8]) -> Result { + let blob = Blob(Bytes::copy_from_slice(bytes)); + let id = blob.id(); + self.repo.put(&blob).await?; + Ok(id) + } + + pub async fn read(&self, id: &RunBlobId) -> Result> { + Ok(self.repo.get(id).await?.map(|blob| blob.0)) + } + + pub async fn exists(&self, id: &RunBlobId) -> Result { + self.repo.exists(id).await + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::time::Duration; + + use bytes::Bytes; + use fabro_types::RunBlobId; + use object_store::memory::InMemory; + + use super::BlobStore; + use crate::Database; + use crate::keys::SlateKey; + + async fn store() -> Arc { + let db = Database::new( + Arc::new(InMemory::new()), + "", + Duration::from_millis(1), + None, + ); + db.blobs().await.unwrap() + } + + #[tokio::test] + async fn writes_reads_and_checks_existence() { + let store = store().await; + let bytes = b"hello world"; + let id = store.write(bytes).await.unwrap(); + + assert_eq!( + store.read(&id).await.unwrap(), + Some(Bytes::from_static(bytes)) + ); + assert_eq!(store.write(bytes).await.unwrap(), id); + assert!(store.exists(&id).await.unwrap()); + assert!(!store.exists(&RunBlobId::new(b"missing")).await.unwrap()); + } + + #[tokio::test] + async fn empty_blobs_round_trip() { + let store = store().await; + let id = store.write(b"").await.unwrap(); + + assert_eq!(store.read(&id).await.unwrap(), Some(Bytes::new())); + } + + #[tokio::test] + async fn raw_db_reads_exact_blob_bytes() { + let raw_db = Arc::new( + slatedb::Db::open("blob-store-tests", Arc::new(InMemory::new())) + .await + .unwrap(), + ); + let store = BlobStore::new(Arc::clone(&raw_db)); + let bytes = b"{\"ok\":true}"; + let id = store.write(bytes).await.unwrap(); + + let saved = raw_db + .get(SlateKey::new("blobs").with("sha256").with(id)) + .await + .unwrap() + .unwrap(); + assert_eq!(saved.as_ref(), bytes); + } +} diff --git a/lib/crates/fabro-store/src/slate/catalog.rs b/lib/crates/fabro-store/src/slate/catalog.rs deleted file mode 100644 index 49181ac08..000000000 --- a/lib/crates/fabro-store/src/slate/catalog.rs +++ /dev/null @@ -1,51 +0,0 @@ -use chrono::{Datelike, Timelike}; -use fabro_types::RunId; -use slatedb::Db; - -use crate::{ListRunsQuery, Result, keys}; - -pub(crate) async fn write_index(db: &Db, run_id: &RunId) -> Result<()> { - db.put(keys::runs_index_by_start_key(run_id), []).await?; - Ok(()) -} - -pub(crate) async fn delete_index(db: &Db, run_id: &RunId) -> Result<()> { - db.delete(keys::runs_index_by_start_key(run_id)).await?; - Ok(()) -} - -pub(crate) async fn list_run_ids(db: &Db, query: &ListRunsQuery) -> Result> { - let mut iter = db.scan_prefix(keys::runs_index_by_start_prefix()).await?; - let mut run_ids = Vec::new(); - while let Some(entry) = iter.next().await? { - let key = String::from_utf8(entry.key.to_vec()) - .map_err(|err| crate::Error::Other(format!("stored key is not valid UTF-8: {err}")))?; - let Some(run_id) = keys::parse_run_id_from_index_key(&key) else { - continue; - }; - let created_at = run_id.created_at(); - if let Some(start) = query.start { - if created_at < start { - continue; - } - } - if let Some(end) = query.end { - if created_at > end { - continue; - } - } - run_ids.push(run_id); - } - run_ids.sort_by_key(|run_id| { - let created_at = run_id.created_at(); - ( - created_at.year(), - created_at.month(), - created_at.day(), - created_at.hour(), - created_at.minute(), - *run_id, - ) - }); - Ok(run_ids) -} diff --git a/lib/crates/fabro-store/src/slate/mod.rs b/lib/crates/fabro-store/src/slate/mod.rs index e368970b0..d14f98aa7 100644 --- a/lib/crates/fabro-store/src/slate/mod.rs +++ b/lib/crates/fabro-store/src/slate/mod.rs @@ -1,6 +1,7 @@ mod auth_codes; mod auth_tokens; -mod catalog; +mod blob_store; +mod run_catalog_index; mod run_store; use std::collections::HashMap; @@ -8,10 +9,12 @@ use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; -pub use auth_codes::{AuthCode, SlateAuthCodeStore}; -pub use auth_tokens::{ConsumeOutcome, RefreshToken, SlateAuthTokenStore}; +pub use auth_codes::{AuthCode, AuthCodeStore}; +pub use auth_tokens::{ConsumeOutcome, RefreshToken, RefreshTokenStore}; +pub use blob_store::{Blob, BlobStore}; use fabro_types::RunId; use object_store::ObjectStore; +pub use run_catalog_index::RunCatalogIndex; pub use run_store::RunDatabase; use run_store::RunDatabaseInner; use slatedb::config::{CompressionCodec, Settings}; @@ -27,8 +30,10 @@ pub struct Database { cache_path: Option, db: Arc>, active_runs: Arc>>>, - auth_codes: Arc>>, - auth_tokens: Arc>>, + blobs: Arc>>, + catalog_index: Arc>>, + auth_codes: Arc>>, + refresh_tokens: Arc>>, } impl std::fmt::Debug for Database { @@ -55,8 +60,10 @@ impl Database { cache_path, db: Arc::new(OnceCell::new()), active_runs: Arc::new(Mutex::new(HashMap::new())), + blobs: Arc::new(OnceCell::new()), + catalog_index: Arc::new(OnceCell::new()), auth_codes: Arc::new(OnceCell::new()), - auth_tokens: Arc::new(OnceCell::new()), + refresh_tokens: Arc::new(OnceCell::new()), } } @@ -116,7 +123,7 @@ impl Database { if run_exists && !active.matches_run(run_id) { return Err(Error::RunAlreadyExists(run_id.to_string())); } - catalog::write_index(&db, run_id).await?; + self.catalog_index().await?.add(run_id).await?; return Ok(active); } @@ -124,7 +131,7 @@ impl Database { return Err(Error::RunAlreadyExists(run_id.to_string())); } - catalog::write_index(&db, run_id).await?; + self.catalog_index().await?.add(run_id).await?; let run_store = RunDatabase::open_writer(*run_id, db).await?; self.cache_active_run(&run_store).await; Ok(run_store) @@ -166,7 +173,7 @@ impl Database { pub async fn list_runs(&self, query: &ListRunsQuery) -> Result> { let db = self.open_db().await?; - let run_ids = catalog::list_run_ids(&db, query).await?; + let run_ids = self.catalog_index().await?.list(query).await?; let mut summaries = Vec::new(); for run_id in run_ids { if let Some(active) = self.get_active_run(&run_id).await { @@ -201,27 +208,49 @@ impl Database { for key in keys_to_delete { db.delete(key).await?; } - catalog::delete_index(&db, run_id).await?; + self.catalog_index().await?.remove(run_id).await?; Ok(()) } - pub async fn auth_codes(&self) -> Result> { + pub async fn auth_codes(&self) -> Result> { let store = self .auth_codes .get_or_try_init(|| async { let db = Arc::new(self.open_db().await?); - Ok::<_, Error>(Arc::new(SlateAuthCodeStore::new(db))) + Ok::<_, Error>(Arc::new(AuthCodeStore::new(db))) }) .await?; Ok(Arc::clone(store)) } - pub async fn auth_tokens(&self) -> Result> { + pub async fn catalog_index(&self) -> Result> { let store = self - .auth_tokens + .catalog_index .get_or_try_init(|| async { let db = Arc::new(self.open_db().await?); - Ok::<_, Error>(Arc::new(SlateAuthTokenStore::new(db))) + Ok::<_, Error>(Arc::new(RunCatalogIndex::new(db))) + }) + .await?; + Ok(Arc::clone(store)) + } + + pub async fn blobs(&self) -> Result> { + let store = self + .blobs + .get_or_try_init(|| async { + let db = Arc::new(self.open_db().await?); + Ok::<_, Error>(Arc::new(BlobStore::new(db))) + }) + .await?; + Ok(Arc::clone(store)) + } + + pub async fn refresh_tokens(&self) -> Result> { + let store = self + .refresh_tokens + .get_or_try_init(|| async { + let db = Arc::new(self.open_db().await?); + Ok::<_, Error>(Arc::new(RefreshTokenStore::new(db))) }) .await?; Ok(Arc::clone(store)) diff --git a/lib/crates/fabro-store/src/slate/run_catalog_index.rs b/lib/crates/fabro-store/src/slate/run_catalog_index.rs new file mode 100644 index 000000000..332c09496 --- /dev/null +++ b/lib/crates/fabro-store/src/slate/run_catalog_index.rs @@ -0,0 +1,150 @@ +use std::sync::Arc; + +use chrono::{Datelike, Timelike}; +use fabro_types::RunId; +use futures::TryStreamExt; + +use crate::record::{MarkerCodec, Record, Repository}; +use crate::{ListRunsQuery, Result}; + +#[derive(Debug, Default)] +pub(crate) struct RunCatalogEntry; + +impl Record for RunCatalogEntry { + type Id = RunId; + type Codec = MarkerCodec; + + const PREFIX: &'static str = "runs/_index/by-start"; + + fn id(&self) -> Self::Id { + unreachable!("marker records must use put_at") + } +} + +pub struct RunCatalogIndex { + repo: Repository, +} + +impl std::fmt::Debug for RunCatalogIndex { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RunCatalogIndex").finish_non_exhaustive() + } +} + +impl RunCatalogIndex { + pub(crate) fn new(db: Arc) -> Self { + Self { + repo: Repository::new(db), + } + } + + pub async fn add(&self, run_id: &RunId) -> Result<()> { + self.repo.put_at(run_id, &RunCatalogEntry).await + } + + pub async fn remove(&self, run_id: &RunId) -> Result<()> { + self.repo.delete(run_id).await + } + + pub async fn list(&self, query: &ListRunsQuery) -> Result> { + let mut run_ids = self.repo.scan_ids_stream().try_collect::>().await?; + run_ids.retain(|run_id| { + let created_at = run_id.created_at(); + if let Some(start) = query.start { + if created_at < start { + return false; + } + } + if let Some(end) = query.end { + if created_at > end { + return false; + } + } + true + }); + run_ids.sort_by_key(|run_id| { + let created_at = run_id.created_at(); + ( + created_at.year(), + created_at.month(), + created_at.day(), + created_at.hour(), + created_at.minute(), + *run_id, + ) + }); + Ok(run_ids) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use chrono::{Duration as ChronoDuration, TimeZone, Utc}; + use object_store::memory::InMemory; + use ulid::Ulid; + + use super::RunCatalogIndex; + use crate::ListRunsQuery; + + async fn index() -> RunCatalogIndex { + let db = Arc::new( + slatedb::Db::open("run-catalog-index-tests", Arc::new(InMemory::new())) + .await + .unwrap(), + ); + RunCatalogIndex::new(db) + } + + #[tokio::test] + async fn add_list_and_remove_round_trip() { + let index = index().await; + let early = fabro_types::RunId::from(Ulid::from_datetime( + Utc.with_ymd_and_hms(2026, 4, 20, 9, 0, 0).unwrap().into(), + )); + let later = fabro_types::RunId::from(Ulid::from_datetime( + Utc.with_ymd_and_hms(2026, 4, 20, 9, 1, 0).unwrap().into(), + )); + + index.add(&later).await.unwrap(); + index.add(&early).await.unwrap(); + + assert_eq!(index.list(&ListRunsQuery::default()).await.unwrap(), vec![ + early, later + ]); + + index.remove(&early).await.unwrap(); + assert_eq!(index.list(&ListRunsQuery::default()).await.unwrap(), vec![ + later + ]); + } + + #[tokio::test] + async fn list_applies_start_and_end_filters() { + let index = index().await; + let first = fabro_types::RunId::from(Ulid::from_datetime( + Utc.with_ymd_and_hms(2026, 4, 20, 9, 0, 0).unwrap().into(), + )); + let second = fabro_types::RunId::from(Ulid::from_datetime( + Utc.with_ymd_and_hms(2026, 4, 20, 9, 1, 0).unwrap().into(), + )); + let third = fabro_types::RunId::from(Ulid::from_datetime( + Utc.with_ymd_and_hms(2026, 4, 20, 9, 2, 0).unwrap().into(), + )); + for run_id in [first, second, third] { + index.add(&run_id).await.unwrap(); + } + + assert_eq!( + index + .list(&ListRunsQuery { + start: Some(second.created_at()), + end: Some(second.created_at() + ChronoDuration::seconds(1)), + }) + .await + .unwrap(), + vec![second] + ); + } +} diff --git a/lib/crates/fabro-store/src/slate/run_store.rs b/lib/crates/fabro-store/src/slate/run_store.rs index 388e5a188..d1379d3e2 100644 --- a/lib/crates/fabro-store/src/slate/run_store.rs +++ b/lib/crates/fabro-store/src/slate/run_store.rs @@ -10,6 +10,7 @@ use slatedb::{Db, DbRead}; use tokio::sync::{Mutex, broadcast, mpsc}; use tokio_stream::wrappers::UnboundedReceiverStream; +use super::blob_store::BlobStore; use crate::run_state::EventProjectionCache; use crate::{Error, EventEnvelope, EventPayload, Result, RunProjection, RunSummary, keys}; @@ -284,17 +285,15 @@ impl RunDatabase { if self.read_only { return Err(Error::ReadOnly); } - let id = RunBlobId::new(data); - self.inner.db.put(keys::blob_key(&id), data).await?; - Ok(id) + BlobStore::new(Arc::new(self.inner.db.clone())) + .write(data) + .await } pub async fn read_blob(&self, id: &RunBlobId) -> Result> { - let global = self.inner.db.get(keys::blob_key(id)).await?; - if global.is_some() { - return Ok(global); - } - Ok(None) + BlobStore::new(Arc::new(self.inner.db.clone())) + .read(id) + .await } pub async fn list_blobs(&self) -> Result> { From 3525be358b92124a627e8ac87214c58f6a4ae3dd Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Tue, 21 Apr 2026 08:33:25 -0400 Subject: [PATCH 10/12] fix(cli): box run command futures for clippy --- lib/crates/fabro-cli/src/commands/run/attach.rs | 17 ++++++++++++----- .../fabro-cli/src/commands/run/command.rs | 4 ++-- lib/crates/fabro-cli/src/commands/run/mod.rs | 9 ++++++--- lib/crates/fabro-cli/src/commands/run/resume.rs | 4 ++-- 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/lib/crates/fabro-cli/src/commands/run/attach.rs b/lib/crates/fabro-cli/src/commands/run/attach.rs index 578fd18fd..0c634fbd9 100644 --- a/lib/crates/fabro-cli/src/commands/run/attach.rs +++ b/lib/crates/fabro-cli/src/commands/run/attach.rs @@ -57,14 +57,14 @@ pub(crate) async fn attach_run( if let (Some(storage_dir), Some(run_id)) = (storage_dir.as_deref(), run_id.as_ref()) { let client = server_client::connect_server(storage_dir).await?; - return attach_run_with_client( + return Box::pin(attach_run_with_client( &client, run_id, kill_on_detach, styles, json_output, Printer::Default, - ) + )) .await; } @@ -535,9 +535,16 @@ mod tests { async fn attach_errors_without_store_context() { let dir = tempfile::tempdir().unwrap(); - let err = attach_run(dir.path(), None, None, false, no_color_styles(), false) - .await - .unwrap_err(); + let err = Box::pin(attach_run( + dir.path(), + None, + None, + false, + no_color_styles(), + false, + )) + .await + .unwrap_err(); assert!( err.to_string() diff --git a/lib/crates/fabro-cli/src/commands/run/command.rs b/lib/crates/fabro-cli/src/commands/run/command.rs index 9d6d2e912..b2866b4be 100644 --- a/lib/crates/fabro-cli/src/commands/run/command.rs +++ b/lib/crates/fabro-cli/src/commands/run/command.rs @@ -58,14 +58,14 @@ pub(crate) async fn execute( fabro_util::printout!(printer, "{}", created_run.run_id); } } else { - let exit_code = super::attach::attach_run_with_client( + let exit_code = Box::pin(super::attach::attach_run_with_client( &client, &created_run.run_id, true, styles, json, printer, - ) + )) .await?; if !json { super::output::print_run_summary_with_client( diff --git a/lib/crates/fabro-cli/src/commands/run/mod.rs b/lib/crates/fabro-cli/src/commands/run/mod.rs index cd43461e2..b9a8eaec0 100644 --- a/lib/crates/fabro-cli/src/commands/run/mod.rs +++ b/lib/crates/fabro-cli/src/commands/run/mod.rs @@ -71,14 +71,14 @@ pub(crate) async fn dispatch( let ctx = CommandContext::for_target(&server, printer, cli.clone(), cli_layer)?; let client = ctx.server().await?; let run_id = client.resolve_run(&run).await?.run_id; - let exit_code = attach::attach_run_with_client( + let exit_code = Box::pin(attach::attach_run_with_client( client.as_ref(), &run_id, false, styles, cli.output.format == OutputFormat::Json, printer, - ) + )) .await?; if exit_code != std::process::ExitCode::SUCCESS { std::process::exit(1); @@ -116,7 +116,10 @@ pub(crate) async fn dispatch( CommandContext::for_target(&args.server, printer, cli.clone(), cli_layer)?; crate::sleep_inhibitor::guard(ctx.cli_settings().exec.prevent_idle_sleep) }; - resume::resume_command(args, styles, cli, cli_layer, printer).await + Box::pin(resume::resume_command( + args, styles, cli, cli_layer, printer, + )) + .await } RunCommands::Rewind(args) => { let styles = Styles::detect_stderr(); diff --git a/lib/crates/fabro-cli/src/commands/run/resume.rs b/lib/crates/fabro-cli/src/commands/run/resume.rs index 1edadc374..1aa723459 100644 --- a/lib/crates/fabro-cli/src/commands/run/resume.rs +++ b/lib/crates/fabro-cli/src/commands/run/resume.rs @@ -33,14 +33,14 @@ pub(crate) async fn resume_command( fabro_util::printout!(printer, "{run_id}"); } } else { - let exit_code = super::attach::attach_run_with_client( + let exit_code = Box::pin(super::attach::attach_run_with_client( client.as_ref(), &run_id, true, styles, json, printer, - ) + )) .await?; if !json { super::output::print_run_summary_with_client( From b862762b29a97d0a1b1a2d4499332bd967e991a7 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Tue, 21 Apr 2026 08:39:44 -0400 Subject: [PATCH 11/12] docs(store): document record repository pattern Add a short record-layer overview plus a concrete example for defining a new record type and wrapping Repository in a domain store, so the internal SlateDB abstraction is easier to discover and reuse. --- lib/crates/fabro-store/src/record/mod.rs | 15 ++++ .../fabro-store/src/record/repository.rs | 71 +++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/lib/crates/fabro-store/src/record/mod.rs b/lib/crates/fabro-store/src/record/mod.rs index 9c455c205..64c956120 100644 --- a/lib/crates/fabro-store/src/record/mod.rs +++ b/lib/crates/fabro-store/src/record/mod.rs @@ -1,3 +1,18 @@ +//! Internal typed key/value helpers for simple SlateDB-backed records. +//! +//! The split of responsibility is: +//! - [`Record`]: declares the key prefix, id type, and codec for one persisted +//! type. +//! - [`RecordId`]: converts the typed id to and from key segments. +//! - [`Repository`]: performs the generic get/put/delete/scan/gc operations. +//! - [`transaction`]: batches multiple typed writes into one atomic SlateDB +//! write. +//! +//! Production callers should add a named domain store on top of this layer +//! rather than exposing `Repository` directly. See `slate/auth_codes.rs`, +//! `slate/auth_tokens.rs`, `slate/blob_store.rs`, and +//! `slate/run_catalog_index.rs` for the intended pattern. + mod codec; mod record_id; mod repository; diff --git a/lib/crates/fabro-store/src/record/repository.rs b/lib/crates/fabro-store/src/record/repository.rs index ae77e103e..f6369ef58 100644 --- a/lib/crates/fabro-store/src/record/repository.rs +++ b/lib/crates/fabro-store/src/record/repository.rs @@ -1,3 +1,68 @@ +//! Thin typed storage wrapper for simple records that live directly in SlateDB. +//! +//! When adding a new persisted record type: +//! 1. Define the data struct. +//! 2. Implement [`Record`] for it with a stable `PREFIX`, `Id`, and `Codec`. +//! 3. Wrap `Repository` in a small domain store that exposes the +//! operations callers should use. +//! +//! Example: +//! +//! ```rust,ignore +//! use std::sync::Arc; +//! +//! use chrono::{DateTime, Utc}; +//! use serde::{Deserialize, Serialize}; +//! +//! use crate::record::{JsonCodec, Record, Repository}; +//! use crate::Result; +//! +//! #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +//! struct Session { +//! id: String, +//! user_id: String, +//! expires_at: DateTime, +//! } +//! +//! impl Record for Session { +//! type Id = String; +//! type Codec = JsonCodec; +//! const PREFIX: &'static str = "auth/session"; +//! +//! fn id(&self) -> Self::Id { +//! self.id.clone() +//! } +//! } +//! +//! struct SessionStore { +//! repo: Repository, +//! } +//! +//! impl SessionStore { +//! fn new(db: Arc) -> Self { +//! Self { +//! repo: Repository::new(db), +//! } +//! } +//! +//! async fn insert(&self, session: Session) -> Result<()> { +//! self.repo.put(&session).await +//! } +//! +//! async fn get(&self, id: &str) -> Result> { +//! self.repo.get(&id.to_string()).await +//! } +//! +//! async fn gc_expired(&self, now: DateTime) -> Result { +//! self.repo.gc(|session| session.expires_at <= now).await +//! } +//! } +//! ``` +//! +//! Keep `Repository` internal. Domain-specific invariants such as consume +//! locks, token rotation, or marker-only behavior belong in the named store +//! that wraps it, not in this generic layer. + use std::marker::PhantomData; use std::pin::Pin; use std::sync::Arc; @@ -9,6 +74,12 @@ use slatedb::{Db, KeyValue, WriteBatch}; use super::{Codec, Record, RecordId}; use crate::{Error, Result, keys}; +/// Generic typed key/value operations shared by the simple record-backed +/// stores. +/// +/// This type is intentionally `pub(crate)`: callers should interact through a +/// named store such as `AuthCodeStore` or `RefreshTokenStore`, which can add +/// domain-specific behavior on top of the generic storage primitives here. pub(crate) struct Repository { db: Arc, prefix_segments: Vec<&'static str>, From 4865efa4998f3b62ff602a8071179ede9f6ab409 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Tue, 21 Apr 2026 09:00:00 -0400 Subject: [PATCH 12/12] refactor(store): simplify Repository and blob wiring - Use hex crate for [u8; 32] RecordId instead of hand-rolled loops - Drop dead prefix_segments cache field; key assembly consumes R::PREFIX.split('/') directly, removing an intermediate Vec<&str> - Cache BlobStore on RunDatabaseInner (built once in open_writer/ open_reader via a new build() helper) instead of per-blob construction - Trim the replay_revocations doc comment to drop a stale plan reference Co-Authored-By: Claude Opus 4.7 (1M context) --- Cargo.lock | 1 + lib/crates/fabro-store/Cargo.toml | 1 + .../fabro-store/src/record/record_id.rs | 25 +----- .../fabro-store/src/record/repository.rs | 76 ++++++++----------- .../fabro-store/src/record/transaction.rs | 13 ++-- .../fabro-store/src/slate/auth_tokens.rs | 5 +- lib/crates/fabro-store/src/slate/run_store.rs | 37 +++------ 7 files changed, 54 insertions(+), 104 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 75efe4397..5a1f79cc0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2102,6 +2102,7 @@ dependencies = [ "dashmap", "fabro-types", "futures", + "hex", "insta", "object_store", "percent-encoding", diff --git a/lib/crates/fabro-store/Cargo.toml b/lib/crates/fabro-store/Cargo.toml index 6b3fe893b..9145a7730 100644 --- a/lib/crates/fabro-store/Cargo.toml +++ b/lib/crates/fabro-store/Cargo.toml @@ -13,6 +13,7 @@ workspace = true [dependencies] fabro-types = { path = "../fabro-types" } +hex.workspace = true slatedb.workspace = true object_store.workspace = true percent-encoding.workspace = true diff --git a/lib/crates/fabro-store/src/record/record_id.rs b/lib/crates/fabro-store/src/record/record_id.rs index 21a483a67..1ce67f069 100644 --- a/lib/crates/fabro-store/src/record/record_id.rs +++ b/lib/crates/fabro-store/src/record/record_id.rs @@ -1,5 +1,3 @@ -use std::fmt::Write; - use fabro_types::{RunBlobId, RunId}; use super::RecordId; @@ -7,11 +5,7 @@ use crate::{Error, Result}; impl RecordId for [u8; 32] { fn key_segments(&self) -> Vec { - let mut encoded = String::with_capacity(self.len() * 2); - for byte in self { - write!(&mut encoded, "{byte:02x}").expect("write to String cannot fail"); - } - vec![encoded] + vec![hex::encode(self)] } fn from_key_segments(segs: &[&str]) -> Result { @@ -21,22 +15,9 @@ impl RecordId for [u8; 32] { segs.len() ))); }; - - if segment.len() != 64 { - return Err(Error::KeyParse(format!( - "expected 64 hex characters for [u8; 32], got {}", - segment.len() - ))); - } - let mut bytes = [0_u8; 32]; - for (index, chunk) in segment.as_bytes().chunks_exact(2).enumerate() { - let chunk = std::str::from_utf8(chunk).map_err(|err| { - Error::KeyParse(format!("hex segment was not valid UTF-8: {err}")) - })?; - bytes[index] = u8::from_str_radix(chunk, 16) - .map_err(|err| Error::KeyParse(format!("invalid hex byte {chunk:?}: {err}")))?; - } + hex::decode_to_slice(segment, &mut bytes) + .map_err(|err| Error::KeyParse(format!("invalid hex segment {segment:?}: {err}")))?; Ok(bytes) } } diff --git a/lib/crates/fabro-store/src/record/repository.rs b/lib/crates/fabro-store/src/record/repository.rs index f6369ef58..bfcc18fdc 100644 --- a/lib/crates/fabro-store/src/record/repository.rs +++ b/lib/crates/fabro-store/src/record/repository.rs @@ -81,16 +81,15 @@ use crate::{Error, Result, keys}; /// named store such as `AuthCodeStore` or `RefreshTokenStore`, which can add /// domain-specific behavior on top of the generic storage primitives here. pub(crate) struct Repository { - db: Arc, - prefix_segments: Vec<&'static str>, - _record: PhantomData, + db: Arc, + _record: PhantomData, } impl Repository { pub(crate) fn new(db: Arc) -> Self { + validate_prefix::(); Self { db, - prefix_segments: prefix_segments::(), _record: PhantomData, } } @@ -141,32 +140,22 @@ impl Repository { extra_segments: &'a [&'a str], ) -> RepositoryStream<'a, (R::Id, R)> { match prefix_key::(extra_segments) { - Ok(prefix) => { - let prefix_segments = self.prefix_segments.clone(); - Box::pin( - scan_entries(Arc::clone(&self.db), &prefix).map(move |result| { - result - .map_err(Into::into) - .and_then(|entry| decode_entry::(&entry, &prefix_segments)) - }), - ) - } + Ok(prefix) => Box::pin(scan_entries(Arc::clone(&self.db), &prefix).map(|result| { + result + .map_err(Into::into) + .and_then(|entry| decode_entry::(&entry)) + })), Err(err) => Box::pin(stream::once(async move { Err(err) })), } } pub(crate) fn scan_ids_stream(&self) -> RepositoryStream<'_, R::Id> { match prefix_key::(&[]) { - Ok(prefix) => { - let prefix_segments = self.prefix_segments.clone(); - Box::pin( - scan_entries(Arc::clone(&self.db), &prefix).map(move |result| { - result - .map_err(Into::into) - .and_then(|entry| parse_entry_id::(&entry, &prefix_segments)) - }), - ) - } + Ok(prefix) => Box::pin(scan_entries(Arc::clone(&self.db), &prefix).map(|result| { + result + .map_err(Into::into) + .and_then(|entry| parse_entry_id::(&entry)) + })), Err(err) => Box::pin(stream::once(async move { Err(err) })), } } @@ -198,52 +187,48 @@ impl Repository { pub(crate) type RepositoryStream<'a, T> = Pin> + Send + 'a>>; pub(super) fn key_for_id(id: &R::Id) -> Result { - let prefix_segments = prefix_segments::(); let id_segments = id.key_segments(); - let id_segments: Vec<&str> = id_segments.iter().map(String::as_str).collect(); key_from_segments( - prefix_segments - .iter() - .copied() - .chain(id_segments.iter().copied()), + R::PREFIX + .split('/') + .chain(id_segments.iter().map(String::as_str)), ) } pub(super) fn prefix_key(extra_segments: &[&str]) -> Result { - let prefix_segments = prefix_segments::(); - prefix_from_segments( - prefix_segments - .iter() - .copied() - .chain(extra_segments.iter().copied()), - ) + prefix_from_segments(R::PREFIX.split('/').chain(extra_segments.iter().copied())) } -fn decode_entry(entry: &KeyValue, prefix_segments: &[&str]) -> Result<(R::Id, R)> { - let id = parse_entry_id::(entry, prefix_segments)?; +fn decode_entry(entry: &KeyValue) -> Result<(R::Id, R)> { + let id = parse_entry_id::(entry)?; let value = R::Codec::decode(&entry.value)?; Ok((id, value)) } -fn parse_entry_id(entry: &KeyValue, prefix_segments: &[&str]) -> Result { +fn parse_entry_id(entry: &KeyValue) -> Result { let raw_key = String::from_utf8(entry.key.to_vec()) .map_err(|err| Error::Other(format!("stored key is not valid UTF-8: {err}")))?; let segments: Vec<&str> = keys::SlateKey::segments(&raw_key).collect(); - if segments.len() < prefix_segments.len() { + let prefix_len = R::PREFIX.split('/').count(); + if segments.len() < prefix_len { return Err(Error::KeyParse(format!( "key {raw_key:?} had {} segments, expected at least {} for prefix {}", segments.len(), - prefix_segments.len(), + prefix_len, R::PREFIX ))); } - if segments[..prefix_segments.len()] != prefix_segments[..] { + if !segments[..prefix_len] + .iter() + .copied() + .eq(R::PREFIX.split('/')) + { return Err(Error::KeyParse(format!( "key {raw_key:?} did not match expected prefix {}", R::PREFIX ))); } - R::Id::from_key_segments(&segments[prefix_segments.len()..]) + R::Id::from_key_segments(&segments[prefix_len..]) } fn scan_entries( @@ -274,7 +259,7 @@ fn scan_entries( ) } -fn prefix_segments() -> Vec<&'static str> { +fn validate_prefix() { debug_assert!( !R::PREFIX.is_empty() && !R::PREFIX.starts_with('/') @@ -283,7 +268,6 @@ fn prefix_segments() -> Vec<&'static str> { "Record::PREFIX must be a non-empty '/'-separated path with no empty segments: {}", R::PREFIX ); - R::PREFIX.split('/').collect() } fn key_from_segments<'a>(segments: impl IntoIterator) -> Result { diff --git a/lib/crates/fabro-store/src/record/transaction.rs b/lib/crates/fabro-store/src/record/transaction.rs index a66e7e80f..8c7c710ac 100644 --- a/lib/crates/fabro-store/src/record/transaction.rs +++ b/lib/crates/fabro-store/src/record/transaction.rs @@ -10,20 +10,21 @@ where { let mut tx = Tx::new(); let value = f(&mut tx)?; - if !tx.has_ops { - return Ok(value); + if tx.has_ops { + db.write(tx.batch).await?; } - db.write(tx.into_batch()).await?; Ok(value) } pub(crate) struct Tx { batch: WriteBatch, + /// SlateDB rejects empty `WriteBatch` commits; skip the write entirely + /// when the closure produced no operations. has_ops: bool, } impl Tx { - pub(crate) fn new() -> Self { + fn new() -> Self { Self { batch: WriteBatch::new(), has_ops: false, @@ -51,10 +52,6 @@ impl Tx { self.has_ops = true; Ok(self) } - - fn into_batch(self) -> WriteBatch { - self.batch - } } #[cfg(test)] diff --git a/lib/crates/fabro-store/src/slate/auth_tokens.rs b/lib/crates/fabro-store/src/slate/auth_tokens.rs index 9357dd5b8..97cb1a8aa 100644 --- a/lib/crates/fabro-store/src/slate/auth_tokens.rs +++ b/lib/crates/fabro-store/src/slate/auth_tokens.rs @@ -49,9 +49,8 @@ pub struct RefreshTokenStore { db: Arc, repo: Repository, consume_locks: KeyedMutex<[u8; 32]>, - /// In-memory only by design (origin R6): persisting attacker-supplied - /// hashes adds an unbounded-growth surface under token-stuffing attack with - /// no security benefit. Do not migrate this into Repository. + /// In-memory only: persisting attacker-supplied hashes would be an + /// unbounded-growth surface under a token-stuffing attack. replay_revocations: DashMap<[u8; 32], DateTime>, } diff --git a/lib/crates/fabro-store/src/slate/run_store.rs b/lib/crates/fabro-store/src/slate/run_store.rs index be565bb63..a334281b9 100644 --- a/lib/crates/fabro-store/src/slate/run_store.rs +++ b/lib/crates/fabro-store/src/slate/run_store.rs @@ -33,6 +33,7 @@ impl std::fmt::Debug for RunDatabase { pub(crate) struct RunDatabaseInner { run_id: RunId, db: Db, + blob_store: BlobStore, event_seq: AtomicU32, close_lock: Mutex<()>, state_lock: Mutex<()>, @@ -44,33 +45,23 @@ pub(crate) struct RunDatabaseInner { impl RunDatabase { pub(crate) async fn open_writer(run_id: RunId, db: Db) -> Result { - let event_seq = - recover_next_seq(&db, keys::run_events_prefix(&run_id), keys::parse_event_seq).await?; - let (event_tx, _) = broadcast::channel(DEFAULT_EVENT_TAIL_LIMIT.max(16)); - Ok(Self { - inner: Arc::new(RunDatabaseInner { - run_id, - db, - event_seq: AtomicU32::new(event_seq), - close_lock: Mutex::new(()), - state_lock: Mutex::new(()), - projection_cache: Mutex::new(EventProjectionCache::default()), - recent_events: Mutex::new(VecDeque::with_capacity(DEFAULT_EVENT_TAIL_LIMIT)), - recent_event_limit: DEFAULT_EVENT_TAIL_LIMIT, - event_tx, - }), - read_only: false, - }) + Self::build(run_id, db, false).await } pub(crate) async fn open_reader(run_id: RunId, db: Db) -> Result { + Self::build(run_id, db, true).await + } + + async fn build(run_id: RunId, db: Db, read_only: bool) -> Result { let event_seq = recover_next_seq(&db, keys::run_events_prefix(&run_id), keys::parse_event_seq).await?; let (event_tx, _) = broadcast::channel(DEFAULT_EVENT_TAIL_LIMIT.max(16)); + let blob_store = BlobStore::new(Arc::new(db.clone())); Ok(Self { - inner: Arc::new(RunDatabaseInner { + inner: Arc::new(RunDatabaseInner { run_id, db, + blob_store, event_seq: AtomicU32::new(event_seq), close_lock: Mutex::new(()), state_lock: Mutex::new(()), @@ -79,7 +70,7 @@ impl RunDatabase { recent_event_limit: DEFAULT_EVENT_TAIL_LIMIT, event_tx, }), - read_only: true, + read_only, }) } @@ -285,15 +276,11 @@ impl RunDatabase { if self.read_only { return Err(Error::ReadOnly); } - BlobStore::new(Arc::new(self.inner.db.clone())) - .write(data) - .await + self.inner.blob_store.write(data).await } pub async fn read_blob(&self, id: &RunBlobId) -> Result> { - BlobStore::new(Arc::new(self.inner.db.clone())) - .read(id) - .await + self.inner.blob_store.read(id).await } pub async fn list_blobs(&self) -> Result> {