From 6460c957514b8451ca7c36af2b3f6b4e9e3d17df Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Sat, 28 Mar 2026 00:13:25 -0400 Subject: [PATCH] Add fabro-store with in-memory and SlateDB backends --- Cargo.lock | 1264 ++++++++++++++--- Cargo.toml | 2 + lib/crates/fabro-store/Cargo.toml | 26 + lib/crates/fabro-store/src/error.rs | 19 + lib/crates/fabro-store/src/keys.rs | 208 +++ lib/crates/fabro-store/src/lib.rs | 106 ++ lib/crates/fabro-store/src/memory.rs | 1052 ++++++++++++++ lib/crates/fabro-store/src/slate/catalog.rs | 172 +++ lib/crates/fabro-store/src/slate/mod.rs | 635 +++++++++ lib/crates/fabro-store/src/slate/run_store.rs | 481 +++++++ lib/crates/fabro-store/src/types.rs | 117 ++ lib/crates/fabro-types/src/lib.rs | 2 + lib/crates/fabro-types/src/node_status.rs | 14 + lib/crates/fabro-workflows/src/run_dir.rs | 42 +- 14 files changed, 3965 insertions(+), 175 deletions(-) create mode 100644 lib/crates/fabro-store/Cargo.toml create mode 100644 lib/crates/fabro-store/src/error.rs create mode 100644 lib/crates/fabro-store/src/keys.rs create mode 100644 lib/crates/fabro-store/src/lib.rs create mode 100644 lib/crates/fabro-store/src/memory.rs create mode 100644 lib/crates/fabro-store/src/slate/catalog.rs create mode 100644 lib/crates/fabro-store/src/slate/mod.rs create mode 100644 lib/crates/fabro-store/src/slate/run_store.rs create mode 100644 lib/crates/fabro-store/src/types.rs create mode 100644 lib/crates/fabro-types/src/node_status.rs diff --git a/Cargo.lock b/Cargo.lock index a96babf69..9e2aa4c6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,6 +40,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + [[package]] name = "allocator-api2" version = "0.2.21" @@ -91,7 +97,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -102,7 +108,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -111,6 +117,21 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arc-swap" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a07d1f37ff60921c83bdfc7407723bdefe89b44b98a9b772f225c8f9d67141a6" +dependencies = [ + "rustversion", +] + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "asn1-rs" version = "0.6.2" @@ -135,7 +156,7 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -147,7 +168,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -175,6 +196,18 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + [[package]] name = "async-lock" version = "3.4.2" @@ -196,6 +229,34 @@ dependencies = [ "event-listener", ] +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + [[package]] name = "async-trait" version = "0.1.89" @@ -204,7 +265,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -216,12 +277,33 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atomic" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89cbf775b137e9b968e67227ef7f775587cde3fd31b0d8599dbd0f598a48340" +dependencies = [ + "bytemuck", +] + [[package]] name = "atomic-waker" version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "auto_enums" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65398a2893f41bce5c9259f6e1a4f03fbae40637c1bdc755b4f387f48c613b03" +dependencies = [ + "derive_utils", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "autocfg" version = "1.5.0" @@ -236,7 +318,7 @@ checksum = "e8b5778837666541195063243828c5b6139221b47dc4ec3ba81738e532469ab1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -314,6 +396,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", + "gloo-timers", + "tokio", +] + [[package]] name = "backtrace" version = "0.3.76" @@ -326,7 +419,7 @@ dependencies = [ "miniz_oxide", "object", "rustc-demangle", - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -341,6 +434,15 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bit-set" version = "0.8.0" @@ -456,6 +558,12 @@ version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" + [[package]] name = "byteorder" version = "1.5.0" @@ -506,7 +614,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -528,7 +636,7 @@ dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim", + "strsim 0.11.1", ] [[package]] @@ -537,10 +645,10 @@ version = "4.5.55" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -568,6 +676,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmsketch" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7ee2cfacbd29706479902b06d75ad8f1362900836aa32799eabc7e004bfd854" + [[package]] name = "colorchoice" version = "1.0.4" @@ -580,7 +694,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -701,6 +815,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "crokey" version = "1.4.0" @@ -724,7 +847,7 @@ dependencies = [ "proc-macro2", "quote", "strict", - "syn", + "syn 2.0.117", ] [[package]] @@ -777,6 +900,16 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-skiplist" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -820,6 +953,16 @@ dependencies = [ "typenum", ] +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core 0.14.4", + "darling_macro 0.14.4", +] + [[package]] name = "darling" version = "0.21.3" @@ -840,6 +983,20 @@ dependencies = [ "darling_macro 0.23.0", ] +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn 1.0.109", +] + [[package]] name = "darling_core" version = "0.21.3" @@ -850,8 +1007,8 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", - "syn", + "strsim 0.11.1", + "syn 2.0.117", ] [[package]] @@ -863,8 +1020,19 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", - "syn", + "strsim 0.11.1", + "syn 2.0.117", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core 0.14.4", + "quote", + "syn 1.0.109", ] [[package]] @@ -875,7 +1043,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core 0.21.3", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -886,7 +1054,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core 0.23.0", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1005,7 +1173,18 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "derive_utils" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "362f47930db19fe7735f527e6595e4900316b893ebf6d48ad3d31be928d57dd6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", ] [[package]] @@ -1056,7 +1235,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -1077,7 +1256,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1095,12 +1274,31 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + [[package]] name = "dunce" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "duration-str" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f88959de2d447fd3eddcf1909d1f19fe084e27a056a6904203dc5d8b9e771c1e" +dependencies = [ + "rust_decimal", + "serde", + "thiserror 2.0.18", + "time", + "winnow 0.6.26", +] + [[package]] name = "dyn-clone" version = "1.0.20" @@ -1164,7 +1362,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1300,7 +1498,7 @@ dependencies = [ "tokio", "tokio-rustls", "tokio-stream", - "toml", + "toml 0.8.23", "tower", "tower-service", "tracing", @@ -1319,7 +1517,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn", + "syn 2.0.117", "typify", "uuid", ] @@ -1386,7 +1584,7 @@ dependencies = [ "shlex", "tempfile", "tokio", - "toml", + "toml 0.8.23", "tracing", "tracing-appender", "tracing-subscriber", @@ -1407,9 +1605,9 @@ dependencies = [ "fabro-util", "serde", "serde_json", - "strsim", + "strsim 0.11.1", "tempfile", - "toml", + "toml 0.8.23", "tracing", ] @@ -1511,7 +1709,7 @@ dependencies = [ "serde_json", "tokio", "tokio-util", - "toml", + "toml 0.8.23", "tracing", ] @@ -1646,7 +1844,7 @@ dependencies = [ "tempfile", "tokio", "tokio-util", - "toml", + "toml 0.8.23", "tracing", "uuid", ] @@ -1665,11 +1863,30 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tokio-tungstenite", - "toml", + "toml 0.8.23", "tracing", "tracing-subscriber", ] +[[package]] +name = "fabro-store" +version = "0.176.2" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "fabro-types", + "futures", + "object_store", + "serde", + "serde_json", + "slatedb", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tracing", +] + [[package]] name = "fabro-telemetry" version = "0.176.2" @@ -1724,7 +1941,7 @@ version = "0.176.2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1743,7 +1960,7 @@ dependencies = [ "tempfile", "termimad", "tokio", - "toml", + "toml 0.8.23", "tracing", "tracing-subscriber", ] @@ -1803,13 +2020,24 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tokio-util", - "toml", + "toml 0.8.23", "tracing", "ulid", "uuid", "walkdir", ] +[[package]] +name = "fail-parallel" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5666e8ca4ec174d896fb742789c29b1bea9319dcfd623c41bececc0a60c4939d" +dependencies = [ + "log", + "once_cell", + "rand 0.8.5", +] + [[package]] name = "fancy-regex" version = "0.17.0" @@ -1827,6 +2055,22 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "figment" +version = "0.10.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cb01cd46b0cf372153850f4c6c272d9cbea2da513e07538405148f95bd789f3" +dependencies = [ + "atomic", + "pear", + "serde", + "serde_json", + "serde_yaml", + "toml 0.8.23", + "uncased", + "version_check", +] + [[package]] name = "filetime" version = "0.2.27" @@ -1844,6 +2088,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags", + "rustc_version", +] + [[package]] name = "float-cmp" version = "0.10.0" @@ -1872,6 +2126,7 @@ checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" dependencies = [ "futures-core", "futures-sink", + "nanorand", "spin", ] @@ -1926,6 +2181,112 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "foyer" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "642093b1a72c4a0ef89862484d669a353e732974781bb9c49a979526d1e30edc" +dependencies = [ + "equivalent", + "foyer-common", + "foyer-memory", + "foyer-storage", + "madsim-tokio", + "mixtrics", + "pin-project", + "serde", + "thiserror 2.0.18", + "tokio", + "tracing", +] + +[[package]] +name = "foyer-common" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9db9c0e4648b13e9216d785b308d43751ca975301aeb83e607ec630b6f956944" +dependencies = [ + "bincode", + "bytes", + "cfg-if", + "itertools", + "madsim-tokio", + "mixtrics", + "parking_lot", + "pin-project", + "serde", + "thiserror 2.0.18", + "tokio", + "twox-hash", +] + +[[package]] +name = "foyer-intrusive-collections" +version = "0.10.0-dev" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4fee46bea69e0596130e3210e65d3424e0ac1e6df3bde6636304bdf1ca4a3b" +dependencies = [ + "memoffset", +] + +[[package]] +name = "foyer-memory" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "040dc38acbfca8f1def26bbbd9e9199090884aabb15de99f7bf4060be66ff608" +dependencies = [ + "arc-swap", + "bitflags", + "cmsketch", + "equivalent", + "foyer-common", + "foyer-intrusive-collections", + "hashbrown 0.15.5", + "itertools", + "madsim-tokio", + "mixtrics", + "parking_lot", + "pin-project", + "serde", + "thiserror 2.0.18", + "tokio", + "tracing", +] + +[[package]] +name = "foyer-storage" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54a77ed888da490e997da6d6d62fcbce3f202ccf28be098c4ea595ca046fc4a9" +dependencies = [ + "allocator-api2", + "anyhow", + "auto_enums", + "bytes", + "equivalent", + "flume", + "foyer-common", + "foyer-memory", + "fs4", + "futures-core", + "futures-util", + "itertools", + "libc", + "lz4", + "madsim-tokio", + "ordered_hash_map", + "parking_lot", + "paste", + "pin-project", + "rand 0.9.2", + "serde", + "thiserror 2.0.18", + "tokio", + "tracing", + "twox-hash", + "zstd", +] + [[package]] name = "fraction" version = "0.15.3" @@ -1936,6 +2297,16 @@ dependencies = [ "num", ] +[[package]] +name = "fs4" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" +dependencies = [ + "rustix", + "windows-sys 0.59.0", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -2019,7 +2390,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2132,6 +2503,18 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "h2" version = "0.4.13" @@ -2157,6 +2540,15 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +dependencies = [ + "ahash", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -2212,6 +2604,12 @@ dependencies = [ "http", ] +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "heck" version = "0.5.0" @@ -2259,7 +2657,7 @@ checksum = "617aaa3557aef3810a6369d0a99fac8a080891b68bd9f9812a1eeda0c0730cbd" dependencies = [ "cfg-if", "libc", - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -2427,6 +2825,7 @@ dependencies = [ "hyper", "hyper-util", "rustls", + "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls", @@ -2502,7 +2901,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core", + "windows-core 0.62.2", ] [[package]] @@ -2664,6 +3063,12 @@ dependencies = [ "web-time", ] +[[package]] +name = "inlinable_string" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" + [[package]] name = "insta" version = "1.46.3" @@ -2717,6 +3122,15 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.17" @@ -2807,7 +3221,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn", + "syn 2.0.117", ] [[package]] @@ -2922,6 +3336,25 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lz4" +version = "1.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" +dependencies = [ + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "mac" version = "0.1.1" @@ -2938,6 +3371,61 @@ dependencies = [ "winapi", ] +[[package]] +name = "madsim" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18351aac4194337d6ea9ffbd25b3d1540ecc0754142af1bff5ba7392d1f6f771" +dependencies = [ + "ahash", + "async-channel", + "async-stream", + "async-task", + "bincode", + "bytes", + "downcast-rs", + "errno 0.3.14", + "futures-util", + "lazy_static", + "libc", + "madsim-macros", + "naive-timer", + "panic-message", + "rand 0.8.5", + "rand_xoshiro 0.6.0", + "rustversion", + "serde", + "spin", + "tokio", + "tokio-util", + "toml 0.9.12+spec-1.1.0", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "madsim-macros" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d248e97b1a48826a12c3828d921e8548e714394bf17274dd0a93910dc946e1" +dependencies = [ + "darling 0.14.4", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "madsim-tokio" +version = "0.2.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d3eb2acc57c82d21d699119b859e2df70a91dbdb84734885a1e72be83bdecb5" +dependencies = [ + "madsim", + "spin", + "tokio", +] + [[package]] name = "markup5ever" version = "0.35.0" @@ -2969,7 +3457,7 @@ checksum = "ac84fd3f360fcc43dc5f5d186f02a94192761a080e8bc58621ad4d12296a58cf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3070,6 +3558,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "mixtrics" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb252c728b9d77c6ef9103f0c81524fa0a3d3b161d0a936295d7fbeff6e04c11" +dependencies = [ + "itertools", + "parking_lot", +] + [[package]] name = "mockito" version = "1.7.2" @@ -3095,6 +3593,21 @@ dependencies = [ "tokio", ] +[[package]] +name = "naive-timer" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "034a0ad7deebf0c2abcf2435950a6666c3c15ea9d8fad0c0f48efa8a7f843fed" + +[[package]] +name = "nanorand" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" +dependencies = [ + "getrandom 0.2.17", +] + [[package]] name = "native-tls" version = "0.2.18" @@ -3171,13 +3684,22 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3387,6 +3909,16 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + [[package]] name = "objc2-io-surface" version = "0.3.2" @@ -3450,6 +3982,42 @@ dependencies = [ "memchr", ] +[[package]] +name = "object_store" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbfbfff40aeccab00ec8a910b57ca8ecf4319b335c542f2edcd19dd25a1e2a00" +dependencies = [ + "async-trait", + "base64", + "bytes", + "chrono", + "form_urlencoded", + "futures", + "http", + "http-body-util", + "humantime", + "hyper", + "itertools", + "md-5", + "parking_lot", + "percent-encoding", + "quick-xml", + "rand 0.9.2", + "reqwest", + "ring", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + [[package]] name = "oid-registry" version = "0.7.1" @@ -3530,7 +4098,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3557,6 +4125,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "ordered_hash_map" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab0e5f22bf6dd04abd854a8874247813a8fa2c8c1260eba6fbb150270ce7c176" +dependencies = [ + "hashbrown 0.13.2", +] + [[package]] name = "os_info" version = "3.14.0" @@ -3580,7 +4157,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" dependencies = [ "libc", - "windows-sys 0.45.0", + "windows-sys 0.61.2", +] + +[[package]] +name = "ouroboros" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0f050db9c44b97a94723127e6be766ac5c340c48f2c4bb3ffa11713744be59" +dependencies = [ + "aliasable", + "ouroboros_macro", + "static_assertions", +] + +[[package]] +name = "ouroboros_macro" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c7028bdd3d43083f6d8d4d5187680d0d3560d54df4cc9d752005268b41e64d0" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.117", ] [[package]] @@ -3589,6 +4190,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" +[[package]] +name = "panic-message" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384e52fd8fbd4cbe3c317e8216260c21a0f9134de108cea8a4dd4e7e152c472d" + [[package]] name = "parking" version = "2.2.1" @@ -3615,7 +4222,7 @@ dependencies = [ "libc", "redox_syscall 0.5.18", "smallvec", - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -3645,6 +4252,29 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" +[[package]] +name = "pear" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdeeaa00ce488657faba8ebf44ab9361f9365a97bd39ffb8a60663f57ff4b467" +dependencies = [ + "inlinable_string", + "pear_codegen", + "yansi", +] + +[[package]] +name = "pear_codegen" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bab5b985dc082b345f812b7df84e1bef27e7207b39e448439ba8bd69c93f147" +dependencies = [ + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.117", +] + [[package]] name = "pem" version = "3.0.6" @@ -3730,7 +4360,7 @@ dependencies = [ "phf_shared 0.13.1", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3751,6 +4381,26 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -3863,7 +4513,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.117", ] [[package]] @@ -3875,6 +4525,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "version_check", + "yansi", +] + [[package]] name = "process-wrap" version = "9.0.3" @@ -3886,7 +4549,17 @@ dependencies = [ "nix 0.31.2", "tokio", "tracing", - "windows", + "windows 0.62.2", +] + +[[package]] +name = "quick-xml" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +dependencies = [ + "memchr", + "serde", ] [[package]] @@ -4018,6 +4691,24 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_xoshiro" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" +dependencies = [ + "rand_core 0.6.4", +] + +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core 0.9.5", +] + [[package]] name = "rayon" version = "1.11.0" @@ -4084,7 +4775,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4170,6 +4861,7 @@ dependencies = [ "pin-project-lite", "quinn", "rustls", + "rustls-native-certs", "rustls-pki-types", "serde", "serde_json", @@ -4255,7 +4947,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn", + "syn 2.0.117", ] [[package]] @@ -4278,6 +4970,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rust_decimal" +version = "1.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ce901f9a19d251159075a4c37af514c3b8ef99c22e02dd8c19161cf397ee94a" +dependencies = [ + "arrayvec", + "num-traits", +] + [[package]] name = "rustc-demangle" version = "0.1.27" @@ -4318,7 +5020,7 @@ dependencies = [ "errno 0.3.14", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4337,6 +5039,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "2.2.0" @@ -4445,7 +5159,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn", + "syn 2.0.117", ] [[package]] @@ -4457,7 +5171,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn", + "syn 2.0.117", ] [[package]] @@ -4607,7 +5321,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4618,7 +5332,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4664,7 +5378,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4725,7 +5439,7 @@ dependencies = [ "darling 0.21.3", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4817,7 +5531,7 @@ version = "1.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" dependencies = [ - "errno 0.2.8", + "errno 0.3.14", "libc", ] @@ -4861,6 +5575,79 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" +[[package]] +name = "slatedb" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d945c6cd6f239873e640c5b7bb204f5638ed5681b02145cdcc2e80b2d3882b8a" +dependencies = [ + "anyhow", + "async-trait", + "atomic", + "backon", + "bitflags", + "bytemuck", + "bytes", + "chrono", + "crc32fast", + "crossbeam-skiplist", + "dotenvy", + "duration-str", + "fail-parallel", + "figment", + "flatbuffers", + "foyer", + "futures", + "log", + "object_store", + "once_cell", + "ouroboros", + "parking_lot", + "rand 0.9.2", + "rand_xoshiro 0.7.0", + "serde", + "serde_json", + "siphasher", + "slatedb-common", + "slatedb-txn-obj", + "sysinfo", + "thiserror 1.0.69", + "thread_local", + "tokio", + "tokio-util", + "tracing", + "ulid", + "url", + "uuid", + "walkdir", +] + +[[package]] +name = "slatedb-common" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2893ff22bb9a1013af0fb5e85380307ccd9d0a4fa8a111e187a48750e14b9a47" +dependencies = [ + "chrono", + "tokio", +] + +[[package]] +name = "slatedb-txn-obj" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c33d5597404e23b9706aa32a1723399f022f0513a289d1059df8810a282dcc4" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "futures", + "log", + "object_store", + "slatedb-common", + "thiserror 1.0.69", +] + [[package]] name = "smallvec" version = "1.15.1" @@ -4988,7 +5775,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn", + "syn 2.0.117", ] [[package]] @@ -4999,7 +5786,7 @@ checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" dependencies = [ "dotenvy", "either", - "heck", + "heck 0.5.0", "hex", "once_cell", "proc-macro2", @@ -5011,7 +5798,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn", + "syn 2.0.117", "tokio", "url", ] @@ -5141,6 +5928,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "strict" version = "0.2.0" @@ -5189,6 +5982,12 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "strsim" version = "0.11.1" @@ -5201,6 +6000,17 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.117" @@ -5229,7 +6039,21 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "sysinfo" +version = "0.35.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3ffa3e4ff2b324a57f7aeb3c349656c7b127c3c189520251a648102a92496e" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows 0.61.3", ] [[package]] @@ -5283,7 +6107,7 @@ dependencies = [ "getrandom 0.4.1", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5354,7 +6178,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5365,7 +6189,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5458,7 +6282,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5518,6 +6342,8 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "futures-util", + "hashbrown 0.15.5", "pin-project-lite", "tokio", ] @@ -5534,6 +6360,21 @@ dependencies = [ "toml_edit 0.22.27", ] +[[package]] +name = "toml" +version = "0.9.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +dependencies = [ + "indexmap 2.13.0", + "serde_core", + "serde_spanned 1.0.4", + "toml_datetime 0.7.5+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 0.7.14", +] + [[package]] name = "toml_datetime" version = "0.6.11" @@ -5563,7 +6404,7 @@ dependencies = [ "serde_spanned 0.6.9", "toml_datetime 0.6.11", "toml_write", - "winnow", + "winnow 0.7.14", ] [[package]] @@ -5578,7 +6419,7 @@ dependencies = [ "toml_datetime 0.7.5+spec-1.1.0", "toml_parser", "toml_writer", - "winnow", + "winnow 0.7.14", ] [[package]] @@ -5587,7 +6428,7 @@ version = "1.0.9+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "702d4415e08923e7e1ef96cd5727c0dfed80b4d2fa25db9647fe5eb6f7c5a4c4" dependencies = [ - "winnow", + "winnow 0.7.14", ] [[package]] @@ -5680,7 +6521,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5765,6 +6606,15 @@ dependencies = [ "utf-8", ] +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +dependencies = [ + "rand 0.9.2", +] + [[package]] name = "typenum" version = "1.19.0" @@ -5786,7 +6636,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2fd0d27608a466d063d23b97cf2d26c25d838f01b4f7d5ff406a7446f16b6e3" dependencies = [ - "heck", + "heck 0.5.0", "log", "proc-macro2", "quote", @@ -5795,7 +6645,7 @@ dependencies = [ "semver", "serde", "serde_json", - "syn", + "syn 2.0.117", "thiserror 2.0.18", "unicode-ident", ] @@ -5807,6 +6657,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" dependencies = [ "rand 0.9.2", + "serde", "web-time", ] @@ -5819,6 +6670,15 @@ dependencies = [ "libc", ] +[[package]] +name = "uncased" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" +dependencies = [ + "version_check", +] + [[package]] name = "unicase" version = "2.9.0" @@ -6102,7 +6962,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -6244,7 +7104,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -6253,16 +7113,38 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.61.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" +dependencies = [ + "windows-collections 0.2.0", + "windows-core 0.61.2", + "windows-future 0.2.1", + "windows-link 0.1.3", + "windows-numerics 0.2.0", +] + [[package]] name = "windows" version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" dependencies = [ - "windows-collections", - "windows-core", - "windows-future", - "windows-numerics", + "windows-collections 0.3.2", + "windows-core 0.62.2", + "windows-future 0.3.2", + "windows-numerics 0.3.1", +] + +[[package]] +name = "windows-collections" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" +dependencies = [ + "windows-core 0.61.2", ] [[package]] @@ -6271,7 +7153,20 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" dependencies = [ - "windows-core", + "windows-core 0.62.2", +] + +[[package]] +name = "windows-core" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link 0.1.3", + "windows-result 0.3.4", + "windows-strings 0.4.2", ] [[package]] @@ -6282,9 +7177,20 @@ checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ "windows-implement", "windows-interface", - "windows-link", - "windows-result", - "windows-strings", + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", +] + +[[package]] +name = "windows-future" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" +dependencies = [ + "windows-core 0.61.2", + "windows-link 0.1.3", + "windows-threading 0.1.0", ] [[package]] @@ -6293,9 +7199,9 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" dependencies = [ - "windows-core", - "windows-link", - "windows-threading", + "windows-core 0.62.2", + "windows-link 0.2.1", + "windows-threading 0.2.1", ] [[package]] @@ -6306,7 +7212,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -6317,23 +7223,39 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-numerics" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" +dependencies = [ + "windows-core 0.61.2", + "windows-link 0.1.3", +] + [[package]] name = "windows-numerics" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" dependencies = [ - "windows-core", - "windows-link", + "windows-core 0.62.2", + "windows-link 0.2.1", ] [[package]] @@ -6342,9 +7264,18 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" dependencies = [ - "windows-link", - "windows-result", - "windows-strings", + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", +] + +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link 0.1.3", ] [[package]] @@ -6353,7 +7284,16 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" dependencies = [ - "windows-link", + "windows-link 0.2.1", +] + +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link 0.1.3", ] [[package]] @@ -6362,16 +7302,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", + "windows-link 0.2.1", ] [[package]] @@ -6416,22 +7347,7 @@ version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", + "windows-link 0.2.1", ] [[package]] @@ -6471,7 +7387,7 @@ version = "0.53.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" dependencies = [ - "windows-link", + "windows-link 0.2.1", "windows_aarch64_gnullvm 0.53.1", "windows_aarch64_msvc 0.53.1", "windows_i686_gnu 0.53.1", @@ -6482,21 +7398,24 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] +[[package]] +name = "windows-threading" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows-threading" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" dependencies = [ - "windows-link", + "windows-link 0.2.1", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -6515,12 +7434,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -6539,12 +7452,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -6575,12 +7482,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -6599,12 +7500,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -6623,12 +7518,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -6647,12 +7536,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -6671,6 +7554,15 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +[[package]] +name = "winnow" +version = "0.6.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e90edd2ac1aa278a5c4599b1d89cf03074b610800f866d4026dc199d7929a28" +dependencies = [ + "memchr", +] + [[package]] name = "winnow" version = "0.7.14" @@ -6696,7 +7588,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" dependencies = [ "anyhow", - "heck", + "heck 0.5.0", "wit-parser", ] @@ -6707,10 +7599,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", - "heck", + "heck 0.5.0", "indexmap 2.13.0", "prettyplease", - "syn", + "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -6726,7 +7618,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -6811,6 +7703,12 @@ dependencies = [ "markup5ever", ] +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + [[package]] name = "yoke" version = "0.8.1" @@ -6830,7 +7728,7 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -6851,7 +7749,7 @@ checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -6871,7 +7769,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -6911,7 +7809,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -6919,3 +7817,31 @@ name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index c1710113e..9f5adefd2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,8 @@ daytona-api-client = { git = "https://github.com/brynary/daytona-sdk-rust", rev sentry = { version = "0.35", default-features = false, features = ["backtrace", "contexts", "ureq", "rustls"] } fork = "0.2" exec = "0.3" +slatedb = "0.11.2" +object_store = "0.12.5" [profile.release] lto = "thin" diff --git a/lib/crates/fabro-store/Cargo.toml b/lib/crates/fabro-store/Cargo.toml new file mode 100644 index 000000000..4daba5d23 --- /dev/null +++ b/lib/crates/fabro-store/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "fabro-store" +edition.workspace = true +version.workspace = true +license.workspace = true + +[lib] +doctest = false + +[dependencies] +fabro-types = { path = "../fabro-types" } +slatedb.workspace = true +object_store.workspace = true +async-trait.workspace = true +tokio = { workspace = true, features = ["full"] } +tokio-stream.workspace = true +serde.workspace = true +serde_json.workspace = true +chrono = { workspace = true, features = ["serde"] } +bytes.workspace = true +thiserror.workspace = true +tracing.workspace = true +futures.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util", "macros"] } diff --git a/lib/crates/fabro-store/src/error.rs b/lib/crates/fabro-store/src/error.rs new file mode 100644 index 000000000..0ea00d1ff --- /dev/null +++ b/lib/crates/fabro-store/src/error.rs @@ -0,0 +1,19 @@ +pub type Result = std::result::Result; + +#[derive(Debug, thiserror::Error)] +pub enum StoreError { + #[error("SlateDB error: {0}")] + Slate(#[from] slatedb::Error), + #[error("Object store error: {0}")] + ObjectStore(#[from] object_store::Error), + #[error("Serialization error: {0}")] + Serde(#[from] serde_json::Error), + #[error("Invalid event payload: {0}")] + InvalidEvent(String), + #[error("Run not found: {0}")] + RunNotFound(String), + #[error("Run already exists: {0}")] + RunAlreadyExists(String), + #[error("{0}")] + Other(String), +} diff --git a/lib/crates/fabro-store/src/keys.rs b/lib/crates/fabro-store/src/keys.rs new file mode 100644 index 000000000..2fa94523a --- /dev/null +++ b/lib/crates/fabro-store/src/keys.rs @@ -0,0 +1,208 @@ +use crate::NodeVisitRef; + +pub const INIT_KEY: &str = "_init.json"; +pub const RUN_KEY: &str = "run.json"; +pub const START_KEY: &str = "start.json"; +pub const STATUS_KEY: &str = "status.json"; +pub const CHECKPOINT_KEY: &str = "checkpoint.json"; +pub const CONCLUSION_KEY: &str = "conclusion.json"; +pub const RETRO_KEY: &str = "retro.json"; +pub const GRAPH_KEY: &str = "graph.fabro"; +pub const SANDBOX_KEY: &str = "sandbox.json"; +pub const RETRO_PROMPT_KEY: &str = "retro/prompt.md"; +pub const RETRO_RESPONSE_KEY: &str = "retro/response.md"; +pub const EVENTS_PREFIX: &str = "events/"; +pub const CHECKPOINTS_PREFIX: &str = "checkpoints/"; +pub const ARTIFACT_VALUES_PREFIX: &str = "artifacts/values/"; +pub const ARTIFACT_NODES_PREFIX: &str = "artifacts/nodes/"; + +pub fn init() -> &'static str { + INIT_KEY +} + +pub fn run() -> &'static str { + RUN_KEY +} + +pub fn start() -> &'static str { + START_KEY +} + +pub fn status() -> &'static str { + STATUS_KEY +} + +pub fn checkpoint() -> &'static str { + CHECKPOINT_KEY +} + +pub fn conclusion() -> &'static str { + CONCLUSION_KEY +} + +pub fn retro() -> &'static str { + RETRO_KEY +} + +pub fn graph() -> &'static str { + GRAPH_KEY +} + +pub fn sandbox() -> &'static str { + SANDBOX_KEY +} + +pub fn node_visit_prefix(node: &NodeVisitRef<'_>) -> String { + format!("nodes/{}/visit-{}", node.node_id, node.visit) +} + +pub fn node_prompt(node: &NodeVisitRef<'_>) -> String { + format!("{}/prompt.md", node_visit_prefix(node)) +} + +pub fn node_response(node: &NodeVisitRef<'_>) -> String { + format!("{}/response.md", node_visit_prefix(node)) +} + +pub fn node_status(node: &NodeVisitRef<'_>) -> String { + format!("{}/status.json", node_visit_prefix(node)) +} + +pub fn node_stdout(node: &NodeVisitRef<'_>) -> String { + format!("{}/stdout.log", node_visit_prefix(node)) +} + +pub fn node_stderr(node: &NodeVisitRef<'_>) -> String { + format!("{}/stderr.log", node_visit_prefix(node)) +} + +pub fn retro_prompt() -> &'static str { + RETRO_PROMPT_KEY +} + +pub fn retro_response() -> &'static str { + RETRO_RESPONSE_KEY +} + +pub fn event_key(seq: u32, epoch_ms: i64) -> String { + format!("{EVENTS_PREFIX}{seq:06}-{epoch_ms}.json") +} + +pub fn checkpoint_history_key(seq: u32, epoch_ms: i64) -> String { + format!("{CHECKPOINTS_PREFIX}{seq:04}-{epoch_ms}.json") +} + +pub fn artifact_value(artifact_id: &str) -> String { + format!("{ARTIFACT_VALUES_PREFIX}{artifact_id}.json") +} + +pub fn node_asset_prefix(node: &NodeVisitRef<'_>) -> String { + format!( + "{ARTIFACT_NODES_PREFIX}{}/visit-{}", + node.node_id, node.visit + ) +} + +pub fn node_asset(node: &NodeVisitRef<'_>, filename: &str) -> String { + format!("{}/{filename}", node_asset_prefix(node)) +} + +pub fn parse_event_seq(key: &str) -> Option { + parse_seq(key, EVENTS_PREFIX) +} + +pub fn parse_checkpoint_seq(key: &str) -> Option { + parse_seq(key, CHECKPOINTS_PREFIX) +} + +pub fn parse_node_key(key: &str) -> Option<(String, u32, String)> { + parse_visit_scoped_key(key, "nodes/") +} + +#[cfg(test)] +pub fn parse_node_asset_key(key: &str) -> Option<(String, u32, String)> { + parse_visit_scoped_key(key, ARTIFACT_NODES_PREFIX) +} + +fn parse_seq(key: &str, prefix: &str) -> Option { + key.strip_prefix(prefix)?.split_once('-')?.0.parse().ok() +} + +fn parse_visit_scoped_key(key: &str, prefix: &str) -> Option<(String, u32, String)> { + let rest = key.strip_prefix(prefix)?; + let (node_id, rest) = rest.split_once("/visit-")?; + let (visit, file) = rest.split_once('/')?; + Some((node_id.to_string(), visit.parse().ok()?, file.to_string())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn top_level_keys_match_spec() { + assert_eq!(init(), "_init.json"); + assert_eq!(run(), "run.json"); + assert_eq!(graph(), "graph.fabro"); + assert_eq!(retro_prompt(), "retro/prompt.md"); + assert_eq!(retro_response(), "retro/response.md"); + } + + #[test] + fn node_keys_match_spec() { + let node = NodeVisitRef { + node_id: "plan", + visit: 3, + }; + assert_eq!(node_visit_prefix(&node), "nodes/plan/visit-3"); + assert_eq!(node_prompt(&node), "nodes/plan/visit-3/prompt.md"); + assert_eq!(node_response(&node), "nodes/plan/visit-3/response.md"); + assert_eq!(node_status(&node), "nodes/plan/visit-3/status.json"); + assert_eq!(node_stdout(&node), "nodes/plan/visit-3/stdout.log"); + assert_eq!(node_stderr(&node), "nodes/plan/visit-3/stderr.log"); + } + + #[test] + fn sequence_keys_are_zero_padded() { + assert_eq!(event_key(7, 123), "events/000007-123.json"); + assert_eq!(checkpoint_history_key(42, 456), "checkpoints/0042-456.json"); + } + + #[test] + fn artifact_keys_match_spec() { + let node = NodeVisitRef { + node_id: "code", + visit: 2, + }; + assert_eq!(artifact_value("summary"), "artifacts/values/summary.json"); + assert_eq!( + node_asset(&node, "src/main.rs"), + "artifacts/nodes/code/visit-2/src/main.rs" + ); + } + + #[test] + fn parse_helpers_extract_sequences_and_node_visits() { + assert_eq!(parse_event_seq("events/000007-123.json"), Some(7)); + assert_eq!(parse_checkpoint_seq("checkpoints/0042-456.json"), Some(42)); + assert_eq!( + parse_node_key("nodes/plan/visit-3/status.json"), + Some(("plan".to_string(), 3, "status.json".to_string())) + ); + assert_eq!( + parse_node_asset_key("artifacts/nodes/code/visit-2/src/main.rs"), + Some(("code".to_string(), 2, "src/main.rs".to_string())) + ); + } + + #[test] + fn parse_helpers_reject_invalid_keys() { + assert_eq!(parse_event_seq("events/not-a-seq.json"), None); + assert_eq!(parse_checkpoint_seq("checkpoints/oops.json"), None); + assert_eq!(parse_node_key("nodes/plan/status.json"), None); + assert_eq!( + parse_node_asset_key("artifacts/nodes/code/status.json"), + None + ); + } +} diff --git a/lib/crates/fabro-store/src/lib.rs b/lib/crates/fabro-store/src/lib.rs new file mode 100644 index 000000000..9ac9f3128 --- /dev/null +++ b/lib/crates/fabro-store/src/lib.rs @@ -0,0 +1,106 @@ +use std::pin::Pin; + +use async_trait::async_trait; +use bytes::Bytes; +use chrono::{DateTime, Utc}; +use futures::Stream; + +mod error; +mod keys; +mod memory; +mod slate; +mod types; + +pub use error::{Result, StoreError}; +pub use memory::InMemoryStore; +pub use slate::SlateStore; +pub use types::{ + CatalogRecord, EventEnvelope, EventPayload, NodeSnapshot, NodeVisitRef, RunSnapshot, RunSummary, +}; + +use fabro_types::{ + Checkpoint, Conclusion, NodeStatusRecord, Retro, RunRecord, RunStatusRecord, SandboxRecord, + StartRecord, +}; + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct ListRunsQuery { + pub start: Option>, + pub end: Option>, +} + +#[async_trait] +pub trait Store: Send + Sync { + async fn create_run( + &self, + run_id: &str, + created_at: DateTime, + ) -> Result>; + async fn open_run(&self, run_id: &str) -> Result>>; + async fn list_runs(&self, query: &ListRunsQuery) -> Result>; + async fn delete_run(&self, run_id: &str) -> Result<()>; +} + +#[async_trait] +pub trait RunStore: Send + Sync { + async fn put_run(&self, record: &RunRecord) -> Result<()>; + async fn get_run(&self) -> Result>; + + async fn put_start(&self, record: &StartRecord) -> Result<()>; + async fn get_start(&self) -> Result>; + + async fn put_status(&self, record: &RunStatusRecord) -> Result<()>; + async fn get_status(&self) -> Result>; + + async fn put_checkpoint(&self, record: &Checkpoint) -> Result<()>; + async fn get_checkpoint(&self) -> Result>; + async fn append_checkpoint(&self, record: &Checkpoint) -> Result; + async fn list_checkpoints(&self) -> Result>; + + async fn put_conclusion(&self, record: &Conclusion) -> Result<()>; + async fn get_conclusion(&self) -> Result>; + + async fn put_retro(&self, retro: &Retro) -> Result<()>; + async fn get_retro(&self) -> Result>; + + async fn put_graph(&self, dot_source: &str) -> Result<()>; + async fn get_graph(&self) -> Result>; + + async fn put_sandbox(&self, record: &SandboxRecord) -> Result<()>; + async fn get_sandbox(&self) -> Result>; + + async fn put_node_prompt(&self, node: &NodeVisitRef<'_>, prompt: &str) -> Result<()>; + async fn put_node_response(&self, node: &NodeVisitRef<'_>, response: &str) -> Result<()>; + async fn put_node_status( + &self, + node: &NodeVisitRef<'_>, + status: &NodeStatusRecord, + ) -> Result<()>; + async fn put_node_stdout(&self, node: &NodeVisitRef<'_>, log: &str) -> Result<()>; + async fn put_node_stderr(&self, node: &NodeVisitRef<'_>, log: &str) -> Result<()>; + + async fn get_node(&self, node: &NodeVisitRef<'_>) -> Result; + async fn list_node_visits(&self, node_id: &str) -> Result>; + + async fn append_event(&self, payload: &EventPayload) -> Result; + async fn list_events(&self) -> Result>; + async fn list_events_from(&self, seq: u32) -> Result>; + async fn watch_events_from( + &self, + seq: u32, + ) -> Result> + Send>>>; + + async fn put_retro_prompt(&self, text: &str) -> Result<()>; + async fn get_retro_prompt(&self) -> Result>; + async fn put_retro_response(&self, text: &str) -> Result<()>; + async fn get_retro_response(&self) -> Result>; + + async fn put_artifact_value(&self, artifact_id: &str, value: &serde_json::Value) -> Result<()>; + async fn get_artifact_value(&self, artifact_id: &str) -> Result>; + + async fn put_asset(&self, node: &NodeVisitRef<'_>, filename: &str, data: &[u8]) -> Result<()>; + async fn get_asset(&self, node: &NodeVisitRef<'_>, filename: &str) -> Result>; + async fn list_assets(&self, node: &NodeVisitRef<'_>) -> Result>; + + async fn get_snapshot(&self) -> Result>; +} diff --git a/lib/crates/fabro-store/src/memory.rs b/lib/crates/fabro-store/src/memory.rs new file mode 100644 index 000000000..377dc45ce --- /dev/null +++ b/lib/crates/fabro-store/src/memory.rs @@ -0,0 +1,1052 @@ +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::Arc; + +use async_trait::async_trait; +use bytes::Bytes; +use chrono::{DateTime, Utc}; +use futures::Stream; +use serde::de::DeserializeOwned; +use serde::Serialize; +use tokio::sync::{mpsc, Mutex}; +use tokio_stream::wrappers::UnboundedReceiverStream; +use tokio_stream::StreamExt as _; + +use crate::keys; +use crate::{ + CatalogRecord, EventEnvelope, EventPayload, ListRunsQuery, NodeSnapshot, NodeVisitRef, Result, + RunSnapshot, RunStore, RunSummary, Store, StoreError, +}; +use fabro_types::{ + Checkpoint, Conclusion, NodeStatusRecord, Retro, RunRecord, RunStatusRecord, SandboxRecord, + StartRecord, +}; + +#[derive(Debug, Default)] +pub struct InMemoryStore { + runs: Mutex>, +} + +#[derive(Debug, Clone)] +struct InMemoryCatalog { + record: CatalogRecord, + run_store: Arc, +} + +#[derive(Debug)] +struct InMemoryRunStore { + run_id: String, + created_at: DateTime, + data: Mutex>>, + event_seq: AtomicU32, + checkpoint_seq: AtomicU32, + watchers: Mutex>>, +} + +impl InMemoryRunStore { + fn new(run_id: &str, created_at: DateTime, db_prefix: String) -> Result { + let record = CatalogRecord { + run_id: run_id.to_string(), + created_at, + db_prefix, + }; + let mut data = BTreeMap::new(); + data.insert(keys::init().to_string(), serde_json::to_vec(&record)?); + Ok(Self { + run_id: run_id.to_string(), + created_at, + data: Mutex::new(data), + event_seq: AtomicU32::new(1), + checkpoint_seq: AtomicU32::new(1), + watchers: Mutex::new(Vec::new()), + }) + } + + async fn put_json(&self, key: String, value: &T) -> Result<()> { + self.data + .lock() + .await + .insert(key, serde_json::to_vec(value)?); + Ok(()) + } + + async fn get_json(&self, key: &str) -> Result> { + let bytes = self.data.lock().await.get(key).cloned(); + bytes + .map(|value| serde_json::from_slice(&value)) + .transpose() + .map_err(Into::into) + } + + async fn put_text(&self, key: String, value: &str) { + self.data + .lock() + .await + .insert(key, value.as_bytes().to_vec()); + } + + async fn get_text(&self, key: &str) -> Result> { + let bytes = self.data.lock().await.get(key).cloned(); + bytes + .map(|value| { + String::from_utf8(value).map_err(|err| { + StoreError::Other(format!("stored text is not valid UTF-8: {err}")) + }) + }) + .transpose() + } + + async fn put_bytes(&self, key: String, value: &[u8]) { + self.data.lock().await.insert(key, value.to_vec()); + } + + async fn get_bytes(&self, key: &str) -> Option> { + self.data.lock().await.get(key).cloned() + } + + async fn snapshot_data(&self) -> BTreeMap> { + self.data.lock().await.clone() + } + + async fn build_node_snapshot_from_data( + &self, + data: &BTreeMap>, + node: &NodeVisitRef<'_>, + ) -> Result { + Ok(NodeSnapshot { + node_id: node.node_id.to_string(), + visit: node.visit, + prompt: read_text(data, &keys::node_prompt(node))?, + response: read_text(data, &keys::node_response(node))?, + status: read_json(data, &keys::node_status(node))?, + stdout: read_text(data, &keys::node_stdout(node))?, + stderr: read_text(data, &keys::node_stderr(node))?, + }) + } + + async fn list_events_from_inner(&self, seq: u32) -> Result> { + let data = self.snapshot_data().await; + let mut events = Vec::new(); + for (key, value) in &data { + let Some(current_seq) = keys::parse_event_seq(key) else { + continue; + }; + if current_seq < seq { + continue; + } + events.push(EventEnvelope { + seq: current_seq, + payload: serde_json::from_slice(value)?, + }); + } + events.sort_by_key(|event| event.seq); + Ok(events) + } + + async fn list_checkpoints_inner(&self) -> Result> { + let data = self.snapshot_data().await; + let mut checkpoints = Vec::new(); + for (key, value) in &data { + let Some(seq) = keys::parse_checkpoint_seq(key) else { + continue; + }; + checkpoints.push((seq, serde_json::from_slice(value)?)); + } + checkpoints.sort_by_key(|(seq, _)| *seq); + Ok(checkpoints) + } + + async fn build_snapshot_from_data( + &self, + data: &BTreeMap>, + ) -> Result> { + let Some(run) = read_json::(data, keys::run())? else { + return Ok(None); + }; + + let mut visits = BTreeSet::new(); + for key in data.keys() { + if let Some((node_id, visit, _)) = keys::parse_node_key(key) { + visits.insert((node_id, visit)); + } + } + + let mut nodes = Vec::new(); + for (node_id, visit) in visits { + let node = NodeVisitRef { + node_id: &node_id, + visit, + }; + nodes.push(self.build_node_snapshot_from_data(data, &node).await?); + } + + Ok(Some(RunSnapshot { + run, + start: read_json(data, keys::start())?, + status: read_json(data, keys::status())?, + checkpoint: read_json(data, keys::checkpoint())?, + conclusion: read_json(data, keys::conclusion())?, + retro: read_json(data, keys::retro())?, + graph: read_text(data, keys::graph())?, + sandbox: read_json(data, keys::sandbox())?, + nodes, + })) + } + + fn validate_run_record(&self, record: &RunRecord) -> Result<()> { + if record.created_at != self.created_at { + return Err(StoreError::Other(format!( + "run record created_at {:?} does not match store created_at {:?}", + record.created_at, self.created_at + ))); + } + if record.run_id != self.run_id { + return Err(StoreError::Other(format!( + "run record run_id {:?} does not match store run_id {:?}", + record.run_id, self.run_id + ))); + } + Ok(()) + } +} + +#[async_trait] +impl Store for InMemoryStore { + async fn create_run( + &self, + run_id: &str, + created_at: DateTime, + ) -> Result> { + let mut runs = self.runs.lock().await; + if let Some(existing) = runs.get(run_id) { + if existing.record.created_at == created_at { + return Ok(Box::new(Arc::clone(&existing.run_store))); + } + return Err(StoreError::RunAlreadyExists(run_id.to_string())); + } + + let db_prefix = format!("memory/runs/{run_id}"); + let run_store = Arc::new(InMemoryRunStore::new( + run_id, + created_at, + db_prefix.clone(), + )?); + let catalog = InMemoryCatalog { + record: CatalogRecord { + run_id: run_id.to_string(), + created_at, + db_prefix, + }, + run_store: Arc::clone(&run_store), + }; + runs.insert(run_id.to_string(), catalog); + Ok(Box::new(run_store)) + } + + async fn open_run(&self, run_id: &str) -> Result>> { + let runs = self.runs.lock().await; + Ok(runs + .get(run_id) + .map(|catalog| Box::new(Arc::clone(&catalog.run_store)) as Box)) + } + + async fn list_runs(&self, query: &ListRunsQuery) -> Result> { + let catalogs = { + let runs = self.runs.lock().await; + runs.values().cloned().collect::>() + }; + + let mut summaries = Vec::new(); + for catalog in catalogs { + if !matches_query(&catalog.record.created_at, query) { + continue; + } + let data = catalog.run_store.snapshot_data().await; + summaries.push(build_run_summary(&catalog.record, &data)?); + } + summaries.sort_by(|a, b| b.created_at.cmp(&a.created_at)); + Ok(summaries) + } + + async fn delete_run(&self, run_id: &str) -> Result<()> { + self.runs.lock().await.remove(run_id); + Ok(()) + } +} + +#[async_trait] +impl RunStore for Arc { + async fn put_run(&self, record: &RunRecord) -> Result<()> { + self.validate_run_record(record)?; + self.put_json(keys::run().to_string(), record).await + } + + async fn get_run(&self) -> Result> { + self.get_json(keys::run()).await + } + + async fn put_start(&self, record: &StartRecord) -> Result<()> { + self.put_json(keys::start().to_string(), record).await + } + + async fn get_start(&self) -> Result> { + self.get_json(keys::start()).await + } + + async fn put_status(&self, record: &RunStatusRecord) -> Result<()> { + self.put_json(keys::status().to_string(), record).await + } + + async fn get_status(&self) -> Result> { + self.get_json(keys::status()).await + } + + async fn put_checkpoint(&self, record: &Checkpoint) -> Result<()> { + self.put_json(keys::checkpoint().to_string(), record).await + } + + async fn get_checkpoint(&self) -> Result> { + self.get_json(keys::checkpoint()).await + } + + async fn append_checkpoint(&self, record: &Checkpoint) -> Result { + let seq = self.checkpoint_seq.fetch_add(1, Ordering::SeqCst); + let now = Utc::now().timestamp_millis(); + self.put_checkpoint(record).await?; + self.put_json(keys::checkpoint_history_key(seq, now), record) + .await?; + Ok(seq) + } + + async fn list_checkpoints(&self) -> Result> { + self.list_checkpoints_inner().await + } + + async fn put_conclusion(&self, record: &Conclusion) -> Result<()> { + self.put_json(keys::conclusion().to_string(), record).await + } + + async fn get_conclusion(&self) -> Result> { + self.get_json(keys::conclusion()).await + } + + async fn put_retro(&self, retro: &Retro) -> Result<()> { + self.put_json(keys::retro().to_string(), retro).await + } + + async fn get_retro(&self) -> Result> { + self.get_json(keys::retro()).await + } + + async fn put_graph(&self, dot_source: &str) -> Result<()> { + self.put_text(keys::graph().to_string(), dot_source).await; + Ok(()) + } + + async fn get_graph(&self) -> Result> { + self.get_text(keys::graph()).await + } + + async fn put_sandbox(&self, record: &SandboxRecord) -> Result<()> { + self.put_json(keys::sandbox().to_string(), record).await + } + + async fn get_sandbox(&self) -> Result> { + self.get_json(keys::sandbox()).await + } + + async fn put_node_prompt(&self, node: &NodeVisitRef<'_>, prompt: &str) -> Result<()> { + self.put_text(keys::node_prompt(node), prompt).await; + Ok(()) + } + + async fn put_node_response(&self, node: &NodeVisitRef<'_>, response: &str) -> Result<()> { + self.put_text(keys::node_response(node), response).await; + Ok(()) + } + + async fn put_node_status( + &self, + node: &NodeVisitRef<'_>, + status: &NodeStatusRecord, + ) -> Result<()> { + self.put_json(keys::node_status(node), status).await + } + + async fn put_node_stdout(&self, node: &NodeVisitRef<'_>, log: &str) -> Result<()> { + self.put_text(keys::node_stdout(node), log).await; + Ok(()) + } + + async fn put_node_stderr(&self, node: &NodeVisitRef<'_>, log: &str) -> Result<()> { + self.put_text(keys::node_stderr(node), log).await; + Ok(()) + } + + async fn get_node(&self, node: &NodeVisitRef<'_>) -> Result { + let data = self.snapshot_data().await; + self.build_node_snapshot_from_data(&data, node).await + } + + async fn list_node_visits(&self, node_id: &str) -> Result> { + let data = self.snapshot_data().await; + let mut visits = BTreeSet::new(); + for key in data.keys() { + let Some((current_node_id, visit, _)) = keys::parse_node_key(key) else { + continue; + }; + if current_node_id == node_id { + visits.insert(visit); + } + } + Ok(visits.into_iter().collect()) + } + + async fn append_event(&self, payload: &EventPayload) -> Result { + payload.validate(&self.run_id)?; + + let seq = self.event_seq.fetch_add(1, Ordering::SeqCst); + let envelope = EventEnvelope { + seq, + payload: payload.clone(), + }; + self.put_json( + keys::event_key(seq, Utc::now().timestamp_millis()), + &envelope.payload, + ) + .await?; + + let mut watchers = self.watchers.lock().await; + watchers.retain(|sender| sender.send(envelope.clone()).is_ok()); + Ok(seq) + } + + async fn list_events(&self) -> Result> { + self.list_events_from_inner(1).await + } + + async fn list_events_from(&self, seq: u32) -> Result> { + self.list_events_from_inner(seq).await + } + + async fn watch_events_from( + &self, + seq: u32, + ) -> Result> + Send>>> { + let (sender, receiver) = mpsc::unbounded_channel(); + + let data = self.data.lock().await; + let mut existing = Vec::new(); + for (key, value) in &*data { + let Some(current_seq) = keys::parse_event_seq(key) else { + continue; + }; + if current_seq < seq { + continue; + } + existing.push(EventEnvelope { + seq: current_seq, + payload: serde_json::from_slice(value)?, + }); + } + existing.sort_by_key(|event| event.seq); + for envelope in existing { + let _ = sender.send(envelope); + } + self.watchers.lock().await.push(sender); + drop(data); + + Ok(Box::pin(UnboundedReceiverStream::new(receiver).map(Ok))) + } + + async fn put_retro_prompt(&self, text: &str) -> Result<()> { + self.put_text(keys::retro_prompt().to_string(), text).await; + Ok(()) + } + + async fn get_retro_prompt(&self) -> Result> { + self.get_text(keys::retro_prompt()).await + } + + async fn put_retro_response(&self, text: &str) -> Result<()> { + self.put_text(keys::retro_response().to_string(), text) + .await; + Ok(()) + } + + async fn get_retro_response(&self) -> Result> { + self.get_text(keys::retro_response()).await + } + + async fn put_artifact_value(&self, artifact_id: &str, value: &serde_json::Value) -> Result<()> { + self.put_json(keys::artifact_value(artifact_id), value) + .await + } + + async fn get_artifact_value(&self, artifact_id: &str) -> Result> { + self.get_json(&keys::artifact_value(artifact_id)).await + } + + async fn put_asset(&self, node: &NodeVisitRef<'_>, filename: &str, data: &[u8]) -> Result<()> { + self.put_bytes(keys::node_asset(node, filename), data).await; + Ok(()) + } + + async fn get_asset(&self, node: &NodeVisitRef<'_>, filename: &str) -> Result> { + Ok(self + .get_bytes(&keys::node_asset(node, filename)) + .await + .map(Bytes::from)) + } + + async fn list_assets(&self, node: &NodeVisitRef<'_>) -> Result> { + let prefix = format!("{}/", keys::node_asset_prefix(node)); + let data = self.snapshot_data().await; + let mut assets = Vec::new(); + for key in data.keys() { + if let Some(path) = key.strip_prefix(&prefix) { + assets.push(path.to_string()); + } + } + assets.sort(); + Ok(assets) + } + + async fn get_snapshot(&self) -> Result> { + let data = self.snapshot_data().await; + self.build_snapshot_from_data(&data).await + } +} + +fn matches_query(created_at: &DateTime, query: &ListRunsQuery) -> bool { + if let Some(start) = query.start { + if *created_at < start { + return false; + } + } + if let Some(end) = query.end { + if *created_at > end { + return false; + } + } + true +} + +fn read_json( + data: &BTreeMap>, + key: &str, +) -> Result> { + data.get(key) + .map(|value| serde_json::from_slice(value)) + .transpose() + .map_err(Into::into) +} + +fn read_text(data: &BTreeMap>, key: &str) -> Result> { + data.get(key) + .map(|value| { + String::from_utf8(value.clone()) + .map_err(|err| StoreError::Other(format!("stored text is not valid UTF-8: {err}"))) + }) + .transpose() +} + +fn build_run_summary( + record: &CatalogRecord, + data: &BTreeMap>, +) -> Result { + let run = read_json::(data, keys::run())?; + let start = read_json::(data, keys::start())?; + let status = read_json::(data, keys::status())?; + let conclusion = read_json::(data, keys::conclusion())?; + + let workflow_name = run.as_ref().map(|run| { + if run.graph.name.is_empty() { + "unnamed".to_string() + } else { + run.graph.name.clone() + } + }); + let goal = run.as_ref().and_then(|run| { + let goal = run.graph.goal(); + (!goal.is_empty()).then(|| goal.to_string()) + }); + + Ok(RunSummary { + run_id: record.run_id.clone(), + created_at: record.created_at, + db_prefix: record.db_prefix.clone(), + workflow_name, + workflow_slug: run.as_ref().and_then(|run| run.workflow_slug.clone()), + goal, + labels: run + .as_ref() + .map(|run| run.labels.clone()) + .unwrap_or_default(), + host_repo_path: run.as_ref().and_then(|run| run.host_repo_path.clone()), + start_time: start.map(|start| start.start_time), + status: status.as_ref().map(|status| status.status), + status_reason: status.and_then(|status| status.reason), + duration_ms: conclusion.as_ref().map(|conclusion| conclusion.duration_ms), + total_cost: conclusion.and_then(|conclusion| conclusion.total_cost), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + use std::path::PathBuf; + use std::time::Duration; + + use chrono::Duration as ChronoDuration; + use fabro_types::{AttrValue, FabroSettings, Graph, RunStatus, StageStatus, StatusReason}; + fn dt(rfc3339: &str) -> DateTime { + DateTime::parse_from_rfc3339(rfc3339) + .unwrap() + .with_timezone(&Utc) + } + + fn sample_run_record(run_id: &str, created_at: DateTime) -> RunRecord { + let mut graph = Graph::new("night-sky"); + graph.attrs.insert( + "goal".to_string(), + AttrValue::String("map the constellations".to_string()), + ); + RunRecord { + run_id: run_id.to_string(), + created_at, + settings: FabroSettings::default(), + graph, + workflow_slug: Some("night-sky".to_string()), + working_directory: PathBuf::from("/tmp/night-sky"), + host_repo_path: Some("github.com/fabro-sh/fabro".to_string()), + base_branch: Some("main".to_string()), + labels: HashMap::from([("team".to_string(), "infra".to_string())]), + } + } + + fn sample_start_record(run_id: &str, created_at: DateTime) -> StartRecord { + StartRecord { + run_id: run_id.to_string(), + start_time: created_at + ChronoDuration::seconds(5), + run_branch: Some("fabro/run/demo".to_string()), + base_sha: Some("abc123".to_string()), + } + } + + fn sample_status(status: RunStatus, reason: Option) -> RunStatusRecord { + RunStatusRecord { + status, + reason, + updated_at: dt("2026-03-27T12:05:00Z"), + } + } + + fn sample_checkpoint() -> Checkpoint { + Checkpoint { + timestamp: dt("2026-03-27T12:10:00Z"), + current_node: "code".to_string(), + completed_nodes: vec!["plan".to_string()], + node_retries: HashMap::from([("code".to_string(), 1)]), + context_values: HashMap::from([( + "artifact".to_string(), + serde_json::json!({"kind": "summary"}), + )]), + node_outcomes: HashMap::new(), + next_node_id: Some("review".to_string()), + git_commit_sha: Some("def456".to_string()), + loop_failure_signatures: HashMap::new(), + restart_failure_signatures: HashMap::new(), + node_visits: HashMap::from([("code".to_string(), 2)]), + } + } + + fn sample_conclusion() -> Conclusion { + Conclusion { + timestamp: dt("2026-03-27T12:15:00Z"), + status: StageStatus::Success, + duration_ms: 3210, + failure_reason: None, + final_git_commit_sha: Some("feedbeef".to_string()), + stages: Vec::new(), + total_cost: Some(1.25), + total_retries: 2, + total_input_tokens: 10, + total_output_tokens: 20, + total_cache_read_tokens: 30, + total_cache_write_tokens: 40, + total_reasoning_tokens: 50, + has_pricing: true, + } + } + + fn sample_retro(run_id: &str) -> Retro { + Retro { + run_id: run_id.to_string(), + workflow_name: "night-sky".to_string(), + goal: "map the constellations".to_string(), + timestamp: dt("2026-03-27T12:20:00Z"), + smoothness: None, + stages: Vec::new(), + stats: fabro_types::AggregateStats { + total_duration_ms: 3210, + total_cost: Some(1.25), + total_retries: 2, + files_touched: vec!["src/lib.rs".to_string()], + stages_completed: 3, + stages_failed: 0, + }, + intent: Some("ship the fix".to_string()), + outcome: Some("done".to_string()), + learnings: None, + friction_points: None, + open_items: None, + } + } + + fn sample_sandbox() -> SandboxRecord { + SandboxRecord { + provider: "local".to_string(), + working_directory: "/tmp/night-sky".to_string(), + identifier: Some("sandbox-1".to_string()), + host_working_directory: Some("/tmp/night-sky".to_string()), + container_mount_point: None, + data_host: None, + } + } + + fn sample_node_status() -> NodeStatusRecord { + NodeStatusRecord { + status: StageStatus::PartialSuccess, + notes: Some("captured output".to_string()), + failure_reason: Some("minor lint".to_string()), + timestamp: dt("2026-03-27T12:12:00Z"), + } + } + + #[tokio::test] + async fn create_run_put_get_and_snapshot_round_trip() { + let store = InMemoryStore::default(); + let created_at = dt("2026-03-27T12:00:00Z"); + let run = store.create_run("run-1", created_at).await.unwrap(); + + let run_record = sample_run_record("run-1", created_at); + let start_record = sample_start_record("run-1", created_at); + let status_record = + sample_status(RunStatus::Running, Some(StatusReason::SandboxInitializing)); + let checkpoint = sample_checkpoint(); + let conclusion = sample_conclusion(); + let retro = sample_retro("run-1"); + let sandbox = sample_sandbox(); + let node = NodeVisitRef { + node_id: "code", + visit: 2, + }; + let node_status = sample_node_status(); + + run.put_run(&run_record).await.unwrap(); + run.put_start(&start_record).await.unwrap(); + run.put_status(&status_record).await.unwrap(); + run.put_checkpoint(&checkpoint).await.unwrap(); + run.put_conclusion(&conclusion).await.unwrap(); + run.put_retro(&retro).await.unwrap(); + run.put_graph("digraph night_sky {}").await.unwrap(); + run.put_sandbox(&sandbox).await.unwrap(); + run.put_node_prompt(&node, "Plan the fix").await.unwrap(); + run.put_node_response(&node, "Implemented").await.unwrap(); + run.put_node_status(&node, &node_status).await.unwrap(); + run.put_node_stdout(&node, "ok").await.unwrap(); + run.put_node_stderr(&node, "").await.unwrap(); + run.put_retro_prompt("How did it go?").await.unwrap(); + run.put_retro_response("Smooth enough").await.unwrap(); + run.put_artifact_value("summary", &serde_json::json!({"done": true})) + .await + .unwrap(); + run.put_asset(&node, "src/lib.rs", b"fn main() {}") + .await + .unwrap(); + + let stored_run = run.get_run().await.unwrap().unwrap(); + assert_eq!(stored_run.run_id, run_record.run_id); + assert_eq!(stored_run.created_at, run_record.created_at); + assert_eq!(stored_run.workflow_slug, run_record.workflow_slug); + assert_eq!(stored_run.graph.name, run_record.graph.name); + + let stored_start = run.get_start().await.unwrap().unwrap(); + assert_eq!(stored_start.run_id, start_record.run_id); + assert_eq!(stored_start.start_time, start_record.start_time); + + let stored_status = run.get_status().await.unwrap().unwrap(); + assert_eq!(stored_status.status, status_record.status); + assert_eq!(stored_status.reason, status_record.reason); + + let stored_checkpoint = run.get_checkpoint().await.unwrap().unwrap(); + assert_eq!(stored_checkpoint.current_node, checkpoint.current_node); + assert_eq!(stored_checkpoint.next_node_id, checkpoint.next_node_id); + + let stored_conclusion = run.get_conclusion().await.unwrap().unwrap(); + assert_eq!(stored_conclusion.status, conclusion.status); + assert_eq!(stored_conclusion.duration_ms, conclusion.duration_ms); + assert_eq!(stored_conclusion.total_cost, conclusion.total_cost); + + let stored_retro = run.get_retro().await.unwrap().unwrap(); + assert_eq!(stored_retro.run_id, retro.run_id); + assert_eq!(stored_retro.intent, retro.intent); + assert_eq!( + run.get_graph().await.unwrap(), + Some("digraph night_sky {}".to_string()) + ); + let stored_sandbox = run.get_sandbox().await.unwrap().unwrap(); + assert_eq!(stored_sandbox.provider, sandbox.provider); + assert_eq!(stored_sandbox.working_directory, sandbox.working_directory); + assert_eq!( + run.get_retro_prompt().await.unwrap(), + Some("How did it go?".to_string()) + ); + assert_eq!( + run.get_retro_response().await.unwrap(), + Some("Smooth enough".to_string()) + ); + assert_eq!( + run.get_artifact_value("summary").await.unwrap(), + Some(serde_json::json!({"done": true})) + ); + assert_eq!( + run.get_asset(&node, "src/lib.rs").await.unwrap(), + Some(Bytes::from_static(b"fn main() {}")) + ); + assert_eq!( + run.list_assets(&node).await.unwrap(), + vec!["src/lib.rs".to_string()] + ); + + let snapshot = run.get_snapshot().await.unwrap().unwrap(); + assert_eq!(snapshot.run.run_id, run_record.run_id); + assert_eq!(snapshot.run.created_at, run_record.created_at); + assert_eq!( + snapshot + .checkpoint + .as_ref() + .map(|checkpoint| checkpoint.current_node.as_str()), + Some("code") + ); + assert_eq!( + snapshot + .conclusion + .as_ref() + .map(|conclusion| conclusion.duration_ms), + Some(3210) + ); + assert_eq!(snapshot.nodes.len(), 1); + assert_eq!(snapshot.nodes[0].node_id, "code"); + assert_eq!(snapshot.nodes[0].visit, 2); + let snapshot_status = snapshot.nodes[0].status.as_ref().unwrap(); + assert_eq!(snapshot_status.status, node_status.status); + assert_eq!(snapshot_status.failure_reason, node_status.failure_reason); + } + + #[tokio::test] + async fn append_event_validates_payload_shape_and_run_id() { + let store = InMemoryStore::default(); + let run = store + .create_run("run-1", dt("2026-03-27T12:00:00Z")) + .await + .unwrap(); + + let invalid_missing: EventPayload = serde_json::from_value(serde_json::json!({ + "run_id": "run-1" + })) + .unwrap(); + let err = run.append_event(&invalid_missing).await.unwrap_err(); + assert!(matches!(err, StoreError::InvalidEvent(_))); + + let invalid_run_id: EventPayload = serde_json::from_value(serde_json::json!({ + "ts": "2026-03-27T12:00:00Z", + "run_id": "other-run", + "event": "StageStarted" + })) + .unwrap(); + let err = run.append_event(&invalid_run_id).await.unwrap_err(); + assert!(matches!(err, StoreError::InvalidEvent(_))); + } + + #[tokio::test] + async fn put_run_rejects_created_at_mismatch() { + let store = InMemoryStore::default(); + let created_at = dt("2026-03-27T12:00:00Z"); + let run = store.create_run("run-1", created_at).await.unwrap(); + let err = run + .put_run(&sample_run_record( + "run-1", + created_at + ChronoDuration::minutes(1), + )) + .await + .unwrap_err(); + assert!(matches!(err, StoreError::Other(_))); + } + + #[tokio::test] + async fn watch_events_from_receives_existing_and_live_events() { + let store = InMemoryStore::default(); + let run = store + .create_run("run-1", dt("2026-03-27T12:00:00Z")) + .await + .unwrap(); + let first = EventPayload::new( + serde_json::json!({ + "ts": "2026-03-27T12:00:00.000Z", + "run_id": "run-1", + "event": "WorkflowRunStarted" + }), + "run-1", + ) + .unwrap(); + let second = EventPayload::new( + serde_json::json!({ + "ts": "2026-03-27T12:00:01.000Z", + "run_id": "run-1", + "event": "StageCompleted" + }), + "run-1", + ) + .unwrap(); + + run.append_event(&first).await.unwrap(); + + let mut stream = run.watch_events_from(1).await.unwrap(); + let existing = tokio::time::timeout( + Duration::from_secs(1), + futures::StreamExt::next(&mut stream), + ) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(existing.seq, 1); + + run.append_event(&second).await.unwrap(); + let live = tokio::time::timeout( + Duration::from_secs(1), + futures::StreamExt::next(&mut stream), + ) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(live.seq, 2); + } + + #[tokio::test] + async fn checkpoint_history_round_trips() { + let store = InMemoryStore::default(); + let run = store + .create_run("run-1", dt("2026-03-27T12:00:00Z")) + .await + .unwrap(); + let checkpoint = sample_checkpoint(); + let seq = run.append_checkpoint(&checkpoint).await.unwrap(); + assert_eq!(seq, 1); + let checkpoints = run.list_checkpoints().await.unwrap(); + assert_eq!(checkpoints.len(), 1); + assert_eq!(checkpoints[0].0, 1); + assert_eq!(checkpoints[0].1.current_node, checkpoint.current_node); + + let latest = run.get_checkpoint().await.unwrap().unwrap(); + assert_eq!(latest.current_node, checkpoint.current_node); + } + + #[tokio::test] + async fn node_visit_storage_round_trips() { + let store = InMemoryStore::default(); + let run = store + .create_run("run-1", dt("2026-03-27T12:00:00Z")) + .await + .unwrap(); + + let first = NodeVisitRef { + node_id: "code", + visit: 1, + }; + let second = NodeVisitRef { + node_id: "code", + visit: 2, + }; + run.put_node_prompt(&first, "first").await.unwrap(); + run.put_node_prompt(&second, "second").await.unwrap(); + run.put_node_status(&second, &sample_node_status()) + .await + .unwrap(); + + let node = run.get_node(&second).await.unwrap(); + assert_eq!(node.prompt, Some("second".to_string())); + assert_eq!(run.list_node_visits("code").await.unwrap(), vec![1, 2]); + } + + #[tokio::test] + async fn list_runs_filters_dates_and_tolerates_missing_status() { + let store = InMemoryStore::default(); + let early = dt("2026-03-27T10:00:00Z"); + let late = dt("2026-03-27T12:00:00Z"); + + let early_run = store.create_run("run-early", early).await.unwrap(); + early_run + .put_run(&sample_run_record("run-early", early)) + .await + .unwrap(); + + let late_run = store.create_run("run-late", late).await.unwrap(); + late_run + .put_run(&sample_run_record("run-late", late)) + .await + .unwrap(); + late_run + .put_start(&sample_start_record("run-late", late)) + .await + .unwrap(); + late_run + .put_status(&sample_status( + RunStatus::Succeeded, + Some(StatusReason::Completed), + )) + .await + .unwrap(); + late_run.put_conclusion(&sample_conclusion()).await.unwrap(); + + let all = store.list_runs(&ListRunsQuery::default()).await.unwrap(); + assert_eq!(all.len(), 2); + assert_eq!(all[0].run_id, "run-late"); + assert_eq!(all[0].workflow_name, Some("night-sky".to_string())); + assert_eq!(all[0].goal, Some("map the constellations".to_string())); + assert_eq!( + all[0].host_repo_path, + Some("github.com/fabro-sh/fabro".to_string()) + ); + assert_eq!(all[0].duration_ms, Some(3210)); + assert_eq!(all[0].total_cost, Some(1.25)); + assert_eq!(all[0].status_reason, Some(StatusReason::Completed)); + assert_eq!(all[1].status, None); + + let filtered = store + .list_runs(&ListRunsQuery { + start: Some(dt("2026-03-27T11:00:00Z")), + end: Some(dt("2026-03-27T13:00:00Z")), + }) + .await + .unwrap(); + assert_eq!(filtered.len(), 1); + assert_eq!(filtered[0].run_id, "run-late"); + } + + #[tokio::test] + async fn delete_run_is_idempotent() { + let store = InMemoryStore::default(); + store + .create_run("run-1", dt("2026-03-27T12:00:00Z")) + .await + .unwrap(); + store.delete_run("run-1").await.unwrap(); + store.delete_run("run-1").await.unwrap(); + assert!(store.open_run("run-1").await.unwrap().is_none()); + } +} diff --git a/lib/crates/fabro-store/src/slate/catalog.rs b/lib/crates/fabro-store/src/slate/catalog.rs new file mode 100644 index 000000000..f05601641 --- /dev/null +++ b/lib/crates/fabro-store/src/slate/catalog.rs @@ -0,0 +1,172 @@ +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use chrono::{DateTime, Utc}; +use futures::TryStreamExt; +use object_store::path::Path; +use object_store::ObjectStore; + +use crate::{CatalogRecord, ListRunsQuery, Result}; + +pub(crate) async fn write_catalog( + store: Arc, + base_prefix: &str, + run_id: &str, + created_at: DateTime, + db_prefix: &str, +) -> Result { + let record = CatalogRecord { + run_id: run_id.to_string(), + created_at, + db_prefix: db_prefix.to_string(), + }; + let bytes = serde_json::to_vec(&record)?; + store + .put(&by_id_path(base_prefix, run_id), bytes.clone().into()) + .await?; + store + .put( + &by_start_path(base_prefix, created_at, run_id), + bytes.into(), + ) + .await?; + Ok(record) +} + +pub(crate) async fn read_locator( + store: Arc, + base_prefix: &str, + run_id: &str, +) -> Result> { + read_catalog_path(store, by_id_path(base_prefix, run_id)).await +} + +pub(crate) async fn list_catalogs( + store: Arc, + base_prefix: &str, + query: &ListRunsQuery, +) -> Result> { + let prefix = Path::from(format!("{base_prefix}by-start")); + let metas = store.list(Some(&prefix)).try_collect::>().await?; + let mut records = Vec::new(); + for meta in metas { + let Some(record) = read_catalog_path(store.clone(), meta.location).await? else { + continue; + }; + if let Some(start) = query.start { + if record.created_at < start { + continue; + } + } + if let Some(end) = query.end { + if record.created_at > end { + continue; + } + } + records.push(record); + } + Ok(records) +} + +pub async fn repair_catalog(store: Arc, base_prefix: &str) -> Result<()> { + let by_id_prefix = Path::from(format!("{base_prefix}by-id")); + let by_start_prefix = Path::from(format!("{base_prefix}by-start")); + + let by_id_metas = store + .list(Some(&by_id_prefix)) + .try_collect::>() + .await?; + let mut canonical = HashMap::new(); + for meta in by_id_metas { + if let Some(record) = read_catalog_path(store.clone(), meta.location).await? { + canonical.insert(record.run_id.clone(), record); + } + } + + for record in canonical.values() { + let path = by_start_path(base_prefix, record.created_at, &record.run_id); + if !object_exists(store.clone(), &path).await? { + store.put(&path, serde_json::to_vec(record)?.into()).await?; + } + } + + let by_start_metas = store + .list(Some(&by_start_prefix)) + .try_collect::>() + .await?; + let mut seen = HashSet::new(); + for meta in by_start_metas { + let location = meta.location.clone(); + let Some(record) = read_catalog_path(store.clone(), location.clone()).await? else { + delete_if_exists(store.clone(), &location).await?; + continue; + }; + let expected = canonical.get(&record.run_id).map(|canonical_record| { + by_start_path(base_prefix, canonical_record.created_at, &record.run_id) + }); + match expected { + Some(expected) if expected == location => { + seen.insert(record.run_id); + } + _ => { + delete_if_exists(store.clone(), &location).await?; + } + } + } + + for record in canonical.values() { + if !seen.contains(&record.run_id) { + store + .put( + &by_start_path(base_prefix, record.created_at, &record.run_id), + serde_json::to_vec(record)?.into(), + ) + .await?; + } + } + Ok(()) +} + +pub(crate) fn db_prefix(base_prefix: &str, created_at: DateTime, run_id: &str) -> String { + format!( + "{base_prefix}db/{}/{run_id}/", + created_at.format("%Y-%m-%d-%H-%M") + ) +} + +pub(crate) fn by_id_path(base_prefix: &str, run_id: &str) -> Path { + Path::from(format!("{base_prefix}by-id/{run_id}.json")) +} + +pub(crate) fn by_start_path(base_prefix: &str, created_at: DateTime, run_id: &str) -> Path { + Path::from(format!( + "{base_prefix}by-start/{}/{run_id}.json", + created_at.format("%Y-%m-%d-%H-%M") + )) +} + +pub(crate) async fn read_catalog_path( + store: Arc, + path: Path, +) -> Result> { + match store.get(&path).await { + Ok(result) => Ok(Some(serde_json::from_slice(&result.bytes().await?)?)), + Err(object_store::Error::NotFound { .. }) => Ok(None), + Err(err) => Err(err.into()), + } +} + +async fn object_exists(store: Arc, path: &Path) -> Result { + match store.head(path).await { + Ok(_) => Ok(true), + Err(object_store::Error::NotFound { .. }) => Ok(false), + Err(err) => Err(err.into()), + } +} + +async fn delete_if_exists(store: Arc, path: &Path) -> Result<()> { + match store.delete(path).await { + Ok(()) | Err(object_store::Error::NotFound { .. }) => Ok(()), + Err(err) => Err(err.into()), + } +} diff --git a/lib/crates/fabro-store/src/slate/mod.rs b/lib/crates/fabro-store/src/slate/mod.rs new file mode 100644 index 000000000..7253d7354 --- /dev/null +++ b/lib/crates/fabro-store/src/slate/mod.rs @@ -0,0 +1,635 @@ +mod catalog; +mod run_store; + +use std::sync::Arc; + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use futures::TryStreamExt; +use object_store::path::Path; +use object_store::ObjectStore; + +use crate::keys; +use crate::{CatalogRecord, ListRunsQuery, Result, RunStore, RunSummary, Store, StoreError}; +use run_store::SlateRunStore; + +#[derive(Clone)] +pub struct SlateStore { + object_store: Arc, + base_prefix: String, +} + +impl std::fmt::Debug for SlateStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SlateStore") + .field("base_prefix", &self.base_prefix) + .finish() + } +} + +impl SlateStore { + pub fn new(object_store: Arc, base_prefix: impl Into) -> Self { + Self { + object_store, + base_prefix: normalize_base_prefix(base_prefix.into()), + } + } + + pub async fn repair_catalog(&self) -> Result<()> { + catalog::repair_catalog(self.object_store.clone(), &self.base_prefix).await + } + + async fn open_db(&self, db_prefix: &str) -> Result { + Ok(slatedb::Db::open(db_prefix.to_string(), self.object_store.clone()).await?) + } + + async fn open_run_store(&self, record: &CatalogRecord) -> Result> { + let db = self.open_db(&record.db_prefix).await?; + if !SlateRunStore::has_init(&db).await? { + let _ = db.close().await; + return Ok(None); + } + Ok(Some( + SlateRunStore::open(record.run_id.clone(), record.created_at, db).await?, + )) + } + + async fn delete_db_prefix(&self, db_prefix: &str) -> Result<()> { + let prefix = Path::from(db_prefix.to_string()); + let metas = self + .object_store + .list(Some(&prefix)) + .try_collect::>() + .await?; + for meta in metas { + delete_path(self.object_store.clone(), &meta.location).await?; + } + Ok(()) + } +} + +#[async_trait] +impl Store for SlateStore { + async fn create_run( + &self, + run_id: &str, + created_at: DateTime, + ) -> Result> { + let locator = + catalog::read_locator(self.object_store.clone(), &self.base_prefix, run_id).await?; + let db_prefix = match locator { + Some(existing) if existing.created_at != created_at => { + return Err(StoreError::RunAlreadyExists(run_id.to_string())); + } + Some(existing) => existing.db_prefix, + None => catalog::db_prefix(&self.base_prefix, created_at, run_id), + }; + + let record = CatalogRecord { + run_id: run_id.to_string(), + created_at, + db_prefix: db_prefix.clone(), + }; + + let db = self.open_db(&db_prefix).await?; + db.put(keys::init(), serde_json::to_vec(&record)?).await?; + catalog::write_catalog( + self.object_store.clone(), + &self.base_prefix, + run_id, + created_at, + &db_prefix, + ) + .await?; + Ok(Box::new( + SlateRunStore::open(run_id.to_string(), created_at, db).await?, + )) + } + + async fn open_run(&self, run_id: &str) -> Result>> { + let Some(locator) = + catalog::read_locator(self.object_store.clone(), &self.base_prefix, run_id).await? + else { + return Ok(None); + }; + + let Some(run_store) = self.open_run_store(&locator).await? else { + return Ok(None); + }; + Ok(Some(Box::new(run_store))) + } + + async fn list_runs(&self, query: &ListRunsQuery) -> Result> { + let catalogs = + catalog::list_catalogs(self.object_store.clone(), &self.base_prefix, query).await?; + let mut summaries = Vec::new(); + for record in catalogs { + let db = self.open_db(&record.db_prefix).await?; + if !SlateRunStore::has_init(&db).await? { + let _ = db.close().await; + continue; + } + let summary = SlateRunStore::build_summary(&db, &record).await?; + let _ = db.close().await; + summaries.push(summary); + } + summaries.sort_by(|a, b| b.created_at.cmp(&a.created_at)); + Ok(summaries) + } + + async fn delete_run(&self, run_id: &str) -> Result<()> { + if let Some(locator) = + catalog::read_locator(self.object_store.clone(), &self.base_prefix, run_id).await? + { + delete_path( + self.object_store.clone(), + &catalog::by_start_path(&self.base_prefix, locator.created_at, run_id), + ) + .await?; + self.delete_db_prefix(&locator.db_prefix).await?; + delete_path( + self.object_store.clone(), + &catalog::by_id_path(&self.base_prefix, run_id), + ) + .await?; + return Ok(()); + } + + let by_start_prefix = Path::from(format!("{}by-start", self.base_prefix)); + let metas = self + .object_store + .list(Some(&by_start_prefix)) + .try_collect::>() + .await?; + let expected_name = format!("{run_id}.json"); + for meta in metas { + if meta.location.filename() != Some(expected_name.as_str()) { + continue; + } + let Some(record) = + catalog::read_catalog_path(self.object_store.clone(), meta.location.clone()) + .await? + else { + delete_path(self.object_store.clone(), &meta.location).await?; + continue; + }; + self.delete_db_prefix(&record.db_prefix).await?; + delete_path(self.object_store.clone(), &meta.location).await?; + } + Ok(()) + } +} + +async fn delete_path(store: Arc, path: &Path) -> Result<()> { + match store.delete(path).await { + Ok(()) | Err(object_store::Error::NotFound { .. }) => Ok(()), + Err(err) => Err(err.into()), + } +} + +pub(crate) fn normalize_base_prefix(prefix: String) -> String { + if prefix.is_empty() { + return String::new(); + } + if prefix.ends_with('/') { + prefix + } else { + format!("{prefix}/") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use std::path::PathBuf; + use std::time::Duration; + + use bytes::Bytes; + use fabro_types::{ + AttrValue, Checkpoint, Conclusion, FabroSettings, Graph, NodeStatusRecord, RunRecord, + RunStatus, RunStatusRecord, StageStatus, StartRecord, StatusReason, + }; + use object_store::memory::InMemory; + + use crate::{EventPayload, NodeVisitRef}; + + fn dt(rfc3339: &str) -> DateTime { + DateTime::parse_from_rfc3339(rfc3339) + .unwrap() + .with_timezone(&Utc) + } + + fn make_store() -> (Arc, SlateStore) { + let object_store: Arc = Arc::new(InMemory::new()); + let store = SlateStore::new(object_store.clone(), "runs/"); + (object_store, store) + } + + fn sample_run_record(run_id: &str, created_at: DateTime) -> RunRecord { + let mut graph = Graph::new("night-sky"); + graph.attrs.insert( + "goal".to_string(), + AttrValue::String("map the constellations".to_string()), + ); + RunRecord { + run_id: run_id.to_string(), + created_at, + settings: FabroSettings::default(), + graph, + workflow_slug: Some("night-sky".to_string()), + working_directory: PathBuf::from("/tmp/night-sky"), + host_repo_path: Some("github.com/fabro-sh/fabro".to_string()), + base_branch: Some("main".to_string()), + labels: std::collections::HashMap::from([("team".to_string(), "infra".to_string())]), + } + } + + fn sample_start_record(run_id: &str, created_at: DateTime) -> StartRecord { + StartRecord { + run_id: run_id.to_string(), + start_time: created_at + chrono::Duration::seconds(5), + run_branch: Some("fabro/run/demo".to_string()), + base_sha: Some("abc123".to_string()), + } + } + + fn sample_status(status: RunStatus, reason: Option) -> RunStatusRecord { + RunStatusRecord { + status, + reason, + updated_at: dt("2026-03-27T12:05:00Z"), + } + } + + fn sample_checkpoint() -> Checkpoint { + Checkpoint { + timestamp: dt("2026-03-27T12:10:00Z"), + current_node: "code".to_string(), + completed_nodes: vec!["plan".to_string()], + node_retries: std::collections::HashMap::from([("code".to_string(), 1)]), + context_values: std::collections::HashMap::new(), + node_outcomes: std::collections::HashMap::new(), + next_node_id: Some("review".to_string()), + git_commit_sha: Some("def456".to_string()), + loop_failure_signatures: std::collections::HashMap::new(), + restart_failure_signatures: std::collections::HashMap::new(), + node_visits: std::collections::HashMap::from([("code".to_string(), 2)]), + } + } + + fn sample_conclusion() -> Conclusion { + Conclusion { + timestamp: dt("2026-03-27T12:15:00Z"), + status: StageStatus::Success, + duration_ms: 3210, + failure_reason: None, + final_git_commit_sha: Some("feedbeef".to_string()), + stages: Vec::new(), + total_cost: Some(1.25), + total_retries: 2, + total_input_tokens: 10, + total_output_tokens: 20, + total_cache_read_tokens: 30, + total_cache_write_tokens: 40, + total_reasoning_tokens: 50, + has_pricing: true, + } + } + + fn sample_node_status() -> NodeStatusRecord { + NodeStatusRecord { + status: StageStatus::Success, + notes: Some("done".to_string()), + failure_reason: None, + timestamp: dt("2026-03-27T12:12:00Z"), + } + } + + fn event_payload(run_id: &str, ts: &str, event: &str) -> EventPayload { + EventPayload::new( + serde_json::json!({ + "ts": ts, + "run_id": run_id, + "event": event + }), + run_id, + ) + .unwrap() + } + + async fn list_paths(store: Arc, prefix: &str) -> Vec { + let mut items = store + .list(Some(&Path::from(prefix.to_string()))) + .map_ok(|meta| meta.location.to_string()) + .try_collect::>() + .await + .unwrap(); + items.sort(); + items + } + + async fn object_exists(store: Arc, path: &Path) -> bool { + store.head(path).await.is_ok() + } + + async fn seed_db( + object_store: Arc, + record: &CatalogRecord, + include_init: bool, + ) -> slatedb::Db { + let db = slatedb::Db::open(record.db_prefix.clone(), object_store) + .await + .unwrap(); + if include_init { + db.put(keys::init(), serde_json::to_vec(record).unwrap()) + .await + .unwrap(); + } + db + } + + #[tokio::test] + async fn create_open_list_and_delete_full_lifecycle() { + let (object_store, store) = make_store(); + let created_at = dt("2026-03-27T12:00:00Z"); + let run = store.create_run("run-1", created_at).await.unwrap(); + + run.put_run(&sample_run_record("run-1", created_at)) + .await + .unwrap(); + run.put_start(&sample_start_record("run-1", created_at)) + .await + .unwrap(); + run.put_status(&sample_status( + RunStatus::Succeeded, + Some(StatusReason::Completed), + )) + .await + .unwrap(); + run.put_conclusion(&sample_conclusion()).await.unwrap(); + + let by_id = catalog::by_id_path("runs/", "run-1"); + let by_start = catalog::by_start_path("runs/", created_at, "run-1"); + assert!(object_exists(object_store.clone(), &by_id).await); + assert!(object_exists(object_store.clone(), &by_start).await); + + let summary = store.list_runs(&ListRunsQuery::default()).await.unwrap(); + assert_eq!(summary.len(), 1); + assert_eq!(summary[0].run_id, "run-1"); + assert_eq!(summary[0].workflow_name, Some("night-sky".to_string())); + assert_eq!(summary[0].goal, Some("map the constellations".to_string())); + assert_eq!(summary[0].status, Some(RunStatus::Succeeded)); + assert_eq!(summary[0].status_reason, Some(StatusReason::Completed)); + + let reopened = store.open_run("run-1").await.unwrap().unwrap(); + let stored = reopened.get_run().await.unwrap().unwrap(); + assert_eq!(stored.run_id, "run-1"); + + store.delete_run("run-1").await.unwrap(); + assert!(store.open_run("run-1").await.unwrap().is_none()); + assert!(!object_exists(object_store.clone(), &by_id).await); + assert!(!object_exists(object_store.clone(), &by_start).await); + assert!(list_paths(object_store, "runs/db").await.is_empty()); + } + + #[tokio::test] + async fn by_id_without_by_start_opens_but_is_omitted_from_list_and_repair_restores_index() { + let (object_store, store) = make_store(); + let created_at = dt("2026-03-27T12:00:00Z"); + let record = CatalogRecord { + run_id: "run-1".to_string(), + created_at, + db_prefix: catalog::db_prefix("runs/", created_at, "run-1"), + }; + + let db = seed_db(object_store.clone(), &record, true).await; + db.put( + keys::run(), + serde_json::to_vec(&sample_run_record("run-1", created_at)).unwrap(), + ) + .await + .unwrap(); + db.close().await.unwrap(); + + object_store + .put( + &catalog::by_id_path("runs/", "run-1"), + serde_json::to_vec(&record).unwrap().into(), + ) + .await + .unwrap(); + + assert!(store.open_run("run-1").await.unwrap().is_some()); + assert!(store + .list_runs(&ListRunsQuery::default()) + .await + .unwrap() + .is_empty()); + + store.repair_catalog().await.unwrap(); + let listed = store.list_runs(&ListRunsQuery::default()).await.unwrap(); + assert_eq!(listed.len(), 1); + assert!( + object_exists( + object_store, + &catalog::by_start_path("runs/", created_at, "run-1") + ) + .await + ); + } + + #[tokio::test] + async fn reopen_recovers_event_and_checkpoint_sequences() { + let (_object_store, store) = make_store(); + let created_at = dt("2026-03-27T12:00:00Z"); + let run = store.create_run("run-1", created_at).await.unwrap(); + run.put_run(&sample_run_record("run-1", created_at)) + .await + .unwrap(); + run.append_event(&event_payload("run-1", "2026-03-27T12:00:00Z", "Started")) + .await + .unwrap(); + run.append_event(&event_payload("run-1", "2026-03-27T12:00:01Z", "Next")) + .await + .unwrap(); + run.append_checkpoint(&sample_checkpoint()).await.unwrap(); + drop(run); + + let reopened = store.open_run("run-1").await.unwrap().unwrap(); + let next_event = reopened + .append_event(&event_payload( + "run-1", + "2026-03-27T12:00:02Z", + "AfterReopen", + )) + .await + .unwrap(); + let next_checkpoint = reopened + .append_checkpoint(&sample_checkpoint()) + .await + .unwrap(); + assert_eq!(next_event, 3); + assert_eq!(next_checkpoint, 2); + } + + #[tokio::test] + async fn open_run_and_list_runs_skip_empty_databases_without_init() { + let (object_store, store) = make_store(); + let created_at = dt("2026-03-27T12:00:00Z"); + let record = CatalogRecord { + run_id: "run-1".to_string(), + created_at, + db_prefix: catalog::db_prefix("runs/", created_at, "run-1"), + }; + + let db = seed_db(object_store.clone(), &record, false).await; + db.close().await.unwrap(); + object_store + .put( + &catalog::by_id_path("runs/", "run-1"), + serde_json::to_vec(&record).unwrap().into(), + ) + .await + .unwrap(); + object_store + .put( + &catalog::by_start_path("runs/", created_at, "run-1"), + serde_json::to_vec(&record).unwrap().into(), + ) + .await + .unwrap(); + + assert!(store.open_run("run-1").await.unwrap().is_none()); + assert!(store + .list_runs(&ListRunsQuery::default()) + .await + .unwrap() + .is_empty()); + } + + #[tokio::test] + async fn create_run_allows_idempotent_retry_and_rejects_conflict() { + let (_object_store, store) = make_store(); + let created_at = dt("2026-03-27T12:00:00Z"); + store.create_run("run-1", created_at).await.unwrap(); + store.create_run("run-1", created_at).await.unwrap(); + + let conflict = store + .create_run("run-1", created_at + chrono::Duration::seconds(1)) + .await; + assert!(matches!(conflict, Err(StoreError::RunAlreadyExists(_)))); + } + + #[tokio::test] + async fn watch_events_from_polls_new_events() { + let (_object_store, store) = make_store(); + let created_at = dt("2026-03-27T12:00:00Z"); + let run = store.create_run("run-1", created_at).await.unwrap(); + let mut stream = run.watch_events_from(1).await.unwrap(); + + run.append_event(&event_payload("run-1", "2026-03-27T12:00:00Z", "Started")) + .await + .unwrap(); + + let event = tokio::time::timeout( + Duration::from_secs(2), + futures::StreamExt::next(&mut stream), + ) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(event.seq, 1); + } + + #[tokio::test] + async fn delete_run_is_idempotent_and_fallback_cleans_by_start_orphans() { + let (object_store, store) = make_store(); + let created_at = dt("2026-03-27T12:00:00Z"); + let record = CatalogRecord { + run_id: "run-1".to_string(), + created_at, + db_prefix: catalog::db_prefix("runs/", created_at, "run-1"), + }; + let db = seed_db(object_store.clone(), &record, true).await; + db.put( + keys::run(), + serde_json::to_vec(&sample_run_record("run-1", created_at)).unwrap(), + ) + .await + .unwrap(); + db.close().await.unwrap(); + object_store + .put( + &catalog::by_start_path("runs/", created_at, "run-1"), + serde_json::to_vec(&record).unwrap().into(), + ) + .await + .unwrap(); + + store.delete_run("run-1").await.unwrap(); + store.delete_run("run-1").await.unwrap(); + assert!(list_paths(object_store, "runs").await.is_empty()); + } + + #[tokio::test] + async fn repair_catalog_removes_stale_wrong_time_prefixes() { + let (object_store, store) = make_store(); + let created_at = dt("2026-03-27T12:00:00Z"); + let wrong_time = dt("2026-03-27T11:00:00Z"); + let run = store.create_run("run-1", created_at).await.unwrap(); + run.put_run(&sample_run_record("run-1", created_at)) + .await + .unwrap(); + + let locator = catalog::read_locator(object_store.clone(), "runs/", "run-1") + .await + .unwrap() + .unwrap(); + object_store + .put( + &catalog::by_start_path("runs/", wrong_time, "run-1"), + serde_json::to_vec(&locator).unwrap().into(), + ) + .await + .unwrap(); + + store.repair_catalog().await.unwrap(); + let paths = list_paths(object_store, "runs/by-start").await; + assert_eq!(paths.len(), 1); + assert!(paths[0].contains("2026-03-27-12-00/run-1.json")); + } + + #[tokio::test] + async fn slate_run_store_round_trips_node_data_and_assets() { + let (_object_store, store) = make_store(); + let created_at = dt("2026-03-27T12:00:00Z"); + let run = store.create_run("run-1", created_at).await.unwrap(); + run.put_run(&sample_run_record("run-1", created_at)) + .await + .unwrap(); + let node = NodeVisitRef { + node_id: "code", + visit: 2, + }; + run.put_node_prompt(&node, "Plan").await.unwrap(); + run.put_node_status(&node, &sample_node_status()) + .await + .unwrap(); + run.put_asset(&node, "src/lib.rs", b"fn main() {}") + .await + .unwrap(); + + let snapshot = run.get_node(&node).await.unwrap(); + assert_eq!(snapshot.prompt, Some("Plan".to_string())); + assert_eq!( + run.get_asset(&node, "src/lib.rs").await.unwrap(), + Some(Bytes::from_static(b"fn main() {}")) + ); + assert_eq!( + run.list_assets(&node).await.unwrap(), + vec!["src/lib.rs".to_string()] + ); + } +} diff --git a/lib/crates/fabro-store/src/slate/run_store.rs b/lib/crates/fabro-store/src/slate/run_store.rs new file mode 100644 index 000000000..0038f6bf9 --- /dev/null +++ b/lib/crates/fabro-store/src/slate/run_store.rs @@ -0,0 +1,481 @@ +use std::collections::BTreeSet; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::time::Duration; + +use async_trait::async_trait; +use bytes::Bytes; +use chrono::{DateTime, Utc}; +use futures::Stream; +use serde::de::DeserializeOwned; +use serde::Serialize; +use tokio::sync::mpsc; +use tokio_stream::wrappers::UnboundedReceiverStream; + +use crate::keys; +use crate::{ + CatalogRecord, EventEnvelope, EventPayload, NodeSnapshot, NodeVisitRef, Result, RunSnapshot, + RunStore, RunSummary, StoreError, +}; +use fabro_types::{ + Checkpoint, Conclusion, NodeStatusRecord, Retro, RunRecord, RunStatusRecord, SandboxRecord, + StartRecord, +}; + +pub(crate) struct SlateRunStore { + run_id: String, + created_at: DateTime, + db: slatedb::Db, + event_seq: AtomicU32, + checkpoint_seq: AtomicU32, +} + +impl SlateRunStore { + pub(crate) async fn open( + run_id: String, + created_at: DateTime, + db: slatedb::Db, + ) -> Result { + let event_seq = recover_next_seq(&db, keys::EVENTS_PREFIX, keys::parse_event_seq).await?; + let checkpoint_seq = + recover_next_seq(&db, keys::CHECKPOINTS_PREFIX, keys::parse_checkpoint_seq).await?; + Ok(Self { + run_id, + created_at, + db, + event_seq: AtomicU32::new(event_seq), + checkpoint_seq: AtomicU32::new(checkpoint_seq), + }) + } + + pub(crate) async fn has_init(db: &slatedb::Db) -> Result { + Ok(db.get(keys::init()).await?.is_some()) + } + + pub(crate) async fn build_summary( + db: &slatedb::Db, + catalog: &CatalogRecord, + ) -> Result { + let run = get_json::(db, keys::run()).await?; + let start = get_json::(db, keys::start()).await?; + let status = get_json::(db, keys::status()).await?; + let conclusion = get_json::(db, keys::conclusion()).await?; + + let workflow_name = run.as_ref().map(|run| { + if run.graph.name.is_empty() { + "unnamed".to_string() + } else { + run.graph.name.clone() + } + }); + let goal = run.as_ref().and_then(|run| { + let goal = run.graph.goal(); + (!goal.is_empty()).then(|| goal.to_string()) + }); + + Ok(RunSummary { + run_id: catalog.run_id.clone(), + created_at: catalog.created_at, + db_prefix: catalog.db_prefix.clone(), + workflow_name, + workflow_slug: run.as_ref().and_then(|run| run.workflow_slug.clone()), + goal, + labels: run + .as_ref() + .map(|run| run.labels.clone()) + .unwrap_or_default(), + host_repo_path: run.as_ref().and_then(|run| run.host_repo_path.clone()), + start_time: start.map(|start| start.start_time), + status: status.as_ref().map(|status| status.status), + status_reason: status.and_then(|status| status.reason), + duration_ms: conclusion.as_ref().map(|conclusion| conclusion.duration_ms), + total_cost: conclusion.and_then(|conclusion| conclusion.total_cost), + }) + } + + fn validate_run_record(&self, record: &RunRecord) -> Result<()> { + if record.created_at != self.created_at { + return Err(StoreError::Other(format!( + "run record created_at {:?} does not match store created_at {:?}", + record.created_at, self.created_at + ))); + } + if record.run_id != self.run_id { + return Err(StoreError::Other(format!( + "run record run_id {:?} does not match store run_id {:?}", + record.run_id, self.run_id + ))); + } + Ok(()) + } + + async fn build_node_snapshot(&self, node: &NodeVisitRef<'_>) -> Result { + Ok(NodeSnapshot { + node_id: node.node_id.to_string(), + visit: node.visit, + prompt: get_text(&self.db, &keys::node_prompt(node)).await?, + response: get_text(&self.db, &keys::node_response(node)).await?, + status: get_json(&self.db, &keys::node_status(node)).await?, + stdout: get_text(&self.db, &keys::node_stdout(node)).await?, + stderr: get_text(&self.db, &keys::node_stderr(node)).await?, + }) + } +} + +#[async_trait] +impl RunStore for SlateRunStore { + async fn put_run(&self, record: &RunRecord) -> Result<()> { + self.validate_run_record(record)?; + put_json(&self.db, keys::run(), record).await + } + + async fn get_run(&self) -> Result> { + get_json(&self.db, keys::run()).await + } + + async fn put_start(&self, record: &StartRecord) -> Result<()> { + put_json(&self.db, keys::start(), record).await + } + + async fn get_start(&self) -> Result> { + get_json(&self.db, keys::start()).await + } + + async fn put_status(&self, record: &RunStatusRecord) -> Result<()> { + put_json(&self.db, keys::status(), record).await + } + + async fn get_status(&self) -> Result> { + get_json(&self.db, keys::status()).await + } + + async fn put_checkpoint(&self, record: &Checkpoint) -> Result<()> { + put_json(&self.db, keys::checkpoint(), record).await + } + + async fn get_checkpoint(&self) -> Result> { + get_json(&self.db, keys::checkpoint()).await + } + + async fn append_checkpoint(&self, record: &Checkpoint) -> Result { + let seq = self.checkpoint_seq.fetch_add(1, Ordering::SeqCst); + self.put_checkpoint(record).await?; + put_json( + &self.db, + &keys::checkpoint_history_key(seq, Utc::now().timestamp_millis()), + record, + ) + .await?; + Ok(seq) + } + + async fn list_checkpoints(&self) -> Result> { + list_checkpoints(&self.db).await + } + + async fn put_conclusion(&self, record: &Conclusion) -> Result<()> { + put_json(&self.db, keys::conclusion(), record).await + } + + async fn get_conclusion(&self) -> Result> { + get_json(&self.db, keys::conclusion()).await + } + + async fn put_retro(&self, retro: &Retro) -> Result<()> { + put_json(&self.db, keys::retro(), retro).await + } + + async fn get_retro(&self) -> Result> { + get_json(&self.db, keys::retro()).await + } + + async fn put_graph(&self, dot_source: &str) -> Result<()> { + put_text(&self.db, keys::graph(), dot_source).await + } + + async fn get_graph(&self) -> Result> { + get_text(&self.db, keys::graph()).await + } + + async fn put_sandbox(&self, record: &SandboxRecord) -> Result<()> { + put_json(&self.db, keys::sandbox(), record).await + } + + async fn get_sandbox(&self) -> Result> { + get_json(&self.db, keys::sandbox()).await + } + + async fn put_node_prompt(&self, node: &NodeVisitRef<'_>, prompt: &str) -> Result<()> { + put_text(&self.db, &keys::node_prompt(node), prompt).await + } + + async fn put_node_response(&self, node: &NodeVisitRef<'_>, response: &str) -> Result<()> { + put_text(&self.db, &keys::node_response(node), response).await + } + + async fn put_node_status( + &self, + node: &NodeVisitRef<'_>, + status: &NodeStatusRecord, + ) -> Result<()> { + put_json(&self.db, &keys::node_status(node), status).await + } + + async fn put_node_stdout(&self, node: &NodeVisitRef<'_>, log: &str) -> Result<()> { + put_text(&self.db, &keys::node_stdout(node), log).await + } + + async fn put_node_stderr(&self, node: &NodeVisitRef<'_>, log: &str) -> Result<()> { + put_text(&self.db, &keys::node_stderr(node), log).await + } + + async fn get_node(&self, node: &NodeVisitRef<'_>) -> Result { + self.build_node_snapshot(node).await + } + + async fn list_node_visits(&self, node_id: &str) -> Result> { + let prefix = format!("nodes/{node_id}/visit-"); + let mut iter = self.db.scan_prefix(prefix.as_bytes()).await?; + let mut visits = BTreeSet::new(); + while let Some(entry) = iter.next().await? { + let key = key_to_string(entry.key)?; + if let Some((current_node_id, visit, _)) = keys::parse_node_key(&key) { + if current_node_id == node_id { + visits.insert(visit); + } + } + } + Ok(visits.into_iter().collect()) + } + + async fn append_event(&self, payload: &EventPayload) -> Result { + payload.validate(&self.run_id)?; + let seq = self.event_seq.fetch_add(1, Ordering::SeqCst); + put_json( + &self.db, + &keys::event_key(seq, Utc::now().timestamp_millis()), + payload, + ) + .await?; + Ok(seq) + } + + async fn list_events(&self) -> Result> { + list_events_from(&self.db, 1).await + } + + async fn list_events_from(&self, seq: u32) -> Result> { + list_events_from(&self.db, seq).await + } + + async fn watch_events_from( + &self, + seq: u32, + ) -> Result> + Send>>> { + let db = self.db.clone(); + let (sender, receiver) = mpsc::unbounded_channel(); + + tokio::spawn(async move { + let mut next_seq = seq; + loop { + if sender.is_closed() { + return; + } + + match list_events_from(&db, next_seq).await { + Ok(events) => { + if events.is_empty() { + tokio::time::sleep(Duration::from_millis(100)).await; + continue; + } + for event in events { + next_seq = event.seq.saturating_add(1); + if sender.send(Ok(event)).is_err() { + return; + } + } + } + Err(err) => { + let _ = sender.send(Err(err)); + return; + } + } + } + }); + + Ok(Box::pin(UnboundedReceiverStream::new(receiver))) + } + + async fn put_retro_prompt(&self, text: &str) -> Result<()> { + put_text(&self.db, keys::retro_prompt(), text).await + } + + async fn get_retro_prompt(&self) -> Result> { + get_text(&self.db, keys::retro_prompt()).await + } + + async fn put_retro_response(&self, text: &str) -> Result<()> { + put_text(&self.db, keys::retro_response(), text).await + } + + async fn get_retro_response(&self) -> Result> { + get_text(&self.db, keys::retro_response()).await + } + + async fn put_artifact_value(&self, artifact_id: &str, value: &serde_json::Value) -> Result<()> { + put_json(&self.db, &keys::artifact_value(artifact_id), value).await + } + + async fn get_artifact_value(&self, artifact_id: &str) -> Result> { + get_json(&self.db, &keys::artifact_value(artifact_id)).await + } + + async fn put_asset(&self, node: &NodeVisitRef<'_>, filename: &str, data: &[u8]) -> Result<()> { + put_bytes(&self.db, &keys::node_asset(node, filename), data).await + } + + async fn get_asset(&self, node: &NodeVisitRef<'_>, filename: &str) -> Result> { + get_bytes(&self.db, &keys::node_asset(node, filename)).await + } + + async fn list_assets(&self, node: &NodeVisitRef<'_>) -> Result> { + let prefix = format!("{}/", keys::node_asset_prefix(node)); + let mut iter = self.db.scan_prefix(prefix.as_bytes()).await?; + let mut assets = Vec::new(); + while let Some(entry) = iter.next().await? { + let key = key_to_string(entry.key)?; + if let Some(asset) = key.strip_prefix(&prefix) { + assets.push(asset.to_string()); + } + } + assets.sort(); + Ok(assets) + } + + async fn get_snapshot(&self) -> Result> { + let Some(run) = self.get_run().await? else { + return Ok(None); + }; + + let mut iter = self.db.scan_prefix(b"nodes/").await?; + let mut visits = BTreeSet::new(); + while let Some(entry) = iter.next().await? { + let key = key_to_string(entry.key)?; + if let Some((node_id, visit, _)) = keys::parse_node_key(&key) { + visits.insert((node_id, visit)); + } + } + + let mut nodes = Vec::new(); + for (node_id, visit) in visits { + let node = NodeVisitRef { + node_id: &node_id, + visit, + }; + nodes.push(self.build_node_snapshot(&node).await?); + } + + Ok(Some(RunSnapshot { + run, + start: self.get_start().await?, + status: self.get_status().await?, + checkpoint: self.get_checkpoint().await?, + conclusion: self.get_conclusion().await?, + retro: self.get_retro().await?, + graph: self.get_graph().await?, + sandbox: self.get_sandbox().await?, + nodes, + })) + } +} + +async fn put_json(db: &slatedb::Db, key: &str, value: &T) -> Result<()> { + db.put(key, serde_json::to_vec(value)?).await?; + Ok(()) +} + +async fn get_json(db: &slatedb::Db, key: &str) -> Result> { + db.get(key) + .await? + .map(|value| serde_json::from_slice(&value)) + .transpose() + .map_err(Into::into) +} + +async fn put_text(db: &slatedb::Db, key: &str, value: &str) -> Result<()> { + db.put(key, value.as_bytes()).await?; + Ok(()) +} + +async fn get_text(db: &slatedb::Db, key: &str) -> Result> { + db.get(key) + .await? + .map(|value| { + String::from_utf8(value.to_vec()) + .map_err(|err| StoreError::Other(format!("stored text is not valid UTF-8: {err}"))) + }) + .transpose() +} + +async fn put_bytes(db: &slatedb::Db, key: &str, value: &[u8]) -> Result<()> { + db.put(key, value).await?; + Ok(()) +} + +async fn get_bytes(db: &slatedb::Db, key: &str) -> Result> { + Ok(db.get(key).await?) +} + +async fn recover_next_seq( + db: &slatedb::Db, + prefix: &str, + parse: fn(&str) -> Option, +) -> Result { + let mut iter = db.scan_prefix(prefix.as_bytes()).await?; + let mut max_seq = 0; + while let Some(entry) = iter.next().await? { + let key = key_to_string(entry.key)?; + if let Some(seq) = parse(&key) { + max_seq = max_seq.max(seq); + } + } + Ok(max_seq.saturating_add(1).max(1)) +} + +async fn list_events_from(db: &slatedb::Db, start_seq: u32) -> Result> { + let mut iter = db.scan_prefix(keys::EVENTS_PREFIX.as_bytes()).await?; + let mut events = Vec::new(); + while let Some(entry) = iter.next().await? { + let key = key_to_string(entry.key)?; + let Some(seq) = keys::parse_event_seq(&key) else { + continue; + }; + if seq < start_seq { + continue; + } + events.push(EventEnvelope { + seq, + payload: serde_json::from_slice(&entry.value)?, + }); + } + events.sort_by_key(|event| event.seq); + Ok(events) +} + +async fn list_checkpoints(db: &slatedb::Db) -> Result> { + let mut iter = db.scan_prefix(keys::CHECKPOINTS_PREFIX.as_bytes()).await?; + let mut checkpoints = Vec::new(); + while let Some(entry) = iter.next().await? { + let key = key_to_string(entry.key)?; + let Some(seq) = keys::parse_checkpoint_seq(&key) else { + continue; + }; + checkpoints.push((seq, serde_json::from_slice(&entry.value)?)); + } + checkpoints.sort_by_key(|(seq, _)| *seq); + Ok(checkpoints) +} + +fn key_to_string(key: Bytes) -> Result { + String::from_utf8(key.to_vec()) + .map_err(|err| StoreError::Other(format!("stored key is not valid UTF-8: {err}"))) +} diff --git a/lib/crates/fabro-store/src/types.rs b/lib/crates/fabro-store/src/types.rs new file mode 100644 index 000000000..a6ecad640 --- /dev/null +++ b/lib/crates/fabro-store/src/types.rs @@ -0,0 +1,117 @@ +use std::collections::HashMap; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +use crate::{Result, StoreError}; +use fabro_types::{ + Checkpoint, Conclusion, NodeStatusRecord, Retro, RunRecord, RunStatus, RunStatusRecord, + SandboxRecord, StartRecord, StatusReason, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct NodeVisitRef<'a> { + pub node_id: &'a str, + pub visit: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CatalogRecord { + pub run_id: String, + pub created_at: DateTime, + pub db_prefix: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RunSummary { + pub run_id: String, + pub created_at: DateTime, + pub db_prefix: String, + pub workflow_name: Option, + pub workflow_slug: Option, + pub goal: Option, + pub labels: HashMap, + pub host_repo_path: Option, + pub start_time: Option>, + pub status: Option, + pub status_reason: Option, + pub duration_ms: Option, + pub total_cost: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RunSnapshot { + pub run: RunRecord, + pub start: Option, + pub status: Option, + pub checkpoint: Option, + pub conclusion: Option, + pub retro: Option, + pub graph: Option, + pub sandbox: Option, + pub nodes: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NodeSnapshot { + pub node_id: String, + pub visit: u32, + pub prompt: Option, + pub response: Option, + pub status: Option, + pub stdout: Option, + pub stderr: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(transparent)] +pub struct EventPayload(serde_json::Value); + +impl EventPayload { + pub fn new(value: serde_json::Value, expected_run_id: &str) -> Result { + let payload = Self(value); + payload.validate(expected_run_id)?; + Ok(payload) + } + + pub fn validate(&self, expected_run_id: &str) -> Result<()> { + let obj = self.0.as_object().ok_or_else(|| { + StoreError::InvalidEvent("event payload must be a JSON object".into()) + })?; + + for field in ["ts", "run_id", "event"] { + match obj.get(field) { + Some(serde_json::Value::String(_)) => {} + _ => { + return Err(StoreError::InvalidEvent(format!( + "missing or non-string required field: {field}" + ))); + } + } + } + + match obj.get("run_id") { + Some(serde_json::Value::String(run_id)) if run_id == expected_run_id => Ok(()), + Some(serde_json::Value::String(run_id)) => Err(StoreError::InvalidEvent(format!( + "payload run_id {run_id:?} does not match store run_id {expected_run_id:?}" + ))), + _ => Err(StoreError::InvalidEvent( + "missing or non-string required field: run_id".into(), + )), + } + } + + pub fn into_inner(self) -> serde_json::Value { + self.0 + } + + pub fn as_value(&self) -> &serde_json::Value { + &self.0 + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct EventEnvelope { + pub seq: u32, + pub payload: EventPayload, +} diff --git a/lib/crates/fabro-types/src/lib.rs b/lib/crates/fabro-types/src/lib.rs index 817516dcb..92621a0b0 100644 --- a/lib/crates/fabro-types/src/lib.rs +++ b/lib/crates/fabro-types/src/lib.rs @@ -5,6 +5,7 @@ pub mod combine; pub mod conclusion; pub mod failure_signature; pub mod graph; +pub mod node_status; pub mod outcome; pub mod retro; pub mod run; @@ -18,6 +19,7 @@ pub use checkpoint::Checkpoint; pub use conclusion::{Conclusion, StageSummary}; pub use failure_signature::FailureSignature; pub use graph::{is_llm_handler_type, shape_to_handler_type, AttrValue, Edge, Graph, Node}; +pub use node_status::NodeStatusRecord; pub use outcome::{FailureCategory, FailureDetail, NodeResult, Outcome, OutcomeMeta, StageStatus}; pub use retro::{ AggregateStats, FrictionKind, FrictionPoint, Learning, LearningCategory, OpenItem, diff --git a/lib/crates/fabro-types/src/node_status.rs b/lib/crates/fabro-types/src/node_status.rs new file mode 100644 index 000000000..835968bcf --- /dev/null +++ b/lib/crates/fabro-types/src/node_status.rs @@ -0,0 +1,14 @@ +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +use crate::outcome::StageStatus; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NodeStatusRecord { + pub status: StageStatus, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub notes: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub failure_reason: Option, + pub timestamp: DateTime, +} diff --git a/lib/crates/fabro-workflows/src/run_dir.rs b/lib/crates/fabro-workflows/src/run_dir.rs index e070d719d..9bc2e402e 100644 --- a/lib/crates/fabro-workflows/src/run_dir.rs +++ b/lib/crates/fabro-workflows/src/run_dir.rs @@ -1,6 +1,7 @@ use std::path::{Path, PathBuf}; use chrono::Utc; +use fabro_types::NodeStatusRecord; use crate::context::Context; use crate::outcome::{Outcome, OutcomeExt}; @@ -50,12 +51,12 @@ pub fn visit_from_context(context: &Context) -> usize { pub(crate) fn write_node_status(run_dir: &Path, node_id: &str, visit: usize, outcome: &Outcome) { let node_dir = node_dir(run_dir, node_id, visit); let _ = std::fs::create_dir_all(&node_dir); - let status = serde_json::json!({ - "status": outcome.status.to_string(), - "notes": outcome.notes, - "failure_reason": outcome.failure_reason(), - "timestamp": Utc::now().to_rfc3339(), - }); + let status = NodeStatusRecord { + status: outcome.status.clone(), + notes: outcome.notes.clone(), + failure_reason: outcome.failure_reason().map(ToOwned::to_owned), + timestamp: Utc::now(), + }; if let Ok(json) = serde_json::to_string_pretty(&status) { let _ = std::fs::write(node_dir.join("status.json"), json); } @@ -66,6 +67,9 @@ mod tests { use super::*; use std::path::Path; + use fabro_types::StageStatus; + use tempfile::TempDir; + use crate::context::Context; #[test] @@ -107,4 +111,30 @@ mod tests { root.join("nodes").join("work-visit_5") ); } + + #[test] + fn write_node_status_uses_typed_record_with_legacy_shape() { + let temp = TempDir::new().unwrap(); + let outcome = Outcome { + status: StageStatus::Fail, + notes: Some("needs retry".to_string()), + failure: Some(crate::outcome::FailureDetail::new( + "boom", + crate::outcome::FailureCategory::Deterministic, + )), + ..Outcome::default() + }; + + write_node_status(temp.path(), "work", 1, &outcome); + + let data = std::fs::read_to_string(temp.path().join("nodes/work/status.json")).unwrap(); + let value: serde_json::Value = serde_json::from_str(&data).unwrap(); + assert_eq!(value.get("status"), Some(&serde_json::json!("fail"))); + assert_eq!(value.get("notes"), Some(&serde_json::json!("needs retry"))); + assert_eq!( + value.get("failure_reason"), + Some(&serde_json::json!("boom")) + ); + assert!(value.get("timestamp").and_then(|v| v.as_str()).is_some()); + } }