mirror of
https://github.com/fabro-sh/fabro.git
synced 2026-09-06 08:18:58 +00:00
Configure SlateDB flush_interval to 5ms for local storage
SlateDB defaults to 100ms flush_interval (tuned for S3 cost). For local/in-memory object stores this adds unnecessary write latency. Pass flush_interval through SlateStore::new() so callers control the setting, and switch open_db() to use Db::builder(). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
55088a00af
commit
200a580009
3 changed files with 43 additions and 8 deletions
|
|
@ -152,7 +152,11 @@ pub async fn serve_command(args: ServeArgs, styles: &'static Styles) -> anyhow::
|
|||
let store_path = data_dir.join("store");
|
||||
std::fs::create_dir_all(&store_path)?;
|
||||
let object_store = Arc::new(LocalFileSystem::new_with_prefix(&store_path)?);
|
||||
let store = Arc::new(fabro_store::SlateStore::new(object_store, ""));
|
||||
let store = Arc::new(fabro_store::SlateStore::new(
|
||||
object_store,
|
||||
"",
|
||||
Duration::from_millis(5),
|
||||
));
|
||||
let state = create_app_state_with_store(
|
||||
db,
|
||||
factory,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::Result;
|
||||
use fabro_store::{RunStore, SlateStore, Store};
|
||||
|
|
@ -9,7 +10,11 @@ pub(crate) fn build_store(storage_dir: &Path) -> Result<Arc<SlateStore>> {
|
|||
let store_path = storage_dir.join("store");
|
||||
std::fs::create_dir_all(&store_path)?;
|
||||
let object_store = Arc::new(LocalFileSystem::new_with_prefix(&store_path)?);
|
||||
Ok(Arc::new(SlateStore::new(object_store, "")))
|
||||
Ok(Arc::new(SlateStore::new(
|
||||
object_store,
|
||||
"",
|
||||
Duration::from_millis(5),
|
||||
)))
|
||||
}
|
||||
|
||||
pub(crate) async fn open_run_reader(
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ mod run_store;
|
|||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
|
|
@ -10,7 +11,7 @@ use futures::TryStreamExt;
|
|||
use object_store::ObjectStore;
|
||||
use object_store::path::Path;
|
||||
use slatedb::DbReader;
|
||||
use slatedb::config::DbReaderOptions;
|
||||
use slatedb::config::{DbReaderOptions, Settings};
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
use crate::keys;
|
||||
|
|
@ -21,6 +22,7 @@ use run_store::{SlateRunStore, SlateRunStoreInner};
|
|||
pub struct SlateStore {
|
||||
object_store: Arc<dyn ObjectStore>,
|
||||
base_prefix: String,
|
||||
flush_interval: Duration,
|
||||
active_runs: Arc<Mutex<HashMap<String, std::sync::Weak<SlateRunStoreInner>>>>,
|
||||
}
|
||||
|
||||
|
|
@ -28,15 +30,21 @@ impl std::fmt::Debug for SlateStore {
|
|||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("SlateStore")
|
||||
.field("base_prefix", &self.base_prefix)
|
||||
.field("flush_interval", &self.flush_interval)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl SlateStore {
|
||||
pub fn new(object_store: Arc<dyn ObjectStore>, base_prefix: impl Into<String>) -> Self {
|
||||
pub fn new(
|
||||
object_store: Arc<dyn ObjectStore>,
|
||||
base_prefix: impl Into<String>,
|
||||
flush_interval: Duration,
|
||||
) -> Self {
|
||||
Self {
|
||||
object_store,
|
||||
base_prefix: normalize_base_prefix(base_prefix.into()),
|
||||
flush_interval,
|
||||
active_runs: Arc::new(Mutex::new(HashMap::new())),
|
||||
}
|
||||
}
|
||||
|
|
@ -46,7 +54,15 @@ impl SlateStore {
|
|||
}
|
||||
|
||||
async fn open_db(&self, db_prefix: &str) -> Result<slatedb::Db> {
|
||||
Ok(slatedb::Db::open(db_prefix.to_string(), self.object_store.clone()).await?)
|
||||
Ok(
|
||||
slatedb::Db::builder(db_prefix.to_string(), self.object_store.clone())
|
||||
.with_settings(Settings {
|
||||
flush_interval: Some(self.flush_interval),
|
||||
..Settings::default()
|
||||
})
|
||||
.build()
|
||||
.await?,
|
||||
)
|
||||
}
|
||||
|
||||
async fn open_reader(&self, db_prefix: &str) -> Result<DbReader> {
|
||||
|
|
@ -382,7 +398,7 @@ mod tests {
|
|||
|
||||
fn make_store() -> (Arc<dyn ObjectStore>, SlateStore) {
|
||||
let object_store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
|
||||
let store = SlateStore::new(object_store.clone(), "runs/");
|
||||
let store = SlateStore::new(object_store.clone(), "runs/", Duration::from_millis(5));
|
||||
(object_store, store)
|
||||
}
|
||||
|
||||
|
|
@ -498,7 +514,12 @@ mod tests {
|
|||
record: &CatalogRecord,
|
||||
include_init: bool,
|
||||
) -> slatedb::Db {
|
||||
let db = slatedb::Db::open(record.db_prefix.clone(), object_store)
|
||||
let db = slatedb::Db::builder(record.db_prefix.clone(), object_store)
|
||||
.with_settings(slatedb::config::Settings {
|
||||
flush_interval: Some(Duration::from_millis(5)),
|
||||
..slatedb::config::Settings::default()
|
||||
})
|
||||
.build()
|
||||
.await
|
||||
.unwrap();
|
||||
if include_init {
|
||||
|
|
@ -858,7 +879,12 @@ mod tests {
|
|||
let (object_store, store) = make_store();
|
||||
let created_at = dt("2026-03-27T12:00:00Z");
|
||||
let db_prefix = catalog::db_prefix("runs/", created_at, "run-1");
|
||||
let db = slatedb::Db::open(db_prefix.clone(), object_store)
|
||||
let db = slatedb::Db::builder(db_prefix.clone(), object_store)
|
||||
.with_settings(slatedb::config::Settings {
|
||||
flush_interval: Some(Duration::from_millis(5)),
|
||||
..slatedb::config::Settings::default()
|
||||
})
|
||||
.build()
|
||||
.await
|
||||
.unwrap();
|
||||
let mismatched = CatalogRecord {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue