From a38d4fa46c078fe519c114799a3d8621f795a0d7 Mon Sep 17 00:00:00 2001 From: "Jason E. Aten" Date: Fri, 6 Nov 2020 15:30:10 +0000 Subject: [PATCH] apply --fsync flag to all tx backend - rename from --rbf-fsync to --fsync, as it now effects bolt, lmdb too. --- bolt.go | 10 ++++++++-- lmdb.go | 22 ++++++++++------------ rbf/cfg/cfg.go | 4 +++- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/bolt.go b/bolt.go index 9772957ab..7980bb3a7 100644 --- a/bolt.go +++ b/bolt.go @@ -164,8 +164,14 @@ func (r *boltRegistrar) OpenDBWrapper(path0 string, doAllocZero bool, rbfcfg *rb // re-sync during recovery. // NoFreelistSync bool - //db.NoSync = true - //db.NoFreelistSync = true + if rbfcfg != nil && !rbfcfg.FsyncEnabled { + db.NoSync = true + db.NoFreelistSync = true + } else { + // default to using fsync on bolt. + db.NoSync = false + db.NoFreelistSync = false + } err = db.Update(func(tx *bolt.Tx) (err error) { _, err = tx.CreateBucketIfNotExists(bucketCT) diff --git a/lmdb.go b/lmdb.go index 4973b2287..28075de2d 100644 --- a/lmdb.go +++ b/lmdb.go @@ -183,13 +183,6 @@ func (r *lmdbRegistrar) OpenDBWrapper(path0 string, doAllocZero bool, rbfcfg *rb // flags = flags | lmdb.WriteMap | lmdb.NoMetaSync | lmdb.NoSync // about the same speed // flags = flags | lmdb.NoMetaSync | lmdb.NoSync // slows things down - // TODO(jea): we got an odd segfault in the roaring/ pkg when we went to read-only mmap - // with the row-cache off; probably means we were trying to write to mmap-ed - // memory. Investigate at some point. reference: 2a77b25d ja/write_map_prevents_segfault - // - // Update: added COW to roaring/roaring.go:3282 func (c *Container) arrayRemove(), - // seems to fix the issue. - // //flags = flags | lmdb.WriteMap // seems faster than without: or maybe not. not sure. // kRemove N= 710401 avg/op: 7.714µs sd: 27.83µs total: 5.480656859s // kAdd N= 722835 avg/op: 9.096µs sd: 105.787µs total: 6.575497725s @@ -202,12 +195,17 @@ func (r *lmdbRegistrar) OpenDBWrapper(path0 string, doAllocZero bool, rbfcfg *rb // On my darwin/OSX laptop with 16GB ram, for instance, we // can have difficulty obtaining this, resulting in // panic: mdb_env_open: no space left on device - lmdb.WriteMap | // Use a writable memory map. + lmdb.WriteMap // Use a writable memory map. - // default ACI (not Durable) transactions; 300% faster write speed results. - lmdb.NoMetaSync | // Don't fsync metapage after commit. - lmdb.NoSync | // Don't fsync after commit. - lmdb.MapAsync // Flush asynchronously when using the WriteMap flag. + if rbfcfg == nil || !rbfcfg.FsyncEnabled { + // default for lmdb: fsync off. + // unsafe, but get upper bound on performance. + flags = flags | + // default ACI (not Durable) transactions; 300% faster write speed results. + lmdb.NoMetaSync | // Don't fsync metapage after commit. + lmdb.NoSync | // Don't fsync after commit. + lmdb.MapAsync // Flush asynchronously when using the WriteMap flag. + } if !DirExists(path) { panicOn(os.MkdirAll(path, 0755)) diff --git a/rbf/cfg/cfg.go b/rbf/cfg/cfg.go index f13e14f4f..733b945ce 100644 --- a/rbf/cfg/cfg.go +++ b/rbf/cfg/cfg.go @@ -58,5 +58,7 @@ func (cfg *Config) DefineFlags(flags *pflag.FlagSet) { flags.DurationVar(&cfg.CheckpointEveryDur, "rbf-checkpoint-dur", default0.CheckpointEveryDur, "RBF checkpoint on the next write that occurs this long or more after the previous write. 0 means checkpoint after every write.") flags.Int64Var(&cfg.MaxSize, "rbf-max-db-size", default0.MaxSize, "RBF maximum size in bytes of a database file (distinct from a WAL file)") - flags.BoolVar(&cfg.FsyncEnabled, "rbf-fsync", default0.FsyncEnabled, "RBF: enable fsync fully safe flush-to-disk at each checkpoint") + + // renamed from --rbf-fsync to just --fsync because now it applies to all Tx backends. + flags.BoolVar(&cfg.FsyncEnabled, "fsync", default0.FsyncEnabled, "enable fsync fully safe flush-to-disk") }