From 509348260fb0c83b10f5d03bc18b933c4d5f79b7 Mon Sep 17 00:00:00 2001 From: "Jason E. Aten" Date: Wed, 28 Oct 2020 22:54:58 +0000 Subject: [PATCH 01/12] better migration logging --- bolt.go | 22 ++++++++++++++++++++++ fragment.go | 2 +- holder.go | 7 ++++--- server.go | 1 + txfactory.go | 10 +++++++++- 5 files changed, 37 insertions(+), 5 deletions(-) diff --git a/bolt.go b/bolt.go index f2f829bde..70d05b179 100644 --- a/bolt.go +++ b/bolt.go @@ -145,6 +145,28 @@ func (r *boltRegistrar) OpenDBWrapper(path0 string, doAllocZero bool, rbfcfg *rb return nil, errors.Wrapf(err, fmt.Sprintf("open bolt path '%v'", path)) } + // docs on fsync from https://godoc.org/github.com/etcd-io/bbolt + // + // Setting the NoSync flag will cause the database to skip fsync() + // calls after each commit. This can be useful when bulk loading data + // into a database and you can restart the bulk load in the event of + // a system failure or database corruption. Do not set this flag for + // normal use. + // + // If the package global IgnoreNoSync constant is true, this value is + // ignored. See the comment on that constant for more details. + // + // THIS IS UNSAFE. PLEASE USE WITH CAUTION. + // NoSync bool + + // When true, skips syncing freelist to disk. This improves the database + // write performance under normal operation, but requires a full database + // re-sync during recovery. + // NoFreelistSync bool + + //db.NoSync = true + //db.NoFreelistSync = true + err = db.Update(func(tx *bolt.Tx) (err error) { _, err = tx.CreateBucketIfNotExists(bucketCT) return diff --git a/fragment.go b/fragment.go index 734d8157d..790b98d7a 100644 --- a/fragment.go +++ b/fragment.go @@ -2064,7 +2064,7 @@ func (f *fragment) Blocks() ([]FragmentBlock, error) { // Cache checksum. chksum := h.Sum() - f.checksums[h.blockID] = chksum + f.checksums[h.blockID] = chksum // the only place checksums is added to. // Append block. a = append(a, FragmentBlock{ diff --git a/holder.go b/holder.go index 6da35f797..e02734be2 100644 --- a/holder.go +++ b/holder.go @@ -203,7 +203,8 @@ type HolderConfig struct { Txsrc string RowcacheOff bool - RBFConfig *rbfcfg.Config + RBFConfig *rbfcfg.Config + AntiEntropyInterval time.Duration } func DefaultHolderConfig() *HolderConfig { @@ -651,8 +652,6 @@ func (h *Holder) Open() error { return errors.Wrap(err, "processing foreign index fields") } - h.Logger.Printf("open holder: complete") - h.Stats.Open() h.opened.Close() @@ -669,6 +668,8 @@ func (h *Holder) Open() error { } h.txf.blueGreenOnIfRunningBlueGreen() + h.Logger.Printf("open holder: complete") + return nil } diff --git a/server.go b/server.go index d543e5088..81cfeffd7 100644 --- a/server.go +++ b/server.go @@ -401,6 +401,7 @@ func NewServer(opts ...ServerOption) (*Server, error) { return nil, errors.Wrap(err, "applying option") } } + s.holderConfig.AntiEntropyInterval = s.antiEntropyInterval // set up executor after server opts have been processed executorOpts := []executorOption{optExecutorInternalQueryClient(s.defaultClient)} diff --git a/txfactory.go b/txfactory.go index c0ba21f0d..83afa31b8 100644 --- a/txfactory.go +++ b/txfactory.go @@ -1308,9 +1308,12 @@ func (f *TxFactory) green2blue(holder *Holder) (err error) { if hasData { verifyInsteadOfCopy = true + } else { + holder.Logger.Printf("bitmap-backend migration starting: populating %v from %v", blueDest, greenSrc) + defer holder.Logger.Printf("bitmap-backend migration done : populated %v from %v", blueDest, greenSrc) } - for _, idx := range idxs { + for k, idx := range idxs { // scan directories blueShards, err := f.dbPerShard.TypedDBPerShardGetShardsForIndex(blueDest, idx, "", false) @@ -1342,6 +1345,7 @@ func (f *TxFactory) green2blue(holder *Holder) (err error) { } } + progressCount := 0 for shard := range greenShards { dbs, err := f.dbPerShard.GetDBShard(idx.name, shard, idx) @@ -1360,6 +1364,10 @@ func (f *TxFactory) green2blue(holder *Holder) (err error) { } } else { // the main copy work + progressCount++ + holder.Logger.Printf("migration progress on index '%v' (%v of %v): on shard %v of %v", + k+1, len(idxs), idx.name, progressCount, len(greenShards)) + err = dbs.populateBlueFromGreen() if err != nil { return errors.Wrap(err, From 85fc785296b07c18193153ee7c8cb39c542b4acb Mon Sep 17 00:00:00 2001 From: "Jason E. Aten" Date: Thu, 29 Oct 2020 01:12:19 +0000 Subject: [PATCH 02/12] timed progress --- txfactory.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/txfactory.go b/txfactory.go index 83afa31b8..84640a3da 100644 --- a/txfactory.go +++ b/txfactory.go @@ -25,6 +25,7 @@ import ( "sync" "syscall" "text/tabwriter" + "time" "github.com/pilosa/pilosa/v2/hash" "github.com/pilosa/pilosa/v2/rbf" @@ -1345,6 +1346,7 @@ func (f *TxFactory) green2blue(holder *Holder) (err error) { } } + lastProgress := time.Now() progressCount := 0 for shard := range greenShards { @@ -1365,9 +1367,11 @@ func (f *TxFactory) green2blue(holder *Holder) (err error) { } else { // the main copy work progressCount++ - holder.Logger.Printf("migration progress on index '%v' (%v of %v): on shard %v of %v", - k+1, len(idxs), idx.name, progressCount, len(greenShards)) - + if progressCount == 1 || time.Since(lastProgress) > time.Second { + holder.Logger.Printf("migration progress on index '%v' (%v of %v): on shard %v of %v", + idx.name, k+1, len(idxs), progressCount, len(greenShards)) + lastProgress = time.Now() + } err = dbs.populateBlueFromGreen() if err != nil { return errors.Wrap(err, From b1ff3791b8f15e81cf6504bfdb39edc1fda71ec1 Mon Sep 17 00:00:00 2001 From: "Jason E. Aten" Date: Thu, 29 Oct 2020 01:20:41 +0000 Subject: [PATCH 03/12] better error when green does not exist on migration --- txfactory.go | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/txfactory.go b/txfactory.go index 84640a3da..1b2ca0328 100644 --- a/txfactory.go +++ b/txfactory.go @@ -1280,6 +1280,17 @@ func (f *TxFactory) blueHasData() (hasData bool, err error) { return f.dbPerShard.HasData(0) } +func (f *TxFactory) greenHasData() (hasData bool, err error) { + n := len(f.types) + switch n { + case 1: + return f.dbPerShard.HasData(0) + case 2: + return f.dbPerShard.HasData(1) + } + panic(fmt.Sprintf("unsupported len(f.types): %v. Must be 1 or 2.", n)) +} + // green2blue is called at the very end of Holder.Open(), so // we know that the holder is ready to go, knowing its holder.Indexes(), fields, // view, shards, and other metadata if any. @@ -1302,12 +1313,20 @@ func (f *TxFactory) green2blue(holder *Holder) (err error) { verifyInsteadOfCopy := false - hasData, err := f.blueHasData() + blueHasData, err := f.blueHasData() if err != nil { - return errors.Wrap(err, "TxFactory.green2blue DataSize(0)") + return errors.Wrap(err, "TxFactory.green2blue f.blueHasData()") } - if hasData { + greenHasData, err := f.greenHasData() + if err != nil { + return errors.Wrap(err, "TxFactory.green2blue f.greenHasData()") + } + if !greenHasData { + return fmt.Errorf("error: cannot migrate from green '%v' because it has no data in it.", greenSrc) + } + + if blueHasData { verifyInsteadOfCopy = true } else { holder.Logger.Printf("bitmap-backend migration starting: populating %v from %v", blueDest, greenSrc) From 6e20e6439fb52a6085f22e636b45535c01af38d4 Mon Sep 17 00:00:00 2001 From: "Jason E. Aten" Date: Thu, 29 Oct 2020 01:30:16 +0000 Subject: [PATCH 04/12] more logging --- txfactory.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/txfactory.go b/txfactory.go index 1b2ca0328..4a6b84e6c 100644 --- a/txfactory.go +++ b/txfactory.go @@ -1322,7 +1322,9 @@ func (f *TxFactory) green2blue(holder *Holder) (err error) { if err != nil { return errors.Wrap(err, "TxFactory.green2blue f.greenHasData()") } + vv("blueHasData = %v, greenHasData = %v", blueHasData, greenHasData) if !greenHasData { + holder.Logger.Printf("error: cannot migrate from green '%v' because it has no data in it.", greenSrc) return fmt.Errorf("error: cannot migrate from green '%v' because it has no data in it.", greenSrc) } From 639e9b3bf041bc942e01c3d0de1c51ec9504b339 Mon Sep 17 00:00:00 2001 From: "Jason E. Aten" Date: Thu, 29 Oct 2020 01:32:04 +0000 Subject: [PATCH 05/12] quiet --- txfactory.go | 1 - 1 file changed, 1 deletion(-) diff --git a/txfactory.go b/txfactory.go index 4a6b84e6c..c797f3415 100644 --- a/txfactory.go +++ b/txfactory.go @@ -1322,7 +1322,6 @@ func (f *TxFactory) green2blue(holder *Holder) (err error) { if err != nil { return errors.Wrap(err, "TxFactory.green2blue f.greenHasData()") } - vv("blueHasData = %v, greenHasData = %v", blueHasData, greenHasData) if !greenHasData { holder.Logger.Printf("error: cannot migrate from green '%v' because it has no data in it.", greenSrc) return fmt.Errorf("error: cannot migrate from green '%v' because it has no data in it.", greenSrc) From 3baa8ea0f6429bc9c1fedd54f507397877545529 Mon Sep 17 00:00:00 2001 From: "Jason E. Aten" Date: Thu, 29 Oct 2020 01:47:26 +0000 Subject: [PATCH 06/12] check if roaring data present with RoaringHasData --- dbshard.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/dbshard.go b/dbshard.go index 6fc057200..ae69c8867 100644 --- a/dbshard.go +++ b/dbshard.go @@ -303,6 +303,10 @@ func newShardSet() *shardSet { func (per *DBPerShard) HasData(which int) (hasData bool, err error) { // has to aggregate across all available DBShard for each index and shard. + if per.types[which] == roaringTxn { + return per.RoaringHasData() + } + for _, v := range per.Flatmap { hasData, err = v.W[which].HasData() if err != nil { @@ -315,6 +319,21 @@ func (per *DBPerShard) HasData(which int) (hasData bool, err error) { return } +func (per *DBPerShard) RoaringHasData() (bool, error) { + idxs := per.holder.Indexes() + const requireData = true + for _, idx := range idxs { + shards, err := per.TypedDBPerShardGetShardsForIndex(roaringTxn, idx, "", requireData) + if err != nil { + return false, err + } + if len(shards) > 0 { + return true, nil + } + } + return false, nil +} + func (per *DBPerShard) ListOpenString() (r string) { for _, v := range per.Flatmap { r += v.HolderPath + " -> " + v.W[per.useOpenList].OpenListString() + "\n" From bceabc9127015218015e00f4618bf4b2f022ed21 Mon Sep 17 00:00:00 2001 From: "Jason E. Aten" Date: Thu, 29 Oct 2020 02:11:28 +0000 Subject: [PATCH 07/12] error on un-implemented migrate to roaring --- txfactory.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/txfactory.go b/txfactory.go index c797f3415..251993ee0 100644 --- a/txfactory.go +++ b/txfactory.go @@ -1309,6 +1309,11 @@ func (f *TxFactory) green2blue(holder *Holder) (err error) { blueDest := f.types[0] greenSrc := f.types[1] + + if blueDest == roaringTxn { + return fmt.Errorf("error: cannot migration to 'roaring': not implemented.") + } + idxs := holder.Indexes() verifyInsteadOfCopy := false From f9f94aab0d0dccbb51f5175c596ba6f8ed840595 Mon Sep 17 00:00:00 2001 From: "Jason E. Aten" Date: Thu, 29 Oct 2020 02:15:43 +0000 Subject: [PATCH 08/12] fix typo --- txfactory.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/txfactory.go b/txfactory.go index 251993ee0..0e44579e3 100644 --- a/txfactory.go +++ b/txfactory.go @@ -1311,7 +1311,7 @@ func (f *TxFactory) green2blue(holder *Holder) (err error) { greenSrc := f.types[1] if blueDest == roaringTxn { - return fmt.Errorf("error: cannot migration to 'roaring': not implemented.") + return fmt.Errorf("error: cannot migrate to 'roaring': not implemented.") } idxs := holder.Indexes() From 0e354aedf23c3b44ce13f617d06eab3715d20d5a Mon Sep 17 00:00:00 2001 From: "Jason E. Aten" Date: Thu, 29 Oct 2020 02:29:39 +0000 Subject: [PATCH 09/12] error expected trying to migration to roaring --- txfactory_internal_test.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/txfactory_internal_test.go b/txfactory_internal_test.go index 15b03b59c..c01f56a1d 100644 --- a/txfactory_internal_test.go +++ b/txfactory_internal_test.go @@ -118,11 +118,18 @@ func Test_TxFactory_UpdateBlueFromGreen_OnStartup(t *testing.T) { checked := []string{"lmdb", "roaring", "rbf"} + expectError := false for _, blue := range checked { for _, green := range checked { if blue == green { continue } + if blue == "roaring" { + // not supported + expectError = true + } else { + expectError = false + } blue_green := blue + "_" + green //vv("setting blue_green to '%v'", blue_green) @@ -230,7 +237,14 @@ func Test_TxFactory_UpdateBlueFromGreen_OnStartup(t *testing.T) { h4 := NewHolder(path, nil) //vv("about to h4.Open we should populate blue from green") - panicOn(h4.Open()) + err = h4.Open() + if expectError { + if err == nil { + panic("expected error since migration to roaring not supported") + } + } else { + panicOn(err) + } testMustHaveBit(t, h4, "i0", "f", rowID, colID) testMustHaveBit(t, h4, "i1", "f", 100, 200) From aa47cf5bf4980896632233c63552ad5ff1ada5ba Mon Sep 17 00:00:00 2001 From: "Jason E. Aten" Date: Thu, 29 Oct 2020 02:35:38 +0000 Subject: [PATCH 10/12] fix verify test --- txfactory_internal_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/txfactory_internal_test.go b/txfactory_internal_test.go index c01f56a1d..26416f95a 100644 --- a/txfactory_internal_test.go +++ b/txfactory_internal_test.go @@ -272,6 +272,10 @@ func Test_TxFactory_verifyBlueEqualsGreen(t *testing.T) { if blue == green { continue } + if blue == "roaring" { + // not supported + continue + } blue_green := blue + "_" + green // ============================= From d4df15721d75f4a6a49e399f2f2266a07db5c2f3 Mon Sep 17 00:00:00 2001 From: "Jason E. Aten" Date: Thu, 29 Oct 2020 02:41:09 +0000 Subject: [PATCH 11/12] allow migration of empty to empty --- txfactory.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/txfactory.go b/txfactory.go index 0e44579e3..3309884e6 100644 --- a/txfactory.go +++ b/txfactory.go @@ -1327,7 +1327,7 @@ func (f *TxFactory) green2blue(holder *Holder) (err error) { if err != nil { return errors.Wrap(err, "TxFactory.green2blue f.greenHasData()") } - if !greenHasData { + if blueHasData && !greenHasData { holder.Logger.Printf("error: cannot migrate from green '%v' because it has no data in it.", greenSrc) return fmt.Errorf("error: cannot migrate from green '%v' because it has no data in it.", greenSrc) } From 2d55ffbd283d9f64d3260c44874bcea864dc9a4d Mon Sep 17 00:00:00 2001 From: "Jason E. Aten" Date: Thu, 29 Oct 2020 02:44:22 +0000 Subject: [PATCH 12/12] short circuit if no data to migrate --- txfactory.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/txfactory.go b/txfactory.go index 3309884e6..17ef34c75 100644 --- a/txfactory.go +++ b/txfactory.go @@ -1327,7 +1327,12 @@ func (f *TxFactory) green2blue(holder *Holder) (err error) { if err != nil { return errors.Wrap(err, "TxFactory.green2blue f.greenHasData()") } - if blueHasData && !greenHasData { + if !blueHasData && !greenHasData { + holder.Logger.Printf("no data in blue or green. No migration or verification to do.") + return nil + } + // INVAR: blue has data. + if !greenHasData { holder.Logger.Printf("error: cannot migrate from green '%v' because it has no data in it.", greenSrc) return fmt.Errorf("error: cannot migrate from green '%v' because it has no data in it.", greenSrc) }