diff --git a/bolt.go b/bolt.go index f2f829bde..70d05b179 100644 --- a/bolt.go +++ b/bolt.go @@ -145,6 +145,28 @@ func (r *boltRegistrar) OpenDBWrapper(path0 string, doAllocZero bool, rbfcfg *rb return nil, errors.Wrapf(err, fmt.Sprintf("open bolt path '%v'", path)) } + // docs on fsync from https://godoc.org/github.com/etcd-io/bbolt + // + // Setting the NoSync flag will cause the database to skip fsync() + // calls after each commit. This can be useful when bulk loading data + // into a database and you can restart the bulk load in the event of + // a system failure or database corruption. Do not set this flag for + // normal use. + // + // If the package global IgnoreNoSync constant is true, this value is + // ignored. See the comment on that constant for more details. + // + // THIS IS UNSAFE. PLEASE USE WITH CAUTION. + // NoSync bool + + // When true, skips syncing freelist to disk. This improves the database + // write performance under normal operation, but requires a full database + // re-sync during recovery. + // NoFreelistSync bool + + //db.NoSync = true + //db.NoFreelistSync = true + err = db.Update(func(tx *bolt.Tx) (err error) { _, err = tx.CreateBucketIfNotExists(bucketCT) return diff --git a/fragment.go b/fragment.go index 734d8157d..790b98d7a 100644 --- a/fragment.go +++ b/fragment.go @@ -2064,7 +2064,7 @@ func (f *fragment) Blocks() ([]FragmentBlock, error) { // Cache checksum. chksum := h.Sum() - f.checksums[h.blockID] = chksum + f.checksums[h.blockID] = chksum // the only place checksums is added to. // Append block. a = append(a, FragmentBlock{ diff --git a/holder.go b/holder.go index 6da35f797..e02734be2 100644 --- a/holder.go +++ b/holder.go @@ -203,7 +203,8 @@ type HolderConfig struct { Txsrc string RowcacheOff bool - RBFConfig *rbfcfg.Config + RBFConfig *rbfcfg.Config + AntiEntropyInterval time.Duration } func DefaultHolderConfig() *HolderConfig { @@ -651,8 +652,6 @@ func (h *Holder) Open() error { return errors.Wrap(err, "processing foreign index fields") } - h.Logger.Printf("open holder: complete") - h.Stats.Open() h.opened.Close() @@ -669,6 +668,8 @@ func (h *Holder) Open() error { } h.txf.blueGreenOnIfRunningBlueGreen() + h.Logger.Printf("open holder: complete") + return nil } diff --git a/server.go b/server.go index d543e5088..81cfeffd7 100644 --- a/server.go +++ b/server.go @@ -401,6 +401,7 @@ func NewServer(opts ...ServerOption) (*Server, error) { return nil, errors.Wrap(err, "applying option") } } + s.holderConfig.AntiEntropyInterval = s.antiEntropyInterval // set up executor after server opts have been processed executorOpts := []executorOption{optExecutorInternalQueryClient(s.defaultClient)} diff --git a/txfactory.go b/txfactory.go index c0ba21f0d..83afa31b8 100644 --- a/txfactory.go +++ b/txfactory.go @@ -1308,9 +1308,12 @@ func (f *TxFactory) green2blue(holder *Holder) (err error) { if hasData { verifyInsteadOfCopy = true + } else { + holder.Logger.Printf("bitmap-backend migration starting: populating %v from %v", blueDest, greenSrc) + defer holder.Logger.Printf("bitmap-backend migration done : populated %v from %v", blueDest, greenSrc) } - for _, idx := range idxs { + for k, idx := range idxs { // scan directories blueShards, err := f.dbPerShard.TypedDBPerShardGetShardsForIndex(blueDest, idx, "", false) @@ -1342,6 +1345,7 @@ func (f *TxFactory) green2blue(holder *Holder) (err error) { } } + progressCount := 0 for shard := range greenShards { dbs, err := f.dbPerShard.GetDBShard(idx.name, shard, idx) @@ -1360,6 +1364,10 @@ func (f *TxFactory) green2blue(holder *Holder) (err error) { } } else { // the main copy work + progressCount++ + holder.Logger.Printf("migration progress on index '%v' (%v of %v): on shard %v of %v", + k+1, len(idxs), idx.name, progressCount, len(greenShards)) + err = dbs.populateBlueFromGreen() if err != nil { return errors.Wrap(err,