better migration logging

bolt.go

@@ -145,6 +145,28 @@ func (r *boltRegistrar) OpenDBWrapper(path0 string, doAllocZero bool, rbfcfg *rb
 		return nil, errors.Wrapf(err, fmt.Sprintf("open bolt path '%v'", path))
 	}
 
+	// docs on fsync from https://godoc.org/github.com/etcd-io/bbolt
+	//
+	// Setting the NoSync flag will cause the database to skip fsync()
+	// calls after each commit. This can be useful when bulk loading data
+	// into a database and you can restart the bulk load in the event of
+	// a system failure or database corruption. Do not set this flag for
+	// normal use.
+	//
+	// If the package global IgnoreNoSync constant is true, this value is
+	// ignored.  See the comment on that constant for more details.
+	//
+	// THIS IS UNSAFE. PLEASE USE WITH CAUTION.
+	//	NoSync bool
+
+	// When true, skips syncing freelist to disk. This improves the database
+	// write performance under normal operation, but requires a full database
+	// re-sync during recovery.
+	// NoFreelistSync bool
+
+	//db.NoSync = true
+	//db.NoFreelistSync = true
+
 	err = db.Update(func(tx *bolt.Tx) (err error) {
 		_, err = tx.CreateBucketIfNotExists(bucketCT)
 		return

fragment.go

@@ -2064,7 +2064,7 @@ func (f *fragment) Blocks() ([]FragmentBlock, error) {
 
 		// Cache checksum.
 		chksum := h.Sum()
-		f.checksums[h.blockID] = chksum
+		f.checksums[h.blockID] = chksum // the only place checksums is added to.
 
 		// Append block.
 		a = append(a, FragmentBlock{

holder.go

@@ -203,7 +203,8 @@ type HolderConfig struct {
 	Txsrc                string
 	RowcacheOff          bool
 
-	RBFConfig *rbfcfg.Config
+	RBFConfig           *rbfcfg.Config
+	AntiEntropyInterval time.Duration
 }
 
 func DefaultHolderConfig() *HolderConfig {
@@ -651,8 +652,6 @@ func (h *Holder) Open() error {
 		return errors.Wrap(err, "processing foreign index fields")
 	}
 
-	h.Logger.Printf("open holder: complete")
-
 	h.Stats.Open()
 
 	h.opened.Close()
@@ -669,6 +668,8 @@ func (h *Holder) Open() error {
 	}
 	h.txf.blueGreenOnIfRunningBlueGreen()
 
+	h.Logger.Printf("open holder: complete")
+
 	return nil
 
 }

server.go

@@ -401,6 +401,7 @@ func NewServer(opts ...ServerOption) (*Server, error) {
 			return nil, errors.Wrap(err, "applying option")
 		}
 	}
+	s.holderConfig.AntiEntropyInterval = s.antiEntropyInterval
 
 	// set up executor after server opts have been processed
 	executorOpts := []executorOption{optExecutorInternalQueryClient(s.defaultClient)}

txfactory.go

@@ -1308,9 +1308,12 @@ func (f *TxFactory) green2blue(holder *Holder) (err error) {
 
 	if hasData {
 		verifyInsteadOfCopy = true
+	} else {
+		holder.Logger.Printf("bitmap-backend migration starting: populating %v from %v", blueDest, greenSrc)
+		defer holder.Logger.Printf("bitmap-backend migration done    : populated  %v from %v", blueDest, greenSrc)
 	}
 
-	for _, idx := range idxs {
+	for k, idx := range idxs {
 
 		// scan directories
 		blueShards, err := f.dbPerShard.TypedDBPerShardGetShardsForIndex(blueDest, idx, "", false)
@@ -1342,6 +1345,7 @@ func (f *TxFactory) green2blue(holder *Holder) (err error) {
 			}
 		}
 
+		progressCount := 0
 		for shard := range greenShards {
 
 			dbs, err := f.dbPerShard.GetDBShard(idx.name, shard, idx)
@@ -1360,6 +1364,10 @@ func (f *TxFactory) green2blue(holder *Holder) (err error) {
 				}
 			} else {
 				// the main copy work
+				progressCount++
+				holder.Logger.Printf("migration progress on index '%v' (%v of %v): on shard %v of %v",
+					k+1, len(idxs), idx.name, progressCount, len(greenShards))
+
 				err = dbs.populateBlueFromGreen()
 				if err != nil {
 					return errors.Wrap(err,