scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 09:05:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

smaller batches of write Tx help boost migration speed

Browse source
This commit is contained in:
Jason E. Aten 2020-12-21 19:00:13 +00:00
parent e290e13d0b
commit 1efff0c99b
1 changed files with 33 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
dbshard.go
View file

@ -342,7 +342,7 @@ func (per *DBPerShard) HasData(which int) (hasData bool, err error) {
// has to aggregate across all available DBShard for each index and shard.
if per.types[which] == roaringTxn {
return per.RoaringHasData()
return per.RoaringHasData() // this needs to be made accurate
}
for _, v := range per.Flatmap {
@ -948,6 +948,8 @@ func (dbs *DBShard) populateBlueFromGreen() (err error) {
panicOn(err)
defer writetx.Rollback()
ctWriteCount := 0
for _, fld := range dbs.idx.Fields() {
field := fld.Name()
for _, vw := range fld.views() {
@ -958,6 +960,7 @@ func (dbs *DBShard) populateBlueFromGreen() (err error) {
if strings.Contains(err.Error(), "fragment not found") {
continue
} else {
writetx.Rollback()
return errors.Wrap(err, "DBShard.populateBlueFromGreen readtx.ContainerIterator")
}
}
@ -967,8 +970,37 @@ func (dbs *DBShard) populateBlueFromGreen() (err error) {
err := writetx.PutContainer(dbs.Index, field, view, dbs.Shard, ckey, rc)
if err != nil {
citer.Close()
writetx.Rollback()
return errors.Wrap(err, "DBShard.populateBlueFromGreen writetx.PutContainer")
}
ctWriteCount++
if ctWriteCount%1000 == 1 {
// regularly commiting smaller batches and the first batch as soon as
// possible massively speeds up writing to bolt.
//
// reference: https://github.com/boltdb/bolt/issues/94
//
// benbjohnson commented on Mar 25, 2014
// "Bulk loading more than 1000 items at a time is very slow. This is because nodes
// are not splitting before commit which causes large memmove() operations during insertion."
// runtime.memmove is taking all of the time in our pprof profile, when copying rbf to bolt, so we suspect it is this.
//
err = writetx.Commit()
if err != nil {
citer.Close()
writetx.Rollback()
return errors.Wrap(err, "DBShard.populateBlueFromGreen writetx.Commit")
}
writetx, err = dest.NewTx(writable, dbs.Index, Txo{Write: writable, Index: dbs.idx, Shard: dbs.Shard})
if err != nil {
citer.Close()
writetx.Rollback()
return errors.Wrap(err, "DBShard.populateBlueFromGreen writetx.NewTx inside citer.Next() loop")
}
}
}
citer.Close()
}