scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 00:55:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #669 from molecula/autocommit2

Browse source 
build-tag out lmdb-go from builds; lower autocommit limits to avoid ErrTxnTooBig; improve txn size estimation
This commit is contained in:
jaten-molecula 2020-08-13 11:52:42 -05:00 • committed by GitHub
commit 05ca83c2f5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 57 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
badger.go
View file

@ -720,7 +720,8 @@ func (tx *BadgerTx) PutContainer(index, field, view string, shard uint64, ckey u
panic(fmt.Sprintf("unknown roaring.Container type: %v", ct))
}
tx.writeCount++
tx.writeByteCount += len(by)
sz := len(by) + len(bkey) + 2
tx.writeByteCount += sz
tx.mu.Lock()
defer tx.mu.Unlock()
@ -728,16 +729,20 @@ func (tx *BadgerTx) PutContainer(index, field, view string, shard uint64, ckey u
// The integration tests do large bit level loads that exceed 10MB.
// So we autocommit and start a new Txn if we are about to
// write too much into one Txn.
//
// The badger defaults limits are currently:
// maxBatchCount:104857, maxBatchSize:10066329
// So we stop a little before to make sure we fit.
if tx.writeCount > 100000 || tx.writeByteCount > 10000000 {
//
// However, emprirically we still get ErrTnTooBig when
// tx.writeByteCount=5884222; or when tx.writeCount=16197.
// So duck under both those thresholds by some margin.
if tx.writeCount > 10000 || tx.writeByteCount > 4000000 {
// avoid ErrTxnTooBig by commiting before going over the limits,
// because then we get a error: "Transaction Conflict. Please retry."
panicOn(tx.tx.Commit())
tx.tx = tx.Db.db.NewTransaction(tx.write)
tx.writeCount = 1
tx.writeByteCount = len(by)
tx.writeByteCount = sz
}
entry := badger.NewEntry(bkey, by).WithMeta(ct)
err := tx.tx.SetEntry(entry)
@ -748,7 +753,7 @@ func (tx *BadgerTx) PutContainer(index, field, view string, shard uint64, ckey u
// is best to commit the transaction and start a new transaction immediately."
//
if err == badger.ErrTxnTooBig {
panic(fmt.Sprintf("got error badger.ErrTxnTooBig, but we shoud never get this now; len(by) = %v; vs limit is 10MB. tx.writeCount=%v; tx.writeByteCount=%v;", len(by), tx.writeCount, tx.writeByteCount))
panic(fmt.Sprintf("got error badger.ErrTxnTooBig, but we shoud never get this now; len(by) = %v; len(bkey)=%v; vs limit is 10MB. tx.writeCount=%v; tx.writeByteCount=%v;", len(by), len(bkey), tx.writeCount, tx.writeByteCount))
}
return err
}

2
badger_test.go
View file

@ -1278,7 +1278,7 @@ func getTestBitmapAsRawRoaring(bitsToSet ...uint64) []byte {
func TestBadger_AutoCommit(t *testing.T) {
// setup
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_DeleteIndex")
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_AutoCommit")
defer clean()
defer dbwrap.Close()

8
bluegreentx.go
View file

@ -192,16 +192,16 @@ func (c *blueGreenTx) Rollback() {
panic(r)
}
}()
fmt.Printf("blueGreenTx.Rollback() about to call (%v) a.Rollback()\n", c.as)
//fmt.Printf("blueGreenTx.Rollback() about to call (%v) a.Rollback()\n", c.as)
c.a.Rollback()
fmt.Printf("blueGreenTx.Rollback() about to call (%v) b.Rollback()\n", c.bs)
//fmt.Printf("blueGreenTx.Rollback() about to call (%v) b.Rollback()\n", c.bs)
c.b.Rollback()
}
func (c *blueGreenTx) Commit() error {
c.mu.Lock()
defer c.mu.Unlock()
fmt.Printf("blueGreenTx.Commit() called.\n")
//fmt.Printf("blueGreenTx.Commit() called.\n")
if c.rollbackOrCommitDone {
return nil
}
@ -766,7 +766,7 @@ type blueGreenChecker struct {
// see would mark a thing as seen.
func (b *blueGreenChecker) see(index, field, view string, shard uint64) {
// keep this next Printf. Useful to see the sequence of Tx operations.
fmt.Printf("blueGreenTx.%v on index='%v'\n", Caller(1), index)
//fmt.Printf("blueGreenTx.%v on index='%v'\n", Caller(1), index)
b.mu.Lock()
defer b.mu.Unlock()

1
cmd/demo-lmdb/lmdb.go
View file

@ -13,6 +13,7 @@
// limitations under the License.
// +build !386
// +build skip_building_lmdb_for_now
package main

1
cmd/demo-lmdb/vprint.go
View file

@ -23,6 +23,7 @@
// SOFTWARE.
// +build !386
// +build skip_building_lmdb_for_now
package main

44
txfactory.go
View file

@ -250,6 +250,18 @@ func (f *TxFactory) DeleteIndex(name string) error {
return f.badgerDB.DeleteIndex(name)
case blueGreenRoaringBadger:
return f.badgerDB.DeleteIndex(name)
case blueGreenRBFBadger:
_ = f.rbfDB.DeleteIndex(name)
return f.badgerDB.DeleteIndex(name)
case blueGreenBadgerRBF:
_ = f.badgerDB.DeleteIndex(name)
return f.rbfDB.DeleteIndex(name)
case blueGreenRBFRoaring:
// roaring already done
return f.rbfDB.DeleteIndex(name)
case blueGreenRoaringRBF:
// roaring already done
return f.rbfDB.DeleteIndex(name)
}
panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
}
@ -260,8 +272,6 @@ func (f *TxFactory) DeleteFieldFromStore(index, field, fieldPath string) error {
return f.roaringDB.DeleteField(index, field, fieldPath)
case badgerTxn:
return f.badgerDB.DeleteField(index, field, fieldPath)
//case lmdbTxn:
//return f.lmDB.DeleteField(index, field, fieldPath)
case rbfTxn:
return f.rbfDB.DeleteField(index, field, fieldPath)
case blueGreenBadgerRoaring:
@ -270,6 +280,20 @@ func (f *TxFactory) DeleteFieldFromStore(index, field, fieldPath string) error {
case blueGreenRoaringBadger:
_ = f.roaringDB.DeleteField(index, field, fieldPath)
return f.badgerDB.DeleteField(index, field, fieldPath)
case blueGreenRBFBadger:
_ = f.badgerDB.DeleteField(index, field, fieldPath)
return f.rbfDB.DeleteField(index, field, fieldPath)
case blueGreenBadgerRBF:
_ = f.rbfDB.DeleteField(index, field, fieldPath)
return f.badgerDB.DeleteField(index, field, fieldPath)
case blueGreenRBFRoaring:
_ = f.rbfDB.DeleteField(index, field, fieldPath)
return f.roaringDB.DeleteField(index, field, fieldPath)
case blueGreenRoaringRBF:
_ = f.roaringDB.DeleteField(index, field, fieldPath)
return f.rbfDB.DeleteField(index, field, fieldPath)
//case lmdbTxn:
//return f.lmDB.DeleteField(index, field, fieldPath)
}
panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
}
@ -282,14 +306,26 @@ func (f *TxFactory) DeleteFragmentFromStore(index, field, view string, shard uin
return f.badgerDB.DeleteFragment(index, field, view, shard, frag)
case rbfTxn:
return f.rbfDB.DeleteFragment(index, field, view, shard, frag)
// case lmdbTxn:
// return f.lmDB.DeleteFragment(index, field, view, shard, frag)
case blueGreenBadgerRoaring:
_ = f.badgerDB.DeleteFragment(index, field, view, shard, frag)
return f.roaringDB.DeleteFragment(index, field, view, shard, frag)
case blueGreenRoaringBadger:
_ = f.roaringDB.DeleteFragment(index, field, view, shard, frag)
return f.badgerDB.DeleteFragment(index, field, view, shard, frag)
case blueGreenRBFBadger:
_ = f.rbfDB.DeleteFragment(index, field, view, shard, frag)
return f.badgerDB.DeleteFragment(index, field, view, shard, frag)
case blueGreenBadgerRBF:
_ = f.badgerDB.DeleteFragment(index, field, view, shard, frag)
return f.rbfDB.DeleteFragment(index, field, view, shard, frag)
case blueGreenRBFRoaring:
_ = f.rbfDB.DeleteFragment(index, field, view, shard, frag)
return f.roaringDB.DeleteFragment(index, field, view, shard, frag)
case blueGreenRoaringRBF:
_ = f.roaringDB.DeleteFragment(index, field, view, shard, frag)
return f.rbfDB.DeleteFragment(index, field, view, shard, frag)
// case lmdbTxn:
// return f.lmDB.DeleteFragment(index, field, view, shard, frag)
}
panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))