diff --git a/badger.go b/badger.go index eb13c5d88..187a320a6 100644 --- a/badger.go +++ b/badger.go @@ -544,6 +544,17 @@ type BadgerTx struct { initialIndexName string DeleteEmptyContainer bool + + // We must avoid writing more than 10MB to badger in + // one transaction. If we go over, then + // we'll get a ErrTxnTooBig error. At that point + // we can't commit more, because the transaction + // will "conflict". So we must monitor + // totals written and auto-commit before going + // over the limits to avoid wedging into an + // unrecoverable state. + writeCount int + writeByteCount int } func (tx *BadgerTx) Type() string { @@ -708,22 +719,36 @@ func (tx *BadgerTx) PutContainer(index, field, view string, shard uint64, ckey u default: panic(fmt.Sprintf("unknown roaring.Container type: %v", ct)) } - entry := badger.NewEntry(bkey, by).WithMeta(ct) + tx.writeCount++ + tx.writeByteCount += len(by) + tx.mu.Lock() - err := tx.tx.SetEntry(entry) defer tx.mu.Unlock() + // The integration tests do large bit level loads that exceed 10MB. + // So we autocommit and start a new Txn if we are about to + // write too much into one Txn. + // The badger defaults limits are currently: + // maxBatchCount:104857, maxBatchSize:10066329 + // So we stop a little before to make sure we fit. + if tx.writeCount > 100000 || tx.writeByteCount > 10000000 { + // avoid ErrTxnTooBig by commiting before going over the limits, + // because then we get a error: "Transaction Conflict. Please retry." + panicOn(tx.tx.Commit()) + tx.tx = tx.Db.db.NewTransaction(tx.write) + tx.writeCount = 1 + tx.writeByteCount = len(by) + } + entry := badger.NewEntry(bkey, by).WithMeta(ct) + err := tx.tx.SetEntry(entry) + // ErrTxnTooBig is returned if too many writes are fit into a single transaction. // badger docs: "An ErrTxnTooBig will be reported in case the number of pending // writes/deletes in the transaction exceeds a certain limit. In that case, it // is best to commit the transaction and start a new transaction immediately." // if err == badger.ErrTxnTooBig { - // The integration tests do large bit level loads that exceed 10MB. - // So we autocommit and start a new Txn. - panicOn(tx.tx.Commit()) - tx.tx = tx.Db.db.NewTransaction(tx.write) - return tx.tx.SetEntry(entry) + panic(fmt.Sprintf("got error badger.ErrTxnTooBig, but we shoud never get this now; len(by) = %v; vs limit is 10MB. tx.writeCount=%v; tx.writeByteCount=%v;", len(by), tx.writeCount, tx.writeByteCount)) } return err } diff --git a/badger_test.go b/badger_test.go index f48a61103..bdeafb0b8 100644 --- a/badger_test.go +++ b/badger_test.go @@ -1275,6 +1275,30 @@ func getTestBitmapAsRawRoaring(bitsToSet ...uint64) []byte { return buf.Bytes() } +func TestBadger_AutoCommit(t *testing.T) { + + // setup + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_DeleteIndex") + defer clean() + defer dbwrap.Close() + + index, field, view, shard := "i", "f", "v", uint64(0) + tx := dbwrap.NewBadgerTx(writable, index, nil) + + // if we go over 100K writes, we should autocommit + // rather than panic. + for v := 0; v < 133444; v++ { + changed, err := tx.Add(index, field, view, shard, doBatched, uint64(v)) + if changed <= 0 { + panic("should have changed") + } + panicOn(err) + } + + err := tx.Commit() + panicOn(err) +} + func TestBadger_DeleteIndex(t *testing.T) { // setup