tested working autocommit approach

badger.go

@@ -544,6 +544,17 @@ type BadgerTx struct {
 	initialIndexName string
 
 	DeleteEmptyContainer bool
+
+	// We must avoid writing more than 10MB to badger in
+	// one transaction. If we go over, then
+	// we'll get a ErrTxnTooBig error. At that point
+	// we can't commit more, because the transaction
+	// will "conflict". So we must monitor
+	// totals written and auto-commit before going
+	// over the limits to avoid wedging into an
+	// unrecoverable state.
+	writeCount     int
+	writeByteCount int
 }
 
 func (tx *BadgerTx) Type() string {
@@ -708,22 +719,36 @@ func (tx *BadgerTx) PutContainer(index, field, view string, shard uint64, ckey u
 	default:
 		panic(fmt.Sprintf("unknown roaring.Container type: %v", ct))
 	}
-	entry := badger.NewEntry(bkey, by).WithMeta(ct)
+	tx.writeCount++
+	tx.writeByteCount += len(by)
+
 	tx.mu.Lock()
-	err := tx.tx.SetEntry(entry)
 	defer tx.mu.Unlock()
 
+	// The integration tests do large bit level loads that exceed 10MB.
+	// So we autocommit and start a new Txn if we are about to
+	// write too much into one Txn.
+	// The badger defaults limits are currently:
+	// maxBatchCount:104857, maxBatchSize:10066329
+	// So we stop a little before to make sure we fit.
+	if tx.writeCount > 100000 || tx.writeByteCount > 10000000 {
+		// avoid ErrTxnTooBig by commiting before going over the limits,
+		// because then we get a error: "Transaction Conflict. Please retry."
+		panicOn(tx.tx.Commit())
+		tx.tx = tx.Db.db.NewTransaction(tx.write)
+		tx.writeCount = 1
+		tx.writeByteCount = len(by)
+	}
+	entry := badger.NewEntry(bkey, by).WithMeta(ct)
+	err := tx.tx.SetEntry(entry)
+
 	// ErrTxnTooBig is returned if too many writes are fit into a single transaction.
 	// badger docs: "An ErrTxnTooBig will be reported in case the number of pending
 	// writes/deletes in the transaction exceeds a certain limit. In that case, it
 	// is best to commit the transaction and start a new transaction immediately."
 	//
 	if err == badger.ErrTxnTooBig {
-		// The integration tests do large bit level loads that exceed 10MB.
-		// So we autocommit and start a new Txn.
-		panicOn(tx.tx.Commit())
-		tx.tx = tx.Db.db.NewTransaction(tx.write)
-		return tx.tx.SetEntry(entry)
+		panic(fmt.Sprintf("got error badger.ErrTxnTooBig, but we shoud never get this now; len(by) = %v; vs limit is 10MB. tx.writeCount=%v; tx.writeByteCount=%v;", len(by), tx.writeCount, tx.writeByteCount))
 	}
 	return err
 }

badger_test.go

@@ -1275,6 +1275,30 @@ func getTestBitmapAsRawRoaring(bitsToSet ...uint64) []byte {
 	return buf.Bytes()
 }
 
+func TestBadger_AutoCommit(t *testing.T) {
+
+	// setup
+	dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_DeleteIndex")
+	defer clean()
+	defer dbwrap.Close()
+
+	index, field, view, shard := "i", "f", "v", uint64(0)
+	tx := dbwrap.NewBadgerTx(writable, index, nil)
+
+	// if we go over 100K writes, we should autocommit
+	// rather than panic.
+	for v := 0; v < 133444; v++ {
+		changed, err := tx.Add(index, field, view, shard, doBatched, uint64(v))
+		if changed <= 0 {
+			panic("should have changed")
+		}
+		panicOn(err)
+	}
+
+	err := tx.Commit()
+	panicOn(err)
+}
+
 func TestBadger_DeleteIndex(t *testing.T) {
 
 	// setup