create naive ApplyFilter

bluegreentx.go

@@ -907,6 +907,10 @@ func (c *blueGreenTx) Sn() int64 {
 	return bsn
 }
 
+func (c *blueGreenTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) {
+	return GenericApplyFilter(c, index, field, view, shard, ckey, filter)
+}
+
 // MultiReaderB is returned by RoaringBitmapReader. It verifies
 // that identical byte streams are read from its two members.
 type MultiReaderB struct {

bolt.go

@@ -1583,3 +1583,7 @@ func (tx *BoltTx) Options() Txo {
 func (tx *BoltTx) Sn() int64 {
 	return tx.sn
 }
+
+func (c *BoltTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) {
+	return GenericApplyFilter(c, index, field, view, shard, ckey, filter)
+}

catcher.go

@@ -307,3 +307,7 @@ func (c *catcherTx) Options() Txo {
 func (c *catcherTx) Sn() int64 {
 	return c.b.Sn()
 }
+
+func (c *catcherTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) {
+	return GenericApplyFilter(c, index, field, view, shard, ckey, filter)
+}

fragment.go

@@ -3179,20 +3179,18 @@ func (f *fragment) rows(ctx context.Context, tx Tx, start uint64, filters ...roa
 // unprotectedRows calls rows without grabbing the mutex.
 func (f *fragment) unprotectedRows(ctx context.Context, tx Tx, start uint64, filters ...roaring.BitmapFilter) ([]uint64, error) {
 	var rows []uint64
-	startKey := rowToKey(start)
-	i, _, err := tx.ContainerIterator(f.index(), f.field(), f.view(), f.shard, startKey)
-	if err != nil {
-		return nil, err
-	} else if i == nil {
-		return rows, nil
-	}
-	callback := func(row uint64) error {
+	cb := func(row uint64) error {
 		rows = append(rows, row)
 		return nil
 	}
-	filter := roaring.NewBitmapRowFilter(callback, filters...)
-	err = roaring.ApplyFilterToIterator(filter, i)
-	return rows, err
+	startKey := rowToKey(start)
+	filter := roaring.NewBitmapRowFilter(cb, filters...)
+	err := tx.ApplyFilter(f.index(), f.field(), f.view(), f.shard, startKey, filter)
+	if err != nil {
+		return nil, err
+	} else {
+		return rows, nil
+	}
 }
 
 // blockToRoaringData converts a fragment block into a roaring.Bitmap
@@ -3361,14 +3359,8 @@ func (f *fragment) intRowIterator(tx Tx, wrap bool, filters ...roaring.BitmapFil
 	return &it, nil
 }
 
-<<<<<<< HEAD
-func (f *fragment) foreachRow(tx Tx, filters []rowFilter, fn func(rid uint64) error) error {
-	var lastRow uint64 = math.MaxUint64
-	i, _, err := tx.ContainerIterator(f.index(), f.field(), f.view(), f.shard, rowToKey(0))
-=======
 func (f *fragment) foreachRow(tx Tx, filters []roaring.BitmapFilter, fn func(rid uint64) error) error {
-	i, _, err := tx.ContainerIterator(f.index, f.field, f.view, f.shard, rowToKey(0))
->>>>>>> Allow arbitrary and potentially more efficient filtering of bitmaps
+	i, _, err := tx.ContainerIterator(f.index, f.field(), f.view(), f.shard, rowToKey(0))
 	if err != nil {
 		return err
 	}

rbf.go

@@ -438,6 +438,10 @@ func (tx *RBFTx) UseRowCache() bool {
 	return rbf.EnableRowCache()
 }
 
+func (c *RBFTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) {
+	return GenericApplyFilter(c, index, field, view, shard, ckey, filter)
+}
+
 // rbfName returns a NULL-separated key used for identifying bitmap maps in RBF.
 func rbfName(index, field, view string, shard uint64) string {
 	return string(txkey.Prefix(index, field, view, shard))

rrtx.go

@@ -150,6 +150,10 @@ func (tx *RoaringTx) IncrementOpN(index, field, view string, shard uint64, chang
 	frag.incrementOpN(changedN)
 }
 
+func (c *RoaringTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) {
+	return GenericApplyFilter(c, index, field, view, shard, ckey, filter)
+}
+
 // Rollback
 func (tx *RoaringTx) Rollback() {
 	tx.w.CleanupTx(tx)

stattx.go

@@ -511,6 +511,10 @@ func (c *statTx) ContainerIterator(index, field, view string, shard uint64, firs
 	return c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey)
 }
 
+func (c *statTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) {
+	return GenericApplyFilter(c, index, field, view, shard, ckey, filter)
+}
+
 func (c *statTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
 	me := kForEach
 

tx.go

@@ -112,6 +112,14 @@ type Tx interface {
 	// citer.Close() must be called when the client is done using it.
 	ContainerIterator(index, field, view string, shard uint64, ckey uint64) (citer roaring.ContainerIterator, found bool, err error)
 
+	// ApplyFilter applies a roaring.BitmapFilter to a specified shard,
+	// starting at the given container key. The filter's ConsiderData
+	// method may be called with transient Container objects which *must
+	// not* be retained or referenced after that function exits. Similarly,
+	// their data must not be retained. If you need the data later, you
+	// must copy it into some other memory.
+	ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error)
+
 	// RoaringBitmap retreives the roaring.Bitmap for the entire shard.
 	RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error)
 
@@ -317,3 +325,14 @@ func (b *TxBitmap) Flush() error {
 	}
 	return nil
 }
+
+// GenericApplyFilter implements ApplyFilter in terms of tx.ContainerIterator,
+// as a convenience if a Tx backend hasn't implemented this new function yet.
+func GenericApplyFilter(tx Tx, index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) {
+	iter, _, err := tx.ContainerIterator(index, field, view, shard, ckey)
+	if err != nil {
+		return err
+	}
+	// ApplyFilterToIterator closes the iterator for us.
+	return roaring.ApplyFilterToIterator(filter, iter)
+}