From 30d5f891c0214b41c6ea26cc33a1ef4d7e492b5a Mon Sep 17 00:00:00 2001 From: Seebs Date: Wed, 9 Dec 2020 13:33:46 -0600 Subject: [PATCH] create naive ApplyFilter --- bluegreentx.go | 4 ++++ bolt.go | 4 ++++ catcher.go | 4 ++++ fragment.go | 28 ++++++++++------------------ rbf.go | 4 ++++ rrtx.go | 4 ++++ stattx.go | 4 ++++ tx.go | 19 +++++++++++++++++++ 8 files changed, 53 insertions(+), 18 deletions(-) diff --git a/bluegreentx.go b/bluegreentx.go index 32f51da4d..738d146c8 100644 --- a/bluegreentx.go +++ b/bluegreentx.go @@ -907,6 +907,10 @@ func (c *blueGreenTx) Sn() int64 { return bsn } +func (c *blueGreenTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { + return GenericApplyFilter(c, index, field, view, shard, ckey, filter) +} + // MultiReaderB is returned by RoaringBitmapReader. It verifies // that identical byte streams are read from its two members. type MultiReaderB struct { diff --git a/bolt.go b/bolt.go index 3c186f3cf..623a1001a 100644 --- a/bolt.go +++ b/bolt.go @@ -1583,3 +1583,7 @@ func (tx *BoltTx) Options() Txo { func (tx *BoltTx) Sn() int64 { return tx.sn } + +func (c *BoltTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { + return GenericApplyFilter(c, index, field, view, shard, ckey, filter) +} diff --git a/catcher.go b/catcher.go index 7ff8b6fd8..960e8527b 100644 --- a/catcher.go +++ b/catcher.go @@ -307,3 +307,7 @@ func (c *catcherTx) Options() Txo { func (c *catcherTx) Sn() int64 { return c.b.Sn() } + +func (c *catcherTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { + return GenericApplyFilter(c, index, field, view, shard, ckey, filter) +} diff --git a/fragment.go b/fragment.go index 306164f25..9214a0eef 100644 --- a/fragment.go +++ b/fragment.go @@ -3179,20 +3179,18 @@ func (f *fragment) rows(ctx context.Context, tx Tx, start uint64, filters ...roa // unprotectedRows calls rows without grabbing the mutex. func (f *fragment) unprotectedRows(ctx context.Context, tx Tx, start uint64, filters ...roaring.BitmapFilter) ([]uint64, error) { var rows []uint64 - startKey := rowToKey(start) - i, _, err := tx.ContainerIterator(f.index(), f.field(), f.view(), f.shard, startKey) - if err != nil { - return nil, err - } else if i == nil { - return rows, nil - } - callback := func(row uint64) error { + cb := func(row uint64) error { rows = append(rows, row) return nil } - filter := roaring.NewBitmapRowFilter(callback, filters...) - err = roaring.ApplyFilterToIterator(filter, i) - return rows, err + startKey := rowToKey(start) + filter := roaring.NewBitmapRowFilter(cb, filters...) + err := tx.ApplyFilter(f.index(), f.field(), f.view(), f.shard, startKey, filter) + if err != nil { + return nil, err + } else { + return rows, nil + } } // blockToRoaringData converts a fragment block into a roaring.Bitmap @@ -3361,14 +3359,8 @@ func (f *fragment) intRowIterator(tx Tx, wrap bool, filters ...roaring.BitmapFil return &it, nil } -<<<<<<< HEAD -func (f *fragment) foreachRow(tx Tx, filters []rowFilter, fn func(rid uint64) error) error { - var lastRow uint64 = math.MaxUint64 - i, _, err := tx.ContainerIterator(f.index(), f.field(), f.view(), f.shard, rowToKey(0)) -======= func (f *fragment) foreachRow(tx Tx, filters []roaring.BitmapFilter, fn func(rid uint64) error) error { - i, _, err := tx.ContainerIterator(f.index, f.field, f.view, f.shard, rowToKey(0)) ->>>>>>> Allow arbitrary and potentially more efficient filtering of bitmaps + i, _, err := tx.ContainerIterator(f.index, f.field(), f.view(), f.shard, rowToKey(0)) if err != nil { return err } diff --git a/rbf.go b/rbf.go index ddddd6316..c0b50d838 100644 --- a/rbf.go +++ b/rbf.go @@ -438,6 +438,10 @@ func (tx *RBFTx) UseRowCache() bool { return rbf.EnableRowCache() } +func (c *RBFTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { + return GenericApplyFilter(c, index, field, view, shard, ckey, filter) +} + // rbfName returns a NULL-separated key used for identifying bitmap maps in RBF. func rbfName(index, field, view string, shard uint64) string { return string(txkey.Prefix(index, field, view, shard)) diff --git a/rrtx.go b/rrtx.go index d8236040e..bd16f04d8 100644 --- a/rrtx.go +++ b/rrtx.go @@ -150,6 +150,10 @@ func (tx *RoaringTx) IncrementOpN(index, field, view string, shard uint64, chang frag.incrementOpN(changedN) } +func (c *RoaringTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { + return GenericApplyFilter(c, index, field, view, shard, ckey, filter) +} + // Rollback func (tx *RoaringTx) Rollback() { tx.w.CleanupTx(tx) diff --git a/stattx.go b/stattx.go index 581533776..be5ea56f1 100644 --- a/stattx.go +++ b/stattx.go @@ -511,6 +511,10 @@ func (c *statTx) ContainerIterator(index, field, view string, shard uint64, firs return c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey) } +func (c *statTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { + return GenericApplyFilter(c, index, field, view, shard, ckey, filter) +} + func (c *statTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error { me := kForEach diff --git a/tx.go b/tx.go index 73834ab43..2d3f77fc9 100644 --- a/tx.go +++ b/tx.go @@ -112,6 +112,14 @@ type Tx interface { // citer.Close() must be called when the client is done using it. ContainerIterator(index, field, view string, shard uint64, ckey uint64) (citer roaring.ContainerIterator, found bool, err error) + // ApplyFilter applies a roaring.BitmapFilter to a specified shard, + // starting at the given container key. The filter's ConsiderData + // method may be called with transient Container objects which *must + // not* be retained or referenced after that function exits. Similarly, + // their data must not be retained. If you need the data later, you + // must copy it into some other memory. + ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) + // RoaringBitmap retreives the roaring.Bitmap for the entire shard. RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) @@ -317,3 +325,14 @@ func (b *TxBitmap) Flush() error { } return nil } + +// GenericApplyFilter implements ApplyFilter in terms of tx.ContainerIterator, +// as a convenience if a Tx backend hasn't implemented this new function yet. +func GenericApplyFilter(tx Tx, index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { + iter, _, err := tx.ContainerIterator(index, field, view, shard, ckey) + if err != nil { + return err + } + // ApplyFilterToIterator closes the iterator for us. + return roaring.ApplyFilterToIterator(filter, iter) +}