Implement BSI Field Range internally

This commit adds `FieldRange()` to the `Frame`, `View`, and
`Fragment` types. It accepts an operator & a predicate and
returns a bitmap of matching field values.

bitmap.go

@@ -97,6 +97,23 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap {
 	return &Bitmap{segments: segments}
 }
 
+// IntersectInverse returns the itersection of b and the inverse other.
+func (b *Bitmap) IntersectInverse(other *Bitmap) *Bitmap {
+	var segments []BitmapSegment
+
+	itr := newMergeSegmentIterator(b.segments, other.segments)
+	for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
+		if s0 == nil {
+			continue
+		} else if s1 == nil {
+			segments = append(segments, *s0)
+		}
+		segments = append(segments, *s0.IntersectInverse(s1))
+	}
+
+	return &Bitmap{segments: segments}
+}
+
 // Union returns the bitwise union of b and other.
 func (b *Bitmap) Union(other *Bitmap) *Bitmap {
 	var segments []BitmapSegment
@@ -320,6 +337,17 @@ func (s *BitmapSegment) Intersect(other *BitmapSegment) *BitmapSegment {
 	}
 }
 
+// IntersectInverse returns the itersection of s and the inverse of other.
+func (s *BitmapSegment) IntersectInverse(other *BitmapSegment) *BitmapSegment {
+	data := s.data.IntersectInverse(&other.data)
+
+	return &BitmapSegment{
+		data:  *data,
+		slice: s.slice,
+		n:     data.Count(),
+	}
+}
+
 // Union returns the bitwise union of s and other.
 func (s *BitmapSegment) Union(other *BitmapSegment) *BitmapSegment {
 	data := s.data.Union(&other.data)

fragment.go

@@ -498,7 +498,7 @@ func (f *Fragment) FieldValue(columnID uint64, bitDepth uint) (value uint64, exi
 	for i := uint(0); i < bitDepth; i++ {
 		if v, err := f.bit(uint64(i), columnID); err != nil {
 			return 0, false, err
-		} else if !v {
+		} else if v {
 			value |= (1 << i)
 		}
 	}
@@ -513,13 +513,13 @@ func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (
 
 	for i := uint(0); i < bitDepth; i++ {
 		if value&(1<<i) != 0 {
-			if c, err := f.clearBit(uint64(i), columnID); err != nil {
+			if c, err := f.setBit(uint64(i), columnID); err != nil {
 				return changed, err
 			} else if c {
 				changed = true
 			}
 		} else {
-			if c, err := f.setBit(uint64(i), columnID); err != nil {
+			if c, err := f.clearBit(uint64(i), columnID); err != nil {
 				return changed, err
 			} else if c {
 				changed = true
@@ -537,6 +537,115 @@ func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (
 	return changed, nil
 }
 
+func (f *Fragment) FieldRange(op string, bitDepth uint, predicate uint64) (*Bitmap, error) {
+	switch op {
+	case RangeOpEQ:
+		return f.fieldRangeEQ(bitDepth, predicate)
+	case RangeOpLT, RangeOpLTE:
+		return f.fieldRangeLT(bitDepth, predicate, op == RangeOpLTE)
+	case RangeOpGT, RangeOpGTE:
+		return f.fieldRangeGT(bitDepth, predicate, op == RangeOpGTE)
+	default:
+		return nil, ErrInvalidRangeOperation
+	}
+}
+
+func (f *Fragment) fieldRangeEQ(bitDepth uint, predicate uint64) (*Bitmap, error) {
+	// Start with set of columns with values set.
+	b := f.Row(uint64(bitDepth))
+
+	// Filter any bits that don't match the current bit value.
+	for i := int(bitDepth - 1); i >= 0; i-- {
+		row := f.Row(uint64(i))
+		bit := (predicate >> uint(i)) & 1
+
+		if bit == 1 {
+			b = b.Intersect(row)
+		} else {
+			b = b.IntersectInverse(row)
+		}
+	}
+
+	return b, nil
+}
+
+func (f *Fragment) fieldRangeLT(bitDepth uint, predicate uint64, allowEquality bool) (*Bitmap, error) {
+	keep := NewBitmap()
+
+	// Start with set of columns with values set.
+	b := f.Row(uint64(bitDepth))
+
+	// Filter any bits that don't match the current bit value.
+	leadingZeros := true
+	for i := int(bitDepth - 1); i >= 0; i-- {
+		row := f.Row(uint64(i))
+		bit := (predicate >> uint(i)) & 1
+
+		// Remove any columns with higher bits set.
+		if leadingZeros {
+			if bit == 0 {
+				b = b.Difference(row)
+				continue
+			} else {
+				leadingZeros = false
+			}
+		}
+
+		// Handle last bit differently.
+		// If bit is zero then return only already kept columns.
+		// If bit is one then remove any one columns.
+		if i == 0 && !allowEquality {
+			if bit == 0 {
+				return keep, nil
+			}
+			return b.Difference(row.Difference(keep)), nil
+		}
+
+		// If bit is zero then remove all set columns not in excluded bitmap.
+		if bit == 0 {
+			b = b.Difference(row.Difference(keep))
+			continue
+		}
+
+		// If bit is set then add columns for set bits to exclude.
+		keep = keep.Union(b.IntersectInverse(row))
+	}
+
+	return b, nil
+}
+
+func (f *Fragment) fieldRangeGT(bitDepth uint, predicate uint64, allowEquality bool) (*Bitmap, error) {
+	b := f.Row(uint64(bitDepth))
+	keep := NewBitmap()
+
+	// Filter any bits that don't match the current bit value.
+	for i := int(bitDepth - 1); i >= 0; i-- {
+		row := f.Row(uint64(i))
+		bit := (predicate >> uint(i)) & 1
+
+		// Handle last bit differently.
+		// If bit is one then return only already kept columns.
+		// If bit is zero then remove any unset columns.
+		if i == 0 && !allowEquality {
+			if bit == 1 {
+				return keep, nil
+			}
+			return b.Difference(b.IntersectInverse(row).Difference(keep)), nil
+		}
+
+		// If bit is set then remove all unset columns not already kept.
+		if bit == 1 {
+			b = b.Difference(b.IntersectInverse(row).Difference(keep))
+			continue
+		}
+
+		// If bit is unset then add columns with set bit to keep.
+		keep = keep.Union(b.Intersect(row))
+	}
+
+	return b, nil
+}
+
 // pos translates the row ID and column ID into a position in the storage bitmap.
 func (f *Fragment) pos(rowID, columnID uint64) (uint64, error) {
 	// Return an error if the column ID is out of the range of the fragment's slice.

fragment_test.go

@@ -216,6 +216,130 @@ func TestFragment_SetFieldValue(t *testing.T) {
 	})
 }
 
+// Ensure a fragment query for matching fields.
+func TestFragment_FieldRange(t *testing.T) {
+	const bitDepth = 16
+
+	t.Run("EQ", func(t *testing.T) {
+		f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
+		defer f.Close()
+
+		// Set values.
+		if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil {
+			t.Fatal(err)
+		}
+
+		// Query for equality.
+		if b, err := f.FieldRange(pilosa.RangeOpEQ, bitDepth, 300); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 4000}) {
+			t.Fatalf("unexpected bits: %+v", b.Bits())
+		}
+	})
+
+	t.Run("LT", func(t *testing.T) {
+		f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
+		defer f.Close()
+
+		// Set values.
+		if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.SetFieldValue(3000, bitDepth, 2817); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.SetFieldValue(4000, bitDepth, 301); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.SetFieldValue(5000, bitDepth, 1); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.SetFieldValue(6000, bitDepth, 0); err != nil {
+			t.Fatal(err)
+		}
+
+		// Query for fields less than (ending with set bit).
+		if b, err := f.FieldRange(pilosa.RangeOpLT, bitDepth, 301); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 5000, 6000}) {
+			t.Fatalf("unexpected bits: %+v", b.Bits())
+		}
+
+		// Query for fields less than (ending with unset bit).
+		if b, err := f.FieldRange(pilosa.RangeOpLT, bitDepth, 300); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(b.Bits(), []uint64{5000, 6000}) {
+			t.Fatalf("unexpected bits: %+v", b.Bits())
+		}
+
+		// Query for fields less than or equal to (ending with set bit).
+		if b, err := f.FieldRange(pilosa.RangeOpLTE, bitDepth, 301); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 4000, 5000, 6000}) {
+			t.Fatalf("unexpected bits: %+v", b.Bits())
+		}
+
+		// Query for fields less than or equal to (ending with unset bit).
+		if b, err := f.FieldRange(pilosa.RangeOpLTE, bitDepth, 300); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 5000, 6000}) {
+			t.Fatalf("unexpected bits: %+v", b.Bits())
+		}
+	})
+
+	t.Run("GT", func(t *testing.T) {
+		f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
+		defer f.Close()
+
+		// Set values.
+		if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.SetFieldValue(3000, bitDepth, 2817); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.SetFieldValue(4000, bitDepth, 301); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.SetFieldValue(5000, bitDepth, 1); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.SetFieldValue(6000, bitDepth, 0); err != nil {
+			t.Fatal(err)
+		}
+
+		// Query for fields greater than (ending with unset bit).
+		if b, err := f.FieldRange(pilosa.RangeOpGT, bitDepth, 300); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) {
+			t.Fatalf("unexpected bits: %+v", b.Bits())
+		}
+
+		// Query for fields greater than (ending with set bit).
+		if b, err := f.FieldRange(pilosa.RangeOpGT, bitDepth, 301); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000}) {
+			t.Fatalf("unexpected bits: %+v", b.Bits())
+		}
+
+		// Query for fields greater than or equal to (ending with unset bit).
+		if b, err := f.FieldRange(pilosa.RangeOpGTE, bitDepth, 300); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 2000, 3000, 4000}) {
+			t.Fatalf("unexpected bits: %+v", b.Bits())
+		}
+
+		// Query for fields greater than or equal to (ending with set bit).
+		if b, err := f.FieldRange(pilosa.RangeOpGTE, bitDepth, 301); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) {
+			t.Fatalf("unexpected bits: %+v", b.Bits())
+		}
+	})
+}
+
 // Ensure a fragment can snapshot correctly.
 func TestFragment_Snapshot(t *testing.T) {
 	f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")

frame.go

@@ -40,6 +40,15 @@ const (
 	DefaultCacheSize = 50000
 )
 
+// List of operators for field range queries.
+const (
+	RangeOpEQ  = "eq"
+	RangeOpLT  = "lt"
+	RangeOpLTE = "lte"
+	RangeOpGT  = "gt"
+	RangeOpGTE = "gte"
+)
+
 // Frame represents a container for views.
 type Frame struct {
 	mu          sync.Mutex
@@ -630,6 +639,27 @@ func (f *Frame) SetFieldValue(columnID uint64, name string, value int64) (change
 	return view.SetFieldValue(columnID, field.BitDepth(), baseValue)
 }
 
+func (f *Frame) FieldRange(name, op string, predicate int64) (*Bitmap, error) {
+	// Retrieve and validate field.
+	field := f.Field(name)
+	if field == nil {
+		return nil, ErrFieldNotFound
+	} else if predicate < field.Min || predicate > field.Max {
+		return nil, nil
+	}
+
+	// Retrieve field's view.
+	view := f.View(ViewFieldPrefix + name)
+	if view == nil {
+		return nil, nil
+	}
+
+	// Adjust predicate to range.
+	baseValue := uint64(predicate - field.Min)
+
+	return view.FieldRange(op, field.BitDepth(), baseValue)
+}
+
 // Import bulk imports data.
 func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) error {
 	// Determine quantum if timestamps are set.

pilosa.go

@@ -57,6 +57,7 @@ var (
 	ErrInvalidFieldValueType  = errors.New("invalid field value type")
 	ErrFieldValueTooLow       = errors.New("field value too low")
 	ErrFieldValueTooHigh      = errors.New("field value too high")
+	ErrInvalidRangeOperation  = errors.New("invalid range operation")
 
 	ErrInvalidView      = errors.New("invalid view")
 	ErrInvalidCacheType = errors.New("invalid cache type")

roaring/roaring.go

@@ -101,6 +101,7 @@ func (b *Bitmap) Add(a ...uint64) (changed bool, err error) {
 
 	return changed, nil
 }
+
 func (b *Bitmap) add(v uint64) bool {
 	hb := highbits(v)
 	i := search64(b.keys, hb)
@@ -373,6 +374,33 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap {
 	return output
 }
 
+// IntersectInverse returns the intersection of b and the inverse of other.
+func (b *Bitmap) IntersectInverse(other *Bitmap) *Bitmap {
+	output := &Bitmap{}
+
+	ki, ci := b.keys, b.containers
+	kj, cj := other.keys, other.containers
+	for {
+		ni, nj := len(ki), len(kj)
+		if ni == 0 && nj == 0 { // eof(i,j)
+			break
+		} else if ni == 0 || (nj != 0 && ki[0] > kj[0]) { // eof(i) or i > j
+			kj, cj = kj[1:], cj[1:]
+		} else if nj == 0 || (ki[0] < kj[0]) { // eof(j) or i < j
+			output.keys = append(output.keys, ki[0])
+			output.containers = append(output.containers, ci[0].clone())
+			ki, ci = ki[1:], ci[1:]
+		} else { // i == j
+			output.keys = append(output.keys, ki[0])
+			output.containers = append(output.containers, intersectInverse(ci[0], cj[0]))
+			ki, ci = ki[1:], ci[1:]
+			kj, cj = kj[1:], cj[1:]
+		}
+	}
+
+	return output
+}
+
 // Union returns the bitwise union of b and other.
 func (b *Bitmap) Union(other *Bitmap) *Bitmap {
 	output := &Bitmap{}
@@ -1429,6 +1457,106 @@ func intersectBitmapBitmap(a, b *container) *container {
 	return output
 }
 
+func intersectInverse(a, b *container) *container {
+	if a.isArray() {
+		if b.isArray() {
+			return intersectInverseArrayArray(a, b)
+		} else {
+			return intersectInverseArrayBitmap(a, b)
+		}
+	} else {
+		if b.isArray() {
+			return intersectInverseBitmapArray(a, b)
+		} else {
+			return intersectInverseBitmapBitmap(a, b)
+		}
+	}
+}
+
+func intersectInverseArrayArray(a, b *container) *container {
+	output := &container{}
+	aa, ab := a.array, b.array
+	for len(aa) > 0 && len(ab) > 0 {
+		if aa[0] < ab[0] {
+			output.array = append(output.array, aa[0])
+			aa = aa[1:]
+		} else if aa[0] > ab[0] {
+			ab = ab[1:]
+		} else {
+			aa, ab = aa[1:], ab[1:]
+		}
+	}
+	output.array = append(output.array, aa...)
+	output.n = len(output.array)
+	return output
+}
+
+func intersectInverseArrayBitmap(a, b *container) *container {
+	output := &container{}
+	aa := a.array
+	itr := newBufIterator(newBitmapIterator(b.bitmap))
+	for len(aa) > 0 {
+		vb, eof := itr.next()
+
+		if aa[0] < vb || eof {
+			output.add(aa[0])
+			aa = aa[1:]
+			itr.unread()
+		} else if aa[0] > vb {
+			// nop
+		} else {
+			aa = aa[1:]
+		}
+	}
+	return output
+}
+
+func intersectInverseBitmapArray(a, b *container) *container {
+	output := &container{}
+	itr := newBufIterator(newBitmapIterator(a.bitmap))
+	ab := b.array
+	for {
+		va, eof := itr.next()
+		if eof {
+			break
+		}
+
+		if len(ab) == 0 {
+			output.add(ab[0])
+			ab = ab[1:]
+		} else if va < ab[0] {
+			output.add(va)
+		} else if va > ab[0] {
+			// nop
+		} else {
+			ab = ab[1:]
+		}
+	}
+	return output
+}
+
+func intersectInverseBitmapBitmap(a, b *container) *container {
+	output := &container{}
+	itr0 := newBufIterator(newBitmapIterator(a.bitmap))
+	itr1 := newBufIterator(newBitmapIterator(b.bitmap))
+	for {
+		va, eof := itr0.next()
+		if eof {
+			break
+		}
+
+		vb, eof := itr1.next()
+
+		if va < vb || eof {
+			output.add(va)
+			itr1.unread()
+		} else if va > vb {
+			itr0.unread()
+		}
+	}
+	return output
+}
+
 func union(a, b *container) *container {
 	if a.isArray() {
 		if b.isArray() {

roaring/roaring_test.go

@@ -15,8 +15,8 @@
 package roaring_test
 
 import (
-	"fmt"
 	"bytes"
+	"fmt"
 	"math"
 	"math/rand"
 	"reflect"
@@ -109,6 +109,19 @@ func TestBitmap_Intersection(t *testing.T) {
 
 }
 
+func TestBitmap_IntersectInverse(t *testing.T) {
+	bm0 := roaring.NewBitmap(200, 2683177, 3000000)
+	bm1 := roaring.NewBitmap()
+	for i := uint64(628); i < 2683301; i++ {
+		bm1.Add(i)
+	}
+
+	result := bm0.IntersectInverse(bm1)
+	if n := result.Count(); n != 2 {
+		t.Fatalf("unexpected n: %d (%#v)", n, result.Slice())
+	}
+}
+
 func TestBitmap_Difference(t *testing.T) {
 	bm0 := roaring.NewBitmap(0, 2683177)
 	bm1 := roaring.NewBitmap()

view.go

@@ -300,6 +300,19 @@ func (v *View) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (chan
 	return frag.SetFieldValue(columnID, bitDepth, value)
 }
 
+// FieldRange returns bitmaps with a field value encoding matching the predicate.
+func (v *View) FieldRange(op string, bitDepth uint, predicate uint64) (*Bitmap, error) {
+	bm := NewBitmap()
+	for _, frag := range v.Fragments() {
+		other, err := frag.FieldRange(op, bitDepth, predicate)
+		if err != nil {
+			return nil, err
+		}
+		bm = bm.Union(other)
+	}
+	return bm, nil
+}
+
 // IsInverseView returns true if the view is used for storing an inverted representation.
 func IsInverseView(name string) bool {
 	return strings.HasPrefix(name, ViewInverse)