Merge pull request #754 from benbjohnson/619-internal-range-field

Implement BSI Field Range internally

fragment.go

@@ -498,7 +498,7 @@ func (f *Fragment) FieldValue(columnID uint64, bitDepth uint) (value uint64, exi
 	for i := uint(0); i < bitDepth; i++ {
 		if v, err := f.bit(uint64(i), columnID); err != nil {
 			return 0, false, err
-		} else if !v {
+		} else if v {
 			value |= (1 << i)
 		}
 	}
@@ -513,13 +513,13 @@ func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (
 
 	for i := uint(0); i < bitDepth; i++ {
 		if value&(1<<i) != 0 {
-			if c, err := f.clearBit(uint64(i), columnID); err != nil {
+			if c, err := f.setBit(uint64(i), columnID); err != nil {
 				return changed, err
 			} else if c {
 				changed = true
 			}
 		} else {
-			if c, err := f.setBit(uint64(i), columnID); err != nil {
+			if c, err := f.clearBit(uint64(i), columnID); err != nil {
 				return changed, err
 			} else if c {
 				changed = true
@@ -537,6 +537,115 @@ func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (
 	return changed, nil
 }
 
+func (f *Fragment) FieldRange(op string, bitDepth uint, predicate uint64) (*Bitmap, error) {
+	switch op {
+	case RangeOpEQ:
+		return f.fieldRangeEQ(bitDepth, predicate)
+	case RangeOpLT, RangeOpLTE:
+		return f.fieldRangeLT(bitDepth, predicate, op == RangeOpLTE)
+	case RangeOpGT, RangeOpGTE:
+		return f.fieldRangeGT(bitDepth, predicate, op == RangeOpGTE)
+	default:
+		return nil, ErrInvalidRangeOperation
+	}
+}
+
+func (f *Fragment) fieldRangeEQ(bitDepth uint, predicate uint64) (*Bitmap, error) {
+	// Start with set of columns with values set.
+	b := f.Row(uint64(bitDepth))
+
+	// Filter any bits that don't match the current bit value.
+	for i := int(bitDepth - 1); i >= 0; i-- {
+		row := f.Row(uint64(i))
+		bit := (predicate >> uint(i)) & 1
+
+		if bit == 1 {
+			b = b.Intersect(row)
+		} else {
+			b = b.Difference(row)
+		}
+	}
+
+	return b, nil
+}
+
+func (f *Fragment) fieldRangeLT(bitDepth uint, predicate uint64, allowEquality bool) (*Bitmap, error) {
+	keep := NewBitmap()
+
+	// Start with set of columns with values set.
+	b := f.Row(uint64(bitDepth))
+
+	// Filter any bits that don't match the current bit value.
+	leadingZeros := true
+	for i := int(bitDepth - 1); i >= 0; i-- {
+		row := f.Row(uint64(i))
+		bit := (predicate >> uint(i)) & 1
+
+		// Remove any columns with higher bits set.
+		if leadingZeros {
+			if bit == 0 {
+				b = b.Difference(row)
+				continue
+			} else {
+				leadingZeros = false
+			}
+		}
+
+		// Handle last bit differently.
+		// If bit is zero then return only already kept columns.
+		// If bit is one then remove any one columns.
+		if i == 0 && !allowEquality {
+			if bit == 0 {
+				return keep, nil
+			}
+			return b.Difference(row.Difference(keep)), nil
+		}
+
+		// If bit is zero then remove all set columns not in excluded bitmap.
+		if bit == 0 {
+			b = b.Difference(row.Difference(keep))
+			continue
+		}
+
+		// If bit is set then add columns for set bits to exclude.
+		keep = keep.Union(b.Difference(row))
+	}
+
+	return b, nil
+}
+
+func (f *Fragment) fieldRangeGT(bitDepth uint, predicate uint64, allowEquality bool) (*Bitmap, error) {
+	b := f.Row(uint64(bitDepth))
+	keep := NewBitmap()
+
+	// Filter any bits that don't match the current bit value.
+	for i := int(bitDepth - 1); i >= 0; i-- {
+		row := f.Row(uint64(i))
+		bit := (predicate >> uint(i)) & 1
+
+		// Handle last bit differently.
+		// If bit is one then return only already kept columns.
+		// If bit is zero then remove any unset columns.
+		if i == 0 && !allowEquality {
+			if bit == 1 {
+				return keep, nil
+			}
+			return b.Difference(b.Difference(row).Difference(keep)), nil
+		}
+
+		// If bit is set then remove all unset columns not already kept.
+		if bit == 1 {
+			b = b.Difference(b.Difference(row).Difference(keep))
+			continue
+		}
+
+		// If bit is unset then add columns with set bit to keep.
+		keep = keep.Union(b.Intersect(row))
+	}
+
+	return b, nil
+}
+
 // pos translates the row ID and column ID into a position in the storage bitmap.
 func (f *Fragment) pos(rowID, columnID uint64) (uint64, error) {
 	// Return an error if the column ID is out of the range of the fragment's slice.

fragment_test.go

@@ -216,6 +216,130 @@ func TestFragment_SetFieldValue(t *testing.T) {
 	})
 }
 
+// Ensure a fragment query for matching fields.
+func TestFragment_FieldRange(t *testing.T) {
+	const bitDepth = 16
+
+	t.Run("EQ", func(t *testing.T) {
+		f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
+		defer f.Close()
+
+		// Set values.
+		if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil {
+			t.Fatal(err)
+		}
+
+		// Query for equality.
+		if b, err := f.FieldRange(pilosa.RangeOpEQ, bitDepth, 300); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 4000}) {
+			t.Fatalf("unexpected bits: %+v", b.Bits())
+		}
+	})
+
+	t.Run("LT", func(t *testing.T) {
+		f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
+		defer f.Close()
+
+		// Set values.
+		if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.SetFieldValue(3000, bitDepth, 2817); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.SetFieldValue(4000, bitDepth, 301); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.SetFieldValue(5000, bitDepth, 1); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.SetFieldValue(6000, bitDepth, 0); err != nil {
+			t.Fatal(err)
+		}
+
+		// Query for fields less than (ending with set bit).
+		if b, err := f.FieldRange(pilosa.RangeOpLT, bitDepth, 301); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 5000, 6000}) {
+			t.Fatalf("unexpected bits: %+v", b.Bits())
+		}
+
+		// Query for fields less than (ending with unset bit).
+		if b, err := f.FieldRange(pilosa.RangeOpLT, bitDepth, 300); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(b.Bits(), []uint64{5000, 6000}) {
+			t.Fatalf("unexpected bits: %+v", b.Bits())
+		}
+
+		// Query for fields less than or equal to (ending with set bit).
+		if b, err := f.FieldRange(pilosa.RangeOpLTE, bitDepth, 301); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 4000, 5000, 6000}) {
+			t.Fatalf("unexpected bits: %+v", b.Bits())
+		}
+
+		// Query for fields less than or equal to (ending with unset bit).
+		if b, err := f.FieldRange(pilosa.RangeOpLTE, bitDepth, 300); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 5000, 6000}) {
+			t.Fatalf("unexpected bits: %+v", b.Bits())
+		}
+	})
+
+	t.Run("GT", func(t *testing.T) {
+		f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
+		defer f.Close()
+
+		// Set values.
+		if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.SetFieldValue(3000, bitDepth, 2817); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.SetFieldValue(4000, bitDepth, 301); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.SetFieldValue(5000, bitDepth, 1); err != nil {
+			t.Fatal(err)
+		} else if _, err := f.SetFieldValue(6000, bitDepth, 0); err != nil {
+			t.Fatal(err)
+		}
+
+		// Query for fields greater than (ending with unset bit).
+		if b, err := f.FieldRange(pilosa.RangeOpGT, bitDepth, 300); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) {
+			t.Fatalf("unexpected bits: %+v", b.Bits())
+		}
+
+		// Query for fields greater than (ending with set bit).
+		if b, err := f.FieldRange(pilosa.RangeOpGT, bitDepth, 301); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000}) {
+			t.Fatalf("unexpected bits: %+v", b.Bits())
+		}
+
+		// Query for fields greater than or equal to (ending with unset bit).
+		if b, err := f.FieldRange(pilosa.RangeOpGTE, bitDepth, 300); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 2000, 3000, 4000}) {
+			t.Fatalf("unexpected bits: %+v", b.Bits())
+		}
+
+		// Query for fields greater than or equal to (ending with set bit).
+		if b, err := f.FieldRange(pilosa.RangeOpGTE, bitDepth, 301); err != nil {
+			t.Fatal(err)
+		} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) {
+			t.Fatalf("unexpected bits: %+v", b.Bits())
+		}
+	})
+}
+
 // Ensure a fragment can snapshot correctly.
 func TestFragment_Snapshot(t *testing.T) {
 	f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")

frame.go

@@ -40,6 +40,15 @@ const (
 	DefaultCacheSize = 50000
 )
 
+// List of operators for field range queries.
+const (
+	RangeOpEQ  = "eq"
+	RangeOpLT  = "lt"
+	RangeOpLTE = "lte"
+	RangeOpGT  = "gt"
+	RangeOpGTE = "gte"
+)
+
 // Frame represents a container for views.
 type Frame struct {
 	mu          sync.Mutex
@@ -630,6 +639,27 @@ func (f *Frame) SetFieldValue(columnID uint64, name string, value int64) (change
 	return view.SetFieldValue(columnID, field.BitDepth(), baseValue)
 }
 
+func (f *Frame) FieldRange(name, op string, predicate int64) (*Bitmap, error) {
+	// Retrieve and validate field.
+	field := f.Field(name)
+	if field == nil {
+		return nil, ErrFieldNotFound
+	} else if predicate < field.Min || predicate > field.Max {
+		return nil, nil
+	}
+
+	// Retrieve field's view.
+	view := f.View(ViewFieldPrefix + name)
+	if view == nil {
+		return nil, nil
+	}
+
+	// Adjust predicate to range.
+	baseValue := uint64(predicate - field.Min)
+
+	return view.FieldRange(op, field.BitDepth(), baseValue)
+}
+
 // Import bulk imports data.
 func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) error {
 	// Determine quantum if timestamps are set.

pilosa.go

@@ -57,6 +57,7 @@ var (
 	ErrInvalidFieldValueType  = errors.New("invalid field value type")
 	ErrFieldValueTooLow       = errors.New("field value too low")
 	ErrFieldValueTooHigh      = errors.New("field value too high")
+	ErrInvalidRangeOperation  = errors.New("invalid range operation")
 
 	ErrInvalidView      = errors.New("invalid view")
 	ErrInvalidCacheType = errors.New("invalid cache type")

roaring/roaring.go

@@ -101,6 +101,7 @@ func (b *Bitmap) Add(a ...uint64) (changed bool, err error) {
 
 	return changed, nil
 }
+
 func (b *Bitmap) add(v uint64) bool {
 	hb := highbits(v)
 	i := search64(b.keys, hb)

roaring/roaring_test.go

@@ -15,8 +15,8 @@
 package roaring_test
 
 import (
-	"fmt"
 	"bytes"
+	"fmt"
 	"math"
 	"math/rand"
 	"reflect"

view.go

@@ -300,6 +300,19 @@ func (v *View) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (chan
 	return frag.SetFieldValue(columnID, bitDepth, value)
 }
 
+// FieldRange returns bitmaps with a field value encoding matching the predicate.
+func (v *View) FieldRange(op string, bitDepth uint, predicate uint64) (*Bitmap, error) {
+	bm := NewBitmap()
+	for _, frag := range v.Fragments() {
+		other, err := frag.FieldRange(op, bitDepth, predicate)
+		if err != nil {
+			return nil, err
+		}
+		bm = bm.Union(other)
+	}
+	return bm, nil
+}
+
 // IsInverseView returns true if the view is used for storing an inverted representation.
 func IsInverseView(name string) bool {
 	return strings.HasPrefix(name, ViewInverse)