diff --git a/fragment.go b/fragment.go index b20dfd9c0..c4fd56764 100644 --- a/fragment.go +++ b/fragment.go @@ -498,7 +498,7 @@ func (f *Fragment) FieldValue(columnID uint64, bitDepth uint) (value uint64, exi for i := uint(0); i < bitDepth; i++ { if v, err := f.bit(uint64(i), columnID); err != nil { return 0, false, err - } else if !v { + } else if v { value |= (1 << i) } } @@ -513,13 +513,13 @@ func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) ( for i := uint(0); i < bitDepth; i++ { if value&(1<= 0; i-- { + row := f.Row(uint64(i)) + bit := (predicate >> uint(i)) & 1 + + if bit == 1 { + b = b.Intersect(row) + } else { + b = b.Difference(row) + } + } + + return b, nil +} + +func (f *Fragment) fieldRangeLT(bitDepth uint, predicate uint64, allowEquality bool) (*Bitmap, error) { + keep := NewBitmap() + + // Start with set of columns with values set. + b := f.Row(uint64(bitDepth)) + + // Filter any bits that don't match the current bit value. + leadingZeros := true + for i := int(bitDepth - 1); i >= 0; i-- { + row := f.Row(uint64(i)) + bit := (predicate >> uint(i)) & 1 + + // Remove any columns with higher bits set. + if leadingZeros { + if bit == 0 { + b = b.Difference(row) + continue + } else { + leadingZeros = false + } + } + + // Handle last bit differently. + // If bit is zero then return only already kept columns. + // If bit is one then remove any one columns. + if i == 0 && !allowEquality { + if bit == 0 { + return keep, nil + } + return b.Difference(row.Difference(keep)), nil + } + + // If bit is zero then remove all set columns not in excluded bitmap. + if bit == 0 { + b = b.Difference(row.Difference(keep)) + continue + } + + // If bit is set then add columns for set bits to exclude. + keep = keep.Union(b.Difference(row)) + } + + return b, nil +} + +func (f *Fragment) fieldRangeGT(bitDepth uint, predicate uint64, allowEquality bool) (*Bitmap, error) { + b := f.Row(uint64(bitDepth)) + keep := NewBitmap() + + // Filter any bits that don't match the current bit value. + for i := int(bitDepth - 1); i >= 0; i-- { + row := f.Row(uint64(i)) + bit := (predicate >> uint(i)) & 1 + + // Handle last bit differently. + // If bit is one then return only already kept columns. + // If bit is zero then remove any unset columns. + if i == 0 && !allowEquality { + if bit == 1 { + return keep, nil + } + return b.Difference(b.Difference(row).Difference(keep)), nil + } + + // If bit is set then remove all unset columns not already kept. + if bit == 1 { + b = b.Difference(b.Difference(row).Difference(keep)) + continue + } + + // If bit is unset then add columns with set bit to keep. + keep = keep.Union(b.Intersect(row)) + } + + return b, nil +} + // pos translates the row ID and column ID into a position in the storage bitmap. func (f *Fragment) pos(rowID, columnID uint64) (uint64, error) { // Return an error if the column ID is out of the range of the fragment's slice. diff --git a/fragment_test.go b/fragment_test.go index b58db2a1a..5c7ae66cb 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -216,6 +216,130 @@ func TestFragment_SetFieldValue(t *testing.T) { }) } +// Ensure a fragment query for matching fields. +func TestFragment_FieldRange(t *testing.T) { + const bitDepth = 16 + + t.Run("EQ", func(t *testing.T) { + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + defer f.Close() + + // Set values. + if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil { + t.Fatal(err) + } + + // Query for equality. + if b, err := f.FieldRange(pilosa.RangeOpEQ, bitDepth, 300); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 4000}) { + t.Fatalf("unexpected bits: %+v", b.Bits()) + } + }) + + t.Run("LT", func(t *testing.T) { + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + defer f.Close() + + // Set values. + if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(3000, bitDepth, 2817); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(4000, bitDepth, 301); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(5000, bitDepth, 1); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(6000, bitDepth, 0); err != nil { + t.Fatal(err) + } + + // Query for fields less than (ending with set bit). + if b, err := f.FieldRange(pilosa.RangeOpLT, bitDepth, 301); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 5000, 6000}) { + t.Fatalf("unexpected bits: %+v", b.Bits()) + } + + // Query for fields less than (ending with unset bit). + if b, err := f.FieldRange(pilosa.RangeOpLT, bitDepth, 300); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(b.Bits(), []uint64{5000, 6000}) { + t.Fatalf("unexpected bits: %+v", b.Bits()) + } + + // Query for fields less than or equal to (ending with set bit). + if b, err := f.FieldRange(pilosa.RangeOpLTE, bitDepth, 301); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 4000, 5000, 6000}) { + t.Fatalf("unexpected bits: %+v", b.Bits()) + } + + // Query for fields less than or equal to (ending with unset bit). + if b, err := f.FieldRange(pilosa.RangeOpLTE, bitDepth, 300); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 5000, 6000}) { + t.Fatalf("unexpected bits: %+v", b.Bits()) + } + }) + + t.Run("GT", func(t *testing.T) { + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + defer f.Close() + + // Set values. + if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(3000, bitDepth, 2817); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(4000, bitDepth, 301); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(5000, bitDepth, 1); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(6000, bitDepth, 0); err != nil { + t.Fatal(err) + } + + // Query for fields greater than (ending with unset bit). + if b, err := f.FieldRange(pilosa.RangeOpGT, bitDepth, 300); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) { + t.Fatalf("unexpected bits: %+v", b.Bits()) + } + + // Query for fields greater than (ending with set bit). + if b, err := f.FieldRange(pilosa.RangeOpGT, bitDepth, 301); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000}) { + t.Fatalf("unexpected bits: %+v", b.Bits()) + } + + // Query for fields greater than or equal to (ending with unset bit). + if b, err := f.FieldRange(pilosa.RangeOpGTE, bitDepth, 300); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 2000, 3000, 4000}) { + t.Fatalf("unexpected bits: %+v", b.Bits()) + } + + // Query for fields greater than or equal to (ending with set bit). + if b, err := f.FieldRange(pilosa.RangeOpGTE, bitDepth, 301); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) { + t.Fatalf("unexpected bits: %+v", b.Bits()) + } + }) +} + // Ensure a fragment can snapshot correctly. func TestFragment_Snapshot(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") diff --git a/frame.go b/frame.go index d102c9ea7..de3dd6fbb 100644 --- a/frame.go +++ b/frame.go @@ -40,6 +40,15 @@ const ( DefaultCacheSize = 50000 ) +// List of operators for field range queries. +const ( + RangeOpEQ = "eq" + RangeOpLT = "lt" + RangeOpLTE = "lte" + RangeOpGT = "gt" + RangeOpGTE = "gte" +) + // Frame represents a container for views. type Frame struct { mu sync.Mutex @@ -630,6 +639,27 @@ func (f *Frame) SetFieldValue(columnID uint64, name string, value int64) (change return view.SetFieldValue(columnID, field.BitDepth(), baseValue) } +func (f *Frame) FieldRange(name, op string, predicate int64) (*Bitmap, error) { + // Retrieve and validate field. + field := f.Field(name) + if field == nil { + return nil, ErrFieldNotFound + } else if predicate < field.Min || predicate > field.Max { + return nil, nil + } + + // Retrieve field's view. + view := f.View(ViewFieldPrefix + name) + if view == nil { + return nil, nil + } + + // Adjust predicate to range. + baseValue := uint64(predicate - field.Min) + + return view.FieldRange(op, field.BitDepth(), baseValue) +} + // Import bulk imports data. func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) error { // Determine quantum if timestamps are set. diff --git a/pilosa.go b/pilosa.go index aae9c0dab..faba8d55a 100644 --- a/pilosa.go +++ b/pilosa.go @@ -57,6 +57,7 @@ var ( ErrInvalidFieldValueType = errors.New("invalid field value type") ErrFieldValueTooLow = errors.New("field value too low") ErrFieldValueTooHigh = errors.New("field value too high") + ErrInvalidRangeOperation = errors.New("invalid range operation") ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type") diff --git a/roaring/roaring.go b/roaring/roaring.go index c5a8b1650..03c4445f9 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -101,6 +101,7 @@ func (b *Bitmap) Add(a ...uint64) (changed bool, err error) { return changed, nil } + func (b *Bitmap) add(v uint64) bool { hb := highbits(v) i := search64(b.keys, hb) diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index d1bb609e2..4166ca880 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -15,8 +15,8 @@ package roaring_test import ( - "fmt" "bytes" + "fmt" "math" "math/rand" "reflect" diff --git a/view.go b/view.go index 2f47fae46..1aa29bbb4 100644 --- a/view.go +++ b/view.go @@ -300,6 +300,19 @@ func (v *View) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (chan return frag.SetFieldValue(columnID, bitDepth, value) } +// FieldRange returns bitmaps with a field value encoding matching the predicate. +func (v *View) FieldRange(op string, bitDepth uint, predicate uint64) (*Bitmap, error) { + bm := NewBitmap() + for _, frag := range v.Fragments() { + other, err := frag.FieldRange(op, bitDepth, predicate) + if err != nil { + return nil, err + } + bm = bm.Union(other) + } + return bm, nil +} + // IsInverseView returns true if the view is used for storing an inverted representation. func IsInverseView(name string) bool { return strings.HasPrefix(name, ViewInverse)