diff --git a/bitmap.go b/bitmap.go index ecdb4c076..78dbc44fb 100644 --- a/bitmap.go +++ b/bitmap.go @@ -97,6 +97,23 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { return &Bitmap{segments: segments} } +// IntersectInverse returns the itersection of b and the inverse other. +func (b *Bitmap) IntersectInverse(other *Bitmap) *Bitmap { + var segments []BitmapSegment + + itr := newMergeSegmentIterator(b.segments, other.segments) + for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() { + if s0 == nil { + continue + } else if s1 == nil { + segments = append(segments, *s0) + } + segments = append(segments, *s0.IntersectInverse(s1)) + } + + return &Bitmap{segments: segments} +} + // Union returns the bitwise union of b and other. func (b *Bitmap) Union(other *Bitmap) *Bitmap { var segments []BitmapSegment @@ -320,6 +337,17 @@ func (s *BitmapSegment) Intersect(other *BitmapSegment) *BitmapSegment { } } +// IntersectInverse returns the itersection of s and the inverse of other. +func (s *BitmapSegment) IntersectInverse(other *BitmapSegment) *BitmapSegment { + data := s.data.IntersectInverse(&other.data) + + return &BitmapSegment{ + data: *data, + slice: s.slice, + n: data.Count(), + } +} + // Union returns the bitwise union of s and other. func (s *BitmapSegment) Union(other *BitmapSegment) *BitmapSegment { data := s.data.Union(&other.data) diff --git a/fragment.go b/fragment.go index b20dfd9c0..b12811d7f 100644 --- a/fragment.go +++ b/fragment.go @@ -498,7 +498,7 @@ func (f *Fragment) FieldValue(columnID uint64, bitDepth uint) (value uint64, exi for i := uint(0); i < bitDepth; i++ { if v, err := f.bit(uint64(i), columnID); err != nil { return 0, false, err - } else if !v { + } else if v { value |= (1 << i) } } @@ -513,13 +513,13 @@ func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) ( for i := uint(0); i < bitDepth; i++ { if value&(1<= 0; i-- { + row := f.Row(uint64(i)) + bit := (predicate >> uint(i)) & 1 + + if bit == 1 { + b = b.Intersect(row) + } else { + b = b.IntersectInverse(row) + } + } + + return b, nil +} + +func (f *Fragment) fieldRangeLT(bitDepth uint, predicate uint64, allowEquality bool) (*Bitmap, error) { + keep := NewBitmap() + + // Start with set of columns with values set. + b := f.Row(uint64(bitDepth)) + + // Filter any bits that don't match the current bit value. + leadingZeros := true + for i := int(bitDepth - 1); i >= 0; i-- { + row := f.Row(uint64(i)) + bit := (predicate >> uint(i)) & 1 + + // Remove any columns with higher bits set. + if leadingZeros { + if bit == 0 { + b = b.Difference(row) + continue + } else { + leadingZeros = false + } + } + + // Handle last bit differently. + // If bit is zero then return only already kept columns. + // If bit is one then remove any one columns. + if i == 0 && !allowEquality { + if bit == 0 { + return keep, nil + } + return b.Difference(row.Difference(keep)), nil + } + + // If bit is zero then remove all set columns not in excluded bitmap. + if bit == 0 { + b = b.Difference(row.Difference(keep)) + continue + } + + // If bit is set then add columns for set bits to exclude. + keep = keep.Union(b.IntersectInverse(row)) + } + + return b, nil +} + +func (f *Fragment) fieldRangeGT(bitDepth uint, predicate uint64, allowEquality bool) (*Bitmap, error) { + b := f.Row(uint64(bitDepth)) + keep := NewBitmap() + + // Filter any bits that don't match the current bit value. + for i := int(bitDepth - 1); i >= 0; i-- { + row := f.Row(uint64(i)) + bit := (predicate >> uint(i)) & 1 + + // Handle last bit differently. + // If bit is one then return only already kept columns. + // If bit is zero then remove any unset columns. + if i == 0 && !allowEquality { + if bit == 1 { + return keep, nil + } + return b.Difference(b.IntersectInverse(row).Difference(keep)), nil + } + + // If bit is set then remove all unset columns not already kept. + if bit == 1 { + b = b.Difference(b.IntersectInverse(row).Difference(keep)) + continue + } + + // If bit is unset then add columns with set bit to keep. + keep = keep.Union(b.Intersect(row)) + } + + return b, nil +} + // pos translates the row ID and column ID into a position in the storage bitmap. func (f *Fragment) pos(rowID, columnID uint64) (uint64, error) { // Return an error if the column ID is out of the range of the fragment's slice. diff --git a/fragment_test.go b/fragment_test.go index b58db2a1a..5c7ae66cb 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -216,6 +216,130 @@ func TestFragment_SetFieldValue(t *testing.T) { }) } +// Ensure a fragment query for matching fields. +func TestFragment_FieldRange(t *testing.T) { + const bitDepth = 16 + + t.Run("EQ", func(t *testing.T) { + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + defer f.Close() + + // Set values. + if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil { + t.Fatal(err) + } + + // Query for equality. + if b, err := f.FieldRange(pilosa.RangeOpEQ, bitDepth, 300); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 4000}) { + t.Fatalf("unexpected bits: %+v", b.Bits()) + } + }) + + t.Run("LT", func(t *testing.T) { + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + defer f.Close() + + // Set values. + if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(3000, bitDepth, 2817); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(4000, bitDepth, 301); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(5000, bitDepth, 1); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(6000, bitDepth, 0); err != nil { + t.Fatal(err) + } + + // Query for fields less than (ending with set bit). + if b, err := f.FieldRange(pilosa.RangeOpLT, bitDepth, 301); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 5000, 6000}) { + t.Fatalf("unexpected bits: %+v", b.Bits()) + } + + // Query for fields less than (ending with unset bit). + if b, err := f.FieldRange(pilosa.RangeOpLT, bitDepth, 300); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(b.Bits(), []uint64{5000, 6000}) { + t.Fatalf("unexpected bits: %+v", b.Bits()) + } + + // Query for fields less than or equal to (ending with set bit). + if b, err := f.FieldRange(pilosa.RangeOpLTE, bitDepth, 301); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 4000, 5000, 6000}) { + t.Fatalf("unexpected bits: %+v", b.Bits()) + } + + // Query for fields less than or equal to (ending with unset bit). + if b, err := f.FieldRange(pilosa.RangeOpLTE, bitDepth, 300); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 5000, 6000}) { + t.Fatalf("unexpected bits: %+v", b.Bits()) + } + }) + + t.Run("GT", func(t *testing.T) { + f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") + defer f.Close() + + // Set values. + if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(3000, bitDepth, 2817); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(4000, bitDepth, 301); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(5000, bitDepth, 1); err != nil { + t.Fatal(err) + } else if _, err := f.SetFieldValue(6000, bitDepth, 0); err != nil { + t.Fatal(err) + } + + // Query for fields greater than (ending with unset bit). + if b, err := f.FieldRange(pilosa.RangeOpGT, bitDepth, 300); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) { + t.Fatalf("unexpected bits: %+v", b.Bits()) + } + + // Query for fields greater than (ending with set bit). + if b, err := f.FieldRange(pilosa.RangeOpGT, bitDepth, 301); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000}) { + t.Fatalf("unexpected bits: %+v", b.Bits()) + } + + // Query for fields greater than or equal to (ending with unset bit). + if b, err := f.FieldRange(pilosa.RangeOpGTE, bitDepth, 300); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 2000, 3000, 4000}) { + t.Fatalf("unexpected bits: %+v", b.Bits()) + } + + // Query for fields greater than or equal to (ending with set bit). + if b, err := f.FieldRange(pilosa.RangeOpGTE, bitDepth, 301); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) { + t.Fatalf("unexpected bits: %+v", b.Bits()) + } + }) +} + // Ensure a fragment can snapshot correctly. func TestFragment_Snapshot(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") diff --git a/frame.go b/frame.go index d102c9ea7..de3dd6fbb 100644 --- a/frame.go +++ b/frame.go @@ -40,6 +40,15 @@ const ( DefaultCacheSize = 50000 ) +// List of operators for field range queries. +const ( + RangeOpEQ = "eq" + RangeOpLT = "lt" + RangeOpLTE = "lte" + RangeOpGT = "gt" + RangeOpGTE = "gte" +) + // Frame represents a container for views. type Frame struct { mu sync.Mutex @@ -630,6 +639,27 @@ func (f *Frame) SetFieldValue(columnID uint64, name string, value int64) (change return view.SetFieldValue(columnID, field.BitDepth(), baseValue) } +func (f *Frame) FieldRange(name, op string, predicate int64) (*Bitmap, error) { + // Retrieve and validate field. + field := f.Field(name) + if field == nil { + return nil, ErrFieldNotFound + } else if predicate < field.Min || predicate > field.Max { + return nil, nil + } + + // Retrieve field's view. + view := f.View(ViewFieldPrefix + name) + if view == nil { + return nil, nil + } + + // Adjust predicate to range. + baseValue := uint64(predicate - field.Min) + + return view.FieldRange(op, field.BitDepth(), baseValue) +} + // Import bulk imports data. func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) error { // Determine quantum if timestamps are set. diff --git a/pilosa.go b/pilosa.go index aae9c0dab..faba8d55a 100644 --- a/pilosa.go +++ b/pilosa.go @@ -57,6 +57,7 @@ var ( ErrInvalidFieldValueType = errors.New("invalid field value type") ErrFieldValueTooLow = errors.New("field value too low") ErrFieldValueTooHigh = errors.New("field value too high") + ErrInvalidRangeOperation = errors.New("invalid range operation") ErrInvalidView = errors.New("invalid view") ErrInvalidCacheType = errors.New("invalid cache type") diff --git a/roaring/roaring.go b/roaring/roaring.go index c5a8b1650..52104977b 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -101,6 +101,7 @@ func (b *Bitmap) Add(a ...uint64) (changed bool, err error) { return changed, nil } + func (b *Bitmap) add(v uint64) bool { hb := highbits(v) i := search64(b.keys, hb) @@ -373,6 +374,33 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { return output } +// IntersectInverse returns the intersection of b and the inverse of other. +func (b *Bitmap) IntersectInverse(other *Bitmap) *Bitmap { + output := &Bitmap{} + + ki, ci := b.keys, b.containers + kj, cj := other.keys, other.containers + for { + ni, nj := len(ki), len(kj) + if ni == 0 && nj == 0 { // eof(i,j) + break + } else if ni == 0 || (nj != 0 && ki[0] > kj[0]) { // eof(i) or i > j + kj, cj = kj[1:], cj[1:] + } else if nj == 0 || (ki[0] < kj[0]) { // eof(j) or i < j + output.keys = append(output.keys, ki[0]) + output.containers = append(output.containers, ci[0].clone()) + ki, ci = ki[1:], ci[1:] + } else { // i == j + output.keys = append(output.keys, ki[0]) + output.containers = append(output.containers, intersectInverse(ci[0], cj[0])) + ki, ci = ki[1:], ci[1:] + kj, cj = kj[1:], cj[1:] + } + } + + return output +} + // Union returns the bitwise union of b and other. func (b *Bitmap) Union(other *Bitmap) *Bitmap { output := &Bitmap{} @@ -1429,6 +1457,106 @@ func intersectBitmapBitmap(a, b *container) *container { return output } +func intersectInverse(a, b *container) *container { + if a.isArray() { + if b.isArray() { + return intersectInverseArrayArray(a, b) + } else { + return intersectInverseArrayBitmap(a, b) + } + } else { + if b.isArray() { + return intersectInverseBitmapArray(a, b) + } else { + return intersectInverseBitmapBitmap(a, b) + } + } +} + +func intersectInverseArrayArray(a, b *container) *container { + output := &container{} + aa, ab := a.array, b.array + for len(aa) > 0 && len(ab) > 0 { + if aa[0] < ab[0] { + output.array = append(output.array, aa[0]) + aa = aa[1:] + } else if aa[0] > ab[0] { + ab = ab[1:] + } else { + aa, ab = aa[1:], ab[1:] + } + } + output.array = append(output.array, aa...) + output.n = len(output.array) + return output +} + +func intersectInverseArrayBitmap(a, b *container) *container { + output := &container{} + aa := a.array + itr := newBufIterator(newBitmapIterator(b.bitmap)) + for len(aa) > 0 { + vb, eof := itr.next() + + if aa[0] < vb || eof { + output.add(aa[0]) + aa = aa[1:] + itr.unread() + } else if aa[0] > vb { + // nop + } else { + aa = aa[1:] + } + } + return output +} + +func intersectInverseBitmapArray(a, b *container) *container { + output := &container{} + itr := newBufIterator(newBitmapIterator(a.bitmap)) + ab := b.array + for { + va, eof := itr.next() + if eof { + break + } + + if len(ab) == 0 { + output.add(ab[0]) + ab = ab[1:] + } else if va < ab[0] { + output.add(va) + } else if va > ab[0] { + // nop + } else { + ab = ab[1:] + } + } + return output +} + +func intersectInverseBitmapBitmap(a, b *container) *container { + output := &container{} + itr0 := newBufIterator(newBitmapIterator(a.bitmap)) + itr1 := newBufIterator(newBitmapIterator(b.bitmap)) + for { + va, eof := itr0.next() + if eof { + break + } + + vb, eof := itr1.next() + + if va < vb || eof { + output.add(va) + itr1.unread() + } else if va > vb { + itr0.unread() + } + } + return output +} + func union(a, b *container) *container { if a.isArray() { if b.isArray() { diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index d1bb609e2..1a29d5278 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -15,8 +15,8 @@ package roaring_test import ( - "fmt" "bytes" + "fmt" "math" "math/rand" "reflect" @@ -109,6 +109,19 @@ func TestBitmap_Intersection(t *testing.T) { } +func TestBitmap_IntersectInverse(t *testing.T) { + bm0 := roaring.NewBitmap(200, 2683177, 3000000) + bm1 := roaring.NewBitmap() + for i := uint64(628); i < 2683301; i++ { + bm1.Add(i) + } + + result := bm0.IntersectInverse(bm1) + if n := result.Count(); n != 2 { + t.Fatalf("unexpected n: %d (%#v)", n, result.Slice()) + } +} + func TestBitmap_Difference(t *testing.T) { bm0 := roaring.NewBitmap(0, 2683177) bm1 := roaring.NewBitmap() diff --git a/view.go b/view.go index 2f47fae46..1aa29bbb4 100644 --- a/view.go +++ b/view.go @@ -300,6 +300,19 @@ func (v *View) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (chan return frag.SetFieldValue(columnID, bitDepth, value) } +// FieldRange returns bitmaps with a field value encoding matching the predicate. +func (v *View) FieldRange(op string, bitDepth uint, predicate uint64) (*Bitmap, error) { + bm := NewBitmap() + for _, frag := range v.Fragments() { + other, err := frag.FieldRange(op, bitDepth, predicate) + if err != nil { + return nil, err + } + bm = bm.Union(other) + } + return bm, nil +} + // IsInverseView returns true if the view is used for storing an inverted representation. func IsInverseView(name string) bool { return strings.HasPrefix(name, ViewInverse)