From 3235d83c9b6debe7266248c2ec854af3d8783a8e Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Wed, 23 Aug 2017 13:12:28 -0500 Subject: [PATCH 1/2] Xor support for PQL --- bitmap.go | 27 +++++++++++++++++++++++++++ executor.go | 21 +++++++++++++++++++++ executor_test.go | 19 +++++++++++++++++++ 3 files changed, 67 insertions(+) diff --git a/bitmap.go b/bitmap.go index ecdb4c076..eb8f7f799 100644 --- a/bitmap.go +++ b/bitmap.go @@ -97,6 +97,22 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { return &Bitmap{segments: segments} } +// Xor returns the xor of b and other. +func (b *Bitmap) Xor(other *Bitmap) *Bitmap { + var segments []BitmapSegment + + itr := newMergeSegmentIterator(b.segments, other.segments) + for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() { + // Ignore non-overlapping segments. + if s0 == nil || s1 == nil { + continue + } + segments = append(segments, *s0.Xor(s1)) + } + + return &Bitmap{segments: segments} +} + // Union returns the bitwise union of b and other. func (b *Bitmap) Union(other *Bitmap) *Bitmap { var segments []BitmapSegment @@ -342,6 +358,17 @@ func (s *BitmapSegment) Difference(other *BitmapSegment) *BitmapSegment { } } +// Xor returns the xor of s and other. +func (s *BitmapSegment) Xor(other *BitmapSegment) *BitmapSegment { + data := s.data.Xor(&other.data) + + return &BitmapSegment{ + data: *data, + slice: s.slice, + n: data.Count(), + } +} + // SetBit sets the i-th bit of the bitmap. func (s *BitmapSegment) SetBit(i uint64) (changed bool) { s.ensureWritable() diff --git a/executor.go b/executor.go index aa5b69034..24777a6f1 100644 --- a/executor.go +++ b/executor.go @@ -316,6 +316,8 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c * return e.executeRangeSlice(ctx, index, c, slice) case "Union": return e.executeUnionSlice(ctx, index, c, slice) + case "Xor": + return e.executeXorSlice(ctx, index, c, slice) default: return nil, fmt.Errorf("unknown call: %s", c.Name) } @@ -721,6 +723,25 @@ func (e *Executor) executeUnionSlice(ctx context.Context, index string, c *pql.C return other, nil } +// executeXorSlice executes a xor() call for a local slice. +func (e *Executor) executeXorSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) { + other := NewBitmap() + for i, input := range c.Children { + bm, err := e.executeBitmapCallSlice(ctx, index, input, slice) + if err != nil { + return nil, err + } + + if i == 0 { + other = bm + } else { + other = other.Xor(bm) + } + } + other.InvalidateCount() + return other, nil +} + // executeCount executes a count() call. func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (uint64, error) { if len(c.Children) == 0 { diff --git a/executor_test.go b/executor_test.go index 265293aef..269747efb 100644 --- a/executor_test.go +++ b/executor_test.go @@ -187,6 +187,25 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) { } } +// Ensure a xor query can be executed. +func TestExecutor_Execute_Xor(t *testing.T) { + hldr := test.MustOpenHolder() + defer hldr.Close() + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) + + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) + + e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil { + t.Fatal(err) + } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1}) { + t.Fatalf("unexpected bits: %+v", bits) + } +} + // Ensure a count query can be executed. func TestExecutor_Execute_Count(t *testing.T) { hldr := test.MustOpenHolder() From cc6dd708f55105d65385d14ae93167577ff7139c Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 24 Aug 2017 15:32:08 -0500 Subject: [PATCH 2/2] increased test coverage for BitmapSegments;fixed bug in Xor --- bitmap.go | 8 +++-- bitmap_test.go | 92 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 bitmap_test.go diff --git a/bitmap.go b/bitmap.go index eb8f7f799..72296efab 100644 --- a/bitmap.go +++ b/bitmap.go @@ -103,10 +103,14 @@ func (b *Bitmap) Xor(other *Bitmap) *Bitmap { itr := newMergeSegmentIterator(b.segments, other.segments) for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() { - // Ignore non-overlapping segments. - if s0 == nil || s1 == nil { + if s1 == nil { + segments = append(segments, *s0) + continue + } else if s0 == nil { + segments = append(segments, *s1) continue } + segments = append(segments, *s0.Xor(s1)) } diff --git a/bitmap_test.go b/bitmap_test.go new file mode 100644 index 000000000..4fcd1b115 --- /dev/null +++ b/bitmap_test.go @@ -0,0 +1,92 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa_test + +import ( + "reflect" + "testing" + + "github.com/pilosa/pilosa" +) + +// Ensure a bitmap can be merged +func TestBitmap_Merge(t *testing.T) { + bm1 := pilosa.NewBitmap(1, 2, 3, SliceWidth+1, 2*SliceWidth) + bm2 := pilosa.NewBitmap(3, 4, 5) + bm1.Merge(bm2) + + if bm1.Count() != 7 { + t.Fatalf("Count after merge %d != 7\n", bm1.Count()) + } + +} + +// Ensure a bitmap can Xor'ed +func TestBitmap_Xor(t *testing.T) { + bm1 := pilosa.NewBitmap(0, 1, SliceWidth) + bm2 := pilosa.NewBitmap(0, 2*SliceWidth) + exp := []uint64{1, SliceWidth, 2 * SliceWidth} + + res := bm1.Xor(bm2) + if res.Count() != 3 { + t.Fatalf("Test 1 Count after xor %d != 3\n", res.Count()) + } + + if !reflect.DeepEqual(res.Bits(), exp) { + t.Fatalf("Test 2 Results %v != expected %v\n", res.Bits(), exp) + } + res = bm2.Xor(bm1) + if res.Count() != 3 { + t.Fatalf("Test 3 Count after xor %d != 3\n", res.Count()) + } + if !reflect.DeepEqual(res.Bits(), exp) { + t.Fatalf("Test 4 Results %v != expected %v\n", res.Bits(), exp) + } + +} +func TestBitmap_Union_Segment(t *testing.T) { + bm1 := pilosa.NewBitmap(0, 1, SliceWidth) + bm2 := pilosa.NewBitmap(0, 2*SliceWidth) + exp := []uint64{0, 1, SliceWidth, 2 * SliceWidth} + res := bm1.Union(bm2) + + if res.Count() != 4 { + t.Fatalf("Test 1 Count after Union %d != 5\n", res.Count()) + } + if !reflect.DeepEqual(res.Bits(), exp) { + t.Fatalf("Test 2 Union Results %v != expected %v\n", res.Bits(), exp) + } + res = bm2.Union(bm1) + if res.Count() != 4 { + t.Fatalf("Test 3 Count after xor %d != 5\n", res.Count()) + } + if !reflect.DeepEqual(res.Bits(), exp) { + t.Fatalf("Test 2 Union Results %v != expected %v\n", res.Bits(), exp) + } +} + +func TestBitmap_Difference_Segment(t *testing.T) { + bm1 := pilosa.NewBitmap(0, 1, SliceWidth) + bm2 := pilosa.NewBitmap(0, 2*SliceWidth) + exp := []uint64{1, SliceWidth} + res := bm1.Difference(bm2) + + if res.Count() != 2 { + t.Fatalf("Test 1 Count after Difference %d != 5\n", res.Count()) + } + if !reflect.DeepEqual(res.Bits(), exp) { + t.Fatalf("Test 2 Difference Results %v != expected %v\n", res.Bits(), exp) + } +}