Merge pull request #789 from tgruben/xor-pql

Xor support for PQL

bitmap.go

@@ -97,6 +97,26 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap {
 	return &Bitmap{segments: segments}
 }
 
+// Xor returns the xor of b and other.
+func (b *Bitmap) Xor(other *Bitmap) *Bitmap {
+	var segments []BitmapSegment
+
+	itr := newMergeSegmentIterator(b.segments, other.segments)
+	for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
+		if s1 == nil {
+			segments = append(segments, *s0)
+			continue
+		} else if s0 == nil {
+			segments = append(segments, *s1)
+			continue
+		}
+
+		segments = append(segments, *s0.Xor(s1))
+	}
+
+	return &Bitmap{segments: segments}
+}
+
 // Union returns the bitwise union of b and other.
 func (b *Bitmap) Union(other *Bitmap) *Bitmap {
 	var segments []BitmapSegment
@@ -342,6 +362,17 @@ func (s *BitmapSegment) Difference(other *BitmapSegment) *BitmapSegment {
 	}
 }
 
+// Xor returns the xor of s and other.
+func (s *BitmapSegment) Xor(other *BitmapSegment) *BitmapSegment {
+	data := s.data.Xor(&other.data)
+
+	return &BitmapSegment{
+		data:  *data,
+		slice: s.slice,
+		n:     data.Count(),
+	}
+}
+
 // SetBit sets the i-th bit of the bitmap.
 func (s *BitmapSegment) SetBit(i uint64) (changed bool) {
 	s.ensureWritable()

bitmap_test.go

@@ -0,0 +1,92 @@
+// Copyright 2017 Pilosa Corp.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pilosa_test
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/pilosa/pilosa"
+)
+
+// Ensure a bitmap can be merged
+func TestBitmap_Merge(t *testing.T) {
+	bm1 := pilosa.NewBitmap(1, 2, 3, SliceWidth+1, 2*SliceWidth)
+	bm2 := pilosa.NewBitmap(3, 4, 5)
+	bm1.Merge(bm2)
+
+	if bm1.Count() != 7 {
+		t.Fatalf("Count after merge %d != 7\n", bm1.Count())
+	}
+
+}
+
+// Ensure a bitmap can Xor'ed
+func TestBitmap_Xor(t *testing.T) {
+	bm1 := pilosa.NewBitmap(0, 1, SliceWidth)
+	bm2 := pilosa.NewBitmap(0, 2*SliceWidth)
+	exp := []uint64{1, SliceWidth, 2 * SliceWidth}
+
+	res := bm1.Xor(bm2)
+	if res.Count() != 3 {
+		t.Fatalf("Test 1 Count after xor %d != 3\n", res.Count())
+	}
+
+	if !reflect.DeepEqual(res.Bits(), exp) {
+		t.Fatalf("Test 2 Results %v != expected %v\n", res.Bits(), exp)
+	}
+	res = bm2.Xor(bm1)
+	if res.Count() != 3 {
+		t.Fatalf("Test 3 Count after xor %d != 3\n", res.Count())
+	}
+	if !reflect.DeepEqual(res.Bits(), exp) {
+		t.Fatalf("Test 4 Results %v != expected %v\n", res.Bits(), exp)
+	}
+
+}
+func TestBitmap_Union_Segment(t *testing.T) {
+	bm1 := pilosa.NewBitmap(0, 1, SliceWidth)
+	bm2 := pilosa.NewBitmap(0, 2*SliceWidth)
+	exp := []uint64{0, 1, SliceWidth, 2 * SliceWidth}
+	res := bm1.Union(bm2)
+
+	if res.Count() != 4 {
+		t.Fatalf("Test 1 Count after Union %d != 5\n", res.Count())
+	}
+	if !reflect.DeepEqual(res.Bits(), exp) {
+		t.Fatalf("Test 2 Union Results %v != expected %v\n", res.Bits(), exp)
+	}
+	res = bm2.Union(bm1)
+	if res.Count() != 4 {
+		t.Fatalf("Test 3 Count after xor %d != 5\n", res.Count())
+	}
+	if !reflect.DeepEqual(res.Bits(), exp) {
+		t.Fatalf("Test 2 Union Results %v != expected %v\n", res.Bits(), exp)
+	}
+}
+
+func TestBitmap_Difference_Segment(t *testing.T) {
+	bm1 := pilosa.NewBitmap(0, 1, SliceWidth)
+	bm2 := pilosa.NewBitmap(0, 2*SliceWidth)
+	exp := []uint64{1, SliceWidth}
+	res := bm1.Difference(bm2)
+
+	if res.Count() != 2 {
+		t.Fatalf("Test 1 Count after Difference %d != 5\n", res.Count())
+	}
+	if !reflect.DeepEqual(res.Bits(), exp) {
+		t.Fatalf("Test 2 Difference Results %v != expected %v\n", res.Bits(), exp)
+	}
+}

executor.go

@@ -323,6 +323,8 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c *
 		return e.executeRangeSlice(ctx, index, c, slice)
 	case "Union":
 		return e.executeUnionSlice(ctx, index, c, slice)
+	case "Xor":
+		return e.executeXorSlice(ctx, index, c, slice)
 	default:
 		return nil, fmt.Errorf("unknown call: %s", c.Name)
 	}
@@ -728,6 +730,25 @@ func (e *Executor) executeUnionSlice(ctx context.Context, index string, c *pql.C
 	return other, nil
 }
 
+// executeXorSlice executes a xor() call for a local slice.
+func (e *Executor) executeXorSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) {
+	other := NewBitmap()
+	for i, input := range c.Children {
+		bm, err := e.executeBitmapCallSlice(ctx, index, input, slice)
+		if err != nil {
+			return nil, err
+		}
+
+		if i == 0 {
+			other = bm
+		} else {
+			other = other.Xor(bm)
+		}
+	}
+	other.InvalidateCount()
+	return other, nil
+}
+
 // executeCount executes a count() call.
 func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (uint64, error) {
 	if len(c.Children) == 0 {

executor_test.go

@@ -206,6 +206,25 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) {
 	}
 }
 
+// Ensure a xor query can be executed.
+func TestExecutor_Execute_Xor(t *testing.T) {
+	hldr := test.MustOpenHolder()
+	defer hldr.Close()
+	hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0)
+	hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
+	hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
+
+	hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2)
+	hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2)
+
+	e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
+	if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil {
+		t.Fatal(err)
+	} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1}) {
+		t.Fatalf("unexpected bits: %+v", bits)
+	}
+}
+
 // Ensure a count query can be executed.
 func TestExecutor_Execute_Count(t *testing.T) {
 	hldr := test.MustOpenHolder()