Merge pull request #97 from tgruben/row-difference-in-place

Difference in place at row level

fragment.go

@@ -1247,7 +1247,6 @@ func (f *fragment) rangeGT(bitDepth uint, predicate int64, allowEquality bool) (
 
 func (f *fragment) rangeGTUnsigned(filter *Row, bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) {
 	keep := NewRow()
-
 	// Filter any bits that don't match the current bit value.
 	for i := int(bitDepth - 1); i >= 0; i-- {
 		row := f.row(uint64(bsiOffsetBit + i))
@@ -1260,12 +1259,12 @@ func (f *fragment) rangeGTUnsigned(filter *Row, bitDepth uint, predicate uint64,
 			if bit == 1 {
 				return keep, nil
 			}
-			return filter.Difference(filter.Difference(row).Difference(keep)), nil
+			return filter.Difference(filter.Difference(row, keep)), nil
 		}
 
 		// If bit is set then remove all unset columns not already kept.
 		if bit == 1 {
-			filter = filter.Difference(filter.Difference(row).Difference(keep))
+			filter = filter.Difference(filter.Difference(row, keep))
 			continue
 		}
 
@@ -1333,7 +1332,7 @@ func (f *fragment) rangeBetweenUnsigned(filter *Row, bitDepth uint, predicateMin
 		// GTE predicateMin
 		// If bit is set then remove all unset columns not already kept.
 		if bit1 == 1 {
-			filter = filter.Difference(filter.Difference(row).Difference(keep1))
+			filter = filter.Difference(filter.Difference(row, keep1))
 		} else {
 			// If bit is unset then add columns with set bit to keep.
 			// Don't bother to compute this on the final iteration.

go.mod

@@ -35,7 +35,6 @@ require (
 	github.com/uber-go/atomic v1.4.0 // indirect
 	github.com/uber/jaeger-client-go v2.16.0+incompatible
 	github.com/uber/jaeger-lib v2.2.0+incompatible // indirect
-	github.com/youtube/vitess v2.1.1+incompatible // indirect
 	go.uber.org/atomic v1.4.0 // indirect
 	golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect
 	golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6
@@ -45,7 +44,6 @@ require (
 	google.golang.org/grpc v1.24.0
 	modernc.org/mathutil v1.0.0
 	modernc.org/strutil v1.0.0
-	vitess.io/vitess v2.1.1+incompatible // indirect
 )
 
 go 1.13

go.sum

@@ -159,8 +159,6 @@ github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/
 github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
-github.com/youtube/vitess v2.1.1+incompatible h1:SE+P7DNX/jw5RHFs5CHRhZQjq402EJFCD33JhzQMdDw=
-github.com/youtube/vitess v2.1.1+incompatible/go.mod h1:hpMim5/30F1r+0P8GGtB29d0gWHr0IZ5unS+CG0zMx8=
 go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -218,5 +216,3 @@ modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I=
 modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
 modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE=
 modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
-vitess.io/vitess v2.1.1+incompatible h1:nuuGHiWYWpudD3gOCLeGzol2EJ25e/u5Wer2wV1O130=
-vitess.io/vitess v2.1.1+incompatible/go.mod h1:h4qvkyNYTOC0xI+vcidSWoka0gQAZc9ZPHbkHo48gP0=

roaring/roaring.go

@@ -995,7 +995,15 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) {
 }
 
 // Difference returns the difference of b and other.
-func (b *Bitmap) Difference(other *Bitmap) *Bitmap {
+func (b *Bitmap) Difference(other ...*Bitmap) *Bitmap {
+	output := b.singleDifference(other[0])
+	if len(other) > 1 {
+		output.DifferenceInPlace(other[1:]...)
+	}
+	return output
+}
+
+func (b *Bitmap) singleDifference(other *Bitmap) *Bitmap {
 	output := NewBitmap()
 	iiter, _ := b.Containers.Iterator(0)
 	jiter, _ := other.Containers.Iterator(0)
@@ -5474,7 +5482,7 @@ func (b *Bitmap) DifferenceInPlace(others ...*Bitmap) {
 		staticHandledIters  = [staticSize]handledIter{}
 		bitmapIters         handledIters
 		target              = b
-		removeContainerKeys = make([]uint64, bSize)
+		removeContainerKeys = make([]uint64, 0, bSize)
 	)
 
 	if requiredSliceSize <= staticSize {
@@ -5495,32 +5503,40 @@ func (b *Bitmap) DifferenceInPlace(others ...*Bitmap) {
 
 	targetItr, _ := target.Containers.Iterator(0)
 	// Go through all the containers and remove the other bits
-	n := 0
 	for targetItr.Next() {
 		targetKey, curContainer := targetItr.Value()
 		// Loop until every iters current value has been handled.
-
 		for _, iIter := range bitmapIters {
 			if !iIter.hasNext {
 				continue
 			}
 			iKey, iContainer := iIter.iter.Value()
-			if targetKey == iKey {
-				curContainer.differenceInPlace(iContainer)
-				if curContainer.N() == 0 { //according to comments N = 1-count, so N should == 1 if 0 elements
-					removeContainerKeys[n] = iKey
-					n++
+			for iKey < targetKey {
+				iIter.hasNext = iIter.iter.Next()
+				if iIter.hasNext {
+					iKey, iContainer = iIter.iter.Value()
+				} else {
+					break
+				}
+			}
+
+			if targetKey == iKey {
+				if curContainer.frozen() {
+					curContainer = curContainer.Clone()
+					b.Containers.Put(targetKey, curContainer)
+				}
+				curContainer.differenceInPlace(iContainer)
+				if curContainer.N() == 0 {
+					removeContainerKeys = append(removeContainerKeys, iKey)
 					break
 				}
-				iIter.hasNext = iIter.iter.Next()
-			} else if targetKey > iKey {
 				iIter.hasNext = iIter.iter.Next()
 			}
 		}
 	}
 
-	for i := 0; i < n; i++ {
-		b.Containers.Remove(removeContainerKeys[i])
+	for _, key := range removeContainerKeys {
+		b.Containers.Remove(key)
 
 	}
 	target.Containers.Repair()

row.go

@@ -304,21 +304,26 @@ func (r *Row) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*Row, args ma
 }
 
 // Difference returns the diff of r and other.
-func (r *Row) Difference(other *Row) *Row {
-	var segments []rowSegment
+func (r *Row) Difference(others ...*Row) *Row {
+	var output []rowSegment
+	o := make(map[uint64][]*rowSegment)
 
-	itr := newMergeSegmentIterator(r.segments, other.segments)
-	for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
-		if s0 == nil {
-			continue
-		} else if s1 == nil {
-			segments = append(segments, *s0)
-			continue
+	for x := range others {
+		for y := range others[x].segments {
+			segment := others[x].segments[y]
+			o[segment.shard] = append(o[segment.shard], &segment)
 		}
-		segments = append(segments, *s0.Difference(s1))
 	}
+	for _, segment := range r.segments {
 
-	return &Row{segments: segments}
+		dest, ok := o[segment.shard]
+		if ok {
+			output = append(output, *segment.Difference(dest...))
+		} else {
+			output = append(output, segment)
+		}
+	}
+	return &Row{segments: output}
 }
 
 // GenericUnary returns the results of a generic op on r.
@@ -576,9 +581,13 @@ func (s *rowSegment) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*rowSe
 }
 
 // Difference returns the diff of s and other.
-func (s *rowSegment) Difference(other *rowSegment) *rowSegment {
-	data := s.data.Difference(other.data)
-	data = data.Freeze()
+func (s *rowSegment) Difference(others ...*rowSegment) *rowSegment {
+	datas := make([]*roaring.Bitmap, len(others))
+	for i, other := range others {
+		datas[i] = other.data
+	}
+	data := s.data.Difference(datas...)
+	data.Freeze()
 
 	return &rowSegment{
 		data:     data,

row_test.go

@@ -138,3 +138,61 @@ func TestRow_Includes(t *testing.T) {
 		t.Fatalf("row should include %d", 2*ShardWidth)
 	}
 }
+
+func TestRow_DifferenceInPlace(t *testing.T) {
+	row0 := pilosa.NewRow(0)
+	row1 := pilosa.NewRow()
+	row2 := pilosa.NewRow(0)
+	res := row0.Difference(row1, row2)
+
+	if !row0.Includes(0) {
+		t.Fatal("row should include 0")
+	}
+	if res.Count() != 0 {
+		t.Fatal("results should be empty")
+	}
+}
+func eq(a, b []uint64) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i, v := range a {
+		if v != b[i] {
+			return false
+		}
+	}
+	return true
+}
+func TestRow_DifferenceInPlace2(t *testing.T) {
+	lucky := []uint64{1, 3, 7, 9, 13, 15, 21, 25, 31, 33, 37, 43, 49, 51, 63, 67, 69, 73, 75, 79, 87, 93, 99, 105, 111, 115, 127, 129, 133, 135, 141, 151, 159, 163, 169, 171, 189, 193, 195, 201, 205, 211, 219, 223, 231, 235, 237, 241, 259, 261, 267, 273, 283, 285, 289, 297}
+	src := pilosa.NewRow(lucky...)
+	row1 := pilosa.NewRow()
+	m := uint64(9)
+	for i := m; i <= lucky[len(lucky)-1]; i += m {
+		row1.SetBit(i)
+	}
+	m = uint64(3)
+	row2 := pilosa.NewRow()
+	for i := m; i <= lucky[len(lucky)-1]; i += m {
+		row2.SetBit(i)
+	}
+	row3 := pilosa.NewRow()
+	m = uint64(5)
+	for i := m; i <= lucky[len(lucky)-1]; i += m {
+		row3.SetBit(i)
+	}
+	row4 := pilosa.NewRow()
+	m = uint64(7)
+	for i := m; i <= lucky[len(lucky)-1]; i += m {
+		row4.SetBit(i)
+	}
+	res5 := src.Difference(row1, row2, row3, row4)
+	res6 := src.Difference(row4, row3, row2, row1)
+	res7 := src.Difference(row4).Difference(row3).Difference(row2).Difference(row1)
+	if !eq(res5.Columns(), res6.Columns()) {
+		t.Fatalf("results do not match: %v, %v", res5.Columns(), res6.Columns())
+	}
+	if !eq(res6.Columns(), res7.Columns()) {
+		t.Fatalf("results do not match: %v, %v", res6.Columns(), res7.Columns())
+	}
+}