diff --git a/fragment.go b/fragment.go index e6c714937..22efb072e 100644 --- a/fragment.go +++ b/fragment.go @@ -1247,7 +1247,6 @@ func (f *fragment) rangeGT(bitDepth uint, predicate int64, allowEquality bool) ( func (f *fragment) rangeGTUnsigned(filter *Row, bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { keep := NewRow() - // Filter any bits that don't match the current bit value. for i := int(bitDepth - 1); i >= 0; i-- { row := f.row(uint64(bsiOffsetBit + i)) @@ -1260,12 +1259,12 @@ func (f *fragment) rangeGTUnsigned(filter *Row, bitDepth uint, predicate uint64, if bit == 1 { return keep, nil } - return filter.Difference(filter.Difference(row).Difference(keep)), nil + return filter.Difference(filter.Difference(row, keep)), nil } // If bit is set then remove all unset columns not already kept. if bit == 1 { - filter = filter.Difference(filter.Difference(row).Difference(keep)) + filter = filter.Difference(filter.Difference(row, keep)) continue } @@ -1333,7 +1332,7 @@ func (f *fragment) rangeBetweenUnsigned(filter *Row, bitDepth uint, predicateMin // GTE predicateMin // If bit is set then remove all unset columns not already kept. if bit1 == 1 { - filter = filter.Difference(filter.Difference(row).Difference(keep1)) + filter = filter.Difference(filter.Difference(row, keep1)) } else { // If bit is unset then add columns with set bit to keep. // Don't bother to compute this on the final iteration. diff --git a/go.mod b/go.mod index 819785c3b..e8f70176e 100644 --- a/go.mod +++ b/go.mod @@ -8,36 +8,42 @@ require ( github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/boltdb/bolt v1.3.1 github.com/cespare/xxhash v1.1.0 + github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect github.com/davecgh/go-spew v1.1.1 + github.com/go-ole/go-ole v1.2.4 // indirect github.com/gogo/protobuf v1.2.0 github.com/golang/protobuf v1.3.2 github.com/google/go-cmp v0.2.0 github.com/gorilla/handlers v1.3.0 github.com/gorilla/mux v1.7.0 github.com/hashicorp/memberlist v0.1.3 + github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/molecula/ext v0.0.0-20200103203257-8a458a73e8c2 github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4 github.com/opentracing/opentracing-go v1.1.0 github.com/pelletier/go-toml v1.2.0 - github.com/pilosa/pilosa v1.4.0 github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v0.9.3 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect github.com/satori/go.uuid v1.2.0 github.com/shirou/gopsutil v2.18.12+incompatible + github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 github.com/spf13/viper v1.3.1 + github.com/uber-go/atomic v1.4.0 // indirect github.com/uber/jaeger-client-go v2.16.0+incompatible github.com/uber/jaeger-lib v2.2.0+incompatible // indirect - github.com/youtube/vitess v2.1.1+incompatible // indirect + go.uber.org/atomic v1.4.0 // indirect + golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 golang.org/x/sync v0.0.0-20190423024810-112230192c58 + golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 // indirect + golang.org/x/text v0.3.2 // indirect google.golang.org/grpc v1.24.0 modernc.org/mathutil v1.0.0 modernc.org/strutil v1.0.0 - vitess.io/vitess v2.1.1+incompatible // indirect ) go 1.13 diff --git a/go.sum b/go.sum index 54cb1e76e..85e0347f6 100644 --- a/go.sum +++ b/go.sum @@ -7,7 +7,6 @@ github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5Vk github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -106,8 +105,6 @@ github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 h1:ERLyN4p3KS5Fk2ADsDENm2cq0+Lx6sF1sG8uwRlySpU= github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/pilosa/pilosa v1.4.0 h1:nqHNIK4nDslFnem3yDp9R+6TgLdlkY9WdJD88Z83T8U= -github.com/pilosa/pilosa v1.4.0/go.mod h1:NSTtTprtb5MSgCs4mcNqeQ2JdIMpInOi4DEImxGJeTs= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -126,7 +123,6 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= @@ -159,13 +155,10 @@ github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o= github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber/jaeger-client-go v2.16.0+incompatible h1:Q2Pp6v3QYiocMxomCaJuwQGFt7E53bPYqEgug/AoBtY= github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= -github.com/uber/jaeger-lib v2.0.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/youtube/vitess v2.1.1+incompatible h1:SE+P7DNX/jw5RHFs5CHRhZQjq402EJFCD33JhzQMdDw= -github.com/youtube/vitess v2.1.1+incompatible/go.mod h1:hpMim5/30F1r+0P8GGtB29d0gWHr0IZ5unS+CG0zMx8= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -223,5 +216,3 @@ modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= -vitess.io/vitess v2.1.1+incompatible h1:nuuGHiWYWpudD3gOCLeGzol2EJ25e/u5Wer2wV1O130= -vitess.io/vitess v2.1.1+incompatible/go.mod h1:h4qvkyNYTOC0xI+vcidSWoka0gQAZc9ZPHbkHo48gP0= diff --git a/roaring/roaring.go b/roaring/roaring.go index 19a671dbf..1ddc5f798 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -833,16 +833,18 @@ func (b *Bitmap) unionIntoTargetSingle(target *Bitmap, other *Bitmap) { // Bitmap 4 |___X_______________________| | |___X_______________________| | |___X_______________________| // _ func (b *Bitmap) unionInPlace(others ...*Bitmap) { + const staticSize = 20 var ( requiredSliceSize = len(others) - // To avoid having to allocate a slice everytime, if the number of bitmaps - // being unioned is small enough we can just use this stack-allocated array. - staticHandledIters = [20]handledIter{} + // To avoid having to allocate a slice every time, if the number of bitmaps + // being unioned is small enough (i.e. smaller than staticSize), we can just + // use this stack-allocated array. + staticHandledIters = [staticSize]handledIter{} bitmapIters handledIters target = b ) - if requiredSliceSize <= 20 { + if requiredSliceSize <= staticSize { bitmapIters = staticHandledIters[:0] } else { bitmapIters = make(handledIters, 0, requiredSliceSize) @@ -993,10 +995,16 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) { } // Difference returns the difference of b and other. -func (b *Bitmap) Difference(other *Bitmap) *Bitmap { - output := NewBitmap() - output.Source = b.Source +func (b *Bitmap) Difference(other ...*Bitmap) *Bitmap { + output := b.singleDifference(other[0]) + if len(other) > 1 { + output.DifferenceInPlace(other[1:]...) + } + return output +} +func (b *Bitmap) singleDifference(other *Bitmap) *Bitmap { + output := NewBitmap() iiter, _ := b.Containers.Iterator(0) jiter, _ := other.Containers.Iterator(0) i, j := iiter.Next(), jiter.Next() @@ -2668,6 +2676,7 @@ func (c *Container) arrayRemove(v uint16) (*Container, bool) { } // removing the last item? we can just return the empty container. if c.N() == 1 { + c.n = 0 return nil, true } c = c.Thaw() @@ -2684,6 +2693,7 @@ func (c *Container) bitmapRemove(v uint16) (*Container, bool) { } // removing the last item? we can just return the empty container. if c.N() == 1 { + c.n = 0 return nil, true } c = c.Thaw() @@ -2709,6 +2719,7 @@ func (c *Container) runRemove(v uint16) (*Container, bool) { } // removing the last item? we can just return the empty container. if c.N() == 1 { + c.n = 0 return nil, true } c = c.Thaw() @@ -5451,3 +5462,459 @@ type containerUnionSummaryStats struct { // extremely dense containers. hasMaxRange bool } + +// DifferenceInPlace returns the bitwise difference of b and others, modifying +// b in place. +func (b *Bitmap) DifferenceInPlace(others ...*Bitmap) { + bSize := b.Size() + + // If b doesn't have any containers then return early. + if bSize == 0 { + return + } + + const staticSize = 20 + var ( + requiredSliceSize = len(others) + // To avoid having to allocate a slice every time, if the number of bitmaps + // being differenced is small enough (i.e. smaller than staticSize), we can + // just use this stack-allocated array. + staticHandledIters = [staticSize]handledIter{} + bitmapIters handledIters + target = b + removeContainerKeys = make([]uint64, 0, bSize) + ) + + if requiredSliceSize <= staticSize { + bitmapIters = staticHandledIters[:0] + } else { + bitmapIters = make(handledIters, 0, requiredSliceSize) + } + + for _, other := range others { + otherIter, _ := other.Containers.Iterator(0) + if otherIter.Next() { + bitmapIters = append(bitmapIters, handledIter{ + iter: otherIter, + hasNext: true, + }) + } + } + + targetItr, _ := target.Containers.Iterator(0) + // Go through all the containers and remove the other bits + for targetItr.Next() { + targetKey, curContainer := targetItr.Value() + // Loop until every iters current value has been handled. + for _, iIter := range bitmapIters { + if !iIter.hasNext { + continue + } + iKey, iContainer := iIter.iter.Value() + for iKey < targetKey { + iIter.hasNext = iIter.iter.Next() + if iIter.hasNext { + iKey, iContainer = iIter.iter.Value() + } else { + break + } + } + + if targetKey == iKey { + if curContainer.frozen() { + curContainer = curContainer.Clone() + b.Containers.Put(targetKey, curContainer) + } + curContainer.differenceInPlace(iContainer) + if curContainer.N() == 0 { + removeContainerKeys = append(removeContainerKeys, iKey) + break + } + iIter.hasNext = iIter.iter.Next() + } + } + } + + for _, key := range removeContainerKeys { + b.Containers.Remove(key) + + } + target.Containers.Repair() +} + +func (c *Container) differenceInPlace(other *Container) { + if other.isArray() { + if c.isArray() { + differenceArrayArrayInPlace(c, other) + } else if c.isBitmap() { + differenceBitmapArrayInPlace(c, other) + } else if c.isRun() { + differenceRunArrayInPlace(c, other) + } + } else if other.isBitmap() { + if c.isArray() { + differenceArrayBitmapInPlace(c, other) + } else if c.isBitmap() { + differenceBitmapBitmapInPlace(c, other) + } else if c.isRun() { + differenceRunBitmapInPlace(c, other) + } + } else if other.isRun() { + if c.isArray() { + differenceArrayRunInPlace(c, other) + } else if c.isBitmap() { + differenceBitmapRunInPlace(c, other) + } else if c.isRun() { + differenceRunRunInPlace(c, other) + } + } +} + +func differenceArrayArrayInPlace(c, other *Container) { + statsHit("differenceInPlace/ArrayArray") + aa, ab := c.array(), other.array() + na, nb := len(aa), len(ab) + if na == 0 || nb == 0 { + return + } + n := 0 + for i, j := 0, 0; i < na; { + va := aa[i] + if j >= nb { + aa[n] = va + n++ + i++ + continue + } + + vb := ab[j] + if va < vb { + aa[n] = va + n++ + i++ + } else if va > vb { + j++ + } else { + i, j = i+1, j+1 + } + } + aa = aa[:n] + c.setArray(aa) +} + +func differenceArrayBitmapInPlace(c, other *Container) { + statsHit("differenceInPlace/ArrayBitmap") + aa := c.array() + n := 0 + bitmap := other.bitmap() + if len(aa) == 0 || len(bitmap) == 0 { + return + } + for _, va := range aa { + bmidx := va / 64 + bidx := va % 64 + mask := uint64(1) << bidx + b := bitmap[bmidx] + + if mask&^b > 0 { + aa[n] = va + n++ + } + } + aa = aa[:n] + c.setArray(aa) +} + +func differenceArrayRunInPlace(c, other *Container) { + statsHit("differenceInPlace/ArrayRun") + + i := 0 // array index + j := 0 // run index + aa, rb := c.array(), other.runs() + if len(aa) == 0 || len(rb) == 0 { + return + } + n := 0 + + // handle overlap + for i < len(aa) { + + // keep all array elements before beginning of runs + if aa[i] < rb[j].start { + aa[n] = aa[i] + n++ + i++ + continue + } + + // if array element in run, skip it + if aa[i] >= rb[j].start && aa[i] <= rb[j].last { + i++ + continue + } + + // if array element larger than current run, check next run + if aa[i] > rb[j].last { + j++ + if j == len(rb) { + break + } + } + } + for ; i < len(aa); i++ { + aa[n] = aa[i] + n++ + } + aa = aa[:n] + c.setArray(aa) +} + +func differenceBitmapArrayInPlace(c, other *Container) { + statsHit("differenceInPlace/BitmapArray") + bitmap := c.bitmap() + ab := other.array() + if len(bitmap) == 0 || len(ab) == 0 { + return + } + + n := c.N() + for _, v := range ab { + if c.bitmapContains(v) { + bitmap[v/64] &^= (uint64(1) << uint(v%64)) + n-- + } + } + c.setN(n) + if n < ArrayMaxSize { + c.bitmapToArray() // With This Work + } +} + +func differenceBitmapBitmapInPlace(c, other *Container) { + statsHit("differenceInPlace/BitmapBitmap") + // local variables added to prevent BCE checks in loop + // see https://go101.org/article/bounds-check-elimination.html + a := c.bitmap() + b := other.bitmap() + if len(a) == 0 || len(b) == 0 { + return + } + + var ( + ab = a[:bitmapN] + bb = b[:bitmapN] + n int32 + ) + + for i := 0; i < bitmapN; i++ { + ab[i] = ab[i] & (^bb[i]) + n += int32(popcount(ab[i])) + } + c.setN(n) + if n < ArrayMaxSize { + c.bitmapToArray() // Will this work? + } +} + +func differenceBitmapRunInPlace(c, other *Container) { + statsHit("differenceInPlace/BitmapRun") + if len(c.bitmap()) == 0 { + return + } + for _, run := range other.runs() { + c.bitmapZeroRange(uint64(run.start), uint64(run.last)+1) + } +} + +func differenceRunArrayInPlace(c, other *Container) { + statsHit("differenceInPlace/RunArray") + ra, ab := c.runs(), other.array() + if len(ra) == 0 || len(ab) == 0 { + return + } + runs := make([]interval16, 0, len(ra)) + bidx := 0 + vb := ab[bidx] + +RUNLOOP: + for _, run := range ra { + start := run.start + for vb < run.start { + bidx++ + if bidx >= len(ab) { + break + } + vb = ab[bidx] + } + for vb >= run.start && vb <= run.last { + if vb == start { + if vb == 65535 { // overflow + break RUNLOOP + } + start++ + bidx++ + if bidx >= len(ab) { + break + } + vb = ab[bidx] + continue + } + runs = append(runs, interval16{start: start, last: vb - 1}) + if vb == 65535 { // overflow + break RUNLOOP + } + start = vb + 1 + bidx++ + if bidx >= len(ab) { + break + } + vb = ab[bidx] + } + + if start <= run.last { + runs = append(runs, interval16{start: start, last: run.last}) + } + } + c.setRuns(runs) + c.n = 0 + for _, run := range runs { + c.n += int32(run.last-run.start) + 1 + } + c.optimize() +} + +func differenceRunBitmapInPlace(c, other *Container) { + statsHit("differenceInPlace/RunBitmap") + ra := c.runs() + if len(ra) == 0 || len(other.bitmap()) == 0 { + return + } + // If a is full, difference is the flip of b. + if len(ra) > 0 && ra[0].start == 0 && ra[0].last == 65535 { + clone := other.Clone() + bitmap := clone.bitmap() + for i, word := range other.bitmap() { + bitmap[i] = ^word + } + c.setTyp(containerBitmap) + c.setMapped(false) + c.setBitmap(bitmap) + c.setN(c.count()) + return + } + runs := make([]interval16, 0, len(ra)) + for _, inputRun := range ra { + run := inputRun + add := true + for bit := inputRun.start; bit <= inputRun.last; bit++ { + if other.bitmapContains(bit) { + if run.start == bit { + if bit == 65535 { //overflow + add = false + } + + run.start++ + } else if bit == run.last { + run.last-- + } else { + run.last = bit - 1 + if run.last >= run.start { + runs = append(runs, run) + } + run.start = bit + 1 + run.last = inputRun.last + } + if run.start > run.last { + break + } + } + + if bit == 65535 { //overflow + break + } + } + if run.start <= run.last { + if add { + runs = append(runs, run) + } + } + } + + c.setRuns(runs) + c.n = 0 + for _, run := range runs { + c.n += int32(run.last-run.start) + 1 + } + if c.N() < ArrayMaxSize && int32(len(runs)) > c.N()/2 { + c.runToArray() + } else if len(runs) > runMaxSize { + c.runToBitmap() + } +} + +func differenceRunRunInPlace(c, other *Container) { + statsHit("differenceInPlace/RunRun") + + ra, rb := c.runs(), other.runs() + if len(ra) == 0 || len(rb) == 0 { + return + } + apos := 0 // current a-run index + bpos := 0 // current b-run index + astart := ra[apos].start + alast := ra[apos].last + bstart := rb[bpos].start + blast := rb[bpos].last + alen := len(ra) + blen := len(rb) + + runs := make([]interval16, 0, alen+blen) // TODO allocate max then truncate? or something else + // cardinality upper bound: sum of number of runs + // each B-run could split an A-run in two, up to len(b.runs) times + + for apos < alen && bpos < blen { + switch { + case alast < bstart: + // current A-run entirely precedes current B-run: keep full A-run, advance to next A-run + runs = append(runs, interval16{start: astart, last: alast}) + apos++ + if apos < alen { + astart = ra[apos].start + alast = ra[apos].last + } + case blast < astart: + // current B-run entirely precedes current A-run: advance to next B-run + bpos++ + if bpos < blen { + bstart = rb[bpos].start + blast = rb[bpos].last + } + default: + // overlap + if astart < bstart { + runs = append(runs, interval16{start: astart, last: bstart - 1}) + } + if alast > blast { + astart = blast + 1 + } else { + apos++ + if apos < alen { + astart = ra[apos].start + alast = ra[apos].last + } + } + } + } + if apos < alen { + runs = append(runs, interval16{start: astart, last: alast}) + apos++ + if apos < alen { + runs = append(runs, ra[apos:]...) + } + } + c.setRuns(runs) + c.n = 0 + for _, run := range runs { + c.n += int32(run.last-run.start) + 1 + } +} diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 0ecd73021..f8476459f 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -2710,6 +2710,15 @@ func unionInPlaceWrapper(a, b *Container) *Container { return out.Containers.Get(0) } +func differenceInPlaceWrapper(a, b *Container) *Container { + out := NewBitmap() + out.Containers.Put(0, a.Clone()) + B := NewBitmap() + B.Containers.Put(0, b) + out.DifferenceInPlace(B) + return out.Containers.Get(0) +} + func TestContainerCombinations(t *testing.T) { cts := setupContainerTests() @@ -3283,6 +3292,117 @@ func TestContainerCombinations(t *testing.T) { {flip, "outerBitsSet", "", "innerBitsSet"}, {flip, "oddBitsSet", "", "evenBitsSet"}, {flip, "evenBitsSet", "", "oddBitsSet"}, + + // differenceInPlace + {differenceInPlaceWrapper, "empty", "empty", "empty"}, + {differenceInPlaceWrapper, "empty", "full", "empty"}, + {differenceInPlaceWrapper, "empty", "firstBitSet", "empty"}, + {differenceInPlaceWrapper, "empty", "lastBitSet", "empty"}, + {differenceInPlaceWrapper, "empty", "firstBitUnset", "empty"}, + {differenceInPlaceWrapper, "empty", "lastBitUnset", "empty"}, + {differenceInPlaceWrapper, "empty", "innerBitsSet", "empty"}, + {differenceInPlaceWrapper, "empty", "outerBitsSet", "empty"}, + {differenceInPlaceWrapper, "empty", "oddBitsSet", "empty"}, + {differenceInPlaceWrapper, "empty", "evenBitsSet", "empty"}, + // + {differenceInPlaceWrapper, "full", "empty", "full"}, + {differenceInPlaceWrapper, "full", "full", "empty"}, + {differenceInPlaceWrapper, "full", "firstBitSet", "firstBitUnset"}, + {differenceInPlaceWrapper, "full", "lastBitSet", "lastBitUnset"}, + {differenceInPlaceWrapper, "full", "firstBitUnset", "firstBitSet"}, + {differenceInPlaceWrapper, "full", "lastBitUnset", "lastBitSet"}, + {differenceInPlaceWrapper, "full", "innerBitsSet", "outerBitsSet"}, + {differenceInPlaceWrapper, "full", "outerBitsSet", "innerBitsSet"}, + {differenceInPlaceWrapper, "full", "oddBitsSet", "evenBitsSet"}, + {differenceInPlaceWrapper, "full", "evenBitsSet", "oddBitsSet"}, + // + {differenceInPlaceWrapper, "firstBitSet", "empty", "firstBitSet"}, + {differenceInPlaceWrapper, "firstBitSet", "full", "empty"}, + {differenceInPlaceWrapper, "firstBitSet", "firstBitSet", "empty"}, + {differenceInPlaceWrapper, "firstBitSet", "lastBitSet", "firstBitSet"}, + {differenceInPlaceWrapper, "firstBitSet", "firstBitUnset", "firstBitSet"}, + {differenceInPlaceWrapper, "firstBitSet", "lastBitUnset", "empty"}, + {differenceInPlaceWrapper, "firstBitSet", "innerBitsSet", "firstBitSet"}, + {differenceInPlaceWrapper, "firstBitSet", "outerBitsSet", "empty"}, + {differenceInPlaceWrapper, "firstBitSet", "oddBitsSet", "firstBitSet"}, + {differenceInPlaceWrapper, "firstBitSet", "evenBitsSet", "empty"}, + // + {differenceInPlaceWrapper, "lastBitSet", "empty", "lastBitSet"}, + {differenceInPlaceWrapper, "lastBitSet", "full", "empty"}, + {differenceInPlaceWrapper, "lastBitSet", "firstBitSet", "lastBitSet"}, + {differenceInPlaceWrapper, "lastBitSet", "lastBitSet", "empty"}, + {differenceInPlaceWrapper, "lastBitSet", "firstBitUnset", "empty"}, + {differenceInPlaceWrapper, "lastBitSet", "lastBitUnset", "lastBitSet"}, + {differenceInPlaceWrapper, "lastBitSet", "innerBitsSet", "lastBitSet"}, + {differenceInPlaceWrapper, "lastBitSet", "outerBitsSet", "empty"}, + {differenceInPlaceWrapper, "lastBitSet", "oddBitsSet", "empty"}, + {differenceInPlaceWrapper, "lastBitSet", "evenBitsSet", "lastBitSet"}, + // + {differenceInPlaceWrapper, "firstBitUnset", "empty", "firstBitUnset"}, + {differenceInPlaceWrapper, "firstBitUnset", "full", "empty"}, + {differenceInPlaceWrapper, "firstBitUnset", "firstBitSet", "firstBitUnset"}, + {differenceInPlaceWrapper, "firstBitUnset", "lastBitSet", "innerBitsSet"}, + {differenceInPlaceWrapper, "firstBitUnset", "firstBitUnset", "empty"}, + {differenceInPlaceWrapper, "firstBitUnset", "lastBitUnset", "lastBitSet"}, + {differenceInPlaceWrapper, "firstBitUnset", "innerBitsSet", "lastBitSet"}, + {differenceInPlaceWrapper, "firstBitUnset", "outerBitsSet", "innerBitsSet"}, + //{differenceInPlaceWrapper, "firstBitUnset", "oddBitsSet", ""}, + {differenceInPlaceWrapper, "firstBitUnset", "evenBitsSet", "oddBitsSet"}, + // + {differenceInPlaceWrapper, "lastBitUnset", "empty", "lastBitUnset"}, + {differenceInPlaceWrapper, "lastBitUnset", "full", "empty"}, + {differenceInPlaceWrapper, "lastBitUnset", "firstBitSet", "innerBitsSet"}, + {differenceInPlaceWrapper, "lastBitUnset", "lastBitSet", "lastBitUnset"}, + {differenceInPlaceWrapper, "lastBitUnset", "firstBitUnset", "firstBitSet"}, + {differenceInPlaceWrapper, "lastBitUnset", "lastBitUnset", "empty"}, + {differenceInPlaceWrapper, "lastBitUnset", "innerBitsSet", "firstBitSet"}, + {differenceInPlaceWrapper, "lastBitUnset", "outerBitsSet", "innerBitsSet"}, + {differenceInPlaceWrapper, "lastBitUnset", "oddBitsSet", "evenBitsSet"}, + //{differenceInPlaceWrapper, "lastBitUnset", "evenBitsSet", ""}, + // + {differenceInPlaceWrapper, "innerBitsSet", "empty", "innerBitsSet"}, + {differenceInPlaceWrapper, "innerBitsSet", "full", "empty"}, + {differenceInPlaceWrapper, "innerBitsSet", "firstBitSet", "innerBitsSet"}, + {differenceInPlaceWrapper, "innerBitsSet", "lastBitSet", "innerBitsSet"}, + {differenceInPlaceWrapper, "innerBitsSet", "firstBitUnset", "empty"}, + {differenceInPlaceWrapper, "innerBitsSet", "lastBitUnset", "empty"}, + {differenceInPlaceWrapper, "innerBitsSet", "innerBitsSet", "empty"}, + {differenceInPlaceWrapper, "innerBitsSet", "outerBitsSet", "innerBitsSet"}, + //{differenceInPlaceWrapper, "innerBitsSet", "oddBitsSet", ""}, + //{differenceInPlaceWrapper, "innerBitsSet", "evenBitsSet", ""}, + // + {differenceInPlaceWrapper, "outerBitsSet", "empty", "outerBitsSet"}, + {differenceInPlaceWrapper, "outerBitsSet", "full", "empty"}, + {differenceInPlaceWrapper, "outerBitsSet", "firstBitSet", "lastBitSet"}, + {differenceInPlaceWrapper, "outerBitsSet", "lastBitSet", "firstBitSet"}, + {differenceInPlaceWrapper, "outerBitsSet", "firstBitUnset", "firstBitSet"}, + {differenceInPlaceWrapper, "outerBitsSet", "lastBitUnset", "lastBitSet"}, + {differenceInPlaceWrapper, "outerBitsSet", "innerBitsSet", "outerBitsSet"}, + {differenceInPlaceWrapper, "outerBitsSet", "outerBitsSet", "empty"}, + {differenceInPlaceWrapper, "outerBitsSet", "oddBitsSet", "firstBitSet"}, + {differenceInPlaceWrapper, "outerBitsSet", "evenBitsSet", "lastBitSet"}, + // + {differenceInPlaceWrapper, "oddBitsSet", "empty", "oddBitsSet"}, + {differenceInPlaceWrapper, "oddBitsSet", "full", "empty"}, + {differenceInPlaceWrapper, "oddBitsSet", "firstBitSet", "oddBitsSet"}, + //{differenceInPlaceWrapper, "oddBitsSet", "lastBitSet", ""}, + {differenceInPlaceWrapper, "oddBitsSet", "firstBitUnset", "empty"}, + {differenceInPlaceWrapper, "oddBitsSet", "lastBitUnset", "lastBitSet"}, + {differenceInPlaceWrapper, "oddBitsSet", "innerBitsSet", "lastBitSet"}, + //{differenceInPlaceWrapper, "oddBitsSet", "outerBitsSet", ""}, + {differenceInPlaceWrapper, "oddBitsSet", "oddBitsSet", "empty"}, + {differenceInPlaceWrapper, "oddBitsSet", "evenBitsSet", "oddBitsSet"}, + // + {differenceInPlaceWrapper, "evenBitsSet", "empty", "evenBitsSet"}, + {differenceInPlaceWrapper, "evenBitsSet", "full", "empty"}, + //{differenceInPlaceWrapper, "evenBitsSet", "firstBitSet", ""}, + {differenceInPlaceWrapper, "evenBitsSet", "lastBitSet", "evenBitsSet"}, + {differenceInPlaceWrapper, "evenBitsSet", "firstBitUnset", "firstBitSet"}, + {differenceInPlaceWrapper, "evenBitsSet", "lastBitUnset", "empty"}, + {differenceInPlaceWrapper, "evenBitsSet", "innerBitsSet", "firstBitSet"}, + //{differenceInPlaceWrapper, "evenBitsSet", "outerBitsSet", ""}, + {differenceInPlaceWrapper, "evenBitsSet", "oddBitsSet", "evenBitsSet"}, + {differenceInPlaceWrapper, "evenBitsSet", "evenBitsSet", "empty"}, } for _, testOp := range testOps { for _, x := range containerTypes { @@ -3896,3 +4016,12 @@ func TestBitmapAny(t *testing.T) { t.Error("shouldn't be any left") } } + +func TestDifferenceInPlace_N(t *testing.T) { + a := doContainer(containerRun, runFull()) + b := doContainer(containerBitmap, bitmapFull()) + r := differenceInPlaceWrapper(a, b) + if r.N() != 0 { + t.Error("expected difference of containers to have n=0") + } +} diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index 836fd8de3..54ea5704a 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -1739,3 +1739,57 @@ func BenchmarkUnionBulk(b *testing.B) { UnionInPlace(data.a1, data.a2, data.b, data.r1, data.r2) } } + +func TestBitmap_DifferenceInPlace(t *testing.T) { + // array + arraybm := roaring.NewSliceBitmap() + for i := uint64(0); i < 1024; i += 4 { + _, _ = arraybm.Add((1 << 16) + i) + } + arraybm.Optimize() + + // bitmap + bitmapbm := roaring.NewSliceBitmap() + for i := uint64(0); i < 16384; i += 2 { + _, _ = bitmapbm.Add((2 << 16) + i) + } + bitmapbm.Optimize() + + // small run + smallrunbm := roaring.NewSliceBitmap() + for i := uint64(0); i < 1024; i++ { + _, _ = smallrunbm.Add((3 << 16) + i) + } + smallrunbm.Optimize() + + // large run + largerunbm := roaring.NewSliceBitmap() + for i := uint64(0); i < 65535; i++ { + _, _ = largerunbm.Add((4 << 16) + i) + } + largerunbm.Optimize() + + // test the difference in place + bm := testBM() + bm.DifferenceInPlace(arraybm, bitmapbm, smallrunbm, largerunbm) + if bm.Count() != 0 { + t.Fatalf("expected bitmap count to be 0, but got: %d", bm.Count()) + } + + bm = testBM() + bm.DifferenceInPlace(bitmapbm, smallrunbm, largerunbm) + if bm.Count() != 256 { + t.Fatalf("expected bitmap count to be 256, but got: %d", bm.Count()) + } +} + +func BenchmarkDifferencInPlace(b *testing.B) { + data := getBenchData(b) + for n := 0; n < b.N; n++ { + bm := roaring.NewBitmap() + bm. + UnionInPlace(data.a1, data.a2, data.b, data.r1, data.r2) + bm. + DifferenceInPlace(data.a1, data.r2, data.b, data.r1) + } +} diff --git a/row.go b/row.go index 82a67cf8f..7c04f60e4 100644 --- a/row.go +++ b/row.go @@ -304,21 +304,26 @@ func (r *Row) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*Row, args ma } // Difference returns the diff of r and other. -func (r *Row) Difference(other *Row) *Row { - var segments []rowSegment +func (r *Row) Difference(others ...*Row) *Row { + var output []rowSegment + o := make(map[uint64][]*rowSegment) - itr := newMergeSegmentIterator(r.segments, other.segments) - for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() { - if s0 == nil { - continue - } else if s1 == nil { - segments = append(segments, *s0) - continue + for x := range others { + for y := range others[x].segments { + segment := others[x].segments[y] + o[segment.shard] = append(o[segment.shard], &segment) } - segments = append(segments, *s0.Difference(s1)) } + for _, segment := range r.segments { - return &Row{segments: segments} + dest, ok := o[segment.shard] + if ok { + output = append(output, *segment.Difference(dest...)) + } else { + output = append(output, segment) + } + } + return &Row{segments: output} } // GenericUnary returns the results of a generic op on r. @@ -576,9 +581,13 @@ func (s *rowSegment) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*rowSe } // Difference returns the diff of s and other. -func (s *rowSegment) Difference(other *rowSegment) *rowSegment { - data := s.data.Difference(other.data) - data = data.Freeze() +func (s *rowSegment) Difference(others ...*rowSegment) *rowSegment { + datas := make([]*roaring.Bitmap, len(others)) + for i, other := range others { + datas[i] = other.data + } + data := s.data.Difference(datas...) + data.Freeze() return &rowSegment{ data: data, diff --git a/row_test.go b/row_test.go index 175faa4e7..a4740c897 100644 --- a/row_test.go +++ b/row_test.go @@ -138,3 +138,61 @@ func TestRow_Includes(t *testing.T) { t.Fatalf("row should include %d", 2*ShardWidth) } } + +func TestRow_DifferenceInPlace(t *testing.T) { + row0 := pilosa.NewRow(0) + row1 := pilosa.NewRow() + row2 := pilosa.NewRow(0) + res := row0.Difference(row1, row2) + + if !row0.Includes(0) { + t.Fatal("row should include 0") + } + if res.Count() != 0 { + t.Fatal("results should be empty") + } +} +func eq(a, b []uint64) bool { + if len(a) != len(b) { + return false + } + for i, v := range a { + if v != b[i] { + return false + } + } + return true +} +func TestRow_DifferenceInPlace2(t *testing.T) { + lucky := []uint64{1, 3, 7, 9, 13, 15, 21, 25, 31, 33, 37, 43, 49, 51, 63, 67, 69, 73, 75, 79, 87, 93, 99, 105, 111, 115, 127, 129, 133, 135, 141, 151, 159, 163, 169, 171, 189, 193, 195, 201, 205, 211, 219, 223, 231, 235, 237, 241, 259, 261, 267, 273, 283, 285, 289, 297} + src := pilosa.NewRow(lucky...) + row1 := pilosa.NewRow() + m := uint64(9) + for i := m; i <= lucky[len(lucky)-1]; i += m { + row1.SetBit(i) + } + m = uint64(3) + row2 := pilosa.NewRow() + for i := m; i <= lucky[len(lucky)-1]; i += m { + row2.SetBit(i) + } + row3 := pilosa.NewRow() + m = uint64(5) + for i := m; i <= lucky[len(lucky)-1]; i += m { + row3.SetBit(i) + } + row4 := pilosa.NewRow() + m = uint64(7) + for i := m; i <= lucky[len(lucky)-1]; i += m { + row4.SetBit(i) + } + res5 := src.Difference(row1, row2, row3, row4) + res6 := src.Difference(row4, row3, row2, row1) + res7 := src.Difference(row4).Difference(row3).Difference(row2).Difference(row1) + if !eq(res5.Columns(), res6.Columns()) { + t.Fatalf("results do not match: %v, %v", res5.Columns(), res6.Columns()) + } + if !eq(res6.Columns(), res7.Columns()) { + t.Fatalf("results do not match: %v, %v", res6.Columns(), res7.Columns()) + } +}