mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-09-07 00:55:55 +00:00
Merge branch 'enterprise' into translation-sharding
This commit is contained in:
Travis
commit
9b4c7610e6
8 changed files with 750 additions and 37 deletions
|
|
@ -1247,7 +1247,6 @@ func (f *fragment) rangeGT(bitDepth uint, predicate int64, allowEquality bool) (
|
|||
|
||||
func (f *fragment) rangeGTUnsigned(filter *Row, bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) {
|
||||
keep := NewRow()
|
||||
|
||||
// Filter any bits that don't match the current bit value.
|
||||
for i := int(bitDepth - 1); i >= 0; i-- {
|
||||
row := f.row(uint64(bsiOffsetBit + i))
|
||||
|
|
@ -1260,12 +1259,12 @@ func (f *fragment) rangeGTUnsigned(filter *Row, bitDepth uint, predicate uint64,
|
|||
if bit == 1 {
|
||||
return keep, nil
|
||||
}
|
||||
return filter.Difference(filter.Difference(row).Difference(keep)), nil
|
||||
return filter.Difference(filter.Difference(row, keep)), nil
|
||||
}
|
||||
|
||||
// If bit is set then remove all unset columns not already kept.
|
||||
if bit == 1 {
|
||||
filter = filter.Difference(filter.Difference(row).Difference(keep))
|
||||
filter = filter.Difference(filter.Difference(row, keep))
|
||||
continue
|
||||
}
|
||||
|
||||
|
|
@ -1333,7 +1332,7 @@ func (f *fragment) rangeBetweenUnsigned(filter *Row, bitDepth uint, predicateMin
|
|||
// GTE predicateMin
|
||||
// If bit is set then remove all unset columns not already kept.
|
||||
if bit1 == 1 {
|
||||
filter = filter.Difference(filter.Difference(row).Difference(keep1))
|
||||
filter = filter.Difference(filter.Difference(row, keep1))
|
||||
} else {
|
||||
// If bit is unset then add columns with set bit to keep.
|
||||
// Don't bother to compute this on the final iteration.
|
||||
|
|
|
|||
12
go.mod
12
go.mod
|
|
@ -8,36 +8,42 @@ require (
|
|||
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
|
||||
github.com/boltdb/bolt v1.3.1
|
||||
github.com/cespare/xxhash v1.1.0
|
||||
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
|
||||
github.com/davecgh/go-spew v1.1.1
|
||||
github.com/go-ole/go-ole v1.2.4 // indirect
|
||||
github.com/gogo/protobuf v1.2.0
|
||||
github.com/golang/protobuf v1.3.2
|
||||
github.com/google/go-cmp v0.2.0
|
||||
github.com/gorilla/handlers v1.3.0
|
||||
github.com/gorilla/mux v1.7.0
|
||||
github.com/hashicorp/memberlist v0.1.3
|
||||
github.com/inconshreveable/mousetrap v1.0.0 // indirect
|
||||
github.com/molecula/ext v0.0.0-20200103203257-8a458a73e8c2
|
||||
github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4
|
||||
github.com/opentracing/opentracing-go v1.1.0
|
||||
github.com/pelletier/go-toml v1.2.0
|
||||
github.com/pilosa/pilosa v1.4.0
|
||||
github.com/pkg/errors v0.8.1
|
||||
github.com/prometheus/client_golang v0.9.3
|
||||
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect
|
||||
github.com/satori/go.uuid v1.2.0
|
||||
github.com/shirou/gopsutil v2.18.12+incompatible
|
||||
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect
|
||||
github.com/spf13/cobra v0.0.3
|
||||
github.com/spf13/pflag v1.0.3
|
||||
github.com/spf13/viper v1.3.1
|
||||
github.com/uber-go/atomic v1.4.0 // indirect
|
||||
github.com/uber/jaeger-client-go v2.16.0+incompatible
|
||||
github.com/uber/jaeger-lib v2.2.0+incompatible // indirect
|
||||
github.com/youtube/vitess v2.1.1+incompatible // indirect
|
||||
go.uber.org/atomic v1.4.0 // indirect
|
||||
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect
|
||||
golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58
|
||||
golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 // indirect
|
||||
golang.org/x/text v0.3.2 // indirect
|
||||
google.golang.org/grpc v1.24.0
|
||||
modernc.org/mathutil v1.0.0
|
||||
modernc.org/strutil v1.0.0
|
||||
vitess.io/vitess v2.1.1+incompatible // indirect
|
||||
)
|
||||
|
||||
go 1.13
|
||||
|
|
|
|||
9
go.sum
9
go.sum
|
|
@ -7,7 +7,6 @@ github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5Vk
|
|||
github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
|
||||
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
|
||||
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
|
||||
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
|
|
@ -106,8 +105,6 @@ github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181
|
|||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 h1:ERLyN4p3KS5Fk2ADsDENm2cq0+Lx6sF1sG8uwRlySpU=
|
||||
github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/pilosa/pilosa v1.4.0 h1:nqHNIK4nDslFnem3yDp9R+6TgLdlkY9WdJD88Z83T8U=
|
||||
github.com/pilosa/pilosa v1.4.0/go.mod h1:NSTtTprtb5MSgCs4mcNqeQ2JdIMpInOi4DEImxGJeTs=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
|
|
@ -126,7 +123,6 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
|
|||
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY=
|
||||
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
|
||||
|
|
@ -159,13 +155,10 @@ github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o=
|
|||
github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
|
||||
github.com/uber/jaeger-client-go v2.16.0+incompatible h1:Q2Pp6v3QYiocMxomCaJuwQGFt7E53bPYqEgug/AoBtY=
|
||||
github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
|
||||
github.com/uber/jaeger-lib v2.0.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
|
||||
github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw=
|
||||
github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
|
||||
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
|
||||
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
||||
github.com/youtube/vitess v2.1.1+incompatible h1:SE+P7DNX/jw5RHFs5CHRhZQjq402EJFCD33JhzQMdDw=
|
||||
github.com/youtube/vitess v2.1.1+incompatible/go.mod h1:hpMim5/30F1r+0P8GGtB29d0gWHr0IZ5unS+CG0zMx8=
|
||||
go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
|
||||
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
|
|
@ -223,5 +216,3 @@ modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I=
|
|||
modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
|
||||
modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE=
|
||||
modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
|
||||
vitess.io/vitess v2.1.1+incompatible h1:nuuGHiWYWpudD3gOCLeGzol2EJ25e/u5Wer2wV1O130=
|
||||
vitess.io/vitess v2.1.1+incompatible/go.mod h1:h4qvkyNYTOC0xI+vcidSWoka0gQAZc9ZPHbkHo48gP0=
|
||||
|
|
|
|||
|
|
@ -833,16 +833,18 @@ func (b *Bitmap) unionIntoTargetSingle(target *Bitmap, other *Bitmap) {
|
|||
// Bitmap 4 |___X_______________________| | |___X_______________________| | |___X_______________________|
|
||||
// _
|
||||
func (b *Bitmap) unionInPlace(others ...*Bitmap) {
|
||||
const staticSize = 20
|
||||
var (
|
||||
requiredSliceSize = len(others)
|
||||
// To avoid having to allocate a slice everytime, if the number of bitmaps
|
||||
// being unioned is small enough we can just use this stack-allocated array.
|
||||
staticHandledIters = [20]handledIter{}
|
||||
// To avoid having to allocate a slice every time, if the number of bitmaps
|
||||
// being unioned is small enough (i.e. smaller than staticSize), we can just
|
||||
// use this stack-allocated array.
|
||||
staticHandledIters = [staticSize]handledIter{}
|
||||
bitmapIters handledIters
|
||||
target = b
|
||||
)
|
||||
|
||||
if requiredSliceSize <= 20 {
|
||||
if requiredSliceSize <= staticSize {
|
||||
bitmapIters = staticHandledIters[:0]
|
||||
} else {
|
||||
bitmapIters = make(handledIters, 0, requiredSliceSize)
|
||||
|
|
@ -993,10 +995,16 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) {
|
|||
}
|
||||
|
||||
// Difference returns the difference of b and other.
|
||||
func (b *Bitmap) Difference(other *Bitmap) *Bitmap {
|
||||
output := NewBitmap()
|
||||
output.Source = b.Source
|
||||
func (b *Bitmap) Difference(other ...*Bitmap) *Bitmap {
|
||||
output := b.singleDifference(other[0])
|
||||
if len(other) > 1 {
|
||||
output.DifferenceInPlace(other[1:]...)
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
func (b *Bitmap) singleDifference(other *Bitmap) *Bitmap {
|
||||
output := NewBitmap()
|
||||
iiter, _ := b.Containers.Iterator(0)
|
||||
jiter, _ := other.Containers.Iterator(0)
|
||||
i, j := iiter.Next(), jiter.Next()
|
||||
|
|
@ -2668,6 +2676,7 @@ func (c *Container) arrayRemove(v uint16) (*Container, bool) {
|
|||
}
|
||||
// removing the last item? we can just return the empty container.
|
||||
if c.N() == 1 {
|
||||
c.n = 0
|
||||
return nil, true
|
||||
}
|
||||
c = c.Thaw()
|
||||
|
|
@ -2684,6 +2693,7 @@ func (c *Container) bitmapRemove(v uint16) (*Container, bool) {
|
|||
}
|
||||
// removing the last item? we can just return the empty container.
|
||||
if c.N() == 1 {
|
||||
c.n = 0
|
||||
return nil, true
|
||||
}
|
||||
c = c.Thaw()
|
||||
|
|
@ -2709,6 +2719,7 @@ func (c *Container) runRemove(v uint16) (*Container, bool) {
|
|||
}
|
||||
// removing the last item? we can just return the empty container.
|
||||
if c.N() == 1 {
|
||||
c.n = 0
|
||||
return nil, true
|
||||
}
|
||||
c = c.Thaw()
|
||||
|
|
@ -5451,3 +5462,459 @@ type containerUnionSummaryStats struct {
|
|||
// extremely dense containers.
|
||||
hasMaxRange bool
|
||||
}
|
||||
|
||||
// DifferenceInPlace returns the bitwise difference of b and others, modifying
|
||||
// b in place.
|
||||
func (b *Bitmap) DifferenceInPlace(others ...*Bitmap) {
|
||||
bSize := b.Size()
|
||||
|
||||
// If b doesn't have any containers then return early.
|
||||
if bSize == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
const staticSize = 20
|
||||
var (
|
||||
requiredSliceSize = len(others)
|
||||
// To avoid having to allocate a slice every time, if the number of bitmaps
|
||||
// being differenced is small enough (i.e. smaller than staticSize), we can
|
||||
// just use this stack-allocated array.
|
||||
staticHandledIters = [staticSize]handledIter{}
|
||||
bitmapIters handledIters
|
||||
target = b
|
||||
removeContainerKeys = make([]uint64, 0, bSize)
|
||||
)
|
||||
|
||||
if requiredSliceSize <= staticSize {
|
||||
bitmapIters = staticHandledIters[:0]
|
||||
} else {
|
||||
bitmapIters = make(handledIters, 0, requiredSliceSize)
|
||||
}
|
||||
|
||||
for _, other := range others {
|
||||
otherIter, _ := other.Containers.Iterator(0)
|
||||
if otherIter.Next() {
|
||||
bitmapIters = append(bitmapIters, handledIter{
|
||||
iter: otherIter,
|
||||
hasNext: true,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
targetItr, _ := target.Containers.Iterator(0)
|
||||
// Go through all the containers and remove the other bits
|
||||
for targetItr.Next() {
|
||||
targetKey, curContainer := targetItr.Value()
|
||||
// Loop until every iters current value has been handled.
|
||||
for _, iIter := range bitmapIters {
|
||||
if !iIter.hasNext {
|
||||
continue
|
||||
}
|
||||
iKey, iContainer := iIter.iter.Value()
|
||||
for iKey < targetKey {
|
||||
iIter.hasNext = iIter.iter.Next()
|
||||
if iIter.hasNext {
|
||||
iKey, iContainer = iIter.iter.Value()
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if targetKey == iKey {
|
||||
if curContainer.frozen() {
|
||||
curContainer = curContainer.Clone()
|
||||
b.Containers.Put(targetKey, curContainer)
|
||||
}
|
||||
curContainer.differenceInPlace(iContainer)
|
||||
if curContainer.N() == 0 {
|
||||
removeContainerKeys = append(removeContainerKeys, iKey)
|
||||
break
|
||||
}
|
||||
iIter.hasNext = iIter.iter.Next()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, key := range removeContainerKeys {
|
||||
b.Containers.Remove(key)
|
||||
|
||||
}
|
||||
target.Containers.Repair()
|
||||
}
|
||||
|
||||
func (c *Container) differenceInPlace(other *Container) {
|
||||
if other.isArray() {
|
||||
if c.isArray() {
|
||||
differenceArrayArrayInPlace(c, other)
|
||||
} else if c.isBitmap() {
|
||||
differenceBitmapArrayInPlace(c, other)
|
||||
} else if c.isRun() {
|
||||
differenceRunArrayInPlace(c, other)
|
||||
}
|
||||
} else if other.isBitmap() {
|
||||
if c.isArray() {
|
||||
differenceArrayBitmapInPlace(c, other)
|
||||
} else if c.isBitmap() {
|
||||
differenceBitmapBitmapInPlace(c, other)
|
||||
} else if c.isRun() {
|
||||
differenceRunBitmapInPlace(c, other)
|
||||
}
|
||||
} else if other.isRun() {
|
||||
if c.isArray() {
|
||||
differenceArrayRunInPlace(c, other)
|
||||
} else if c.isBitmap() {
|
||||
differenceBitmapRunInPlace(c, other)
|
||||
} else if c.isRun() {
|
||||
differenceRunRunInPlace(c, other)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func differenceArrayArrayInPlace(c, other *Container) {
|
||||
statsHit("differenceInPlace/ArrayArray")
|
||||
aa, ab := c.array(), other.array()
|
||||
na, nb := len(aa), len(ab)
|
||||
if na == 0 || nb == 0 {
|
||||
return
|
||||
}
|
||||
n := 0
|
||||
for i, j := 0, 0; i < na; {
|
||||
va := aa[i]
|
||||
if j >= nb {
|
||||
aa[n] = va
|
||||
n++
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
vb := ab[j]
|
||||
if va < vb {
|
||||
aa[n] = va
|
||||
n++
|
||||
i++
|
||||
} else if va > vb {
|
||||
j++
|
||||
} else {
|
||||
i, j = i+1, j+1
|
||||
}
|
||||
}
|
||||
aa = aa[:n]
|
||||
c.setArray(aa)
|
||||
}
|
||||
|
||||
func differenceArrayBitmapInPlace(c, other *Container) {
|
||||
statsHit("differenceInPlace/ArrayBitmap")
|
||||
aa := c.array()
|
||||
n := 0
|
||||
bitmap := other.bitmap()
|
||||
if len(aa) == 0 || len(bitmap) == 0 {
|
||||
return
|
||||
}
|
||||
for _, va := range aa {
|
||||
bmidx := va / 64
|
||||
bidx := va % 64
|
||||
mask := uint64(1) << bidx
|
||||
b := bitmap[bmidx]
|
||||
|
||||
if mask&^b > 0 {
|
||||
aa[n] = va
|
||||
n++
|
||||
}
|
||||
}
|
||||
aa = aa[:n]
|
||||
c.setArray(aa)
|
||||
}
|
||||
|
||||
func differenceArrayRunInPlace(c, other *Container) {
|
||||
statsHit("differenceInPlace/ArrayRun")
|
||||
|
||||
i := 0 // array index
|
||||
j := 0 // run index
|
||||
aa, rb := c.array(), other.runs()
|
||||
if len(aa) == 0 || len(rb) == 0 {
|
||||
return
|
||||
}
|
||||
n := 0
|
||||
|
||||
// handle overlap
|
||||
for i < len(aa) {
|
||||
|
||||
// keep all array elements before beginning of runs
|
||||
if aa[i] < rb[j].start {
|
||||
aa[n] = aa[i]
|
||||
n++
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
// if array element in run, skip it
|
||||
if aa[i] >= rb[j].start && aa[i] <= rb[j].last {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
// if array element larger than current run, check next run
|
||||
if aa[i] > rb[j].last {
|
||||
j++
|
||||
if j == len(rb) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
for ; i < len(aa); i++ {
|
||||
aa[n] = aa[i]
|
||||
n++
|
||||
}
|
||||
aa = aa[:n]
|
||||
c.setArray(aa)
|
||||
}
|
||||
|
||||
func differenceBitmapArrayInPlace(c, other *Container) {
|
||||
statsHit("differenceInPlace/BitmapArray")
|
||||
bitmap := c.bitmap()
|
||||
ab := other.array()
|
||||
if len(bitmap) == 0 || len(ab) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
n := c.N()
|
||||
for _, v := range ab {
|
||||
if c.bitmapContains(v) {
|
||||
bitmap[v/64] &^= (uint64(1) << uint(v%64))
|
||||
n--
|
||||
}
|
||||
}
|
||||
c.setN(n)
|
||||
if n < ArrayMaxSize {
|
||||
c.bitmapToArray() // With This Work
|
||||
}
|
||||
}
|
||||
|
||||
func differenceBitmapBitmapInPlace(c, other *Container) {
|
||||
statsHit("differenceInPlace/BitmapBitmap")
|
||||
// local variables added to prevent BCE checks in loop
|
||||
// see https://go101.org/article/bounds-check-elimination.html
|
||||
a := c.bitmap()
|
||||
b := other.bitmap()
|
||||
if len(a) == 0 || len(b) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
var (
|
||||
ab = a[:bitmapN]
|
||||
bb = b[:bitmapN]
|
||||
n int32
|
||||
)
|
||||
|
||||
for i := 0; i < bitmapN; i++ {
|
||||
ab[i] = ab[i] & (^bb[i])
|
||||
n += int32(popcount(ab[i]))
|
||||
}
|
||||
c.setN(n)
|
||||
if n < ArrayMaxSize {
|
||||
c.bitmapToArray() // Will this work?
|
||||
}
|
||||
}
|
||||
|
||||
func differenceBitmapRunInPlace(c, other *Container) {
|
||||
statsHit("differenceInPlace/BitmapRun")
|
||||
if len(c.bitmap()) == 0 {
|
||||
return
|
||||
}
|
||||
for _, run := range other.runs() {
|
||||
c.bitmapZeroRange(uint64(run.start), uint64(run.last)+1)
|
||||
}
|
||||
}
|
||||
|
||||
func differenceRunArrayInPlace(c, other *Container) {
|
||||
statsHit("differenceInPlace/RunArray")
|
||||
ra, ab := c.runs(), other.array()
|
||||
if len(ra) == 0 || len(ab) == 0 {
|
||||
return
|
||||
}
|
||||
runs := make([]interval16, 0, len(ra))
|
||||
bidx := 0
|
||||
vb := ab[bidx]
|
||||
|
||||
RUNLOOP:
|
||||
for _, run := range ra {
|
||||
start := run.start
|
||||
for vb < run.start {
|
||||
bidx++
|
||||
if bidx >= len(ab) {
|
||||
break
|
||||
}
|
||||
vb = ab[bidx]
|
||||
}
|
||||
for vb >= run.start && vb <= run.last {
|
||||
if vb == start {
|
||||
if vb == 65535 { // overflow
|
||||
break RUNLOOP
|
||||
}
|
||||
start++
|
||||
bidx++
|
||||
if bidx >= len(ab) {
|
||||
break
|
||||
}
|
||||
vb = ab[bidx]
|
||||
continue
|
||||
}
|
||||
runs = append(runs, interval16{start: start, last: vb - 1})
|
||||
if vb == 65535 { // overflow
|
||||
break RUNLOOP
|
||||
}
|
||||
start = vb + 1
|
||||
bidx++
|
||||
if bidx >= len(ab) {
|
||||
break
|
||||
}
|
||||
vb = ab[bidx]
|
||||
}
|
||||
|
||||
if start <= run.last {
|
||||
runs = append(runs, interval16{start: start, last: run.last})
|
||||
}
|
||||
}
|
||||
c.setRuns(runs)
|
||||
c.n = 0
|
||||
for _, run := range runs {
|
||||
c.n += int32(run.last-run.start) + 1
|
||||
}
|
||||
c.optimize()
|
||||
}
|
||||
|
||||
func differenceRunBitmapInPlace(c, other *Container) {
|
||||
statsHit("differenceInPlace/RunBitmap")
|
||||
ra := c.runs()
|
||||
if len(ra) == 0 || len(other.bitmap()) == 0 {
|
||||
return
|
||||
}
|
||||
// If a is full, difference is the flip of b.
|
||||
if len(ra) > 0 && ra[0].start == 0 && ra[0].last == 65535 {
|
||||
clone := other.Clone()
|
||||
bitmap := clone.bitmap()
|
||||
for i, word := range other.bitmap() {
|
||||
bitmap[i] = ^word
|
||||
}
|
||||
c.setTyp(containerBitmap)
|
||||
c.setMapped(false)
|
||||
c.setBitmap(bitmap)
|
||||
c.setN(c.count())
|
||||
return
|
||||
}
|
||||
runs := make([]interval16, 0, len(ra))
|
||||
for _, inputRun := range ra {
|
||||
run := inputRun
|
||||
add := true
|
||||
for bit := inputRun.start; bit <= inputRun.last; bit++ {
|
||||
if other.bitmapContains(bit) {
|
||||
if run.start == bit {
|
||||
if bit == 65535 { //overflow
|
||||
add = false
|
||||
}
|
||||
|
||||
run.start++
|
||||
} else if bit == run.last {
|
||||
run.last--
|
||||
} else {
|
||||
run.last = bit - 1
|
||||
if run.last >= run.start {
|
||||
runs = append(runs, run)
|
||||
}
|
||||
run.start = bit + 1
|
||||
run.last = inputRun.last
|
||||
}
|
||||
if run.start > run.last {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if bit == 65535 { //overflow
|
||||
break
|
||||
}
|
||||
}
|
||||
if run.start <= run.last {
|
||||
if add {
|
||||
runs = append(runs, run)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
c.setRuns(runs)
|
||||
c.n = 0
|
||||
for _, run := range runs {
|
||||
c.n += int32(run.last-run.start) + 1
|
||||
}
|
||||
if c.N() < ArrayMaxSize && int32(len(runs)) > c.N()/2 {
|
||||
c.runToArray()
|
||||
} else if len(runs) > runMaxSize {
|
||||
c.runToBitmap()
|
||||
}
|
||||
}
|
||||
|
||||
func differenceRunRunInPlace(c, other *Container) {
|
||||
statsHit("differenceInPlace/RunRun")
|
||||
|
||||
ra, rb := c.runs(), other.runs()
|
||||
if len(ra) == 0 || len(rb) == 0 {
|
||||
return
|
||||
}
|
||||
apos := 0 // current a-run index
|
||||
bpos := 0 // current b-run index
|
||||
astart := ra[apos].start
|
||||
alast := ra[apos].last
|
||||
bstart := rb[bpos].start
|
||||
blast := rb[bpos].last
|
||||
alen := len(ra)
|
||||
blen := len(rb)
|
||||
|
||||
runs := make([]interval16, 0, alen+blen) // TODO allocate max then truncate? or something else
|
||||
// cardinality upper bound: sum of number of runs
|
||||
// each B-run could split an A-run in two, up to len(b.runs) times
|
||||
|
||||
for apos < alen && bpos < blen {
|
||||
switch {
|
||||
case alast < bstart:
|
||||
// current A-run entirely precedes current B-run: keep full A-run, advance to next A-run
|
||||
runs = append(runs, interval16{start: astart, last: alast})
|
||||
apos++
|
||||
if apos < alen {
|
||||
astart = ra[apos].start
|
||||
alast = ra[apos].last
|
||||
}
|
||||
case blast < astart:
|
||||
// current B-run entirely precedes current A-run: advance to next B-run
|
||||
bpos++
|
||||
if bpos < blen {
|
||||
bstart = rb[bpos].start
|
||||
blast = rb[bpos].last
|
||||
}
|
||||
default:
|
||||
// overlap
|
||||
if astart < bstart {
|
||||
runs = append(runs, interval16{start: astart, last: bstart - 1})
|
||||
}
|
||||
if alast > blast {
|
||||
astart = blast + 1
|
||||
} else {
|
||||
apos++
|
||||
if apos < alen {
|
||||
astart = ra[apos].start
|
||||
alast = ra[apos].last
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if apos < alen {
|
||||
runs = append(runs, interval16{start: astart, last: alast})
|
||||
apos++
|
||||
if apos < alen {
|
||||
runs = append(runs, ra[apos:]...)
|
||||
}
|
||||
}
|
||||
c.setRuns(runs)
|
||||
c.n = 0
|
||||
for _, run := range runs {
|
||||
c.n += int32(run.last-run.start) + 1
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2710,6 +2710,15 @@ func unionInPlaceWrapper(a, b *Container) *Container {
|
|||
return out.Containers.Get(0)
|
||||
}
|
||||
|
||||
func differenceInPlaceWrapper(a, b *Container) *Container {
|
||||
out := NewBitmap()
|
||||
out.Containers.Put(0, a.Clone())
|
||||
B := NewBitmap()
|
||||
B.Containers.Put(0, b)
|
||||
out.DifferenceInPlace(B)
|
||||
return out.Containers.Get(0)
|
||||
}
|
||||
|
||||
func TestContainerCombinations(t *testing.T) {
|
||||
|
||||
cts := setupContainerTests()
|
||||
|
|
@ -3283,6 +3292,117 @@ func TestContainerCombinations(t *testing.T) {
|
|||
{flip, "outerBitsSet", "", "innerBitsSet"},
|
||||
{flip, "oddBitsSet", "", "evenBitsSet"},
|
||||
{flip, "evenBitsSet", "", "oddBitsSet"},
|
||||
|
||||
// differenceInPlace
|
||||
{differenceInPlaceWrapper, "empty", "empty", "empty"},
|
||||
{differenceInPlaceWrapper, "empty", "full", "empty"},
|
||||
{differenceInPlaceWrapper, "empty", "firstBitSet", "empty"},
|
||||
{differenceInPlaceWrapper, "empty", "lastBitSet", "empty"},
|
||||
{differenceInPlaceWrapper, "empty", "firstBitUnset", "empty"},
|
||||
{differenceInPlaceWrapper, "empty", "lastBitUnset", "empty"},
|
||||
{differenceInPlaceWrapper, "empty", "innerBitsSet", "empty"},
|
||||
{differenceInPlaceWrapper, "empty", "outerBitsSet", "empty"},
|
||||
{differenceInPlaceWrapper, "empty", "oddBitsSet", "empty"},
|
||||
{differenceInPlaceWrapper, "empty", "evenBitsSet", "empty"},
|
||||
//
|
||||
{differenceInPlaceWrapper, "full", "empty", "full"},
|
||||
{differenceInPlaceWrapper, "full", "full", "empty"},
|
||||
{differenceInPlaceWrapper, "full", "firstBitSet", "firstBitUnset"},
|
||||
{differenceInPlaceWrapper, "full", "lastBitSet", "lastBitUnset"},
|
||||
{differenceInPlaceWrapper, "full", "firstBitUnset", "firstBitSet"},
|
||||
{differenceInPlaceWrapper, "full", "lastBitUnset", "lastBitSet"},
|
||||
{differenceInPlaceWrapper, "full", "innerBitsSet", "outerBitsSet"},
|
||||
{differenceInPlaceWrapper, "full", "outerBitsSet", "innerBitsSet"},
|
||||
{differenceInPlaceWrapper, "full", "oddBitsSet", "evenBitsSet"},
|
||||
{differenceInPlaceWrapper, "full", "evenBitsSet", "oddBitsSet"},
|
||||
//
|
||||
{differenceInPlaceWrapper, "firstBitSet", "empty", "firstBitSet"},
|
||||
{differenceInPlaceWrapper, "firstBitSet", "full", "empty"},
|
||||
{differenceInPlaceWrapper, "firstBitSet", "firstBitSet", "empty"},
|
||||
{differenceInPlaceWrapper, "firstBitSet", "lastBitSet", "firstBitSet"},
|
||||
{differenceInPlaceWrapper, "firstBitSet", "firstBitUnset", "firstBitSet"},
|
||||
{differenceInPlaceWrapper, "firstBitSet", "lastBitUnset", "empty"},
|
||||
{differenceInPlaceWrapper, "firstBitSet", "innerBitsSet", "firstBitSet"},
|
||||
{differenceInPlaceWrapper, "firstBitSet", "outerBitsSet", "empty"},
|
||||
{differenceInPlaceWrapper, "firstBitSet", "oddBitsSet", "firstBitSet"},
|
||||
{differenceInPlaceWrapper, "firstBitSet", "evenBitsSet", "empty"},
|
||||
//
|
||||
{differenceInPlaceWrapper, "lastBitSet", "empty", "lastBitSet"},
|
||||
{differenceInPlaceWrapper, "lastBitSet", "full", "empty"},
|
||||
{differenceInPlaceWrapper, "lastBitSet", "firstBitSet", "lastBitSet"},
|
||||
{differenceInPlaceWrapper, "lastBitSet", "lastBitSet", "empty"},
|
||||
{differenceInPlaceWrapper, "lastBitSet", "firstBitUnset", "empty"},
|
||||
{differenceInPlaceWrapper, "lastBitSet", "lastBitUnset", "lastBitSet"},
|
||||
{differenceInPlaceWrapper, "lastBitSet", "innerBitsSet", "lastBitSet"},
|
||||
{differenceInPlaceWrapper, "lastBitSet", "outerBitsSet", "empty"},
|
||||
{differenceInPlaceWrapper, "lastBitSet", "oddBitsSet", "empty"},
|
||||
{differenceInPlaceWrapper, "lastBitSet", "evenBitsSet", "lastBitSet"},
|
||||
//
|
||||
{differenceInPlaceWrapper, "firstBitUnset", "empty", "firstBitUnset"},
|
||||
{differenceInPlaceWrapper, "firstBitUnset", "full", "empty"},
|
||||
{differenceInPlaceWrapper, "firstBitUnset", "firstBitSet", "firstBitUnset"},
|
||||
{differenceInPlaceWrapper, "firstBitUnset", "lastBitSet", "innerBitsSet"},
|
||||
{differenceInPlaceWrapper, "firstBitUnset", "firstBitUnset", "empty"},
|
||||
{differenceInPlaceWrapper, "firstBitUnset", "lastBitUnset", "lastBitSet"},
|
||||
{differenceInPlaceWrapper, "firstBitUnset", "innerBitsSet", "lastBitSet"},
|
||||
{differenceInPlaceWrapper, "firstBitUnset", "outerBitsSet", "innerBitsSet"},
|
||||
//{differenceInPlaceWrapper, "firstBitUnset", "oddBitsSet", ""},
|
||||
{differenceInPlaceWrapper, "firstBitUnset", "evenBitsSet", "oddBitsSet"},
|
||||
//
|
||||
{differenceInPlaceWrapper, "lastBitUnset", "empty", "lastBitUnset"},
|
||||
{differenceInPlaceWrapper, "lastBitUnset", "full", "empty"},
|
||||
{differenceInPlaceWrapper, "lastBitUnset", "firstBitSet", "innerBitsSet"},
|
||||
{differenceInPlaceWrapper, "lastBitUnset", "lastBitSet", "lastBitUnset"},
|
||||
{differenceInPlaceWrapper, "lastBitUnset", "firstBitUnset", "firstBitSet"},
|
||||
{differenceInPlaceWrapper, "lastBitUnset", "lastBitUnset", "empty"},
|
||||
{differenceInPlaceWrapper, "lastBitUnset", "innerBitsSet", "firstBitSet"},
|
||||
{differenceInPlaceWrapper, "lastBitUnset", "outerBitsSet", "innerBitsSet"},
|
||||
{differenceInPlaceWrapper, "lastBitUnset", "oddBitsSet", "evenBitsSet"},
|
||||
//{differenceInPlaceWrapper, "lastBitUnset", "evenBitsSet", ""},
|
||||
//
|
||||
{differenceInPlaceWrapper, "innerBitsSet", "empty", "innerBitsSet"},
|
||||
{differenceInPlaceWrapper, "innerBitsSet", "full", "empty"},
|
||||
{differenceInPlaceWrapper, "innerBitsSet", "firstBitSet", "innerBitsSet"},
|
||||
{differenceInPlaceWrapper, "innerBitsSet", "lastBitSet", "innerBitsSet"},
|
||||
{differenceInPlaceWrapper, "innerBitsSet", "firstBitUnset", "empty"},
|
||||
{differenceInPlaceWrapper, "innerBitsSet", "lastBitUnset", "empty"},
|
||||
{differenceInPlaceWrapper, "innerBitsSet", "innerBitsSet", "empty"},
|
||||
{differenceInPlaceWrapper, "innerBitsSet", "outerBitsSet", "innerBitsSet"},
|
||||
//{differenceInPlaceWrapper, "innerBitsSet", "oddBitsSet", ""},
|
||||
//{differenceInPlaceWrapper, "innerBitsSet", "evenBitsSet", ""},
|
||||
//
|
||||
{differenceInPlaceWrapper, "outerBitsSet", "empty", "outerBitsSet"},
|
||||
{differenceInPlaceWrapper, "outerBitsSet", "full", "empty"},
|
||||
{differenceInPlaceWrapper, "outerBitsSet", "firstBitSet", "lastBitSet"},
|
||||
{differenceInPlaceWrapper, "outerBitsSet", "lastBitSet", "firstBitSet"},
|
||||
{differenceInPlaceWrapper, "outerBitsSet", "firstBitUnset", "firstBitSet"},
|
||||
{differenceInPlaceWrapper, "outerBitsSet", "lastBitUnset", "lastBitSet"},
|
||||
{differenceInPlaceWrapper, "outerBitsSet", "innerBitsSet", "outerBitsSet"},
|
||||
{differenceInPlaceWrapper, "outerBitsSet", "outerBitsSet", "empty"},
|
||||
{differenceInPlaceWrapper, "outerBitsSet", "oddBitsSet", "firstBitSet"},
|
||||
{differenceInPlaceWrapper, "outerBitsSet", "evenBitsSet", "lastBitSet"},
|
||||
//
|
||||
{differenceInPlaceWrapper, "oddBitsSet", "empty", "oddBitsSet"},
|
||||
{differenceInPlaceWrapper, "oddBitsSet", "full", "empty"},
|
||||
{differenceInPlaceWrapper, "oddBitsSet", "firstBitSet", "oddBitsSet"},
|
||||
//{differenceInPlaceWrapper, "oddBitsSet", "lastBitSet", ""},
|
||||
{differenceInPlaceWrapper, "oddBitsSet", "firstBitUnset", "empty"},
|
||||
{differenceInPlaceWrapper, "oddBitsSet", "lastBitUnset", "lastBitSet"},
|
||||
{differenceInPlaceWrapper, "oddBitsSet", "innerBitsSet", "lastBitSet"},
|
||||
//{differenceInPlaceWrapper, "oddBitsSet", "outerBitsSet", ""},
|
||||
{differenceInPlaceWrapper, "oddBitsSet", "oddBitsSet", "empty"},
|
||||
{differenceInPlaceWrapper, "oddBitsSet", "evenBitsSet", "oddBitsSet"},
|
||||
//
|
||||
{differenceInPlaceWrapper, "evenBitsSet", "empty", "evenBitsSet"},
|
||||
{differenceInPlaceWrapper, "evenBitsSet", "full", "empty"},
|
||||
//{differenceInPlaceWrapper, "evenBitsSet", "firstBitSet", ""},
|
||||
{differenceInPlaceWrapper, "evenBitsSet", "lastBitSet", "evenBitsSet"},
|
||||
{differenceInPlaceWrapper, "evenBitsSet", "firstBitUnset", "firstBitSet"},
|
||||
{differenceInPlaceWrapper, "evenBitsSet", "lastBitUnset", "empty"},
|
||||
{differenceInPlaceWrapper, "evenBitsSet", "innerBitsSet", "firstBitSet"},
|
||||
//{differenceInPlaceWrapper, "evenBitsSet", "outerBitsSet", ""},
|
||||
{differenceInPlaceWrapper, "evenBitsSet", "oddBitsSet", "evenBitsSet"},
|
||||
{differenceInPlaceWrapper, "evenBitsSet", "evenBitsSet", "empty"},
|
||||
}
|
||||
for _, testOp := range testOps {
|
||||
for _, x := range containerTypes {
|
||||
|
|
@ -3896,3 +4016,12 @@ func TestBitmapAny(t *testing.T) {
|
|||
t.Error("shouldn't be any left")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDifferenceInPlace_N(t *testing.T) {
|
||||
a := doContainer(containerRun, runFull())
|
||||
b := doContainer(containerBitmap, bitmapFull())
|
||||
r := differenceInPlaceWrapper(a, b)
|
||||
if r.N() != 0 {
|
||||
t.Error("expected difference of containers to have n=0")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1739,3 +1739,57 @@ func BenchmarkUnionBulk(b *testing.B) {
|
|||
UnionInPlace(data.a1, data.a2, data.b, data.r1, data.r2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBitmap_DifferenceInPlace(t *testing.T) {
|
||||
// array
|
||||
arraybm := roaring.NewSliceBitmap()
|
||||
for i := uint64(0); i < 1024; i += 4 {
|
||||
_, _ = arraybm.Add((1 << 16) + i)
|
||||
}
|
||||
arraybm.Optimize()
|
||||
|
||||
// bitmap
|
||||
bitmapbm := roaring.NewSliceBitmap()
|
||||
for i := uint64(0); i < 16384; i += 2 {
|
||||
_, _ = bitmapbm.Add((2 << 16) + i)
|
||||
}
|
||||
bitmapbm.Optimize()
|
||||
|
||||
// small run
|
||||
smallrunbm := roaring.NewSliceBitmap()
|
||||
for i := uint64(0); i < 1024; i++ {
|
||||
_, _ = smallrunbm.Add((3 << 16) + i)
|
||||
}
|
||||
smallrunbm.Optimize()
|
||||
|
||||
// large run
|
||||
largerunbm := roaring.NewSliceBitmap()
|
||||
for i := uint64(0); i < 65535; i++ {
|
||||
_, _ = largerunbm.Add((4 << 16) + i)
|
||||
}
|
||||
largerunbm.Optimize()
|
||||
|
||||
// test the difference in place
|
||||
bm := testBM()
|
||||
bm.DifferenceInPlace(arraybm, bitmapbm, smallrunbm, largerunbm)
|
||||
if bm.Count() != 0 {
|
||||
t.Fatalf("expected bitmap count to be 0, but got: %d", bm.Count())
|
||||
}
|
||||
|
||||
bm = testBM()
|
||||
bm.DifferenceInPlace(bitmapbm, smallrunbm, largerunbm)
|
||||
if bm.Count() != 256 {
|
||||
t.Fatalf("expected bitmap count to be 256, but got: %d", bm.Count())
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkDifferencInPlace(b *testing.B) {
|
||||
data := getBenchData(b)
|
||||
for n := 0; n < b.N; n++ {
|
||||
bm := roaring.NewBitmap()
|
||||
bm.
|
||||
UnionInPlace(data.a1, data.a2, data.b, data.r1, data.r2)
|
||||
bm.
|
||||
DifferenceInPlace(data.a1, data.r2, data.b, data.r1)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
37
row.go
37
row.go
|
|
@ -304,21 +304,26 @@ func (r *Row) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*Row, args ma
|
|||
}
|
||||
|
||||
// Difference returns the diff of r and other.
|
||||
func (r *Row) Difference(other *Row) *Row {
|
||||
var segments []rowSegment
|
||||
func (r *Row) Difference(others ...*Row) *Row {
|
||||
var output []rowSegment
|
||||
o := make(map[uint64][]*rowSegment)
|
||||
|
||||
itr := newMergeSegmentIterator(r.segments, other.segments)
|
||||
for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
|
||||
if s0 == nil {
|
||||
continue
|
||||
} else if s1 == nil {
|
||||
segments = append(segments, *s0)
|
||||
continue
|
||||
for x := range others {
|
||||
for y := range others[x].segments {
|
||||
segment := others[x].segments[y]
|
||||
o[segment.shard] = append(o[segment.shard], &segment)
|
||||
}
|
||||
segments = append(segments, *s0.Difference(s1))
|
||||
}
|
||||
for _, segment := range r.segments {
|
||||
|
||||
return &Row{segments: segments}
|
||||
dest, ok := o[segment.shard]
|
||||
if ok {
|
||||
output = append(output, *segment.Difference(dest...))
|
||||
} else {
|
||||
output = append(output, segment)
|
||||
}
|
||||
}
|
||||
return &Row{segments: output}
|
||||
}
|
||||
|
||||
// GenericUnary returns the results of a generic op on r.
|
||||
|
|
@ -576,9 +581,13 @@ func (s *rowSegment) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*rowSe
|
|||
}
|
||||
|
||||
// Difference returns the diff of s and other.
|
||||
func (s *rowSegment) Difference(other *rowSegment) *rowSegment {
|
||||
data := s.data.Difference(other.data)
|
||||
data = data.Freeze()
|
||||
func (s *rowSegment) Difference(others ...*rowSegment) *rowSegment {
|
||||
datas := make([]*roaring.Bitmap, len(others))
|
||||
for i, other := range others {
|
||||
datas[i] = other.data
|
||||
}
|
||||
data := s.data.Difference(datas...)
|
||||
data.Freeze()
|
||||
|
||||
return &rowSegment{
|
||||
data: data,
|
||||
|
|
|
|||
58
row_test.go
58
row_test.go
|
|
@ -138,3 +138,61 @@ func TestRow_Includes(t *testing.T) {
|
|||
t.Fatalf("row should include %d", 2*ShardWidth)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRow_DifferenceInPlace(t *testing.T) {
|
||||
row0 := pilosa.NewRow(0)
|
||||
row1 := pilosa.NewRow()
|
||||
row2 := pilosa.NewRow(0)
|
||||
res := row0.Difference(row1, row2)
|
||||
|
||||
if !row0.Includes(0) {
|
||||
t.Fatal("row should include 0")
|
||||
}
|
||||
if res.Count() != 0 {
|
||||
t.Fatal("results should be empty")
|
||||
}
|
||||
}
|
||||
func eq(a, b []uint64) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i, v := range a {
|
||||
if v != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
func TestRow_DifferenceInPlace2(t *testing.T) {
|
||||
lucky := []uint64{1, 3, 7, 9, 13, 15, 21, 25, 31, 33, 37, 43, 49, 51, 63, 67, 69, 73, 75, 79, 87, 93, 99, 105, 111, 115, 127, 129, 133, 135, 141, 151, 159, 163, 169, 171, 189, 193, 195, 201, 205, 211, 219, 223, 231, 235, 237, 241, 259, 261, 267, 273, 283, 285, 289, 297}
|
||||
src := pilosa.NewRow(lucky...)
|
||||
row1 := pilosa.NewRow()
|
||||
m := uint64(9)
|
||||
for i := m; i <= lucky[len(lucky)-1]; i += m {
|
||||
row1.SetBit(i)
|
||||
}
|
||||
m = uint64(3)
|
||||
row2 := pilosa.NewRow()
|
||||
for i := m; i <= lucky[len(lucky)-1]; i += m {
|
||||
row2.SetBit(i)
|
||||
}
|
||||
row3 := pilosa.NewRow()
|
||||
m = uint64(5)
|
||||
for i := m; i <= lucky[len(lucky)-1]; i += m {
|
||||
row3.SetBit(i)
|
||||
}
|
||||
row4 := pilosa.NewRow()
|
||||
m = uint64(7)
|
||||
for i := m; i <= lucky[len(lucky)-1]; i += m {
|
||||
row4.SetBit(i)
|
||||
}
|
||||
res5 := src.Difference(row1, row2, row3, row4)
|
||||
res6 := src.Difference(row4, row3, row2, row1)
|
||||
res7 := src.Difference(row4).Difference(row3).Difference(row2).Difference(row1)
|
||||
if !eq(res5.Columns(), res6.Columns()) {
|
||||
t.Fatalf("results do not match: %v, %v", res5.Columns(), res6.Columns())
|
||||
}
|
||||
if !eq(res6.Columns(), res7.Columns()) {
|
||||
t.Fatalf("results do not match: %v, %v", res6.Columns(), res7.Columns())
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue