mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 10:54:59 +00:00
high level support for difference in place
This commit is contained in:
Todd Gruben
parent
32b92abd6e
commit
8de32fd62a
6 changed files with 121 additions and 40 deletions
|
|
@ -1247,7 +1247,6 @@ func (f *fragment) rangeGT(bitDepth uint, predicate int64, allowEquality bool) (
|
|||
|
||||
func (f *fragment) rangeGTUnsigned(filter *Row, bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) {
|
||||
keep := NewRow()
|
||||
|
||||
// Filter any bits that don't match the current bit value.
|
||||
for i := int(bitDepth - 1); i >= 0; i-- {
|
||||
row := f.row(uint64(bsiOffsetBit + i))
|
||||
|
|
@ -1260,12 +1259,12 @@ func (f *fragment) rangeGTUnsigned(filter *Row, bitDepth uint, predicate uint64,
|
|||
if bit == 1 {
|
||||
return keep, nil
|
||||
}
|
||||
return filter.Difference(filter.Difference(row).Difference(keep)), nil
|
||||
return filter.Difference(filter.Difference(row, keep)), nil
|
||||
}
|
||||
|
||||
// If bit is set then remove all unset columns not already kept.
|
||||
if bit == 1 {
|
||||
filter = filter.Difference(filter.Difference(row).Difference(keep))
|
||||
filter = filter.Difference(filter.Difference(row, keep))
|
||||
continue
|
||||
}
|
||||
|
||||
|
|
@ -1333,7 +1332,7 @@ func (f *fragment) rangeBetweenUnsigned(filter *Row, bitDepth uint, predicateMin
|
|||
// GTE predicateMin
|
||||
// If bit is set then remove all unset columns not already kept.
|
||||
if bit1 == 1 {
|
||||
filter = filter.Difference(filter.Difference(row).Difference(keep1))
|
||||
filter = filter.Difference(filter.Difference(row, keep1))
|
||||
} else {
|
||||
// If bit is unset then add columns with set bit to keep.
|
||||
// Don't bother to compute this on the final iteration.
|
||||
|
|
|
|||
10
go.mod
10
go.mod
|
|
@ -8,40 +8,32 @@ require (
|
|||
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
|
||||
github.com/boltdb/bolt v1.3.1
|
||||
github.com/cespare/xxhash v1.1.0
|
||||
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
|
||||
github.com/davecgh/go-spew v1.1.1
|
||||
github.com/go-ole/go-ole v1.2.4 // indirect
|
||||
github.com/gogo/protobuf v1.2.0
|
||||
github.com/golang/protobuf v1.3.2
|
||||
github.com/google/go-cmp v0.2.0
|
||||
github.com/gorilla/handlers v1.3.0
|
||||
github.com/gorilla/mux v1.7.0
|
||||
github.com/hashicorp/memberlist v0.1.3
|
||||
github.com/inconshreveable/mousetrap v1.0.0 // indirect
|
||||
github.com/molecula/ext v0.0.0-20200103203257-8a458a73e8c2
|
||||
github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4
|
||||
github.com/opentracing/opentracing-go v1.1.0
|
||||
github.com/pelletier/go-toml v1.2.0
|
||||
github.com/pilosa/pilosa v1.4.0
|
||||
github.com/pkg/errors v0.8.1
|
||||
github.com/prometheus/client_golang v0.9.3
|
||||
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect
|
||||
github.com/satori/go.uuid v1.2.0
|
||||
github.com/shirou/gopsutil v2.18.12+incompatible
|
||||
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect
|
||||
github.com/spf13/cobra v0.0.3
|
||||
github.com/spf13/pflag v1.0.3
|
||||
github.com/spf13/viper v1.3.1
|
||||
github.com/uber-go/atomic v1.4.0 // indirect
|
||||
github.com/uber/jaeger-client-go v2.16.0+incompatible
|
||||
github.com/uber/jaeger-lib v2.2.0+incompatible // indirect
|
||||
github.com/youtube/vitess v2.1.1+incompatible // indirect
|
||||
go.uber.org/atomic v1.4.0 // indirect
|
||||
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect
|
||||
golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58
|
||||
golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 // indirect
|
||||
golang.org/x/text v0.3.2 // indirect
|
||||
google.golang.org/grpc v1.24.0
|
||||
modernc.org/mathutil v1.0.0
|
||||
modernc.org/strutil v1.0.0
|
||||
|
|
|
|||
5
go.sum
5
go.sum
|
|
@ -7,6 +7,7 @@ github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5Vk
|
|||
github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
|
||||
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
|
||||
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
|
||||
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
|
|
@ -105,6 +106,8 @@ github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181
|
|||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 h1:ERLyN4p3KS5Fk2ADsDENm2cq0+Lx6sF1sG8uwRlySpU=
|
||||
github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/pilosa/pilosa v1.4.0 h1:nqHNIK4nDslFnem3yDp9R+6TgLdlkY9WdJD88Z83T8U=
|
||||
github.com/pilosa/pilosa v1.4.0/go.mod h1:NSTtTprtb5MSgCs4mcNqeQ2JdIMpInOi4DEImxGJeTs=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
|
|
@ -123,6 +126,7 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
|
|||
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY=
|
||||
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
|
||||
|
|
@ -155,6 +159,7 @@ github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o=
|
|||
github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
|
||||
github.com/uber/jaeger-client-go v2.16.0+incompatible h1:Q2Pp6v3QYiocMxomCaJuwQGFt7E53bPYqEgug/AoBtY=
|
||||
github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
|
||||
github.com/uber/jaeger-lib v2.0.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
|
||||
github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw=
|
||||
github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
|
||||
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
|
||||
|
|
|
|||
|
|
@ -995,7 +995,15 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) {
|
|||
}
|
||||
|
||||
// Difference returns the difference of b and other.
|
||||
func (b *Bitmap) Difference(other *Bitmap) *Bitmap {
|
||||
func (b *Bitmap) Difference(other ...*Bitmap) *Bitmap {
|
||||
output := b.singleDifference(other[0])
|
||||
if len(other) > 1 {
|
||||
output.DifferenceInPlace(other[1:]...)
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
func (b *Bitmap) singleDifference(other *Bitmap) *Bitmap {
|
||||
output := NewBitmap()
|
||||
iiter, _ := b.Containers.Iterator(0)
|
||||
jiter, _ := other.Containers.Iterator(0)
|
||||
|
|
@ -5474,7 +5482,7 @@ func (b *Bitmap) DifferenceInPlace(others ...*Bitmap) {
|
|||
staticHandledIters = [staticSize]handledIter{}
|
||||
bitmapIters handledIters
|
||||
target = b
|
||||
removeContainerKeys = make([]uint64, bSize)
|
||||
removeContainerKeys = make([]uint64, 0, bSize)
|
||||
)
|
||||
|
||||
if requiredSliceSize <= staticSize {
|
||||
|
|
@ -5495,32 +5503,41 @@ func (b *Bitmap) DifferenceInPlace(others ...*Bitmap) {
|
|||
|
||||
targetItr, _ := target.Containers.Iterator(0)
|
||||
// Go through all the containers and remove the other bits
|
||||
n := 0
|
||||
for targetItr.Next() {
|
||||
targetKey, curContainer := targetItr.Value()
|
||||
// Loop until every iters current value has been handled.
|
||||
|
||||
for _, iIter := range bitmapIters {
|
||||
if !iIter.hasNext {
|
||||
continue
|
||||
}
|
||||
iKey, iContainer := iIter.iter.Value()
|
||||
if targetKey == iKey {
|
||||
curContainer.differenceInPlace(iContainer)
|
||||
if curContainer.N() == 0 { //according to comments N = 1-count, so N should == 1 if 0 elements
|
||||
removeContainerKeys[n] = iKey
|
||||
n++
|
||||
for iKey < targetKey {
|
||||
iIter.hasNext = iIter.iter.Next()
|
||||
if iIter.hasNext {
|
||||
iKey, iContainer = iIter.iter.Value()
|
||||
} else {
|
||||
break
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if targetKey == iKey {
|
||||
if curContainer.frozen() {
|
||||
curContainer = curContainer.Clone()
|
||||
b.Containers.Put(targetKey, curContainer)
|
||||
}
|
||||
curContainer.differenceInPlace(iContainer)
|
||||
if curContainer.N() == 0 {
|
||||
removeContainerKeys = append(removeContainerKeys, iKey)
|
||||
break
|
||||
}
|
||||
iIter.hasNext = iIter.iter.Next()
|
||||
} else if targetKey > iKey {
|
||||
iIter.hasNext = iIter.iter.Next()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for i := 0; i < n; i++ {
|
||||
b.Containers.Remove(removeContainerKeys[i])
|
||||
for _, key := range removeContainerKeys {
|
||||
b.Containers.Remove(key)
|
||||
|
||||
}
|
||||
target.Containers.Repair()
|
||||
|
|
|
|||
37
row.go
37
row.go
|
|
@ -304,21 +304,26 @@ func (r *Row) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*Row, args ma
|
|||
}
|
||||
|
||||
// Difference returns the diff of r and other.
|
||||
func (r *Row) Difference(other *Row) *Row {
|
||||
var segments []rowSegment
|
||||
func (r *Row) Difference(others ...*Row) *Row {
|
||||
var output []rowSegment
|
||||
o := make(map[uint64][]*rowSegment)
|
||||
|
||||
itr := newMergeSegmentIterator(r.segments, other.segments)
|
||||
for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
|
||||
if s0 == nil {
|
||||
continue
|
||||
} else if s1 == nil {
|
||||
segments = append(segments, *s0)
|
||||
continue
|
||||
for x := range others {
|
||||
for y := range others[x].segments {
|
||||
segment := others[x].segments[y]
|
||||
o[segment.shard] = append(o[segment.shard], &segment)
|
||||
}
|
||||
segments = append(segments, *s0.Difference(s1))
|
||||
}
|
||||
for _, segment := range r.segments {
|
||||
|
||||
return &Row{segments: segments}
|
||||
dest, ok := o[segment.shard]
|
||||
if ok {
|
||||
output = append(output, *segment.Difference(dest...))
|
||||
} else {
|
||||
output = append(output, segment)
|
||||
}
|
||||
}
|
||||
return &Row{segments: output}
|
||||
}
|
||||
|
||||
// GenericUnary returns the results of a generic op on r.
|
||||
|
|
@ -576,9 +581,13 @@ func (s *rowSegment) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*rowSe
|
|||
}
|
||||
|
||||
// Difference returns the diff of s and other.
|
||||
func (s *rowSegment) Difference(other *rowSegment) *rowSegment {
|
||||
data := s.data.Difference(other.data)
|
||||
data = data.Freeze()
|
||||
func (s *rowSegment) Difference(others ...*rowSegment) *rowSegment {
|
||||
datas := make([]*roaring.Bitmap, len(others))
|
||||
for i, other := range others {
|
||||
datas[i] = other.data
|
||||
}
|
||||
data := s.data.Difference(datas...)
|
||||
data.Freeze()
|
||||
|
||||
return &rowSegment{
|
||||
data: data,
|
||||
|
|
|
|||
59
row_test.go
59
row_test.go
|
|
@ -138,3 +138,62 @@ func TestRow_Includes(t *testing.T) {
|
|||
t.Fatalf("row should include %d", 2*ShardWidth)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRow_DifferenceInPlace(t *testing.T) {
|
||||
row0 := pilosa.NewRow(0)
|
||||
row1 := pilosa.NewRow()
|
||||
row2 := pilosa.NewRow(0)
|
||||
res := row0.Difference(row1, row2)
|
||||
|
||||
if !row0.Includes(0) {
|
||||
t.Fatal("row should include 0")
|
||||
}
|
||||
if res.Count() != 0 {
|
||||
t.Fatal("results should be empty")
|
||||
}
|
||||
}
|
||||
func eq(a, b []uint64) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i, v := range a {
|
||||
if v != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
func TestRow_DifferenceInPlace2(t *testing.T) {
|
||||
lucky := []uint64{1, 3, 7, 9, 13, 15, 21, 25, 31, 33, 37, 43, 49, 51, 63, 67, 69, 73, 75, 79, 87, 93, 99, 105, 111, 115, 127, 129, 133, 135, 141, 151, 159, 163, 169, 171, 189, 193, 195, 201, 205, 211, 219, 223, 231, 235, 237, 241, 259, 261, 267, 273, 283, 285, 289, 297}
|
||||
src := pilosa.NewRow(lucky...)
|
||||
fmt.Println(src.Columns())
|
||||
row1 := pilosa.NewRow()
|
||||
m := uint64(9)
|
||||
for i := m; i <= lucky[len(lucky)-1]; i += m {
|
||||
row1.SetBit(i)
|
||||
}
|
||||
m = uint64(3)
|
||||
row2 := pilosa.NewRow()
|
||||
for i := m; i <= lucky[len(lucky)-1]; i += m {
|
||||
row2.SetBit(i)
|
||||
}
|
||||
row3 := pilosa.NewRow()
|
||||
m = uint64(5)
|
||||
for i := m; i <= lucky[len(lucky)-1]; i += m {
|
||||
row3.SetBit(i)
|
||||
}
|
||||
row4 := pilosa.NewRow()
|
||||
m = uint64(7)
|
||||
for i := m; i <= lucky[len(lucky)-1]; i += m {
|
||||
row4.SetBit(i)
|
||||
}
|
||||
res5 := src.Difference(row1, row2, row3, row4)
|
||||
res6 := src.Difference(row4, row3, row2, row1)
|
||||
res7 := src.Difference(row4).Difference(row3).Difference(row2).Difference(row1)
|
||||
if eq(res5.Columns(), res6.Columns()) {
|
||||
t.Fatal("should match")
|
||||
}
|
||||
if eq(res6.Columns(), res7.Columns()) {
|
||||
t.Fatal("should match")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue