From 933d5e28e7bafcceac061cc6881c729a319eb510 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 20 Nov 2018 15:12:02 -0600 Subject: [PATCH 1/6] initial bit shift functions for all container type --- roaring/roaring.go | 79 ++++++++++++++++++++++ roaring/roaring_helpers_test.go | 19 ++++++ roaring/roaring_internal_test.go | 112 +++++++++++++++++++++++++++++++ 3 files changed, 210 insertions(+) diff --git a/roaring/roaring.go b/roaring/roaring.go index 9c4274df9..a94121dc9 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -3006,6 +3006,85 @@ func xorBitmapBitmap(a, b *Container) *Container { return output } +func shift(a *Container) (*Container, bool) { + if a.isArray() { + return shiftArray(a) + } else if a.isRun() { + return shiftRun(a) + } + return shiftBitmap(a) +} + +func shiftArray(a *Container) (*Container, bool) { + statsHit("shift/Array") + carry := false + output := &Container{containerType: containerArray} + output.array = make([]uint16, len(a.array)) + output.array = output.array[:0] + output.n = a.n + for _, v := range a.array { + fmt.Println(v,v+1) + if v+1 == 0 { //overflow + carry = true + output.n -= 1 + } else { + output.array = append(output.array, v+1) + } + } + return output, carry +} + +func shiftBitmap(a *Container) (*Container, bool) { + statsHit("shift/Bitmap") + carry := false + output := &Container{containerType: containerBitmap} + output.bitmap = make([]uint64, len(a.bitmap)) + output.bitmap = output.bitmap[:0] + output.n = a.n + lastcarry:=false + for i, v := range a.bitmap { + carry = (v&(1<<63))!= 0 + v = v << 1 + if i != 0 { + if lastcarry { + v |= 1 + } + } + output.bitmap = append(output.bitmap, v) + lastcarry = carry + } + if carry { + output.n -= 1 + } + return output, carry +} + +func shiftRun(a *Container) (*Container, bool) { + statsHit("shift/Run") + carry := false + output := &Container{containerType: containerRun} + output.runs = make([]interval16, len(a.runs)) + output.runs = output.runs[:0] + for _, v := range a.runs { + if v.start+1 == 0 { + carry = true + output.n -= 1 + break + } else if v.last+1 == 0 { + v.start += 1 + carry = true + output.n -= 1 + } else { + v.start += 1 + v.last += 1 + carry = false + } + output.runs = append(output.runs, v) + } + + return output, carry +} + // opType represents a type of operation. type opType uint8 diff --git a/roaring/roaring_helpers_test.go b/roaring/roaring_helpers_test.go index cd22978fb..36963425e 100644 --- a/roaring/roaring_helpers_test.go +++ b/roaring/roaring_helpers_test.go @@ -106,6 +106,25 @@ func bitmapFirstBitSet() []uint64 { return bitmap } +func bitmapSecondBitSet() []uint64 { + bitmap := make([]uint64, bitmapN) + bitmap[0] = 0x0000000000000002 + return bitmap +} + +func bitmapLastBitFirstRowSet() []uint64 { + bitmap := make([]uint64, bitmapN) + bitmap[0] = 0x8000000000000000 + return bitmap +} + +func bitmapFirstBitSecoundRowSet() []uint64 { + bitmap := make([]uint64, bitmapN) + bitmap[1] = 0x0000000000000001 + return bitmap +} + + func bitmapLastBitSet() []uint64 { bitmap := make([]uint64, bitmapN) bitmap[bitmapN-1] = 0x8000000000000000 diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 78b686371..ffa8be812 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -3289,3 +3289,115 @@ func TestEquals(t *testing.T) { } } */ +func TestShiftArray(t *testing.T) { + a := &Container{ + containerType: containerArray, +} + tests := []struct { + array []uint16 + exp []uint16 + }{ + { + array: []uint16{1}, + exp: []uint16{2}, + }, + { + array: []uint16{}, + exp: []uint16{}, + }, + { + array: []uint16{1, 2, 3, 4, 5, 11, 12}, + exp: []uint16{2, 3, 4, 5, 6, 12,13}, + }, + { + array: []uint16{65535}, + exp: []uint16{}, + }, + } + + for i, test := range tests { + a.array = test.array + a.n = int32(len(a.array)) + ret,_ := shiftArray(a) + if !reflect.DeepEqual(ret.array, test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array) + } + } +} + + +func TestShiftBitmap(t *testing.T) { + a := &Container{ + containerType: containerBitmap, + } + tests := []struct { + bitmap []uint64 + exp []uint64 + }{ + { + bitmap:bitmapFirstBitSet() , + exp: bitmapSecondBitSet(), + }, + { + bitmap:bitmapLastBitSet() , + exp: bitmapEmpty(), + }, + { + bitmap:bitmapLastBitFirstRowSet() , + exp: bitmapFirstBitSecoundRowSet (), + }, + } + + for i, test := range tests { + a.bitmap = test.bitmap + a.n = 1 + ret,_ := shiftBitmap(a) + if !reflect.DeepEqual(ret.bitmap, test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.bitmap) + } + } +} +func TestShiftRun(t *testing.T) { + a := &Container{ + containerType: containerRun, + } + + tests := []struct { + runs []interval16 + n int32 + en int32 + exp []interval16 + carry bool + }{ + { + runs: []interval16{{start: 5, last: 10}}, + n: 5, + en: 5, + exp: []interval16{{start: 6, last: 11}}, + carry: false, + }, + { + runs: []interval16{{start: 5, last:65535 }}, + n: 65530, + en: 65529, + exp: []interval16{{start: 6, last: 65535}}, + carry: true, + }, + { + runs: []interval16{{start: 65535, last:65535 }}, + n: 1, + en: 0, + exp: []interval16{}, + carry: true, + }, + } + + for i, test := range tests { + a.runs = test.runs + a.n = test.n + ret,c := shiftRun(a) + if !reflect.DeepEqual(ret.runs, test.exp) && c == test.carry && ret.n == test.en { + t.Fatalf("test #%v expected %v, but got %v %d", i, test.exp, ret.runs,ret.n) + } + } +} From 0404354faaabbf343197462f1d37031174ce38ea Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 26 Nov 2018 11:29:05 -0600 Subject: [PATCH 2/6] add shift operater to pql --- executor.go | 18 +++++++++++ executor_test.go | 75 ++++++++++++++++++++++++++++++++++++++++++++++ roaring/roaring.go | 31 +++++++++++++++++-- row.go | 20 +++++++++++++ 4 files changed, 141 insertions(+), 3 deletions(-) diff --git a/executor.go b/executor.go index c4153ab61..918fe9611 100644 --- a/executor.go +++ b/executor.go @@ -515,6 +515,8 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c * return e.executeXorShard(ctx, index, c, shard) case "Not": return e.executeNotShard(ctx, index, c, shard) + case "Shift": + return e.executeShiftShard(ctx, index, c, shard) default: return nil, fmt.Errorf("unknown call: %s", c.Name) } @@ -1419,6 +1421,22 @@ func (e *executor) executeNotShard(ctx context.Context, index string, c *pql.Cal return existenceRow.Difference(row), nil } +// executeShiftShard executes a shift() call for a local shard. +func (e *executor) executeShiftShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + if len(c.Children) == 0 { + return nil, errors.New("Shift() requires an input row") + } else if len(c.Children) > 1 { + return nil, errors.New("Shift() only accepts a single row input") + } + + row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) + if err != nil { + return nil, err + } + + return row.Shift(), nil +} + // executeCount executes a count() call. func (e *executor) executeCount(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (uint64, error) { if len(c.Children) == 0 { diff --git a/executor_test.go b/executor_test.go index 87b72689c..9553192ee 100644 --- a/executor_test.go +++ b/executor_test.go @@ -3163,3 +3163,78 @@ func runCallTest(t *testing.T, writeQuery string, readQueries []string, indexOpt return responses } + +func TestExecutor_Execute_Shift(t *testing.T) { + t.Run("Shift Bit 0", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "general", 10, 0) + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10))`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1}) { + t.Fatalf("unexpected columns: %+v", columns) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Shift(Row(general=10)))`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + t.Run("Shift container boundary", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "general", 10, 65535) + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10))`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{65536}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("Shift shard boundary", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "general", 10, ShardWidth-1) + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10))`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{ShardWidth}) { + t.Fatalf("unexpected columns: %+v", columns) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Shift(Row(general=10)))`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{ShardWidth+1}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("Shift shard boundary no create", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "general", 10, ShardWidth-2) //shardwidth -1 + hldr.SetBit("i", "general", 10, ShardWidth-1) //shardwidth + hldr.SetBit("i", "general", 10, ShardWidth) //shardwidth +1 + hldr.SetBit("i", "general", 10, ShardWidth+2) //shardwidth +3 + + exp:=[]uint64{ ShardWidth-1,ShardWidth,ShardWidth+1,ShardWidth+3} + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10))`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, exp) { + t.Fatalf("unexpected columns: %+v", columns) + } + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Shift(Row(general=10)))`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns,[]uint64{ ShardWidth,ShardWidth+1,ShardWidth+2,ShardWidth+4} ) { + t.Fatalf("unexpected columns: \n%+v\n%+v", columns,exp) + } + }) + +} diff --git a/roaring/roaring.go b/roaring/roaring.go index a94121dc9..77483c84f 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -478,6 +478,32 @@ func (b *Bitmap) Xor(other *Bitmap) *Bitmap { return output } +func (b *Bitmap) Shift() (*Bitmap) { + output := NewBitmap() + iiter, _ := b.Containers.Iterator(0) + last:=false + lastKey:= uint64(0) + for iiter.Next() { + ki, ci := iiter.Value() + o, carry := shift(ci) + if last { + o.add(0) + } + if o.n>0{ + output.Containers.Put(ki, o) + } + last = carry + lastKey=ki + } + if last { //handle the overflow + extra:= NewContainer() + extra.add(0) + output.Containers.Put(lastKey+1, extra) + } + + return output +} + // removeEmptyContainers deletes all containers that have a count of zero. func (b *Bitmap) removeEmptyContainers() { citer, _ := b.Containers.Iterator(0) @@ -3023,7 +3049,6 @@ func shiftArray(a *Container) (*Container, bool) { output.array = output.array[:0] output.n = a.n for _, v := range a.array { - fmt.Println(v,v+1) if v+1 == 0 { //overflow carry = true output.n -= 1 @@ -3041,9 +3066,9 @@ func shiftBitmap(a *Container) (*Container, bool) { output.bitmap = make([]uint64, len(a.bitmap)) output.bitmap = output.bitmap[:0] output.n = a.n - lastcarry:=false + lastcarry := false for i, v := range a.bitmap { - carry = (v&(1<<63))!= 0 + carry = (v & (1 << 63)) != 0 v = v << 1 if i != 0 { if lastcarry { diff --git a/row.go b/row.go index 9f8f9a403..fcf0b2438 100644 --- a/row.go +++ b/row.go @@ -153,6 +153,15 @@ func (r *Row) Difference(other *Row) *Row { return &Row{segments: segments} } +// Shift returns the bitwise shift of r by 1 bit. +func (r *Row) Shift() *Row { + var segments []rowSegment + for _,segment := range r.segments { + segments = append(segments, *segment.Shift()) + } + + return &Row{segments: segments} +} // SetBit sets the i-th column of the row. func (r *Row) SetBit(i uint64) (changed bool) { @@ -327,6 +336,17 @@ func (s *rowSegment) Xor(other *rowSegment) *rowSegment { n: data.Count(), } } +// Shift returns s shifted by 1 bit. +func (s *rowSegment) Shift() *rowSegment { + //TODO deal with overflow + data := s.data.Shift() + + return &rowSegment{ + data: *data, + shard: s.shard, + n: data.Count(), + } +} // SetBit sets the i-th column of the row. func (s *rowSegment) SetBit(i uint64) (changed bool) { From f6d34b42ba3d874cc207ca9d0787cc2ce9a521e8 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 26 Nov 2018 15:05:43 -0600 Subject: [PATCH 3/6] gofmt --- executor_test.go | 10 ++--- roaring/roaring_helpers_test.go | 1 - roaring/roaring_internal_test.go | 65 ++++++++++++++++---------------- 3 files changed, 37 insertions(+), 39 deletions(-) diff --git a/executor_test.go b/executor_test.go index 9553192ee..5b7623006 100644 --- a/executor_test.go +++ b/executor_test.go @@ -3210,7 +3210,7 @@ func TestExecutor_Execute_Shift(t *testing.T) { if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Shift(Row(general=10)))`}); err != nil { t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{ShardWidth+1}) { + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{ShardWidth + 1}) { t.Fatalf("unexpected columns: %+v", columns) } }) @@ -3221,10 +3221,10 @@ func TestExecutor_Execute_Shift(t *testing.T) { hldr := test.Holder{Holder: c[0].Server.Holder()} hldr.SetBit("i", "general", 10, ShardWidth-2) //shardwidth -1 hldr.SetBit("i", "general", 10, ShardWidth-1) //shardwidth - hldr.SetBit("i", "general", 10, ShardWidth) //shardwidth +1 + hldr.SetBit("i", "general", 10, ShardWidth) //shardwidth +1 hldr.SetBit("i", "general", 10, ShardWidth+2) //shardwidth +3 - exp:=[]uint64{ ShardWidth-1,ShardWidth,ShardWidth+1,ShardWidth+3} + exp := []uint64{ShardWidth - 1, ShardWidth, ShardWidth + 1, ShardWidth + 3} if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10))`}); err != nil { t.Fatal(err) } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, exp) { @@ -3232,8 +3232,8 @@ func TestExecutor_Execute_Shift(t *testing.T) { } if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Shift(Row(general=10)))`}); err != nil { t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns,[]uint64{ ShardWidth,ShardWidth+1,ShardWidth+2,ShardWidth+4} ) { - t.Fatalf("unexpected columns: \n%+v\n%+v", columns,exp) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{ShardWidth, ShardWidth + 1, ShardWidth + 2, ShardWidth + 4}) { + t.Fatalf("unexpected columns: \n%+v\n%+v", columns, exp) } }) diff --git a/roaring/roaring_helpers_test.go b/roaring/roaring_helpers_test.go index 36963425e..f99d922e8 100644 --- a/roaring/roaring_helpers_test.go +++ b/roaring/roaring_helpers_test.go @@ -124,7 +124,6 @@ func bitmapFirstBitSecoundRowSet() []uint64 { return bitmap } - func bitmapLastBitSet() []uint64 { bitmap := make([]uint64, bitmapN) bitmap[bitmapN-1] = 0x8000000000000000 diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index ffa8be812..21c2b2088 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -3292,7 +3292,7 @@ func TestEquals(t *testing.T) { func TestShiftArray(t *testing.T) { a := &Container{ containerType: containerArray, -} + } tests := []struct { array []uint16 exp []uint16 @@ -3307,7 +3307,7 @@ func TestShiftArray(t *testing.T) { }, { array: []uint16{1, 2, 3, 4, 5, 11, 12}, - exp: []uint16{2, 3, 4, 5, 6, 12,13}, + exp: []uint16{2, 3, 4, 5, 6, 12, 13}, }, { array: []uint16{65535}, @@ -3318,13 +3318,12 @@ func TestShiftArray(t *testing.T) { for i, test := range tests { a.array = test.array a.n = int32(len(a.array)) - ret,_ := shiftArray(a) + ret, _ := shiftArray(a) if !reflect.DeepEqual(ret.array, test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array) } } -} - +} func TestShiftBitmap(t *testing.T) { a := &Container{ @@ -3332,72 +3331,72 @@ func TestShiftBitmap(t *testing.T) { } tests := []struct { bitmap []uint64 - exp []uint64 + exp []uint64 }{ { - bitmap:bitmapFirstBitSet() , - exp: bitmapSecondBitSet(), + bitmap: bitmapFirstBitSet(), + exp: bitmapSecondBitSet(), }, { - bitmap:bitmapLastBitSet() , - exp: bitmapEmpty(), + bitmap: bitmapLastBitSet(), + exp: bitmapEmpty(), }, { - bitmap:bitmapLastBitFirstRowSet() , - exp: bitmapFirstBitSecoundRowSet (), + bitmap: bitmapLastBitFirstRowSet(), + exp: bitmapFirstBitSecoundRowSet(), }, } for i, test := range tests { a.bitmap = test.bitmap a.n = 1 - ret,_ := shiftBitmap(a) + ret, _ := shiftBitmap(a) if !reflect.DeepEqual(ret.bitmap, test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.bitmap) } } -} +} func TestShiftRun(t *testing.T) { a := &Container{ containerType: containerRun, } tests := []struct { - runs []interval16 - n int32 - en int32 + runs []interval16 + n int32 + en int32 exp []interval16 carry bool }{ { runs: []interval16{{start: 5, last: 10}}, - n: 5, - en: 5, - exp: []interval16{{start: 6, last: 11}}, + n: 5, + en: 5, + exp: []interval16{{start: 6, last: 11}}, carry: false, }, { - runs: []interval16{{start: 5, last:65535 }}, - n: 65530, - en: 65529, - exp: []interval16{{start: 6, last: 65535}}, + runs: []interval16{{start: 5, last: 65535}}, + n: 65530, + en: 65529, + exp: []interval16{{start: 6, last: 65535}}, carry: true, }, { - runs: []interval16{{start: 65535, last:65535 }}, - n: 1, - en: 0, - exp: []interval16{}, - carry: true, + runs: []interval16{{start: 65535, last: 65535}}, + n: 1, + en: 0, + exp: []interval16{}, + carry: true, }, } for i, test := range tests { a.runs = test.runs a.n = test.n - ret,c := shiftRun(a) - if !reflect.DeepEqual(ret.runs, test.exp) && c == test.carry && ret.n == test.en { - t.Fatalf("test #%v expected %v, but got %v %d", i, test.exp, ret.runs,ret.n) + ret, c := shiftRun(a) + if !reflect.DeepEqual(ret.runs, test.exp) && c == test.carry && ret.n == test.en { + t.Fatalf("test #%v expected %v, but got %v %d", i, test.exp, ret.runs, ret.n) } } -} +} From de1b9b4de2bf7eb4d158b18cf2154b909269d71b Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 26 Nov 2018 15:21:37 -0600 Subject: [PATCH 4/6] yata gofmt --- roaring/roaring.go | 12 ++++++------ row.go | 4 +++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 77483c84f..8c3cbc52e 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -478,25 +478,25 @@ func (b *Bitmap) Xor(other *Bitmap) *Bitmap { return output } -func (b *Bitmap) Shift() (*Bitmap) { +func (b *Bitmap) Shift() *Bitmap { output := NewBitmap() iiter, _ := b.Containers.Iterator(0) - last:=false - lastKey:= uint64(0) + last := false + lastKey := uint64(0) for iiter.Next() { ki, ci := iiter.Value() o, carry := shift(ci) if last { o.add(0) } - if o.n>0{ + if o.n > 0 { output.Containers.Put(ki, o) } last = carry - lastKey=ki + lastKey = ki } if last { //handle the overflow - extra:= NewContainer() + extra := NewContainer() extra.add(0) output.Containers.Put(lastKey+1, extra) } diff --git a/row.go b/row.go index fcf0b2438..6a5f541c4 100644 --- a/row.go +++ b/row.go @@ -153,10 +153,11 @@ func (r *Row) Difference(other *Row) *Row { return &Row{segments: segments} } + // Shift returns the bitwise shift of r by 1 bit. func (r *Row) Shift() *Row { var segments []rowSegment - for _,segment := range r.segments { + for _, segment := range r.segments { segments = append(segments, *segment.Shift()) } @@ -336,6 +337,7 @@ func (s *rowSegment) Xor(other *rowSegment) *rowSegment { n: data.Count(), } } + // Shift returns s shifted by 1 bit. func (s *rowSegment) Shift() *rowSegment { //TODO deal with overflow From 8fe966e8a0551800fb3f30329b069eefb108dbac Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 24 Jan 2019 14:56:39 -0600 Subject: [PATCH 5/6] Modifying some of the logic around Shift() add some comments to the shift() logic improve test coverage fix full bitmap overflow add support to specify shift-by amount --- executor.go | 7 +++- executor_test.go | 37 ++++++++++++------ roaring/roaring.go | 64 +++++++++++++++++++------------- roaring/roaring_internal_test.go | 27 +++++++++----- roaring/roaring_test.go | 12 ++++++ row.go | 38 ++++++++++++++----- 6 files changed, 129 insertions(+), 56 deletions(-) diff --git a/executor.go b/executor.go index b765a5ac9..a5c9b4391 100644 --- a/executor.go +++ b/executor.go @@ -1531,6 +1531,11 @@ func (e *executor) executeNotShard(ctx context.Context, index string, c *pql.Cal // executeShiftShard executes a shift() call for a local shard. func (e *executor) executeShiftShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + n, _, err := c.IntArg("n") + if err != nil { + return nil, fmt.Errorf("executeShiftShard: %v", err) + } + if len(c.Children) == 0 { return nil, errors.New("Shift() requires an input row") } else if len(c.Children) > 1 { @@ -1542,7 +1547,7 @@ func (e *executor) executeShiftShard(ctx context.Context, index string, c *pql.C return nil, err } - return row.Shift(), nil + return row.Shift(n) } // executeCount executes a count() call. diff --git a/executor_test.go b/executor_test.go index 47b8ec9b3..3d17a14e9 100644 --- a/executor_test.go +++ b/executor_test.go @@ -3724,25 +3724,26 @@ func TestExecutor_Execute_Shift(t *testing.T) { hldr := test.Holder{Holder: c[0].Server.Holder()} hldr.SetBit("i", "general", 10, 0) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10))`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10), n=1)`}); err != nil { t.Fatal(err) } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1}) { t.Fatalf("unexpected columns: %+v", columns) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Shift(Row(general=10)))`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Shift(Row(general=10), n=1), n=1)`}); err != nil { t.Fatal(err) } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2}) { t.Fatalf("unexpected columns: %+v", columns) } }) + t.Run("Shift container boundary", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() hldr := test.Holder{Holder: c[0].Server.Holder()} hldr.SetBit("i", "general", 10, 65535) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10))`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10), n=1)`}); err != nil { t.Fatal(err) } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{65536}) { t.Fatalf("unexpected columns: %+v", columns) @@ -3753,18 +3754,31 @@ func TestExecutor_Execute_Shift(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() hldr := test.Holder{Holder: c[0].Server.Holder()} - hldr.SetBit("i", "general", 10, ShardWidth-1) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10))`}); err != nil { + orig := []uint64{1, ShardWidth - 1, ShardWidth + 1} + shift1 := []uint64{2, ShardWidth, ShardWidth + 2} + shift2 := []uint64{3, ShardWidth + 1, ShardWidth + 3} + + for _, bit := range orig { + hldr.SetBit("i", "general", 10, bit) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10), n=1)`}); err != nil { t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{ShardWidth}) { - t.Fatalf("unexpected columns: %+v", columns) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, shift1) { + t.Fatalf("unexpected shift by 1: expected: %+v, but got: %+v", shift1, columns) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10), n=2)`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, shift2) { + t.Fatalf("unexpected shift by 2: expected: %+v, but got: %+v", shift2, columns) } if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Shift(Row(general=10)))`}); err != nil { t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{ShardWidth + 1}) { - t.Fatalf("unexpected columns: %+v", columns) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, orig) { + t.Fatalf("unexpected shift by 0: expected: %+v, but got: %+v", orig, columns) } }) @@ -3778,16 +3792,15 @@ func TestExecutor_Execute_Shift(t *testing.T) { hldr.SetBit("i", "general", 10, ShardWidth+2) //shardwidth +3 exp := []uint64{ShardWidth - 1, ShardWidth, ShardWidth + 1, ShardWidth + 3} - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10))`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10), n=1)`}); err != nil { t.Fatal(err) } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, exp) { t.Fatalf("unexpected columns: %+v", columns) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Shift(Row(general=10)))`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Shift(Row(general=10), n=1), n=1)`}); err != nil { t.Fatal(err) } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{ShardWidth, ShardWidth + 1, ShardWidth + 2, ShardWidth + 4}) { t.Fatalf("unexpected columns: \n%+v\n%+v", columns, exp) } }) - } diff --git a/roaring/roaring.go b/roaring/roaring.go index a0568dbc2..abc4e4d2f 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -52,16 +52,20 @@ const ( // bitmapN is the number of values in a container.bitmap. bitmapN = (1 << 16) / 64 - //containerArray indicates a container of bit position values + // containerArray indicates a container of bit position values containerArray = byte(1) - //containerBitmap indicates a container of bits packed in a uint64 array block + // containerBitmap indicates a container of bits packed in a uint64 array block containerBitmap = byte(2) - //containerRun indicates a container of run encoded bits + // containerRun indicates a container of run encoded bits containerRun = byte(3) maxContainerVal = 0xffff + + // maxContainerKey is the key representing the last container in a full row. + // It is the full bitmap space (2^64) divided by container width (2^16). + maxContainerKey = (1 << 48) - 1 ) type Containers interface { @@ -753,30 +757,36 @@ func (b *Bitmap) Xor(other *Bitmap) *Bitmap { return output } -func (b *Bitmap) Shift() *Bitmap { +// Shift shifts the contents of b by 1. +func (b *Bitmap) Shift(n int) (*Bitmap, error) { + if n != 1 { + return nil, errors.New("cannot shift by a value other than 1") + } output := NewBitmap() iiter, _ := b.Containers.Iterator(0) - last := false + lastCarry := false lastKey := uint64(0) for iiter.Next() { ki, ci := iiter.Value() o, carry := shift(ci) - if last { + if lastCarry { o.add(0) } if o.n > 0 { output.Containers.Put(ki, o) } - last = carry + lastCarry = carry lastKey = ki } - if last { //handle the overflow + // As long as the carry wasn't from the max container, + // append a new container and add the carried bit. + if lastCarry && lastKey != maxContainerKey { extra := NewContainer() extra.add(0) output.Containers.Put(lastKey+1, extra) } - return output + return output, nil } // removeEmptyContainers deletes all containers that have a count of zero. @@ -3387,15 +3397,19 @@ func xorBitmapBitmap(a, b *Container) *Container { return output } -func shift(a *Container) (*Container, bool) { - if a.isArray() { - return shiftArray(a) - } else if a.isRun() { - return shiftRun(a) +// shift() shifts the contents of c by one. It returns +// the new container and a bool indicating whether a +// carry bit was shifted out. +func shift(c *Container) (*Container, bool) { + if c.isArray() { + return shiftArray(c) + } else if c.isRun() { + return shiftRun(c) } - return shiftBitmap(a) + return shiftBitmap(c) } +// shiftArray is an array-specific implementation of shift(). func shiftArray(a *Container) (*Container, bool) { statsHit("shift/Array") carry := false @@ -3404,7 +3418,7 @@ func shiftArray(a *Container) (*Container, bool) { output.array = output.array[:0] output.n = a.n for _, v := range a.array { - if v+1 == 0 { //overflow + if v+1 == 0 { // overflow carry = true output.n -= 1 } else { @@ -3414,6 +3428,7 @@ func shiftArray(a *Container) (*Container, bool) { return output, carry } +// shiftBitmap is a bitmap-specific implementation of shift(). func shiftBitmap(a *Container) (*Container, bool) { statsHit("shift/Bitmap") carry := false @@ -3421,17 +3436,15 @@ func shiftBitmap(a *Container) (*Container, bool) { output.bitmap = make([]uint64, len(a.bitmap)) output.bitmap = output.bitmap[:0] output.n = a.n - lastcarry := false - for i, v := range a.bitmap { + lastCarry := false + for _, v := range a.bitmap { carry = (v & (1 << 63)) != 0 v = v << 1 - if i != 0 { - if lastcarry { - v |= 1 - } + if lastCarry { + v |= 1 } output.bitmap = append(output.bitmap, v) - lastcarry = carry + lastCarry = carry } if carry { output.n -= 1 @@ -3439,6 +3452,7 @@ func shiftBitmap(a *Container) (*Container, bool) { return output, carry } +// shiftRun is a run-specific implementation of shift(). func shiftRun(a *Container) (*Container, bool) { statsHit("shift/Run") carry := false @@ -3446,11 +3460,11 @@ func shiftRun(a *Container) (*Container, bool) { output.runs = make([]interval16, len(a.runs)) output.runs = output.runs[:0] for _, v := range a.runs { - if v.start+1 == 0 { + if v.start+1 == 0 { // final run was 1 bit on container edge carry = true output.n -= 1 break - } else if v.last+1 == 0 { + } else if v.last+1 == 0 { // final run ends on container edge v.start += 1 carry = true output.n -= 1 diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 2dda414b3..5ca0edb2b 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -3346,9 +3346,12 @@ func TestShiftArray(t *testing.T) { for i, test := range tests { a.array = test.array a.n = int32(len(a.array)) - ret, _ := shiftArray(a) - if !reflect.DeepEqual(ret.array, test.exp) { - t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array) + ret1, _ := shift(a) // test generic shift function + ret2, _ := shiftArray(a) // test array-specific shift function + if !reflect.DeepEqual(ret1.array, test.exp) { + t.Fatalf("test #%v shift() expected %v, but got %v", i, test.exp, ret1.array) + } else if !reflect.DeepEqual(ret2.array, test.exp) { + t.Fatalf("test #%v shiftArray() expected %v, but got %v", i, test.exp, ret2.array) } } } @@ -3378,9 +3381,12 @@ func TestShiftBitmap(t *testing.T) { for i, test := range tests { a.bitmap = test.bitmap a.n = 1 - ret, _ := shiftBitmap(a) - if !reflect.DeepEqual(ret.bitmap, test.exp) { - t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.bitmap) + ret1, _ := shift(a) // test generic shift function + ret2, _ := shiftBitmap(a) // test bitmap-specific shift function + if !reflect.DeepEqual(ret1.bitmap, test.exp) { + t.Fatalf("test #%v shift() expected %v, but got %v", i, test.exp, ret1.bitmap) + } else if !reflect.DeepEqual(ret2.bitmap, test.exp) { + t.Fatalf("test #%v shiftBitmap() expected %v, but got %v", i, test.exp, ret2.bitmap) } } } @@ -3422,9 +3428,12 @@ func TestShiftRun(t *testing.T) { for i, test := range tests { a.runs = test.runs a.n = test.n - ret, c := shiftRun(a) - if !reflect.DeepEqual(ret.runs, test.exp) && c == test.carry && ret.n == test.en { - t.Fatalf("test #%v expected %v, but got %v %d", i, test.exp, ret.runs, ret.n) + ret1, c1 := shift(a) // test generic shift function + ret2, c2 := shiftRun(a) // test run-specific shift function + if !reflect.DeepEqual(ret1.runs, test.exp) && c1 == test.carry && ret1.n == test.en { + t.Fatalf("test #%v shift() expected %v, but got %v %d", i, test.exp, ret1.runs, ret1.n) + } else if !reflect.DeepEqual(ret2.runs, test.exp) && c2 == test.carry && ret2.n == test.en { + t.Fatalf("test #%v shiftRun() expected %v, but got %v %d", i, test.exp, ret2.runs, ret2.n) } } } diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index b165042a9..a4333d0fc 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -985,6 +985,18 @@ func TestBitmap_IntersectionCount_Mixed(t *testing.T) { } } +func TestBitmap_Shift(t *testing.T) { + var max uint64 = math.MaxUint64 + bm1 := roaring.NewFileBitmap(0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 65536, max) + bm2 := roaring.NewFileBitmap(1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 65537) + + if got, err := bm1.Shift(1); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(got.Slice(), bm2.Slice()) { + t.Fatalf("unexpected bitmap: expected %v, but got %v", bm2.Slice(), got.Slice()) + } +} + func TestBitmap_Quick_Array1(t *testing.T) { testBitmapQuick(t, 1000, 1000, 2000) } func TestBitmap_Quick_Array2(t *testing.T) { testBitmapQuick(t, 10000, 0, 1000) } func TestBitmap_Quick_Bitmap1(t *testing.T) { testBitmapQuick(t, 10000, 0, 10000) } diff --git a/row.go b/row.go index 2a0eddf58..0a8fc0acd 100644 --- a/row.go +++ b/row.go @@ -19,6 +19,7 @@ import ( "sort" "github.com/pilosa/pilosa/roaring" + "github.com/pkg/errors" ) // Row is a set of integers (the associated columns), and attributes which are @@ -167,14 +168,30 @@ func (r *Row) Difference(other *Row) *Row { return &Row{segments: segments} } -// Shift returns the bitwise shift of r by 1 bit. -func (r *Row) Shift() *Row { - var segments []rowSegment - for _, segment := range r.segments { - segments = append(segments, *segment.Shift()) +// Shift returns the bitwise shift of r by n bits. +// Currently only positive shift values are supported. +func (r *Row) Shift(n int64) (*Row, error) { + if n < 0 { + return nil, errors.New("cannot shift by negative values") + } else if n == 0 { + return r, nil } - return &Row{segments: segments} + work := r + var segments []rowSegment + for i := int64(0); i < n; i++ { + segments = segments[:0] + for _, segment := range work.segments { + shifted, err := segment.Shift() + if err != nil { + return nil, errors.Wrap(err, "shifting row segment") + } + segments = append(segments, *shifted) + } + work = &Row{segments: segments} + } + + return work, nil } // SetBit sets the i-th column of the row. @@ -352,15 +369,18 @@ func (s *rowSegment) Xor(other *rowSegment) *rowSegment { } // Shift returns s shifted by 1 bit. -func (s *rowSegment) Shift() *rowSegment { +func (s *rowSegment) Shift() (*rowSegment, error) { //TODO deal with overflow - data := s.data.Shift() + data, err := s.data.Shift(1) + if err != nil { + return nil, errors.Wrap(err, "shifting roaring data") + } return &rowSegment{ data: *data, shard: s.shard, n: data.Count(), - } + }, nil } // SetBit sets the i-th column of the row. From 113c6998be50931d40935fb183a0c201f1d546d0 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 25 Jan 2019 14:08:49 -0600 Subject: [PATCH 6/6] add Shift() to query docs --- docs/query-language.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/query-language.md b/docs/query-language.md index 82ffecd2a..4dd838727 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -594,6 +594,34 @@ Count(Row(stargazer=1)) * Result is the number of repositories that user 1 has starred. +#### Shift +**Spec:** + +``` +Shift(, [n=UINT]) +``` + +**Description:** + +Returns the row specified by `ROW_CALL` shifted by `n` bits. + +**Result Type:** object with attrs and columns + +attrs will always be empty + +**Examples:** + +Query all columns with a bit set in row 1 of the field `stargazer` +and shift the result by 2: +```request +Shift(Row(stargazer=1), n=2) +``` +```response +{"attrs":{},"columns":[12, 22]} +``` + +* columns are the repositories which user 1 has starred shifted by 2 bits. + #### TopN **Spec:**