From 0404354faaabbf343197462f1d37031174ce38ea Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 26 Nov 2018 11:29:05 -0600 Subject: [PATCH] add shift operater to pql --- executor.go | 18 +++++++++++ executor_test.go | 75 ++++++++++++++++++++++++++++++++++++++++++++++ roaring/roaring.go | 31 +++++++++++++++++-- row.go | 20 +++++++++++++ 4 files changed, 141 insertions(+), 3 deletions(-) diff --git a/executor.go b/executor.go index c4153ab61..918fe9611 100644 --- a/executor.go +++ b/executor.go @@ -515,6 +515,8 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c * return e.executeXorShard(ctx, index, c, shard) case "Not": return e.executeNotShard(ctx, index, c, shard) + case "Shift": + return e.executeShiftShard(ctx, index, c, shard) default: return nil, fmt.Errorf("unknown call: %s", c.Name) } @@ -1419,6 +1421,22 @@ func (e *executor) executeNotShard(ctx context.Context, index string, c *pql.Cal return existenceRow.Difference(row), nil } +// executeShiftShard executes a shift() call for a local shard. +func (e *executor) executeShiftShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + if len(c.Children) == 0 { + return nil, errors.New("Shift() requires an input row") + } else if len(c.Children) > 1 { + return nil, errors.New("Shift() only accepts a single row input") + } + + row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) + if err != nil { + return nil, err + } + + return row.Shift(), nil +} + // executeCount executes a count() call. func (e *executor) executeCount(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (uint64, error) { if len(c.Children) == 0 { diff --git a/executor_test.go b/executor_test.go index 87b72689c..9553192ee 100644 --- a/executor_test.go +++ b/executor_test.go @@ -3163,3 +3163,78 @@ func runCallTest(t *testing.T, writeQuery string, readQueries []string, indexOpt return responses } + +func TestExecutor_Execute_Shift(t *testing.T) { + t.Run("Shift Bit 0", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "general", 10, 0) + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10))`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1}) { + t.Fatalf("unexpected columns: %+v", columns) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Shift(Row(general=10)))`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + t.Run("Shift container boundary", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "general", 10, 65535) + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10))`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{65536}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("Shift shard boundary", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "general", 10, ShardWidth-1) + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10))`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{ShardWidth}) { + t.Fatalf("unexpected columns: %+v", columns) + } + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Shift(Row(general=10)))`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{ShardWidth+1}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("Shift shard boundary no create", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "general", 10, ShardWidth-2) //shardwidth -1 + hldr.SetBit("i", "general", 10, ShardWidth-1) //shardwidth + hldr.SetBit("i", "general", 10, ShardWidth) //shardwidth +1 + hldr.SetBit("i", "general", 10, ShardWidth+2) //shardwidth +3 + + exp:=[]uint64{ ShardWidth-1,ShardWidth,ShardWidth+1,ShardWidth+3} + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Row(general=10))`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, exp) { + t.Fatalf("unexpected columns: %+v", columns) + } + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Shift(Shift(Row(general=10)))`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns,[]uint64{ ShardWidth,ShardWidth+1,ShardWidth+2,ShardWidth+4} ) { + t.Fatalf("unexpected columns: \n%+v\n%+v", columns,exp) + } + }) + +} diff --git a/roaring/roaring.go b/roaring/roaring.go index a94121dc9..77483c84f 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -478,6 +478,32 @@ func (b *Bitmap) Xor(other *Bitmap) *Bitmap { return output } +func (b *Bitmap) Shift() (*Bitmap) { + output := NewBitmap() + iiter, _ := b.Containers.Iterator(0) + last:=false + lastKey:= uint64(0) + for iiter.Next() { + ki, ci := iiter.Value() + o, carry := shift(ci) + if last { + o.add(0) + } + if o.n>0{ + output.Containers.Put(ki, o) + } + last = carry + lastKey=ki + } + if last { //handle the overflow + extra:= NewContainer() + extra.add(0) + output.Containers.Put(lastKey+1, extra) + } + + return output +} + // removeEmptyContainers deletes all containers that have a count of zero. func (b *Bitmap) removeEmptyContainers() { citer, _ := b.Containers.Iterator(0) @@ -3023,7 +3049,6 @@ func shiftArray(a *Container) (*Container, bool) { output.array = output.array[:0] output.n = a.n for _, v := range a.array { - fmt.Println(v,v+1) if v+1 == 0 { //overflow carry = true output.n -= 1 @@ -3041,9 +3066,9 @@ func shiftBitmap(a *Container) (*Container, bool) { output.bitmap = make([]uint64, len(a.bitmap)) output.bitmap = output.bitmap[:0] output.n = a.n - lastcarry:=false + lastcarry := false for i, v := range a.bitmap { - carry = (v&(1<<63))!= 0 + carry = (v & (1 << 63)) != 0 v = v << 1 if i != 0 { if lastcarry { diff --git a/row.go b/row.go index 9f8f9a403..fcf0b2438 100644 --- a/row.go +++ b/row.go @@ -153,6 +153,15 @@ func (r *Row) Difference(other *Row) *Row { return &Row{segments: segments} } +// Shift returns the bitwise shift of r by 1 bit. +func (r *Row) Shift() *Row { + var segments []rowSegment + for _,segment := range r.segments { + segments = append(segments, *segment.Shift()) + } + + return &Row{segments: segments} +} // SetBit sets the i-th column of the row. func (r *Row) SetBit(i uint64) (changed bool) { @@ -327,6 +336,17 @@ func (s *rowSegment) Xor(other *rowSegment) *rowSegment { n: data.Count(), } } +// Shift returns s shifted by 1 bit. +func (s *rowSegment) Shift() *rowSegment { + //TODO deal with overflow + data := s.data.Shift() + + return &rowSegment{ + data: *data, + shard: s.shard, + n: data.Count(), + } +} // SetBit sets the i-th column of the row. func (s *rowSegment) SetBit(i uint64) (changed bool) {