From 54ce53732713eee358b231d1689df9f1900585a1 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 17 Sep 2018 16:01:51 -0500 Subject: [PATCH 01/39] copy non roaring-import code from Todds's row-iterate PR tests passing --- encoding/proto/proto.go | 26 ++ executor.go | 391 +++++++++++++++++++++++++++- executor_test.go | 143 +++++++++++ fragment.go | 49 +++- internal/public.pb.go | 552 ++++++++++++++++++++++++++++++++-------- internal/public.proto | 9 +- pilosa.go | 4 +- 7 files changed, 1051 insertions(+), 123 deletions(-) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 0fcd9df3a..e133de7fc 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -360,6 +360,12 @@ func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse { case bool: pb.Results[i].Type = queryResultTypeBool pb.Results[i].Changed = result + case pilosa.RowIDs: + pb.Results[i].Type = queryResultTypeRowIDs + pb.Results[i].RowIDs = result + case pilosa.GroupByCounts: + pb.Results[i].Type = queryResultTypeGroupByCounts + pb.Results[i].GroupByCounts = encodeGroupByCount(result) case nil: pb.Results[i].Type = queryResultTypeNil } @@ -922,6 +928,8 @@ const ( queryResultTypeValCount queryResultTypeUint64 queryResultTypeBool + queryResultTypeRowIDs + queryResultTypeGroupByCounts ) func decodeQueryResult(pb *internal.QueryResult) interface{} { @@ -938,6 +946,8 @@ func decodeQueryResult(pb *internal.QueryResult) interface{} { return pb.Changed case queryResultTypeNil: return nil + case queryResultTypeGroupByCounts: + return decodeGroupByCounts(pb.GroupByCounts) } panic(fmt.Sprintf("unknown type: %d", pb.Type)) } @@ -988,6 +998,14 @@ func decodeAttr(attr *internal.Attr) (key string, value interface{}) { } } +func decodeGroupByCounts(a []*internal.GroupLine) pilosa.GroupByCounts { + gbc := make(pilosa.GroupByCounts, 0) + for i := range a { + gbc = append(gbc, pilosa.GroupLine{a[i].Groups, a[i].Total}) + } + return gbc +} + func decodePairs(a []*internal.Pair) []pilosa.Pair { other := make([]pilosa.Pair, len(a)) for i := range a { @@ -1039,6 +1057,14 @@ func encodeRow(r *pilosa.Row) *internal.Row { } } +func encodeGroupByCount(counts pilosa.GroupByCounts) []*internal.GroupLine { + result := make([]*internal.GroupLine, len(counts)) + for i := range counts { + result[i] = &internal.GroupLine{Groups: counts[i].Groups, Total: counts[i].Total} + } + return result +} + func encodePairs(a pilosa.Pairs) []*internal.Pair { other := make([]*internal.Pair, len(a)) for i := range a { diff --git a/executor.go b/executor.go index b0083a514..56ab05269 100644 --- a/executor.go +++ b/executor.go @@ -18,6 +18,8 @@ import ( "context" "fmt" "sort" + "strconv" + "strings" "time" "github.com/pilosa/pilosa/pql" @@ -194,6 +196,12 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s case "TopN": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeTopN(ctx, index, c, shards, opt) + case "Rows": + e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) + return e.executeRows(ctx, index, c, shards, opt) + case "GroupBy": + e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) + return e.executeGroupBy(ctx, index, c, shards, opt) case "Options": return e.executeOptionsCall(ctx, index, c, shards, opt) default: @@ -714,14 +722,355 @@ func (e *executor) executeDifferenceShard(ctx context.Context, index string, c * return other, nil } +type RowIDs []uint64 + +func (r RowIDs) Merge(other RowIDs) RowIDs { + i, j := 0, 0 + result := make(RowIDs, 0) + for i < len(r) && j < len(other) { + av, bv := r[i], other[j] + if av < bv { + result = append(result, av) + i++ + } else if av > bv { + result = append(result, bv) + j++ + } else { + result = append(result, bv) + i++ + j++ + } + } + for i < len(r) { + result = append(result, r[i]) + i++ + } + for j < len(other) { + result = append(result, other[j]) + j++ + } + return result +} +func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (GroupByCounts, error) { + // Execute calls in bulk on each remote node and merge. + mapFn := func(shard uint64) (interface{}, error) { + return e.executeGroupByShard(ctx, index, c, shard) + } + // Merge returned results at coordinating node. + reduceFn := func(prev, v interface{}) interface{} { + other, _ := prev.(GroupByCounts) + return other.Merge(v.(GroupByCounts)) + } + // Get full result set. + other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) + if err != nil { + return nil, err + } + results, _ := other.(GroupByCounts) + // Apply offset. + if offset, hasOffset, err := c.UintArg("offset"); err != nil { + return nil, err + } else if hasOffset { + if int(offset) < len(results) { + results = results[offset:] + } + } + // Apply limit. + if limit, hasLimit, err := c.UintArg("limit"); err != nil { + return nil, err + } else if hasLimit { + if int(limit) < len(results) { + results = results[:limit] + } + } + return results, nil +} + +// gbi is a groupBy item. +type gbi struct { + row *Row + fieldKey string + rowID uint64 +} +type GroupLine struct { + Groups []string + Total uint64 +} +type GroupByCounts []GroupLine + +func (gbc GroupByCounts) Merge(other GroupByCounts) GroupByCounts { + m := make(map[string]struct { + i int + total uint64 + }) + for i := range gbc { + m[strings.Join(gbc[i].Groups, "-")] = struct { + i int + total uint64 + }{total: gbc[i].Total, i: i} + } + for i := range other { + key := strings.Join(other[i].Groups, "-") + o, found := m[key] + if found { + gbc[o.i].Total += other[i].Total + } else { + gbc = append(gbc, other[i]) + } + } + return gbc +} +func makeGroup(parts []gbi) GroupLine { + var other *Row + line := GroupLine{} + for i, o := range parts { + if i == 0 { + other = o.row + } else { + other = other.Intersect(o.row) + } + line.Groups = append(line.Groups, o.fieldKey) + } + line.Total = other.Count() + return line +} +func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql.Call, shard uint64) (GroupByCounts, error) { + // Fetch index. + idx := e.Holder.Index(index) + if idx == nil { + return nil, ErrIndexNotFound + } + // fieldDirective is a combination of field + // instructions, represented as a string with + // the form [fieldName:offset:limit] or + // [fieldName:limit]. + fieldDirectives, ok := c.Args["fields"] + if !ok { + return nil, errors.Wrap(ErrFieldsArgumentRequired, "executeGroupBy") + } + // Ensure that fieldDirectives is a list. + if _, ok := fieldDirectives.([]interface{}); !ok { + return nil, errors.Wrap(ErrExpectedFieldListArgument, "executeGroupBy") + } + // getFieldName extracts the fieldName portion of the + // fieldDirective. + getFieldName := func(s string) string { + parts := strings.Split(s, ":") + return parts[0] + } + // Ensure that all of the fields exist. + for _, fieldDirective := range fieldDirectives.([]interface{}) { + fieldName := getFieldName(fieldDirective.(string)) + f := e.Holder.Field(index, fieldName) + if f == nil { + return nil, errors.Wrap(ErrFieldNotFound, fmt.Sprintf("executeGroupBy: %s", fieldDirective.(string))) + } + } + results := make(GroupByCounts, 0) + var work listOfGBILists + for _, fieldDirective := range fieldDirectives.([]interface{}) { + fieldName := getFieldName(fieldDirective.(string)) + // Fetch fragment. + frag := e.Holder.fragment(index, fieldName, viewStandard, shard) + if frag == nil { // this means this whole shard doesn't have all it needs to continue + return results, nil + } + // Get filter based on the field directive. + filter, err := getGroupByFilterFunction(fieldDirective.(string)) + if err != nil { + return nil, err + } + set := make(gbiList, 0) + for _, rowID := range frag.rowsWithFilter(filter) { + set = append(set, gbi{ + row: frag.row(rowID), + rowID: rowID, + fieldKey: fmt.Sprintf("%s.%d", fieldName, rowID), + }) + } + work = append(work, set) + } + for _, group := range product(work) { + group.gl.Total = group.row.Count() + if group.gl.Total > 0 { + results = append(results, group.gl) + } + } + return results, nil +} + +type gbiList []gbi +type listOfGBILists []gbiList + +// pi is a product process item. +type pi struct { + row *Row + gl GroupLine +} +type piList []pi + +// product generates the cartiesian product of the input +// using tail recursion +func product(input listOfGBILists) piList { + res := make(piList, 0) + if len(input) == 0 { //base return empty list + res = append(res, pi{gl: GroupLine{Groups: make([]string, 0)}}) + } else { + res = productHelper(input, res) + } + return res +} +func productHelper(lists listOfGBILists, res piList) piList { + head := lists[0] //take first element of the list + tail := product(lists[1:]) //invoke product on remaining element + for h := range head { // for each head + for t := range tail { //iterate over the tail + s := pi{gl: GroupLine{Groups: make([]string, 0)}} + s.gl.Groups = append([]string{head[h].fieldKey}, tail[t].gl.Groups...) //had to insert at the front to match input order + if tail[t].row != nil { //first time around nothing to intersect + s.row = head[h].row.Intersect(tail[t].row) + } else { + s.row = head[h].row + } + res = append(res, s) + } + } + return res +} +func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (RowIDs, error) { + // Execute calls in bulk on each remote node and merge. + mapFn := func(shard uint64) (interface{}, error) { + return e.executeRowsShard(ctx, index, c, shard) + } + // Merge returned results at coordinating node. + reduceFn := func(prev, v interface{}) interface{} { + other, _ := prev.(RowIDs) + return other.Merge(v.(RowIDs)) + } + // Get full result set. + other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) + if err != nil { + return nil, err + } + results, _ := other.(RowIDs) + // Apply offset. + if offset, hasOffset, err := c.UintArg("offset"); err != nil { + return nil, err + } else if hasOffset { + if int(offset) < len(results) { + results = results[offset:] + } + } + // Apply limit. + if limit, hasLimit, err := c.UintArg("limit"); err != nil { + return nil, err + } else if hasLimit { + if int(limit) < len(results) { + results = results[:limit] + } + } + return results, nil +} +func (e *executor) executeRowsShard(ctx context.Context, index string, c *pql.Call, shard uint64) (RowIDs, error) { + // Fetch index. + idx := e.Holder.Index(index) + if idx == nil { + return nil, ErrIndexNotFound + } + // Fetch field name from argument. + fieldName, ok := c.Args["field"].(string) + if !ok { + return nil, errors.New("Rows() argument required: field") + } + // Fetch field. + f := e.Holder.Field(index, fieldName) + if f == nil { + return nil, ErrFieldNotFound + } + frag := e.Holder.fragment(index, fieldName, viewStandard, shard) + if frag == nil { + return make(RowIDs, 0), nil + } + if columnID, ok, err := c.UintArg("column"); err != nil { + return nil, err + } else if ok { + // TODO: it's possible that filters could be applied here, so this returns too early. + return frag.rowsForColumn(columnID), nil + } + filter := getFilterFunction(c) + return frag.rowsWithFilter(filter), nil +} + +// getGroupByFilterFunction returns a rowFilter based on the +// field directive provided. +func getGroupByFilterFunction(fieldDirective string) (rowFilter, error) { + parts := strings.Split(fieldDirective, ":") + hasLimit := false + hasOffset := false + limit := uint64(0) + offset := uint64(0) + var err error + // fieldDirective can have one of the following forms: + // [fieldName] + // [fieldName:limit] + // [fieldName:offset:limit] + // + // Note that a field directive with the form + // [fieldName:offset:limit:extra] will be treated as + // [fieldName:offset:limit] (i.e. `extra` is ignored). + if len(parts) == 1 { + return noFilter, nil + } else if len(parts) == 2 { + hasLimit = true + if limit, err = strconv.ParseUint(parts[1], 10, 64); err != nil { + return nil, errors.Wrap(err, "getting groupby field limit only value") + } + } else { + hasOffset = true + if offset, err = strconv.ParseUint(parts[1], 10, 64); err != nil { + return nil, errors.Wrap(err, "getting groupby field offset value") + } + if parts[2] != "" { + hasLimit = true + if limit, err = strconv.ParseUint(parts[2], 10, 64); err != nil { + return nil, errors.Wrap(err, "getting groupby field limit value") + } + } + } + if hasOffset && hasLimit { + f := filterWithOffsetLimit{offset: offset, limit: limit} + return f.filter, nil + } else if hasLimit { + f := filterWithLimit{limit: limit} + return f.filter, nil + } + f := filterWithOffset{offset: offset} + return f.filter, nil +} +func getFilterFunction(c *pql.Call) rowFilter { + offset, hasOffset, _ := c.UintArg("shardoffset") + limit, hasLimit, _ := c.UintArg("shardlimit") + if hasOffset && hasLimit { + f := filterWithOffsetLimit{offset: offset, limit: limit} + return f.filter + } else if hasOffset { + f := filterWithOffset{offset: offset} + return f.filter + } else if hasLimit { + f := filterWithLimit{limit: limit} + return f.filter + } + return noFilter +} + func (e *executor) executeBitmapShard(_ context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { - // Fetch column label from index. + // Fetch index. idx := e.Holder.Index(index) if idx == nil { return nil, ErrIndexNotFound } - // Fetch field & row label based on argument. + // Fetch field name from argument. fieldName, err := c.FieldArg() if err != nil { return nil, errors.New("Row() argument required: field") @@ -1787,7 +2136,7 @@ func needsShards(calls []*pql.Call) bool { switch call.Name { case "Clear", "Set", "SetRowAttrs", "SetColumnAttrs": continue - case "Count", "TopN": + case "Count", "TopN", "Rows": return true // default catches Bitmap calls default: @@ -1845,3 +2194,39 @@ func isString(v interface{}) bool { _, ok := v.(string) return ok } + +// Filters to be used with RowsWithFilter queries. +type filterWithOffsetLimit struct { + offset, limit uint64 +} + +func (fol *filterWithOffsetLimit) filter(rowID uint64) (bool, bool) { + if rowID >= fol.offset { + if fol.limit > 0 { + fol.limit-- + return true, false + } + return false, true + } + return false, false +} + +type filterWithOffset struct { + offset uint64 +} + +func (fo *filterWithOffset) filter(rowID uint64) (bool, bool) { + return rowID >= fo.offset, false +} + +type filterWithLimit struct { + limit uint64 +} + +func (fl *filterWithLimit) filter(rowID uint64) (bool, bool) { // nolint: unparam + if fl.limit > 0 { + fl.limit-- + return true, false + } + return false, true +} diff --git a/executor_test.go b/executor_test.go index ee919f001..84e00936d 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1225,6 +1225,22 @@ Set(4500001, fn=4) t.Fatalf("wrong attrs: %v", attrst) } }) + + t.Run("remote groupBy", func(t *testing.T) { + if res, err := c[1].API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "i", + Query: `GroupBy(fields=[f])`, + }); err != nil { + t.Fatalf("GroupBy querying: %v", err) + } else { + expected := pilosa.GroupByCounts{ + {Groups: []string{"f.10"}, Total: 4}, + {Groups: []string{"f.7"}, Total: 1}, + } + results := res.Results[0].(pilosa.GroupByCounts) + checkGroupBy(expected, results, t) + } + }) } // Ensure executor returns an error if too many writes are in a single request. @@ -1622,3 +1638,130 @@ func benchmarkExistence(nn bool, b *testing.B) { func BenchmarkExecutor_Existence_True(b *testing.B) { benchmarkExistence(true, b) } func BenchmarkExecutor_Existence_False(b *testing.B) { benchmarkExistence(false, b) } + +func TestExecutor_Execute_Rows(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "general", 10, 0) + hldr.SetBit("i", "general", 10, ShardWidth+1) + hldr.SetBit("i", "general", 11, 2) + hldr.SetBit("i", "general", 11, ShardWidth+2) + hldr.SetBit("i", "general", 12, 2) + hldr.SetBit("i", "general", 12, ShardWidth+2) + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=general)`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(pilosa.RowIDs); !reflect.DeepEqual(columns, pilosa.RowIDs{10, 11, 12}) { + t.Fatalf("unexpected columns: %+v", columns) + } + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=general, limit=2)`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(pilosa.RowIDs); !reflect.DeepEqual(columns, pilosa.RowIDs{10, 11}) { + t.Fatalf("unexpected columns: %+v", columns) + } + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=general, offset=1,limit=2)`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(pilosa.RowIDs); !reflect.DeepEqual(columns, pilosa.RowIDs{11, 12}) { + t.Fatalf("unexpected columns: %+v", columns) + } + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=general, column=2)`}); err != nil { + t.Fatal(err) + } else if columns := res.Results[0].(pilosa.RowIDs); !reflect.DeepEqual(columns, pilosa.RowIDs{11, 12}) { + t.Fatalf("unexpected columns: %+v", columns) + } +} +func TestExecutor_Execute_GroupBy(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + hldr.SetBit("i", "general", 10, 0) + hldr.SetBit("i", "general", 10, 1) + hldr.SetBit("i", "general", 10, ShardWidth+1) + hldr.SetBit("i", "general", 11, 2) + hldr.SetBit("i", "general", 11, ShardWidth+2) + hldr.SetBit("i", "general", 12, 2) + hldr.SetBit("i", "general", 12, ShardWidth+2) + hldr.SetBit("i", "sub", 10, 0) + hldr.SetBit("i", "sub", 10, 1) + hldr.SetBit("i", "sub", 10, 3) + hldr.SetBit("i", "sub", 11, 2) + hldr.SetBit("i", "sub", 11, 0) + expected := pilosa.GroupByCounts{ + {Groups: []string{"general.10", "sub.11"}, Total: 1}, + {Groups: []string{"general.11", "sub.11"}, Total: 1}, + {Groups: []string{"general.12", "sub.11"}, Total: 1}, + {Groups: []string{"general.10", "sub.10"}, Total: 2}, + } + t.Run("No Field List Arguments", func(t *testing.T) { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy()`}); err != nil { + if errors.Cause(err) != pilosa.ErrFieldsArgumentRequired { + t.Fatalf("unexpected error\n\"%s\" not returned instead \n\"%s\"", pilosa.ErrFieldsArgumentRequired, err) + } + } + }) + t.Run("Unknown Field ", func(t *testing.T) { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(fields=[missing])`}); err != nil { + if errors.Cause(err) != pilosa.ErrFieldNotFound { + t.Fatalf("unexpected error\n\"%s\" not returned instead \n\"%s\"", pilosa.ErrFieldNotFound, err) + } + } + }) + t.Run("Bad Field Format", func(t *testing.T) { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(fields=missing)`}); err != nil { + if errors.Cause(err) != pilosa.ErrExpectedFieldListArgument { + t.Fatalf("unexpected error\n\"%s\" not returned instead \n\"%s\"", pilosa.ErrExpectedFieldListArgument, err) + } + } + }) + t.Run("Basic", func(t *testing.T) { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(fields=[general,sub])`}); err != nil { + t.Fatal(err) + } else { + results := res.Results[0].(pilosa.GroupByCounts) + checkGroupBy(expected, results, t) + } + }) + expected = pilosa.GroupByCounts{ + {Groups: []string{"general.11"}, Total: 2}, + {Groups: []string{"general.12"}, Total: 2}, + } + t.Run("check field offset no limit", func(t *testing.T) { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(fields=[general:11:])`}); err != nil { + t.Fatal(err) + } else { + results := res.Results[0].(pilosa.GroupByCounts) + checkGroupBy(expected, results, t) + } + }) + expected = pilosa.GroupByCounts{ + {Groups: []string{"general.11"}, Total: 2}, + } + t.Run("check field offset limit", func(t *testing.T) { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(fields=[general:11:1])`}); err != nil { + t.Fatal(err) + } else { + results := res.Results[0].(pilosa.GroupByCounts) + checkGroupBy(expected, results, t) + } + }) +} +func checkGroupBy(expected, results pilosa.GroupByCounts, t *testing.T) { + notIn := func(item pilosa.GroupLine, expected pilosa.GroupByCounts) bool { + for i := range expected { + if item.Total == expected[i].Total { + if reflect.DeepEqual(item.Groups, expected[i].Groups) { + return false + } + } + } + return true + } + if len(results) != len(expected) { + t.Fatalf("number of groupings mismatch: \n%+v\n%+v\n", results, expected) + } + for _, result := range results { + if notIn(result, expected) { + t.Fatalf("unexpected grouping: \n%+v\n\n\n%+v\n", result, expected) + } + } +} diff --git a/fragment.go b/fragment.go index 7f651b135..bcd8e9b13 100644 --- a/fragment.go +++ b/fragment.go @@ -1758,10 +1758,27 @@ func (f *fragment) readCacheFromArchive(r io.Reader) error { return nil } +// rowFilter is a filter function which takes a rowID +// and determines if that row should be included in +// the result set. Additionally, it signals whether +// to halt processing any more rows. The two bool +// returned are (1) include row, (2) break further +// processing. +type rowFilter func(rowID uint64) (bool, bool) + +// noFilter is a filter function which has no restrictions. +var noFilter = func(rowID uint64) (bool, bool) { return true, false } + +// rows returns all rows by calling rowsWithFilter() +// with a completely unrestrictive filter. + func (f *fragment) rows() []uint64 { + return f.rowsWithFilter(noFilter) +} + +func (f *fragment) rowsWithFilter(filter rowFilter) []uint64 { i, _ := f.storage.Containers.Iterator(0) rows := make([]uint64, 0) - var lastRow uint64 = math.MaxUint64 // Loop over the existing containers. @@ -1776,22 +1793,33 @@ func (f *fragment) rows() []uint64 { continue } - rows = append(rows, vRow) + // apply filter + if addRow, breakOut := filter(vRow); breakOut { + break + } else if addRow { + rows = append(rows, vRow) + } + lastRow = vRow } return rows } +// rowsForColumn is similar to the rows method, but isolated +// to a single column. func (f *fragment) rowsForColumn(columnID uint64) []uint64 { - var colKey uint64 + return f.rowsForColumnWithFilter(columnID, noFilter) +} + +func (f *fragment) rowsForColumnWithFilter(columnID uint64, filter rowFilter) []uint64 { + i, _ := f.storage.Containers.Iterator(0) + rows := make([]uint64, 0) colID := columnID % ShardWidth - i, _ := f.storage.Containers.Iterator(0) + colVal := uint16(colID & 0xFFFF) // columnID within the container - colVal := uint16(colID & 0xFFFF) - - rows := make([]uint64, 0) + var colKey uint64 // Loop over the existing containers. for i.Next() { @@ -1807,8 +1835,13 @@ func (f *fragment) rowsForColumn(columnID uint64) []uint64 { continue } + // apply filter if c.Contains(colVal) { - rows = append(rows, vRow) + if addRow, breakOut := filter(vRow); breakOut { + break + } else if addRow { + rows = append(rows, vRow) + } } } return rows diff --git a/internal/public.pb.go b/internal/public.pb.go index cceea4806..8139d3a28 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -10,6 +10,7 @@ It has these top-level messages: Row Pair + GroupLine ValCount Bit ColumnAttrSet @@ -106,6 +107,30 @@ func (m *Pair) GetCount() uint64 { return 0 } +type GroupLine struct { + Groups []string `protobuf:"bytes,1,rep,name=Groups" json:"Groups,omitempty"` + Total uint64 `protobuf:"varint,2,opt,name=Total,proto3" json:"Total,omitempty"` +} + +func (m *GroupLine) Reset() { *m = GroupLine{} } +func (m *GroupLine) String() string { return proto.CompactTextString(m) } +func (*GroupLine) ProtoMessage() {} +func (*GroupLine) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} } + +func (m *GroupLine) GetGroups() []string { + if m != nil { + return m.Groups + } + return nil +} + +func (m *GroupLine) GetTotal() uint64 { + if m != nil { + return m.Total + } + return 0 +} + type ValCount struct { Val int64 `protobuf:"varint,1,opt,name=Val,proto3" json:"Val,omitempty"` Count int64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` @@ -114,7 +139,7 @@ type ValCount struct { func (m *ValCount) Reset() { *m = ValCount{} } func (m *ValCount) String() string { return proto.CompactTextString(m) } func (*ValCount) ProtoMessage() {} -func (*ValCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} } +func (*ValCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} } func (m *ValCount) GetVal() int64 { if m != nil { @@ -139,7 +164,7 @@ type Bit struct { func (m *Bit) Reset() { *m = Bit{} } func (m *Bit) String() string { return proto.CompactTextString(m) } func (*Bit) ProtoMessage() {} -func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} } +func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{4} } func (m *Bit) GetRowID() uint64 { if m != nil { @@ -171,7 +196,7 @@ type ColumnAttrSet struct { func (m *ColumnAttrSet) Reset() { *m = ColumnAttrSet{} } func (m *ColumnAttrSet) String() string { return proto.CompactTextString(m) } func (*ColumnAttrSet) ProtoMessage() {} -func (*ColumnAttrSet) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{4} } +func (*ColumnAttrSet) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{5} } func (m *ColumnAttrSet) GetID() uint64 { if m != nil { @@ -206,7 +231,7 @@ type Attr struct { func (m *Attr) Reset() { *m = Attr{} } func (m *Attr) String() string { return proto.CompactTextString(m) } func (*Attr) ProtoMessage() {} -func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{5} } +func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{6} } func (m *Attr) GetKey() string { if m != nil { @@ -257,7 +282,7 @@ type AttrMap struct { func (m *AttrMap) Reset() { *m = AttrMap{} } func (m *AttrMap) String() string { return proto.CompactTextString(m) } func (*AttrMap) ProtoMessage() {} -func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{6} } +func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{7} } func (m *AttrMap) GetAttrs() []*Attr { if m != nil { @@ -278,7 +303,7 @@ type QueryRequest struct { func (m *QueryRequest) Reset() { *m = QueryRequest{} } func (m *QueryRequest) String() string { return proto.CompactTextString(m) } func (*QueryRequest) ProtoMessage() {} -func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{7} } +func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{8} } func (m *QueryRequest) GetQuery() string { if m != nil { @@ -331,7 +356,7 @@ type QueryResponse struct { func (m *QueryResponse) Reset() { *m = QueryResponse{} } func (m *QueryResponse) String() string { return proto.CompactTextString(m) } func (*QueryResponse) ProtoMessage() {} -func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{8} } +func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{9} } func (m *QueryResponse) GetErr() string { if m != nil { @@ -355,18 +380,20 @@ func (m *QueryResponse) GetColumnAttrSets() []*ColumnAttrSet { } type QueryResult struct { - Type uint32 `protobuf:"varint,6,opt,name=Type,proto3" json:"Type,omitempty"` - Row *Row `protobuf:"bytes,1,opt,name=Row" json:"Row,omitempty"` - N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` - Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` - ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount" json:"ValCount,omitempty"` - Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` + Type uint32 `protobuf:"varint,6,opt,name=Type,proto3" json:"Type,omitempty"` + Row *Row `protobuf:"bytes,1,opt,name=Row" json:"Row,omitempty"` + N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` + Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` + Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` + ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount" json:"ValCount,omitempty"` + RowIDs []uint64 `protobuf:"varint,7,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` + GroupByCounts []*GroupLine `protobuf:"bytes,8,rep,name=GroupByCounts" json:"GroupByCounts,omitempty"` } func (m *QueryResult) Reset() { *m = QueryResult{} } func (m *QueryResult) String() string { return proto.CompactTextString(m) } func (*QueryResult) ProtoMessage() {} -func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{9} } +func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{10} } func (m *QueryResult) GetType() uint32 { if m != nil { @@ -396,6 +423,13 @@ func (m *QueryResult) GetPairs() []*Pair { return nil } +func (m *QueryResult) GetChanged() bool { + if m != nil { + return m.Changed + } + return false +} + func (m *QueryResult) GetValCount() *ValCount { if m != nil { return m.ValCount @@ -403,11 +437,18 @@ func (m *QueryResult) GetValCount() *ValCount { return nil } -func (m *QueryResult) GetChanged() bool { +func (m *QueryResult) GetRowIDs() []uint64 { if m != nil { - return m.Changed + return m.RowIDs } - return false + return nil +} + +func (m *QueryResult) GetGroupByCounts() []*GroupLine { + if m != nil { + return m.GroupByCounts + } + return nil } type ImportRequest struct { @@ -424,7 +465,7 @@ type ImportRequest struct { func (m *ImportRequest) Reset() { *m = ImportRequest{} } func (m *ImportRequest) String() string { return proto.CompactTextString(m) } func (*ImportRequest) ProtoMessage() {} -func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{10} } +func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{11} } func (m *ImportRequest) GetIndex() string { if m != nil { @@ -494,7 +535,7 @@ type ImportValueRequest struct { func (m *ImportValueRequest) Reset() { *m = ImportValueRequest{} } func (m *ImportValueRequest) String() string { return proto.CompactTextString(m) } func (*ImportValueRequest) ProtoMessage() {} -func (*ImportValueRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{11} } +func (*ImportValueRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{12} } func (m *ImportValueRequest) GetIndex() string { if m != nil { @@ -541,6 +582,7 @@ func (m *ImportValueRequest) GetValues() []int64 { func init() { proto.RegisterType((*Row)(nil), "internal.Row") proto.RegisterType((*Pair)(nil), "internal.Pair") + proto.RegisterType((*GroupLine)(nil), "internal.GroupLine") proto.RegisterType((*ValCount)(nil), "internal.ValCount") proto.RegisterType((*Bit)(nil), "internal.Bit") proto.RegisterType((*ColumnAttrSet)(nil), "internal.ColumnAttrSet") @@ -648,6 +690,44 @@ func (m *Pair) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *GroupLine) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *GroupLine) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Groups) > 0 { + for _, s := range m.Groups { + dAtA[i] = 0xa + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + if m.Total != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.Total)) + } + return i, nil +} + func (m *ValCount) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -1032,6 +1112,35 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPublic(dAtA, i, uint64(m.Type)) } + if len(m.RowIDs) > 0 { + dAtA8 := make([]byte, len(m.RowIDs)*10) + var j7 int + for _, num := range m.RowIDs { + for num >= 1<<7 { + dAtA8[j7] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j7++ + } + dAtA8[j7] = uint8(num) + j7++ + } + dAtA[i] = 0x3a + i++ + i = encodeVarintPublic(dAtA, i, uint64(j7)) + i += copy(dAtA[i:], dAtA8[:j7]) + } + if len(m.GroupByCounts) > 0 { + for _, msg := range m.GroupByCounts { + dAtA[i] = 0x42 + i++ + i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } return i, nil } @@ -1068,26 +1177,9 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(m.Shard)) } if len(m.RowIDs) > 0 { - dAtA8 := make([]byte, len(m.RowIDs)*10) - var j7 int - for _, num := range m.RowIDs { - for num >= 1<<7 { - dAtA8[j7] = uint8(uint64(num)&0x7f | 0x80) - num >>= 7 - j7++ - } - dAtA8[j7] = uint8(num) - j7++ - } - dAtA[i] = 0x22 - i++ - i = encodeVarintPublic(dAtA, i, uint64(j7)) - i += copy(dAtA[i:], dAtA8[:j7]) - } - if len(m.ColumnIDs) > 0 { - dAtA10 := make([]byte, len(m.ColumnIDs)*10) + dAtA10 := make([]byte, len(m.RowIDs)*10) var j9 int - for _, num := range m.ColumnIDs { + for _, num := range m.RowIDs { for num >= 1<<7 { dAtA10[j9] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -1096,16 +1188,15 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { dAtA10[j9] = uint8(num) j9++ } - dAtA[i] = 0x2a + dAtA[i] = 0x22 i++ i = encodeVarintPublic(dAtA, i, uint64(j9)) i += copy(dAtA[i:], dAtA10[:j9]) } - if len(m.Timestamps) > 0 { - dAtA12 := make([]byte, len(m.Timestamps)*10) + if len(m.ColumnIDs) > 0 { + dAtA12 := make([]byte, len(m.ColumnIDs)*10) var j11 int - for _, num1 := range m.Timestamps { - num := uint64(num1) + for _, num := range m.ColumnIDs { for num >= 1<<7 { dAtA12[j11] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -1114,11 +1205,29 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { dAtA12[j11] = uint8(num) j11++ } - dAtA[i] = 0x32 + dAtA[i] = 0x2a i++ i = encodeVarintPublic(dAtA, i, uint64(j11)) i += copy(dAtA[i:], dAtA12[:j11]) } + if len(m.Timestamps) > 0 { + dAtA14 := make([]byte, len(m.Timestamps)*10) + var j13 int + for _, num1 := range m.Timestamps { + num := uint64(num1) + for num >= 1<<7 { + dAtA14[j13] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j13++ + } + dAtA14[j13] = uint8(num) + j13++ + } + dAtA[i] = 0x32 + i++ + i = encodeVarintPublic(dAtA, i, uint64(j13)) + i += copy(dAtA[i:], dAtA14[:j13]) + } if len(m.RowKeys) > 0 { for _, s := range m.RowKeys { dAtA[i] = 0x3a @@ -1185,27 +1294,9 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(m.Shard)) } if len(m.ColumnIDs) > 0 { - dAtA14 := make([]byte, len(m.ColumnIDs)*10) - var j13 int - for _, num := range m.ColumnIDs { - for num >= 1<<7 { - dAtA14[j13] = uint8(uint64(num)&0x7f | 0x80) - num >>= 7 - j13++ - } - dAtA14[j13] = uint8(num) - j13++ - } - dAtA[i] = 0x2a - i++ - i = encodeVarintPublic(dAtA, i, uint64(j13)) - i += copy(dAtA[i:], dAtA14[:j13]) - } - if len(m.Values) > 0 { - dAtA16 := make([]byte, len(m.Values)*10) + dAtA16 := make([]byte, len(m.ColumnIDs)*10) var j15 int - for _, num1 := range m.Values { - num := uint64(num1) + for _, num := range m.ColumnIDs { for num >= 1<<7 { dAtA16[j15] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -1214,11 +1305,29 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { dAtA16[j15] = uint8(num) j15++ } - dAtA[i] = 0x32 + dAtA[i] = 0x2a i++ i = encodeVarintPublic(dAtA, i, uint64(j15)) i += copy(dAtA[i:], dAtA16[:j15]) } + if len(m.Values) > 0 { + dAtA18 := make([]byte, len(m.Values)*10) + var j17 int + for _, num1 := range m.Values { + num := uint64(num1) + for num >= 1<<7 { + dAtA18[j17] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j17++ + } + dAtA18[j17] = uint8(num) + j17++ + } + dAtA[i] = 0x32 + i++ + i = encodeVarintPublic(dAtA, i, uint64(j17)) + i += copy(dAtA[i:], dAtA18[:j17]) + } if len(m.ColumnKeys) > 0 { for _, s := range m.ColumnKeys { dAtA[i] = 0x3a @@ -1287,6 +1396,21 @@ func (m *Pair) Size() (n int) { return n } +func (m *GroupLine) Size() (n int) { + var l int + _ = l + if len(m.Groups) > 0 { + for _, s := range m.Groups { + l = len(s) + n += 1 + l + sovPublic(uint64(l)) + } + } + if m.Total != 0 { + n += 1 + sovPublic(uint64(m.Total)) + } + return n +} + func (m *ValCount) Size() (n int) { var l int _ = l @@ -1448,6 +1572,19 @@ func (m *QueryResult) Size() (n int) { if m.Type != 0 { n += 1 + sovPublic(uint64(m.Type)) } + if len(m.RowIDs) > 0 { + l = 0 + for _, e := range m.RowIDs { + l += sovPublic(uint64(e)) + } + n += 1 + sovPublic(uint64(l)) + l + } + if len(m.GroupByCounts) > 0 { + for _, e := range m.GroupByCounts { + l = e.Size() + n += 1 + l + sovPublic(uint64(l)) + } + } return n } @@ -1840,6 +1977,104 @@ func (m *Pair) Unmarshal(dAtA []byte) error { } return nil } +func (m *GroupLine) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: GroupLine: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: GroupLine: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Groups", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Groups = append(m.Groups, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Total", wireType) + } + m.Total = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Total |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipPublic(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPublic + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *ValCount) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -2968,6 +3203,99 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { break } } + case 7: + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.RowIDs = append(m.RowIDs, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.RowIDs = append(m.RowIDs, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field RowIDs", wireType) + } + case 8: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field GroupByCounts", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.GroupByCounts = append(m.GroupByCounts, &GroupLine{}) + if err := m.GroupByCounts[len(m.GroupByCounts)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -3748,49 +4076,53 @@ var ( func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ - // 701 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd3, 0x4c, - 0x14, 0xfd, 0x26, 0x76, 0xfe, 0x6e, 0x9a, 0x7c, 0xd5, 0x08, 0x8a, 0x85, 0x50, 0xb0, 0x2c, 0x84, - 0xbc, 0x4a, 0xa5, 0xb0, 0x07, 0xd1, 0x3f, 0x29, 0xaa, 0xa8, 0xe0, 0xb6, 0x14, 0xb1, 0x74, 0x9b, - 0x51, 0x1b, 0xc9, 0xf1, 0x18, 0x7b, 0xac, 0x34, 0xcf, 0xc1, 0x86, 0x47, 0x60, 0xc1, 0x43, 0xb0, - 0xec, 0x92, 0x47, 0x80, 0xf2, 0x22, 0x68, 0xee, 0x78, 0x62, 0x37, 0x95, 0x2a, 0x16, 0xec, 0xe6, - 0x9c, 0x33, 0x73, 0x67, 0xce, 0xcc, 0xb9, 0x36, 0x6c, 0xa4, 0xc5, 0x59, 0x3c, 0x3b, 0x1f, 0xa5, - 0x99, 0x54, 0x92, 0x77, 0x66, 0x89, 0x12, 0x59, 0x12, 0xc5, 0xc1, 0x47, 0x70, 0x50, 0x2e, 0xb8, - 0x07, 0xed, 0x5d, 0x19, 0x17, 0xf3, 0x24, 0xf7, 0x98, 0xef, 0x84, 0x2e, 0x5a, 0xc8, 0x9f, 0x41, - 0xf3, 0xb5, 0x52, 0x59, 0xee, 0x35, 0x7c, 0x27, 0xec, 0x8d, 0x07, 0x23, 0xbb, 0x74, 0xa4, 0x69, - 0x34, 0x22, 0xe7, 0xe0, 0x1e, 0x8a, 0x65, 0xee, 0x39, 0xbe, 0x13, 0x76, 0x91, 0xc6, 0xc1, 0x4b, - 0x70, 0xdf, 0x46, 0xb3, 0x8c, 0x0f, 0xa0, 0x31, 0xd9, 0xf3, 0x98, 0xcf, 0x42, 0x17, 0x1b, 0x93, - 0x3d, 0xfe, 0x00, 0x9a, 0xbb, 0xb2, 0x48, 0x94, 0xd7, 0x20, 0xca, 0x00, 0xbe, 0x09, 0xce, 0xa1, - 0x58, 0x7a, 0x8e, 0xcf, 0xc2, 0x2e, 0xea, 0x61, 0x30, 0x86, 0xce, 0x69, 0x14, 0xaf, 0xd4, 0xd3, - 0x28, 0xa6, 0x22, 0x0e, 0xea, 0xe1, 0xed, 0x2a, 0x4e, 0x59, 0x25, 0x78, 0x0f, 0xce, 0xce, 0x4c, - 0x69, 0x11, 0xe5, 0x62, 0xb5, 0xab, 0x01, 0xfc, 0x31, 0x74, 0x8c, 0xab, 0xc9, 0x5e, 0xb9, 0xf7, - 0x0a, 0xf3, 0x27, 0xd0, 0x3d, 0x99, 0xcd, 0x45, 0xae, 0xa2, 0x79, 0x4a, 0x87, 0x70, 0xb0, 0x22, - 0x82, 0x0f, 0xd0, 0x37, 0x33, 0xb5, 0xdb, 0x63, 0xa1, 0xee, 0x78, 0xfa, 0xbb, 0x5b, 0xba, 0xeb, - 0xf1, 0x2b, 0x03, 0x57, 0x6b, 0x56, 0x62, 0x2b, 0x49, 0x5f, 0xe9, 0xc9, 0x32, 0x15, 0xe5, 0x49, - 0x69, 0xcc, 0x7d, 0xe8, 0x1d, 0xab, 0x6c, 0x96, 0x5c, 0x9c, 0x46, 0x71, 0x21, 0xca, 0x42, 0x75, - 0x4a, 0x7b, 0x9c, 0x24, 0xca, 0xc8, 0x2e, 0xd9, 0x58, 0x61, 0xed, 0x71, 0x47, 0xca, 0xd8, 0x88, - 0x4d, 0x9f, 0x85, 0x1d, 0xac, 0x08, 0x3e, 0x04, 0x38, 0x88, 0x65, 0x54, 0xae, 0x6d, 0xf9, 0x2c, - 0x64, 0x58, 0x63, 0x82, 0x6d, 0x68, 0xeb, 0x93, 0xbe, 0x89, 0xd2, 0xca, 0x2d, 0xbb, 0xc7, 0x6d, - 0x70, 0xcd, 0x60, 0xe3, 0x5d, 0x21, 0xb2, 0x25, 0x8a, 0x4f, 0x85, 0xc8, 0xe9, 0x55, 0x08, 0x97, - 0x2e, 0x0d, 0xe0, 0x5b, 0xd0, 0x3a, 0xbe, 0x8c, 0xb2, 0xa9, 0xb9, 0x3b, 0x17, 0x4b, 0xa4, 0xbd, - 0x56, 0x77, 0x9e, 0x93, 0xd7, 0x0e, 0xd6, 0x29, 0xbd, 0x12, 0xc5, 0x5c, 0x2a, 0x6b, 0xa6, 0x44, - 0x3c, 0x84, 0xff, 0xf7, 0xaf, 0xce, 0xe3, 0x62, 0x2a, 0x50, 0x2e, 0xcc, 0xea, 0x16, 0x4d, 0x58, - 0xa7, 0xf9, 0x73, 0x18, 0x94, 0x94, 0x4d, 0x7f, 0x9b, 0x26, 0xae, 0xb1, 0xc1, 0x67, 0x06, 0xfd, - 0xd2, 0x4a, 0x9e, 0xca, 0x24, 0x17, 0xfa, 0xbd, 0xf6, 0xb3, 0xcc, 0xbe, 0xd7, 0x7e, 0x96, 0xf1, - 0x6d, 0x68, 0xa3, 0xc8, 0x8b, 0x58, 0xd9, 0x10, 0x3c, 0xac, 0xae, 0xc5, 0xae, 0x2d, 0x62, 0x85, - 0x76, 0x16, 0x7f, 0x05, 0x83, 0x5b, 0xa1, 0x32, 0xdd, 0xd3, 0x1b, 0x3f, 0xaa, 0xd6, 0xdd, 0xd2, - 0x71, 0x6d, 0x7a, 0xf0, 0x9d, 0x41, 0xaf, 0x56, 0x99, 0x3f, 0xa5, 0x5e, 0xa6, 0x33, 0xf5, 0xc6, - 0xfd, 0xaa, 0x0a, 0xca, 0x05, 0x52, 0x97, 0x6f, 0x00, 0x3b, 0x2a, 0xf3, 0xc4, 0x8e, 0xf4, 0x2b, - 0xea, 0xfe, 0xb4, 0xdb, 0xd6, 0x5e, 0x51, 0xd3, 0x68, 0x44, 0xfa, 0x32, 0x5c, 0x46, 0xc9, 0x85, - 0x98, 0x52, 0x9e, 0x3a, 0x68, 0x21, 0x1f, 0x55, 0xfd, 0x49, 0x0f, 0xd0, 0x1b, 0xf3, 0xaa, 0x84, - 0x55, 0xb0, 0xea, 0x61, 0x1b, 0x68, 0xfd, 0x16, 0x7d, 0x13, 0xe8, 0xe0, 0x17, 0x83, 0xfe, 0x64, - 0x9e, 0xca, 0x4c, 0xd5, 0x42, 0x32, 0x49, 0xa6, 0xe2, 0xca, 0x86, 0x84, 0x80, 0x66, 0x0f, 0x66, - 0x22, 0x9e, 0xd2, 0xe9, 0xbb, 0x68, 0x80, 0x66, 0x29, 0x2c, 0x14, 0x0e, 0x17, 0x0d, 0xa0, 0x58, - 0xe8, 0x7e, 0xcf, 0x3d, 0xd7, 0x04, 0xca, 0x20, 0x1d, 0x7f, 0xdb, 0xee, 0xb9, 0xd7, 0x24, 0xa9, - 0x22, 0x74, 0xfc, 0x57, 0xfd, 0xae, 0xf3, 0xe2, 0x84, 0x0e, 0xd6, 0x18, 0x7d, 0x0f, 0x28, 0x17, - 0xf4, 0x91, 0x6b, 0xd3, 0x47, 0xce, 0x42, 0xbd, 0xd2, 0x94, 0x21, 0xb1, 0x43, 0x62, 0x8d, 0x09, - 0xbe, 0x31, 0xe0, 0xc6, 0x23, 0x35, 0xd2, 0xbf, 0x33, 0x7a, 0xbf, 0xa1, 0x2d, 0x68, 0xd1, 0x7e, - 0xd6, 0x4c, 0x89, 0xd6, 0x8e, 0xdb, 0x5e, 0x3f, 0xee, 0xce, 0xe6, 0xf5, 0xcd, 0x90, 0xfd, 0xb8, - 0x19, 0xb2, 0x9f, 0x37, 0x43, 0xf6, 0xe5, 0xf7, 0xf0, 0xbf, 0xb3, 0x16, 0xfd, 0x34, 0x5e, 0xfc, - 0x09, 0x00, 0x00, 0xff, 0xff, 0x67, 0xca, 0x55, 0x5d, 0x44, 0x06, 0x00, 0x00, + // 760 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd3, 0x4a, + 0x14, 0xbe, 0x13, 0x3b, 0x89, 0x73, 0xd2, 0xe4, 0x56, 0x73, 0xef, 0xed, 0xb5, 0x50, 0x15, 0x2c, + 0x0b, 0x21, 0xaf, 0x52, 0x29, 0xac, 0xba, 0x01, 0x91, 0xfe, 0xa0, 0xa8, 0x50, 0xc1, 0xb4, 0x14, + 0xb1, 0x74, 0x9b, 0x51, 0x6b, 0xc9, 0xf1, 0x18, 0x7b, 0xac, 0x34, 0xcf, 0xd1, 0x0d, 0x8f, 0xc0, + 0x82, 0x07, 0xe9, 0x92, 0x47, 0x80, 0xf2, 0x22, 0x68, 0xce, 0x78, 0x62, 0x27, 0x95, 0x2a, 0x16, + 0xec, 0xfc, 0x7d, 0x67, 0xe6, 0xcc, 0xf9, 0xce, 0x9f, 0x61, 0x23, 0x2d, 0xce, 0xe3, 0xe8, 0x62, + 0x98, 0x66, 0x42, 0x0a, 0xea, 0x44, 0x89, 0xe4, 0x59, 0x12, 0xc6, 0xfe, 0x47, 0xb0, 0x98, 0x98, + 0x53, 0x17, 0xda, 0x7b, 0x22, 0x2e, 0x66, 0x49, 0xee, 0x12, 0xcf, 0x0a, 0x6c, 0x66, 0x20, 0x7d, + 0x02, 0xcd, 0x97, 0x52, 0x66, 0xb9, 0xdb, 0xf0, 0xac, 0xa0, 0x3b, 0xea, 0x0f, 0xcd, 0xd5, 0xa1, + 0xa2, 0x99, 0x36, 0x52, 0x0a, 0xf6, 0x11, 0x5f, 0xe4, 0xae, 0xe5, 0x59, 0x41, 0x87, 0xe1, 0xb7, + 0xff, 0x1c, 0xec, 0xb7, 0x61, 0x94, 0xd1, 0x3e, 0x34, 0x26, 0xfb, 0x2e, 0xf1, 0x48, 0x60, 0xb3, + 0xc6, 0x64, 0x9f, 0xfe, 0x0b, 0xcd, 0x3d, 0x51, 0x24, 0xd2, 0x6d, 0x20, 0xa5, 0x01, 0xdd, 0x04, + 0xeb, 0x88, 0x2f, 0x5c, 0xcb, 0x23, 0x41, 0x87, 0xa9, 0x4f, 0x7f, 0x17, 0x3a, 0xaf, 0x32, 0x51, + 0xa4, 0xaf, 0xa3, 0x84, 0xd3, 0x2d, 0x68, 0x21, 0xd0, 0xf1, 0x75, 0x58, 0x89, 0x94, 0xb3, 0x53, + 0x21, 0xc3, 0xd8, 0x38, 0x43, 0xe0, 0x8f, 0xc0, 0x39, 0x0b, 0xe3, 0xa5, 0xe3, 0xb3, 0x30, 0xc6, + 0xf7, 0x2d, 0xa6, 0x3e, 0x57, 0x03, 0xb0, 0xca, 0x00, 0xfc, 0xf7, 0x60, 0x8d, 0x23, 0xa9, 0x8c, + 0x4c, 0xcc, 0x97, 0x01, 0x6b, 0x40, 0x1f, 0x81, 0xa3, 0x13, 0x32, 0xd9, 0x2f, 0x5f, 0x5a, 0x62, + 0xba, 0x0d, 0x9d, 0xd3, 0x68, 0xc6, 0x73, 0x19, 0xce, 0x52, 0x8c, 0xdf, 0x62, 0x15, 0xe1, 0x7f, + 0x80, 0x9e, 0x3e, 0xa9, 0x12, 0x75, 0xc2, 0xe5, 0xbd, 0x74, 0xfc, 0x5e, 0x82, 0xef, 0xa7, 0xe7, + 0x0b, 0x01, 0x5b, 0xd9, 0x8c, 0x89, 0x2c, 0x4d, 0xaa, 0x1a, 0xa7, 0x8b, 0x94, 0x97, 0x91, 0xe2, + 0x37, 0xf5, 0xa0, 0x7b, 0x22, 0xb3, 0x28, 0xb9, 0x3c, 0x0b, 0xe3, 0x82, 0x97, 0x8e, 0xea, 0x94, + 0xd2, 0x38, 0x49, 0xa4, 0x36, 0xdb, 0x28, 0x63, 0x89, 0x95, 0xc6, 0xb1, 0x10, 0xb1, 0x36, 0x36, + 0x3d, 0x12, 0x38, 0xac, 0x22, 0xe8, 0x00, 0xe0, 0x30, 0x16, 0x61, 0x79, 0xb7, 0xe5, 0x91, 0x80, + 0xb0, 0x1a, 0xe3, 0xef, 0x40, 0x5b, 0x45, 0xfa, 0x26, 0x4c, 0x2b, 0xb5, 0xe4, 0x01, 0xb5, 0xfe, + 0x2d, 0x81, 0x8d, 0x77, 0x05, 0xcf, 0x16, 0x8c, 0x7f, 0x2a, 0x78, 0x8e, 0x55, 0x41, 0x5c, 0xaa, + 0xd4, 0x40, 0x35, 0xc5, 0xc9, 0x55, 0x98, 0x4d, 0x75, 0xee, 0x6c, 0x56, 0x22, 0xa5, 0xb5, 0xca, + 0x79, 0x8e, 0x5a, 0x1d, 0x56, 0xa7, 0xd4, 0x4d, 0xc6, 0x67, 0x42, 0x1a, 0x31, 0x25, 0xa2, 0x01, + 0xfc, 0x7d, 0x70, 0x7d, 0x11, 0x17, 0x53, 0xce, 0xc4, 0x5c, 0xdf, 0x6e, 0xe1, 0x81, 0x75, 0x9a, + 0x3e, 0x85, 0x7e, 0x49, 0x99, 0xc1, 0x69, 0xe3, 0xc1, 0x35, 0xd6, 0xbf, 0x21, 0xd0, 0x2b, 0xa5, + 0xe4, 0xa9, 0x48, 0x72, 0xae, 0xea, 0x75, 0x90, 0x65, 0xa6, 0x5e, 0x07, 0x59, 0x46, 0x77, 0xa0, + 0xcd, 0x78, 0x5e, 0xc4, 0xd2, 0x34, 0xc1, 0x7f, 0x55, 0x5a, 0xcc, 0xdd, 0x22, 0x96, 0xcc, 0x9c, + 0xa2, 0x2f, 0xa0, 0xbf, 0xd2, 0x54, 0x7a, 0xf0, 0xba, 0xa3, 0xff, 0xab, 0x7b, 0x2b, 0x76, 0xb6, + 0x76, 0xdc, 0xbf, 0x69, 0x40, 0xb7, 0xe6, 0x99, 0x3e, 0xc6, 0x35, 0x80, 0x31, 0x75, 0x47, 0xbd, + 0xca, 0x0b, 0x13, 0x73, 0x86, 0x0b, 0x62, 0x03, 0xc8, 0x71, 0xd9, 0x4f, 0xe4, 0x58, 0x55, 0x51, + 0x8d, 0xb6, 0x79, 0xb6, 0x56, 0x45, 0x45, 0x33, 0x6d, 0xc4, 0xa5, 0x72, 0x15, 0x26, 0x97, 0x7c, + 0x8a, 0xfd, 0xe4, 0x30, 0x03, 0xe9, 0xb0, 0x9a, 0x4f, 0x2c, 0x40, 0x77, 0x44, 0x2b, 0x17, 0xc6, + 0xc2, 0xaa, 0x19, 0x36, 0x0d, 0xad, 0x6a, 0xd1, 0x2b, 0x1b, 0x5a, 0x95, 0x50, 0xcd, 0xa6, 0x4a, + 0x3c, 0x16, 0x5f, 0x23, 0xba, 0x0b, 0x3d, 0xdc, 0x0d, 0xe3, 0x05, 0xde, 0xcd, 0x5d, 0x07, 0x63, + 0xfc, 0xa7, 0x7a, 0x60, 0xb9, 0x55, 0xd8, 0xea, 0x49, 0xff, 0x07, 0x81, 0xde, 0x64, 0x96, 0x8a, + 0x4c, 0xd6, 0xfa, 0x6e, 0x92, 0x4c, 0xf9, 0xb5, 0xe9, 0x3b, 0x04, 0x8a, 0x3d, 0x8c, 0x78, 0x3c, + 0xc5, 0x84, 0x74, 0x98, 0x06, 0x8a, 0xc5, 0xfe, 0xc3, 0x7e, 0xb3, 0x99, 0x06, 0xb5, 0x30, 0xed, + 0x95, 0x30, 0xb7, 0xa1, 0x63, 0x36, 0x48, 0xee, 0x36, 0xd1, 0x54, 0x11, 0x6a, 0xa2, 0x96, 0x2b, + 0x44, 0xb5, 0xa0, 0x15, 0x58, 0xac, 0xc6, 0xa8, 0xd4, 0x32, 0x31, 0xc7, 0x95, 0xdb, 0xc6, 0x7d, + 0x68, 0xa0, 0xba, 0xa9, 0xdd, 0xa0, 0xd1, 0x41, 0x63, 0x8d, 0xf1, 0xbf, 0x12, 0xa0, 0x5a, 0x23, + 0xce, 0xe6, 0x9f, 0x13, 0xfa, 0xb0, 0xa0, 0x2d, 0x68, 0xe1, 0x7b, 0x46, 0x4c, 0x89, 0xd6, 0xc2, + 0x6d, 0xaf, 0x87, 0x3b, 0xde, 0xbc, 0xbd, 0x1b, 0x90, 0x6f, 0x77, 0x03, 0xf2, 0xfd, 0x6e, 0x40, + 0x3e, 0xff, 0x1c, 0xfc, 0x75, 0xde, 0xc2, 0x5f, 0xd8, 0xb3, 0x5f, 0x01, 0x00, 0x00, 0xff, 0xff, + 0x22, 0x44, 0x8c, 0x98, 0xd2, 0x06, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index 04c98d070..f3f362fd1 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -14,6 +14,11 @@ message Pair { uint64 Count = 2; } +message GroupLine{ + repeated string Groups = 1; + uint64 Total=2; +} + message ValCount { int64 Val = 1; int64 Count = 2; @@ -64,8 +69,10 @@ message QueryResult { Row Row = 1; uint64 N = 2; repeated Pair Pairs = 3; - ValCount ValCount = 5; bool Changed = 4; + ValCount ValCount = 5; + repeated uint64 RowIDs = 7; + repeated GroupLine GroupByCounts = 8; } message ImportRequest { diff --git a/pilosa.go b/pilosa.go index 7378eb628..96b06d9e6 100644 --- a/pilosa.go +++ b/pilosa.go @@ -62,7 +62,9 @@ var ( ErrNodeNotCoordinator = errors.New("node is not the coordinator") ErrResizeNotRunning = errors.New("no resize job currently running") - ErrNotImplemented = errors.New("not implemented") + ErrNotImplemented = errors.New("not implemented") + ErrFieldsArgumentRequired = errors.New("fields argument required") + ErrExpectedFieldListArgument = errors.New("expected field list argument") ) // apiMethodNotAllowedError wraps an error value indicating that a particular From e1b938e52c7187d3bc266f24c1ece678a7e24a05 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 22 Aug 2018 12:18:27 -0500 Subject: [PATCH 02/39] add FieldRow struct to replace the groupBy string key --- encoding/proto/proto.go | 44 +++++- executor.go | 102 +++++++------ executor_test.go | 60 +++++--- internal/public.pb.go | 326 ++++++++++++++++++++++++++++++---------- internal/public.proto | 9 +- 5 files changed, 381 insertions(+), 160 deletions(-) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index e133de7fc..4f886acce 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -999,11 +999,29 @@ func decodeAttr(attr *internal.Attr) (key string, value interface{}) { } func decodeGroupByCounts(a []*internal.GroupLine) pilosa.GroupByCounts { - gbc := make(pilosa.GroupByCounts, 0) + other := make([]pilosa.GroupLine, len(a)) for i := range a { - gbc = append(gbc, pilosa.GroupLine{a[i].Groups, a[i].Total}) + other[i] = pilosa.GroupLine{ + decodeFieldRows(a[i].Groups), + a[i].Total, + } + } + return pilosa.GroupByCounts(other) +} + +func decodeFieldRows(a []*internal.FieldRow) []pilosa.FieldRow { + other := make([]pilosa.FieldRow, len(a)) + for i := range a { + other[i] = decodeFieldRow(a[i]) + } + return other +} + +func decodeFieldRow(pb *internal.FieldRow) pilosa.FieldRow { + return pilosa.FieldRow{ + Field: pb.Field, + RowID: pb.RowID, } - return gbc } func decodePairs(a []*internal.Pair) []pilosa.Pair { @@ -1060,11 +1078,29 @@ func encodeRow(r *pilosa.Row) *internal.Row { func encodeGroupByCount(counts pilosa.GroupByCounts) []*internal.GroupLine { result := make([]*internal.GroupLine, len(counts)) for i := range counts { - result[i] = &internal.GroupLine{Groups: counts[i].Groups, Total: counts[i].Total} + result[i] = &internal.GroupLine{ + Groups: encodeFieldRows(counts[i].Groups), + Total: counts[i].Total, + } } return result } +func encodeFieldRows(a []pilosa.FieldRow) []*internal.FieldRow { + other := make([]*internal.FieldRow, len(a)) + for i := range a { + other[i] = encodeFieldRow(a[i]) + } + return other +} + +func encodeFieldRow(p pilosa.FieldRow) *internal.FieldRow { + return &internal.FieldRow{ + Field: p.Field, + RowID: p.RowID, + } +} + func encodePairs(a pilosa.Pairs) []*internal.Pair { other := make([]*internal.Pair, len(a)) for i := range a { diff --git a/executor.go b/executor.go index 56ab05269..35740866b 100644 --- a/executor.go +++ b/executor.go @@ -786,16 +786,37 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call return results, nil } +// FieldRow is used to distinguish rows in a group by result. +type FieldRow struct { + Field string + RowID uint64 + RowKey string +} + +func (fr FieldRow) String() string { + return fmt.Sprintf("%s.%d", fr.Field, fr.RowID) +} + +// TODO: we shouldn't need to string this +func uniqueGroupString(fr []FieldRow) string { + s := []string{} + for _, f := range fr { + s = append(s, f.String()) + } + return strings.Join(s, "-") +} + // gbi is a groupBy item. type gbi struct { row *Row - fieldKey string - rowID uint64 + fieldRow FieldRow } type GroupLine struct { - Groups []string + Groups []FieldRow Total uint64 } + +// GroupByCounts is the return type for GroupBy queries. type GroupByCounts []GroupLine func (gbc GroupByCounts) Merge(other GroupByCounts) GroupByCounts { @@ -804,14 +825,13 @@ func (gbc GroupByCounts) Merge(other GroupByCounts) GroupByCounts { total uint64 }) for i := range gbc { - m[strings.Join(gbc[i].Groups, "-")] = struct { + m[uniqueGroupString(gbc[i].Groups)] = struct { i int total uint64 - }{total: gbc[i].Total, i: i} + }{i, gbc[i].Total} } for i := range other { - key := strings.Join(other[i].Groups, "-") - o, found := m[key] + o, found := m[uniqueGroupString(other[i].Groups)] if found { gbc[o.i].Total += other[i].Total } else { @@ -820,20 +840,7 @@ func (gbc GroupByCounts) Merge(other GroupByCounts) GroupByCounts { } return gbc } -func makeGroup(parts []gbi) GroupLine { - var other *Row - line := GroupLine{} - for i, o := range parts { - if i == 0 { - other = o.row - } else { - other = other.Intersect(o.row) - } - line.Groups = append(line.Groups, o.fieldKey) - } - line.Total = other.Count() - return line -} + func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql.Call, shard uint64) (GroupByCounts, error) { // Fetch index. idx := e.Holder.Index(index) @@ -867,7 +874,7 @@ func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql } } results := make(GroupByCounts, 0) - var work listOfGBILists + var work [][]gbi for _, fieldDirective := range fieldDirectives.([]interface{}) { fieldName := getFieldName(fieldDirective.(string)) // Fetch fragment. @@ -880,12 +887,15 @@ func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql if err != nil { return nil, err } - set := make(gbiList, 0) + + set := make([]gbi, 0) for _, rowID := range frag.rowsWithFilter(filter) { set = append(set, gbi{ - row: frag.row(rowID), - rowID: rowID, - fieldKey: fmt.Sprintf("%s.%d", fieldName, rowID), + row: frag.row(rowID), + fieldRow: FieldRow{ + Field: fieldName, + RowID: rowID, + }, }) } work = append(work, set) @@ -899,35 +909,29 @@ func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql return results, nil } -type gbiList []gbi -type listOfGBILists []gbiList - -// pi is a product process item. -type pi struct { +// ppi is a product process item. +type ppi struct { row *Row gl GroupLine } -type piList []pi -// product generates the cartiesian product of the input -// using tail recursion -func product(input listOfGBILists) piList { - res := make(piList, 0) - if len(input) == 0 { //base return empty list - res = append(res, pi{gl: GroupLine{Groups: make([]string, 0)}}) - } else { - res = productHelper(input, res) +// product generates the cartesian product of the input +// using tail recursion. +func product(input [][]gbi) []ppi { + if len(input) == 0 { // base return empty list + return []ppi{ + {gl: GroupLine{Groups: make([]FieldRow, 0)}}, + } } - return res -} -func productHelper(lists listOfGBILists, res piList) piList { - head := lists[0] //take first element of the list - tail := product(lists[1:]) //invoke product on remaining element + + res := make([]ppi, 0) + head := input[0] // take first element of the list + tail := product(input[1:]) // invoke product on remaining element for h := range head { // for each head - for t := range tail { //iterate over the tail - s := pi{gl: GroupLine{Groups: make([]string, 0)}} - s.gl.Groups = append([]string{head[h].fieldKey}, tail[t].gl.Groups...) //had to insert at the front to match input order - if tail[t].row != nil { //first time around nothing to intersect + for t := range tail { // iterate over the tail + s := ppi{gl: GroupLine{Groups: make([]FieldRow, 0)}} + s.gl.Groups = append([]FieldRow{head[h].fieldRow}, tail[t].gl.Groups...) // had to insert at the front to match input order + if tail[t].row != nil { // first time around nothing to intersect s.row = head[h].row.Intersect(tail[t].row) } else { s.row = head[h].row diff --git a/executor_test.go b/executor_test.go index 84e00936d..e3afda3ed 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1234,11 +1234,11 @@ Set(4500001, fn=4) t.Fatalf("GroupBy querying: %v", err) } else { expected := pilosa.GroupByCounts{ - {Groups: []string{"f.10"}, Total: 4}, - {Groups: []string{"f.7"}, Total: 1}, + {Groups: []pilosa.FieldRow{{Field: "f", RowID: 10}}, Total: 4}, + {Groups: []pilosa.FieldRow{{Field: "f", RowID: 7}}, Total: 1}, } results := res.Results[0].(pilosa.GroupByCounts) - checkGroupBy(expected, results, t) + checkGroupBy(t, expected, results) } }) } @@ -1681,17 +1681,15 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { hldr.SetBit("i", "general", 11, ShardWidth+2) hldr.SetBit("i", "general", 12, 2) hldr.SetBit("i", "general", 12, ShardWidth+2) - hldr.SetBit("i", "sub", 10, 0) - hldr.SetBit("i", "sub", 10, 1) - hldr.SetBit("i", "sub", 10, 3) - hldr.SetBit("i", "sub", 11, 2) - hldr.SetBit("i", "sub", 11, 0) - expected := pilosa.GroupByCounts{ - {Groups: []string{"general.10", "sub.11"}, Total: 1}, - {Groups: []string{"general.11", "sub.11"}, Total: 1}, - {Groups: []string{"general.12", "sub.11"}, Total: 1}, - {Groups: []string{"general.10", "sub.10"}, Total: 2}, - } + + hldr.SetBit("i", "sub", 100, 0) + hldr.SetBit("i", "sub", 100, 1) + hldr.SetBit("i", "sub", 100, 3) + hldr.SetBit("i", "sub", 100, ShardWidth+1) + + hldr.SetBit("i", "sub", 110, 2) + hldr.SetBit("i", "sub", 110, 0) + t.Run("No Field List Arguments", func(t *testing.T) { if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy()`}); err != nil { if errors.Cause(err) != pilosa.ErrFieldsArgumentRequired { @@ -1714,38 +1712,50 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { } }) t.Run("Basic", func(t *testing.T) { + expected := pilosa.GroupByCounts{ + {Groups: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Total: 1}, + {Groups: []pilosa.FieldRow{{Field: "general", RowID: 11}, {Field: "sub", RowID: 110}}, Total: 1}, + {Groups: []pilosa.FieldRow{{Field: "general", RowID: 12}, {Field: "sub", RowID: 110}}, Total: 1}, + {Groups: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Total: 3}, + } + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(fields=[general,sub])`}); err != nil { t.Fatal(err) } else { results := res.Results[0].(pilosa.GroupByCounts) - checkGroupBy(expected, results, t) + checkGroupBy(t, expected, results) } }) - expected = pilosa.GroupByCounts{ - {Groups: []string{"general.11"}, Total: 2}, - {Groups: []string{"general.12"}, Total: 2}, - } + t.Run("check field offset no limit", func(t *testing.T) { + expected := pilosa.GroupByCounts{ + {Groups: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Total: 2}, + {Groups: []pilosa.FieldRow{{Field: "general", RowID: 12}}, Total: 2}, + } + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(fields=[general:11:])`}); err != nil { t.Fatal(err) } else { results := res.Results[0].(pilosa.GroupByCounts) - checkGroupBy(expected, results, t) + checkGroupBy(t, expected, results) } }) - expected = pilosa.GroupByCounts{ - {Groups: []string{"general.11"}, Total: 2}, - } + t.Run("check field offset limit", func(t *testing.T) { + expected := pilosa.GroupByCounts{ + {Groups: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Total: 2}, + } + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(fields=[general:11:1])`}); err != nil { t.Fatal(err) } else { results := res.Results[0].(pilosa.GroupByCounts) - checkGroupBy(expected, results, t) + checkGroupBy(t, expected, results) } }) } -func checkGroupBy(expected, results pilosa.GroupByCounts, t *testing.T) { + +func checkGroupBy(t *testing.T, expected, results pilosa.GroupByCounts) { notIn := func(item pilosa.GroupLine, expected pilosa.GroupByCounts) bool { for i := range expected { if item.Total == expected[i].Total { diff --git a/internal/public.pb.go b/internal/public.pb.go index 8139d3a28..2203c87d6 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -10,6 +10,7 @@ It has these top-level messages: Row Pair + FieldRow GroupLine ValCount Bit @@ -107,17 +108,41 @@ func (m *Pair) GetCount() uint64 { return 0 } +type FieldRow struct { + Field string `protobuf:"bytes,1,opt,name=Field,proto3" json:"Field,omitempty"` + RowID uint64 `protobuf:"varint,2,opt,name=RowID,proto3" json:"RowID,omitempty"` +} + +func (m *FieldRow) Reset() { *m = FieldRow{} } +func (m *FieldRow) String() string { return proto.CompactTextString(m) } +func (*FieldRow) ProtoMessage() {} +func (*FieldRow) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} } + +func (m *FieldRow) GetField() string { + if m != nil { + return m.Field + } + return "" +} + +func (m *FieldRow) GetRowID() uint64 { + if m != nil { + return m.RowID + } + return 0 +} + type GroupLine struct { - Groups []string `protobuf:"bytes,1,rep,name=Groups" json:"Groups,omitempty"` - Total uint64 `protobuf:"varint,2,opt,name=Total,proto3" json:"Total,omitempty"` + Groups []*FieldRow `protobuf:"bytes,1,rep,name=Groups" json:"Groups,omitempty"` + Total uint64 `protobuf:"varint,2,opt,name=Total,proto3" json:"Total,omitempty"` } func (m *GroupLine) Reset() { *m = GroupLine{} } func (m *GroupLine) String() string { return proto.CompactTextString(m) } func (*GroupLine) ProtoMessage() {} -func (*GroupLine) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} } +func (*GroupLine) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} } -func (m *GroupLine) GetGroups() []string { +func (m *GroupLine) GetGroups() []*FieldRow { if m != nil { return m.Groups } @@ -139,7 +164,7 @@ type ValCount struct { func (m *ValCount) Reset() { *m = ValCount{} } func (m *ValCount) String() string { return proto.CompactTextString(m) } func (*ValCount) ProtoMessage() {} -func (*ValCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} } +func (*ValCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{4} } func (m *ValCount) GetVal() int64 { if m != nil { @@ -164,7 +189,7 @@ type Bit struct { func (m *Bit) Reset() { *m = Bit{} } func (m *Bit) String() string { return proto.CompactTextString(m) } func (*Bit) ProtoMessage() {} -func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{4} } +func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{5} } func (m *Bit) GetRowID() uint64 { if m != nil { @@ -196,7 +221,7 @@ type ColumnAttrSet struct { func (m *ColumnAttrSet) Reset() { *m = ColumnAttrSet{} } func (m *ColumnAttrSet) String() string { return proto.CompactTextString(m) } func (*ColumnAttrSet) ProtoMessage() {} -func (*ColumnAttrSet) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{5} } +func (*ColumnAttrSet) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{6} } func (m *ColumnAttrSet) GetID() uint64 { if m != nil { @@ -231,7 +256,7 @@ type Attr struct { func (m *Attr) Reset() { *m = Attr{} } func (m *Attr) String() string { return proto.CompactTextString(m) } func (*Attr) ProtoMessage() {} -func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{6} } +func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{7} } func (m *Attr) GetKey() string { if m != nil { @@ -282,7 +307,7 @@ type AttrMap struct { func (m *AttrMap) Reset() { *m = AttrMap{} } func (m *AttrMap) String() string { return proto.CompactTextString(m) } func (*AttrMap) ProtoMessage() {} -func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{7} } +func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{8} } func (m *AttrMap) GetAttrs() []*Attr { if m != nil { @@ -303,7 +328,7 @@ type QueryRequest struct { func (m *QueryRequest) Reset() { *m = QueryRequest{} } func (m *QueryRequest) String() string { return proto.CompactTextString(m) } func (*QueryRequest) ProtoMessage() {} -func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{8} } +func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{9} } func (m *QueryRequest) GetQuery() string { if m != nil { @@ -356,7 +381,7 @@ type QueryResponse struct { func (m *QueryResponse) Reset() { *m = QueryResponse{} } func (m *QueryResponse) String() string { return proto.CompactTextString(m) } func (*QueryResponse) ProtoMessage() {} -func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{9} } +func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{10} } func (m *QueryResponse) GetErr() string { if m != nil { @@ -393,7 +418,7 @@ type QueryResult struct { func (m *QueryResult) Reset() { *m = QueryResult{} } func (m *QueryResult) String() string { return proto.CompactTextString(m) } func (*QueryResult) ProtoMessage() {} -func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{10} } +func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{11} } func (m *QueryResult) GetType() uint32 { if m != nil { @@ -465,7 +490,7 @@ type ImportRequest struct { func (m *ImportRequest) Reset() { *m = ImportRequest{} } func (m *ImportRequest) String() string { return proto.CompactTextString(m) } func (*ImportRequest) ProtoMessage() {} -func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{11} } +func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{12} } func (m *ImportRequest) GetIndex() string { if m != nil { @@ -535,7 +560,7 @@ type ImportValueRequest struct { func (m *ImportValueRequest) Reset() { *m = ImportValueRequest{} } func (m *ImportValueRequest) String() string { return proto.CompactTextString(m) } func (*ImportValueRequest) ProtoMessage() {} -func (*ImportValueRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{12} } +func (*ImportValueRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{13} } func (m *ImportValueRequest) GetIndex() string { if m != nil { @@ -582,6 +607,7 @@ func (m *ImportValueRequest) GetValues() []int64 { func init() { proto.RegisterType((*Row)(nil), "internal.Row") proto.RegisterType((*Pair)(nil), "internal.Pair") + proto.RegisterType((*FieldRow)(nil), "internal.FieldRow") proto.RegisterType((*GroupLine)(nil), "internal.GroupLine") proto.RegisterType((*ValCount)(nil), "internal.ValCount") proto.RegisterType((*Bit)(nil), "internal.Bit") @@ -690,6 +716,35 @@ func (m *Pair) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *FieldRow) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FieldRow) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Field) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) + } + if m.RowID != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.RowID)) + } + return i, nil +} + func (m *GroupLine) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -706,18 +761,15 @@ func (m *GroupLine) MarshalTo(dAtA []byte) (int, error) { var l int _ = l if len(m.Groups) > 0 { - for _, s := range m.Groups { + for _, msg := range m.Groups { dAtA[i] = 0xa i++ - l = len(s) - for l >= 1<<7 { - dAtA[i] = uint8(uint64(l)&0x7f | 0x80) - l >>= 7 - i++ + i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err } - dAtA[i] = uint8(l) - i++ - i += copy(dAtA[i:], s) + i += n } } if m.Total != 0 { @@ -1396,12 +1448,25 @@ func (m *Pair) Size() (n int) { return n } +func (m *FieldRow) Size() (n int) { + var l int + _ = l + l = len(m.Field) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } + if m.RowID != 0 { + n += 1 + sovPublic(uint64(m.RowID)) + } + return n +} + func (m *GroupLine) Size() (n int) { var l int _ = l if len(m.Groups) > 0 { - for _, s := range m.Groups { - l = len(s) + for _, e := range m.Groups { + l = e.Size() n += 1 + l + sovPublic(uint64(l)) } } @@ -1977,6 +2042,104 @@ func (m *Pair) Unmarshal(dAtA []byte) error { } return nil } +func (m *FieldRow) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FieldRow: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FieldRow: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Field = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field RowID", wireType) + } + m.RowID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.RowID |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipPublic(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPublic + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *GroupLine) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -2010,7 +2173,7 @@ func (m *GroupLine) Unmarshal(dAtA []byte) error { if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Groups", wireType) } - var stringLen uint64 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPublic @@ -2020,20 +2183,22 @@ func (m *GroupLine) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { + if msglen < 0 { return ErrInvalidLengthPublic } - postIndex := iNdEx + intStringLen + postIndex := iNdEx + msglen if postIndex > l { return io.ErrUnexpectedEOF } - m.Groups = append(m.Groups, string(dAtA[iNdEx:postIndex])) + m.Groups = append(m.Groups, &FieldRow{}) + if err := m.Groups[len(m.Groups)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } iNdEx = postIndex case 2: if wireType != 0 { @@ -4076,53 +4241,54 @@ var ( func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ - // 760 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd3, 0x4a, - 0x14, 0xbe, 0x13, 0x3b, 0x89, 0x73, 0xd2, 0xe4, 0x56, 0x73, 0xef, 0xed, 0xb5, 0x50, 0x15, 0x2c, - 0x0b, 0x21, 0xaf, 0x52, 0x29, 0xac, 0xba, 0x01, 0x91, 0xfe, 0xa0, 0xa8, 0x50, 0xc1, 0xb4, 0x14, - 0xb1, 0x74, 0x9b, 0x51, 0x6b, 0xc9, 0xf1, 0x18, 0x7b, 0xac, 0x34, 0xcf, 0xd1, 0x0d, 0x8f, 0xc0, - 0x82, 0x07, 0xe9, 0x92, 0x47, 0x80, 0xf2, 0x22, 0x68, 0xce, 0x78, 0x62, 0x27, 0x95, 0x2a, 0x16, - 0xec, 0xfc, 0x7d, 0x67, 0xe6, 0xcc, 0xf9, 0xce, 0x9f, 0x61, 0x23, 0x2d, 0xce, 0xe3, 0xe8, 0x62, - 0x98, 0x66, 0x42, 0x0a, 0xea, 0x44, 0x89, 0xe4, 0x59, 0x12, 0xc6, 0xfe, 0x47, 0xb0, 0x98, 0x98, - 0x53, 0x17, 0xda, 0x7b, 0x22, 0x2e, 0x66, 0x49, 0xee, 0x12, 0xcf, 0x0a, 0x6c, 0x66, 0x20, 0x7d, - 0x02, 0xcd, 0x97, 0x52, 0x66, 0xb9, 0xdb, 0xf0, 0xac, 0xa0, 0x3b, 0xea, 0x0f, 0xcd, 0xd5, 0xa1, - 0xa2, 0x99, 0x36, 0x52, 0x0a, 0xf6, 0x11, 0x5f, 0xe4, 0xae, 0xe5, 0x59, 0x41, 0x87, 0xe1, 0xb7, - 0xff, 0x1c, 0xec, 0xb7, 0x61, 0x94, 0xd1, 0x3e, 0x34, 0x26, 0xfb, 0x2e, 0xf1, 0x48, 0x60, 0xb3, - 0xc6, 0x64, 0x9f, 0xfe, 0x0b, 0xcd, 0x3d, 0x51, 0x24, 0xd2, 0x6d, 0x20, 0xa5, 0x01, 0xdd, 0x04, - 0xeb, 0x88, 0x2f, 0x5c, 0xcb, 0x23, 0x41, 0x87, 0xa9, 0x4f, 0x7f, 0x17, 0x3a, 0xaf, 0x32, 0x51, - 0xa4, 0xaf, 0xa3, 0x84, 0xd3, 0x2d, 0x68, 0x21, 0xd0, 0xf1, 0x75, 0x58, 0x89, 0x94, 0xb3, 0x53, - 0x21, 0xc3, 0xd8, 0x38, 0x43, 0xe0, 0x8f, 0xc0, 0x39, 0x0b, 0xe3, 0xa5, 0xe3, 0xb3, 0x30, 0xc6, - 0xf7, 0x2d, 0xa6, 0x3e, 0x57, 0x03, 0xb0, 0xca, 0x00, 0xfc, 0xf7, 0x60, 0x8d, 0x23, 0xa9, 0x8c, - 0x4c, 0xcc, 0x97, 0x01, 0x6b, 0x40, 0x1f, 0x81, 0xa3, 0x13, 0x32, 0xd9, 0x2f, 0x5f, 0x5a, 0x62, - 0xba, 0x0d, 0x9d, 0xd3, 0x68, 0xc6, 0x73, 0x19, 0xce, 0x52, 0x8c, 0xdf, 0x62, 0x15, 0xe1, 0x7f, - 0x80, 0x9e, 0x3e, 0xa9, 0x12, 0x75, 0xc2, 0xe5, 0xbd, 0x74, 0xfc, 0x5e, 0x82, 0xef, 0xa7, 0xe7, - 0x0b, 0x01, 0x5b, 0xd9, 0x8c, 0x89, 0x2c, 0x4d, 0xaa, 0x1a, 0xa7, 0x8b, 0x94, 0x97, 0x91, 0xe2, - 0x37, 0xf5, 0xa0, 0x7b, 0x22, 0xb3, 0x28, 0xb9, 0x3c, 0x0b, 0xe3, 0x82, 0x97, 0x8e, 0xea, 0x94, - 0xd2, 0x38, 0x49, 0xa4, 0x36, 0xdb, 0x28, 0x63, 0x89, 0x95, 0xc6, 0xb1, 0x10, 0xb1, 0x36, 0x36, - 0x3d, 0x12, 0x38, 0xac, 0x22, 0xe8, 0x00, 0xe0, 0x30, 0x16, 0x61, 0x79, 0xb7, 0xe5, 0x91, 0x80, - 0xb0, 0x1a, 0xe3, 0xef, 0x40, 0x5b, 0x45, 0xfa, 0x26, 0x4c, 0x2b, 0xb5, 0xe4, 0x01, 0xb5, 0xfe, - 0x2d, 0x81, 0x8d, 0x77, 0x05, 0xcf, 0x16, 0x8c, 0x7f, 0x2a, 0x78, 0x8e, 0x55, 0x41, 0x5c, 0xaa, - 0xd4, 0x40, 0x35, 0xc5, 0xc9, 0x55, 0x98, 0x4d, 0x75, 0xee, 0x6c, 0x56, 0x22, 0xa5, 0xb5, 0xca, - 0x79, 0x8e, 0x5a, 0x1d, 0x56, 0xa7, 0xd4, 0x4d, 0xc6, 0x67, 0x42, 0x1a, 0x31, 0x25, 0xa2, 0x01, - 0xfc, 0x7d, 0x70, 0x7d, 0x11, 0x17, 0x53, 0xce, 0xc4, 0x5c, 0xdf, 0x6e, 0xe1, 0x81, 0x75, 0x9a, - 0x3e, 0x85, 0x7e, 0x49, 0x99, 0xc1, 0x69, 0xe3, 0xc1, 0x35, 0xd6, 0xbf, 0x21, 0xd0, 0x2b, 0xa5, - 0xe4, 0xa9, 0x48, 0x72, 0xae, 0xea, 0x75, 0x90, 0x65, 0xa6, 0x5e, 0x07, 0x59, 0x46, 0x77, 0xa0, - 0xcd, 0x78, 0x5e, 0xc4, 0xd2, 0x34, 0xc1, 0x7f, 0x55, 0x5a, 0xcc, 0xdd, 0x22, 0x96, 0xcc, 0x9c, - 0xa2, 0x2f, 0xa0, 0xbf, 0xd2, 0x54, 0x7a, 0xf0, 0xba, 0xa3, 0xff, 0xab, 0x7b, 0x2b, 0x76, 0xb6, - 0x76, 0xdc, 0xbf, 0x69, 0x40, 0xb7, 0xe6, 0x99, 0x3e, 0xc6, 0x35, 0x80, 0x31, 0x75, 0x47, 0xbd, - 0xca, 0x0b, 0x13, 0x73, 0x86, 0x0b, 0x62, 0x03, 0xc8, 0x71, 0xd9, 0x4f, 0xe4, 0x58, 0x55, 0x51, - 0x8d, 0xb6, 0x79, 0xb6, 0x56, 0x45, 0x45, 0x33, 0x6d, 0xc4, 0xa5, 0x72, 0x15, 0x26, 0x97, 0x7c, - 0x8a, 0xfd, 0xe4, 0x30, 0x03, 0xe9, 0xb0, 0x9a, 0x4f, 0x2c, 0x40, 0x77, 0x44, 0x2b, 0x17, 0xc6, - 0xc2, 0xaa, 0x19, 0x36, 0x0d, 0xad, 0x6a, 0xd1, 0x2b, 0x1b, 0x5a, 0x95, 0x50, 0xcd, 0xa6, 0x4a, - 0x3c, 0x16, 0x5f, 0x23, 0xba, 0x0b, 0x3d, 0xdc, 0x0d, 0xe3, 0x05, 0xde, 0xcd, 0x5d, 0x07, 0x63, - 0xfc, 0xa7, 0x7a, 0x60, 0xb9, 0x55, 0xd8, 0xea, 0x49, 0xff, 0x07, 0x81, 0xde, 0x64, 0x96, 0x8a, - 0x4c, 0xd6, 0xfa, 0x6e, 0x92, 0x4c, 0xf9, 0xb5, 0xe9, 0x3b, 0x04, 0x8a, 0x3d, 0x8c, 0x78, 0x3c, - 0xc5, 0x84, 0x74, 0x98, 0x06, 0x8a, 0xc5, 0xfe, 0xc3, 0x7e, 0xb3, 0x99, 0x06, 0xb5, 0x30, 0xed, - 0x95, 0x30, 0xb7, 0xa1, 0x63, 0x36, 0x48, 0xee, 0x36, 0xd1, 0x54, 0x11, 0x6a, 0xa2, 0x96, 0x2b, - 0x44, 0xb5, 0xa0, 0x15, 0x58, 0xac, 0xc6, 0xa8, 0xd4, 0x32, 0x31, 0xc7, 0x95, 0xdb, 0xc6, 0x7d, - 0x68, 0xa0, 0xba, 0xa9, 0xdd, 0xa0, 0xd1, 0x41, 0x63, 0x8d, 0xf1, 0xbf, 0x12, 0xa0, 0x5a, 0x23, - 0xce, 0xe6, 0x9f, 0x13, 0xfa, 0xb0, 0xa0, 0x2d, 0x68, 0xe1, 0x7b, 0x46, 0x4c, 0x89, 0xd6, 0xc2, - 0x6d, 0xaf, 0x87, 0x3b, 0xde, 0xbc, 0xbd, 0x1b, 0x90, 0x6f, 0x77, 0x03, 0xf2, 0xfd, 0x6e, 0x40, - 0x3e, 0xff, 0x1c, 0xfc, 0x75, 0xde, 0xc2, 0x5f, 0xd8, 0xb3, 0x5f, 0x01, 0x00, 0x00, 0xff, 0xff, - 0x22, 0x44, 0x8c, 0x98, 0xd2, 0x06, 0x00, 0x00, + // 783 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xdd, 0x6a, 0xdb, 0x48, + 0x14, 0xde, 0xb1, 0x64, 0x5b, 0x3e, 0x8e, 0xbd, 0x61, 0x36, 0x9b, 0x15, 0x4b, 0xf0, 0x0a, 0xb1, + 0x2c, 0x62, 0x2f, 0x1c, 0xf0, 0xc2, 0x42, 0x6f, 0x5a, 0xea, 0xfc, 0x14, 0x93, 0x26, 0xb4, 0x93, + 0x34, 0xa5, 0x97, 0x4a, 0x3c, 0x24, 0x02, 0x59, 0xa3, 0x4a, 0x23, 0x1c, 0x3f, 0x47, 0x6e, 0xfa, + 0x08, 0xbd, 0xe8, 0x83, 0xe4, 0xb2, 0x8f, 0xd0, 0xa6, 0x2f, 0x52, 0xe6, 0x8c, 0xc6, 0x92, 0x1d, + 0x08, 0xbd, 0xe8, 0xdd, 0x7c, 0xe7, 0xcc, 0x1c, 0x7d, 0xdf, 0xf9, 0x13, 0x6c, 0xa4, 0xc5, 0x45, + 0x1c, 0x5d, 0x0e, 0xd3, 0x4c, 0x48, 0x41, 0x9d, 0x28, 0x91, 0x3c, 0x4b, 0xc2, 0xd8, 0x7f, 0x07, + 0x16, 0x13, 0x73, 0xea, 0x42, 0x7b, 0x4f, 0xc4, 0xc5, 0x2c, 0xc9, 0x5d, 0xe2, 0x59, 0x81, 0xcd, + 0x0c, 0xa4, 0x7f, 0x43, 0xf3, 0xb9, 0x94, 0x59, 0xee, 0x36, 0x3c, 0x2b, 0xe8, 0x8e, 0xfa, 0x43, + 0xf3, 0x74, 0xa8, 0xcc, 0x4c, 0x3b, 0x29, 0x05, 0xfb, 0x88, 0x2f, 0x72, 0xd7, 0xf2, 0xac, 0xa0, + 0xc3, 0xf0, 0xec, 0x3f, 0x05, 0xfb, 0x55, 0x18, 0x65, 0xb4, 0x0f, 0x8d, 0xc9, 0xbe, 0x4b, 0x3c, + 0x12, 0xd8, 0xac, 0x31, 0xd9, 0xa7, 0x5b, 0xd0, 0xdc, 0x13, 0x45, 0x22, 0xdd, 0x06, 0x9a, 0x34, + 0xa0, 0x9b, 0x60, 0x1d, 0xf1, 0x85, 0x6b, 0x79, 0x24, 0xe8, 0x30, 0x75, 0xf4, 0xff, 0x07, 0xe7, + 0x30, 0xe2, 0xf1, 0x54, 0xf1, 0xdb, 0x82, 0x26, 0x9e, 0x31, 0x4c, 0x87, 0x69, 0xa0, 0xac, 0x4c, + 0xcc, 0x27, 0xfb, 0x26, 0x12, 0x02, 0xff, 0x18, 0x3a, 0x2f, 0x32, 0x51, 0xa4, 0x2f, 0xa3, 0x84, + 0xd3, 0x7f, 0xa1, 0x85, 0x40, 0xeb, 0xea, 0x8e, 0x68, 0xc5, 0xdf, 0x04, 0x67, 0xe5, 0x0d, 0x15, + 0xee, 0x4c, 0xc8, 0x30, 0x36, 0xe1, 0x10, 0xf8, 0x23, 0x70, 0xce, 0xc3, 0x78, 0x49, 0xf2, 0x3c, + 0x8c, 0x91, 0x84, 0xc5, 0xd4, 0x71, 0x55, 0x8c, 0x55, 0x8a, 0xf1, 0xdf, 0x80, 0x35, 0x8e, 0x64, + 0xc5, 0x8f, 0xd4, 0xf8, 0xd1, 0x3f, 0xc1, 0xd1, 0xc9, 0x5d, 0x12, 0x5f, 0x62, 0xba, 0x03, 0x9d, + 0xb3, 0x68, 0xc6, 0x73, 0x19, 0xce, 0x52, 0xcc, 0x85, 0xc5, 0x2a, 0x83, 0xff, 0x16, 0x7a, 0xfa, + 0xa6, 0x4a, 0xfa, 0x29, 0x97, 0x0f, 0x52, 0xfb, 0x63, 0xc5, 0x7a, 0x98, 0xea, 0x8f, 0x04, 0x6c, + 0xe5, 0x33, 0x2e, 0xb2, 0x74, 0xa9, 0xca, 0x9e, 0x2d, 0x52, 0x5e, 0x32, 0xc5, 0x33, 0xf5, 0xa0, + 0x7b, 0x2a, 0xb3, 0x28, 0xb9, 0x3a, 0x0f, 0xe3, 0x82, 0x97, 0x81, 0xea, 0x26, 0xa5, 0x71, 0x92, + 0x48, 0xed, 0xb6, 0x51, 0xc6, 0x12, 0x2b, 0x8d, 0x63, 0x21, 0x62, 0xed, 0x6c, 0x7a, 0x24, 0x70, + 0x58, 0x65, 0xa0, 0x03, 0x80, 0xc3, 0x58, 0x84, 0xe5, 0xdb, 0x96, 0x47, 0x02, 0xc2, 0x6a, 0x16, + 0x7f, 0x17, 0xda, 0x8a, 0xe9, 0x71, 0x98, 0x56, 0x6a, 0xc9, 0x23, 0x6a, 0xfd, 0x3b, 0x02, 0x1b, + 0xaf, 0x0b, 0x9e, 0x2d, 0x18, 0x7f, 0x5f, 0xf0, 0x1c, 0xab, 0x82, 0xd8, 0xf4, 0x12, 0x02, 0xba, + 0x0d, 0xad, 0xd3, 0xeb, 0x30, 0x9b, 0xea, 0xdc, 0xd9, 0xac, 0x44, 0x4a, 0x6b, 0x95, 0xf3, 0x1c, + 0xb5, 0x3a, 0xac, 0x6e, 0x52, 0x2f, 0x19, 0x9f, 0x09, 0x69, 0xc4, 0x94, 0x88, 0x06, 0xf0, 0xeb, + 0xc1, 0xcd, 0x65, 0x5c, 0x4c, 0x39, 0x13, 0x73, 0xfd, 0xba, 0x85, 0x17, 0xd6, 0xcd, 0xf4, 0x1f, + 0xe8, 0x97, 0x26, 0x33, 0x84, 0x6d, 0xbc, 0xb8, 0x66, 0xf5, 0x6f, 0x09, 0xf4, 0x4a, 0x29, 0x79, + 0x2a, 0x92, 0x9c, 0xab, 0x7a, 0x1d, 0x64, 0x99, 0xa9, 0xd7, 0x41, 0x96, 0xd1, 0x5d, 0x68, 0x33, + 0x9e, 0x17, 0xb1, 0x34, 0x4d, 0xf0, 0x7b, 0x95, 0x16, 0xf3, 0xb6, 0x88, 0x25, 0x33, 0xb7, 0xe8, + 0x33, 0xe8, 0xaf, 0x34, 0x95, 0x1e, 0xe2, 0xee, 0xe8, 0x8f, 0xea, 0xdd, 0x8a, 0x9f, 0xad, 0x5d, + 0xf7, 0x6f, 0x1b, 0xd0, 0xad, 0x45, 0xa6, 0x7f, 0xe1, 0x4a, 0x41, 0x4e, 0xdd, 0x51, 0xaf, 0x8a, + 0xa2, 0x46, 0x0d, 0x97, 0xcd, 0x06, 0x90, 0x93, 0xb2, 0x9f, 0xc8, 0x89, 0xaa, 0xa2, 0x5a, 0x13, + 0xe6, 0xb3, 0xb5, 0x2a, 0x2a, 0x33, 0xd3, 0x4e, 0x5c, 0x50, 0xd7, 0x61, 0x72, 0xc5, 0xa7, 0xd8, + 0x4f, 0x0e, 0x33, 0x90, 0x0e, 0xab, 0xf9, 0xc4, 0x02, 0xac, 0xcc, 0xb8, 0xf1, 0xb0, 0x6a, 0x86, + 0x4d, 0x43, 0xab, 0x5a, 0xf4, 0xca, 0x86, 0x56, 0x25, 0x54, 0xb3, 0xa9, 0x12, 0x8f, 0xc5, 0xd7, + 0x88, 0x3e, 0x81, 0x1e, 0xee, 0x86, 0xf1, 0x02, 0xdf, 0xe6, 0xae, 0x83, 0x1c, 0x7f, 0xab, 0x3e, + 0xb0, 0xdc, 0x34, 0x6c, 0xf5, 0xa6, 0xff, 0x95, 0x40, 0x6f, 0x32, 0x4b, 0x45, 0x26, 0x6b, 0x7d, + 0x37, 0x49, 0xa6, 0xfc, 0xc6, 0xf4, 0x1d, 0x82, 0x6a, 0xb3, 0x35, 0xd6, 0x36, 0x1b, 0xf6, 0x1f, + 0xf6, 0x9b, 0xcd, 0x34, 0xa8, 0xd1, 0xb4, 0x57, 0x68, 0xee, 0x40, 0xc7, 0x6c, 0x90, 0xdc, 0x6d, + 0xa2, 0xab, 0x32, 0xa8, 0x89, 0x5a, 0xae, 0x10, 0xd5, 0x82, 0x56, 0x60, 0xb1, 0x9a, 0x45, 0xa5, + 0x96, 0x89, 0x39, 0xae, 0xef, 0x36, 0xae, 0x6f, 0x03, 0xd5, 0x4b, 0x1d, 0x06, 0x9d, 0x0e, 0x3a, + 0x6b, 0x16, 0xff, 0x13, 0x01, 0xaa, 0x35, 0xe2, 0x6c, 0xfe, 0x3c, 0xa1, 0x8f, 0x0b, 0xda, 0x86, + 0x16, 0x7e, 0xcf, 0x88, 0x29, 0xd1, 0x1a, 0xdd, 0xf6, 0x3a, 0xdd, 0xf1, 0xe6, 0xdd, 0xfd, 0x80, + 0x7c, 0xbe, 0x1f, 0x90, 0x2f, 0xf7, 0x03, 0xf2, 0xe1, 0xdb, 0xe0, 0x97, 0x8b, 0x16, 0xfe, 0x0e, + 0xff, 0xfb, 0x1e, 0x00, 0x00, 0xff, 0xff, 0x1c, 0xa1, 0x99, 0xa7, 0x1e, 0x07, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index f3f362fd1..a7a6a20f0 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -14,9 +14,14 @@ message Pair { uint64 Count = 2; } +message FieldRow{ + string Field = 1; + uint64 RowID = 2; +} + message GroupLine{ - repeated string Groups = 1; - uint64 Total=2; + repeated FieldRow Groups = 1; + uint64 Total = 2; } message ValCount { From de071d548a59f9f7c10651e85b88beb185fc0056 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 24 Aug 2018 15:28:18 -0500 Subject: [PATCH 03/39] remove additional decodeFieldRow (and hopefully allocation) --- encoding/proto/proto.go | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 4f886acce..ba89a1b95 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -1012,18 +1012,12 @@ func decodeGroupByCounts(a []*internal.GroupLine) pilosa.GroupByCounts { func decodeFieldRows(a []*internal.FieldRow) []pilosa.FieldRow { other := make([]pilosa.FieldRow, len(a)) for i := range a { - other[i] = decodeFieldRow(a[i]) + other[i].Field = a[i].Field + other[i].RowID = a[i].RowID } return other } -func decodeFieldRow(pb *internal.FieldRow) pilosa.FieldRow { - return pilosa.FieldRow{ - Field: pb.Field, - RowID: pb.RowID, - } -} - func decodePairs(a []*internal.Pair) []pilosa.Pair { other := make([]pilosa.Pair, len(a)) for i := range a { @@ -1089,18 +1083,14 @@ func encodeGroupByCount(counts pilosa.GroupByCounts) []*internal.GroupLine { func encodeFieldRows(a []pilosa.FieldRow) []*internal.FieldRow { other := make([]*internal.FieldRow, len(a)) for i := range a { - other[i] = encodeFieldRow(a[i]) + other[i] = &internal.FieldRow{ + Field: a[i].Field, + RowID: a[i].RowID, + } } return other } -func encodeFieldRow(p pilosa.FieldRow) *internal.FieldRow { - return &internal.FieldRow{ - Field: p.Field, - RowID: p.RowID, - } -} - func encodePairs(a pilosa.Pairs) []*internal.Pair { other := make([]*internal.Pair, len(a)) for i := range a { From 96b408636065691998eaca077293de5a076383b4 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 22 Aug 2018 14:58:50 -0500 Subject: [PATCH 04/39] plug in in translation. adjust output format. --- encoding/proto/proto.go | 8 +-- executor.go | 58 +++++++++++++---- internal/public.pb.go | 140 ++++++++++++++++++++-------------------- internal/public.proto | 4 +- 4 files changed, 120 insertions(+), 90 deletions(-) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index ba89a1b95..4df552307 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -1002,8 +1002,8 @@ func decodeGroupByCounts(a []*internal.GroupLine) pilosa.GroupByCounts { other := make([]pilosa.GroupLine, len(a)) for i := range a { other[i] = pilosa.GroupLine{ - decodeFieldRows(a[i].Groups), - a[i].Total, + decodeFieldRows(a[i].Group), + a[i].Count, } } return pilosa.GroupByCounts(other) @@ -1073,8 +1073,8 @@ func encodeGroupByCount(counts pilosa.GroupByCounts) []*internal.GroupLine { result := make([]*internal.GroupLine, len(counts)) for i := range counts { result[i] = &internal.GroupLine{ - Groups: encodeFieldRows(counts[i].Groups), - Total: counts[i].Total, + Group: encodeFieldRows(counts[i].Group), + Count: counts[i].Count, } } return result diff --git a/executor.go b/executor.go index 35740866b..6759ba3ea 100644 --- a/executor.go +++ b/executor.go @@ -788,9 +788,9 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call // FieldRow is used to distinguish rows in a group by result. type FieldRow struct { - Field string - RowID uint64 - RowKey string + Field string `json:"field"` + RowID uint64 `json:"rowID"` + RowKey string `json:"rowKey,omitempty"` } func (fr FieldRow) String() string { @@ -812,8 +812,8 @@ type gbi struct { fieldRow FieldRow } type GroupLine struct { - Groups []FieldRow - Total uint64 + Group []FieldRow `json:"group"` + Count uint64 `json:"count"` } // GroupByCounts is the return type for GroupBy queries. @@ -825,15 +825,15 @@ func (gbc GroupByCounts) Merge(other GroupByCounts) GroupByCounts { total uint64 }) for i := range gbc { - m[uniqueGroupString(gbc[i].Groups)] = struct { + m[uniqueGroupString(gbc[i].Group)] = struct { i int total uint64 }{i, gbc[i].Total} } for i := range other { - o, found := m[uniqueGroupString(other[i].Groups)] + o, found := m[uniqueGroupString(other[i].Group)] if found { - gbc[o.i].Total += other[i].Total + gbc[o.i].Count += other[i].Count } else { gbc = append(gbc, other[i]) } @@ -901,8 +901,8 @@ func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql work = append(work, set) } for _, group := range product(work) { - group.gl.Total = group.row.Count() - if group.gl.Total > 0 { + group.gl.Count = group.row.Count() + if group.gl.Count > 0 { results = append(results, group.gl) } } @@ -920,7 +920,7 @@ type ppi struct { func product(input [][]gbi) []ppi { if len(input) == 0 { // base return empty list return []ppi{ - {gl: GroupLine{Groups: make([]FieldRow, 0)}}, + {gl: GroupLine{Group: make([]FieldRow, 0)}}, } } @@ -929,9 +929,9 @@ func product(input [][]gbi) []ppi { tail := product(input[1:]) // invoke product on remaining element for h := range head { // for each head for t := range tail { // iterate over the tail - s := ppi{gl: GroupLine{Groups: make([]FieldRow, 0)}} - s.gl.Groups = append([]FieldRow{head[h].fieldRow}, tail[t].gl.Groups...) // had to insert at the front to match input order - if tail[t].row != nil { // first time around nothing to intersect + s := ppi{gl: GroupLine{Group: make([]FieldRow, 0)}} + s.gl.Group = append([]FieldRow{head[h].fieldRow}, tail[t].gl.Group...) // had to insert at the front to match input order + if tail[t].row != nil { // first time around nothing to intersect s.row = head[h].row.Intersect(tail[t].row) } else { s.row = head[h].row @@ -2091,7 +2091,37 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res return other, nil } } + + case GroupByCounts: + other := make([]GroupLine, 0) + for _, gl := range result { + + group := make([]FieldRow, len(gl.Group)) + for i, g := range gl.Group { + group[i] = g + + // TODO: It may be useful to cache this field lookup. + field := idx.Field(g.Field) + if field == nil { + return nil, ErrFieldNotFound + } + if field.keys() { + key, err := e.TranslateStore.TranslateRowToString(index, g.Field, g.RowID) + if err != nil { + return nil, err + } + group[i].RowKey = key + } + } + + other = append(other, GroupLine{ + Group: group, + Count: gl.Count, + }) + } + return other, nil } + return result, nil } diff --git a/internal/public.pb.go b/internal/public.pb.go index 2203c87d6..29254543a 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -133,8 +133,8 @@ func (m *FieldRow) GetRowID() uint64 { } type GroupLine struct { - Groups []*FieldRow `protobuf:"bytes,1,rep,name=Groups" json:"Groups,omitempty"` - Total uint64 `protobuf:"varint,2,opt,name=Total,proto3" json:"Total,omitempty"` + Group []*FieldRow `protobuf:"bytes,1,rep,name=Group" json:"Group,omitempty"` + Count uint64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` } func (m *GroupLine) Reset() { *m = GroupLine{} } @@ -142,16 +142,16 @@ func (m *GroupLine) String() string { return proto.CompactTextString( func (*GroupLine) ProtoMessage() {} func (*GroupLine) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} } -func (m *GroupLine) GetGroups() []*FieldRow { +func (m *GroupLine) GetGroup() []*FieldRow { if m != nil { - return m.Groups + return m.Group } return nil } -func (m *GroupLine) GetTotal() uint64 { +func (m *GroupLine) GetCount() uint64 { if m != nil { - return m.Total + return m.Count } return 0 } @@ -760,8 +760,8 @@ func (m *GroupLine) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.Groups) > 0 { - for _, msg := range m.Groups { + if len(m.Group) > 0 { + for _, msg := range m.Group { dAtA[i] = 0xa i++ i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) @@ -772,10 +772,10 @@ func (m *GroupLine) MarshalTo(dAtA []byte) (int, error) { i += n } } - if m.Total != 0 { + if m.Count != 0 { dAtA[i] = 0x10 i++ - i = encodeVarintPublic(dAtA, i, uint64(m.Total)) + i = encodeVarintPublic(dAtA, i, uint64(m.Count)) } return i, nil } @@ -1464,14 +1464,14 @@ func (m *FieldRow) Size() (n int) { func (m *GroupLine) Size() (n int) { var l int _ = l - if len(m.Groups) > 0 { - for _, e := range m.Groups { + if len(m.Group) > 0 { + for _, e := range m.Group { l = e.Size() n += 1 + l + sovPublic(uint64(l)) } } - if m.Total != 0 { - n += 1 + sovPublic(uint64(m.Total)) + if m.Count != 0 { + n += 1 + sovPublic(uint64(m.Count)) } return n } @@ -2171,7 +2171,7 @@ func (m *GroupLine) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Groups", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Group", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -2195,16 +2195,16 @@ func (m *GroupLine) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Groups = append(m.Groups, &FieldRow{}) - if err := m.Groups[len(m.Groups)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.Group = append(m.Group, &FieldRow{}) + if err := m.Group[len(m.Group)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Total", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Count", wireType) } - m.Total = 0 + m.Count = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPublic @@ -2214,7 +2214,7 @@ func (m *GroupLine) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Total |= (uint64(b) & 0x7F) << shift + m.Count |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -4241,54 +4241,54 @@ var ( func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ - // 783 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xdd, 0x6a, 0xdb, 0x48, - 0x14, 0xde, 0xb1, 0x64, 0x5b, 0x3e, 0x8e, 0xbd, 0x61, 0x36, 0x9b, 0x15, 0x4b, 0xf0, 0x0a, 0xb1, - 0x2c, 0x62, 0x2f, 0x1c, 0xf0, 0xc2, 0x42, 0x6f, 0x5a, 0xea, 0xfc, 0x14, 0x93, 0x26, 0xb4, 0x93, - 0x34, 0xa5, 0x97, 0x4a, 0x3c, 0x24, 0x02, 0x59, 0xa3, 0x4a, 0x23, 0x1c, 0x3f, 0x47, 0x6e, 0xfa, - 0x08, 0xbd, 0xe8, 0x83, 0xe4, 0xb2, 0x8f, 0xd0, 0xa6, 0x2f, 0x52, 0xe6, 0x8c, 0xc6, 0x92, 0x1d, - 0x08, 0xbd, 0xe8, 0xdd, 0x7c, 0xe7, 0xcc, 0x1c, 0x7d, 0xdf, 0xf9, 0x13, 0x6c, 0xa4, 0xc5, 0x45, - 0x1c, 0x5d, 0x0e, 0xd3, 0x4c, 0x48, 0x41, 0x9d, 0x28, 0x91, 0x3c, 0x4b, 0xc2, 0xd8, 0x7f, 0x07, - 0x16, 0x13, 0x73, 0xea, 0x42, 0x7b, 0x4f, 0xc4, 0xc5, 0x2c, 0xc9, 0x5d, 0xe2, 0x59, 0x81, 0xcd, - 0x0c, 0xa4, 0x7f, 0x43, 0xf3, 0xb9, 0x94, 0x59, 0xee, 0x36, 0x3c, 0x2b, 0xe8, 0x8e, 0xfa, 0x43, - 0xf3, 0x74, 0xa8, 0xcc, 0x4c, 0x3b, 0x29, 0x05, 0xfb, 0x88, 0x2f, 0x72, 0xd7, 0xf2, 0xac, 0xa0, - 0xc3, 0xf0, 0xec, 0x3f, 0x05, 0xfb, 0x55, 0x18, 0x65, 0xb4, 0x0f, 0x8d, 0xc9, 0xbe, 0x4b, 0x3c, - 0x12, 0xd8, 0xac, 0x31, 0xd9, 0xa7, 0x5b, 0xd0, 0xdc, 0x13, 0x45, 0x22, 0xdd, 0x06, 0x9a, 0x34, - 0xa0, 0x9b, 0x60, 0x1d, 0xf1, 0x85, 0x6b, 0x79, 0x24, 0xe8, 0x30, 0x75, 0xf4, 0xff, 0x07, 0xe7, - 0x30, 0xe2, 0xf1, 0x54, 0xf1, 0xdb, 0x82, 0x26, 0x9e, 0x31, 0x4c, 0x87, 0x69, 0xa0, 0xac, 0x4c, - 0xcc, 0x27, 0xfb, 0x26, 0x12, 0x02, 0xff, 0x18, 0x3a, 0x2f, 0x32, 0x51, 0xa4, 0x2f, 0xa3, 0x84, - 0xd3, 0x7f, 0xa1, 0x85, 0x40, 0xeb, 0xea, 0x8e, 0x68, 0xc5, 0xdf, 0x04, 0x67, 0xe5, 0x0d, 0x15, - 0xee, 0x4c, 0xc8, 0x30, 0x36, 0xe1, 0x10, 0xf8, 0x23, 0x70, 0xce, 0xc3, 0x78, 0x49, 0xf2, 0x3c, - 0x8c, 0x91, 0x84, 0xc5, 0xd4, 0x71, 0x55, 0x8c, 0x55, 0x8a, 0xf1, 0xdf, 0x80, 0x35, 0x8e, 0x64, - 0xc5, 0x8f, 0xd4, 0xf8, 0xd1, 0x3f, 0xc1, 0xd1, 0xc9, 0x5d, 0x12, 0x5f, 0x62, 0xba, 0x03, 0x9d, - 0xb3, 0x68, 0xc6, 0x73, 0x19, 0xce, 0x52, 0xcc, 0x85, 0xc5, 0x2a, 0x83, 0xff, 0x16, 0x7a, 0xfa, - 0xa6, 0x4a, 0xfa, 0x29, 0x97, 0x0f, 0x52, 0xfb, 0x63, 0xc5, 0x7a, 0x98, 0xea, 0x8f, 0x04, 0x6c, - 0xe5, 0x33, 0x2e, 0xb2, 0x74, 0xa9, 0xca, 0x9e, 0x2d, 0x52, 0x5e, 0x32, 0xc5, 0x33, 0xf5, 0xa0, - 0x7b, 0x2a, 0xb3, 0x28, 0xb9, 0x3a, 0x0f, 0xe3, 0x82, 0x97, 0x81, 0xea, 0x26, 0xa5, 0x71, 0x92, - 0x48, 0xed, 0xb6, 0x51, 0xc6, 0x12, 0x2b, 0x8d, 0x63, 0x21, 0x62, 0xed, 0x6c, 0x7a, 0x24, 0x70, - 0x58, 0x65, 0xa0, 0x03, 0x80, 0xc3, 0x58, 0x84, 0xe5, 0xdb, 0x96, 0x47, 0x02, 0xc2, 0x6a, 0x16, - 0x7f, 0x17, 0xda, 0x8a, 0xe9, 0x71, 0x98, 0x56, 0x6a, 0xc9, 0x23, 0x6a, 0xfd, 0x3b, 0x02, 0x1b, - 0xaf, 0x0b, 0x9e, 0x2d, 0x18, 0x7f, 0x5f, 0xf0, 0x1c, 0xab, 0x82, 0xd8, 0xf4, 0x12, 0x02, 0xba, - 0x0d, 0xad, 0xd3, 0xeb, 0x30, 0x9b, 0xea, 0xdc, 0xd9, 0xac, 0x44, 0x4a, 0x6b, 0x95, 0xf3, 0x1c, - 0xb5, 0x3a, 0xac, 0x6e, 0x52, 0x2f, 0x19, 0x9f, 0x09, 0x69, 0xc4, 0x94, 0x88, 0x06, 0xf0, 0xeb, - 0xc1, 0xcd, 0x65, 0x5c, 0x4c, 0x39, 0x13, 0x73, 0xfd, 0xba, 0x85, 0x17, 0xd6, 0xcd, 0xf4, 0x1f, - 0xe8, 0x97, 0x26, 0x33, 0x84, 0x6d, 0xbc, 0xb8, 0x66, 0xf5, 0x6f, 0x09, 0xf4, 0x4a, 0x29, 0x79, - 0x2a, 0x92, 0x9c, 0xab, 0x7a, 0x1d, 0x64, 0x99, 0xa9, 0xd7, 0x41, 0x96, 0xd1, 0x5d, 0x68, 0x33, - 0x9e, 0x17, 0xb1, 0x34, 0x4d, 0xf0, 0x7b, 0x95, 0x16, 0xf3, 0xb6, 0x88, 0x25, 0x33, 0xb7, 0xe8, - 0x33, 0xe8, 0xaf, 0x34, 0x95, 0x1e, 0xe2, 0xee, 0xe8, 0x8f, 0xea, 0xdd, 0x8a, 0x9f, 0xad, 0x5d, - 0xf7, 0x6f, 0x1b, 0xd0, 0xad, 0x45, 0xa6, 0x7f, 0xe1, 0x4a, 0x41, 0x4e, 0xdd, 0x51, 0xaf, 0x8a, - 0xa2, 0x46, 0x0d, 0x97, 0xcd, 0x06, 0x90, 0x93, 0xb2, 0x9f, 0xc8, 0x89, 0xaa, 0xa2, 0x5a, 0x13, - 0xe6, 0xb3, 0xb5, 0x2a, 0x2a, 0x33, 0xd3, 0x4e, 0x5c, 0x50, 0xd7, 0x61, 0x72, 0xc5, 0xa7, 0xd8, - 0x4f, 0x0e, 0x33, 0x90, 0x0e, 0xab, 0xf9, 0xc4, 0x02, 0xac, 0xcc, 0xb8, 0xf1, 0xb0, 0x6a, 0x86, - 0x4d, 0x43, 0xab, 0x5a, 0xf4, 0xca, 0x86, 0x56, 0x25, 0x54, 0xb3, 0xa9, 0x12, 0x8f, 0xc5, 0xd7, - 0x88, 0x3e, 0x81, 0x1e, 0xee, 0x86, 0xf1, 0x02, 0xdf, 0xe6, 0xae, 0x83, 0x1c, 0x7f, 0xab, 0x3e, - 0xb0, 0xdc, 0x34, 0x6c, 0xf5, 0xa6, 0xff, 0x95, 0x40, 0x6f, 0x32, 0x4b, 0x45, 0x26, 0x6b, 0x7d, - 0x37, 0x49, 0xa6, 0xfc, 0xc6, 0xf4, 0x1d, 0x82, 0x6a, 0xb3, 0x35, 0xd6, 0x36, 0x1b, 0xf6, 0x1f, - 0xf6, 0x9b, 0xcd, 0x34, 0xa8, 0xd1, 0xb4, 0x57, 0x68, 0xee, 0x40, 0xc7, 0x6c, 0x90, 0xdc, 0x6d, - 0xa2, 0xab, 0x32, 0xa8, 0x89, 0x5a, 0xae, 0x10, 0xd5, 0x82, 0x56, 0x60, 0xb1, 0x9a, 0x45, 0xa5, - 0x96, 0x89, 0x39, 0xae, 0xef, 0x36, 0xae, 0x6f, 0x03, 0xd5, 0x4b, 0x1d, 0x06, 0x9d, 0x0e, 0x3a, - 0x6b, 0x16, 0xff, 0x13, 0x01, 0xaa, 0x35, 0xe2, 0x6c, 0xfe, 0x3c, 0xa1, 0x8f, 0x0b, 0xda, 0x86, - 0x16, 0x7e, 0xcf, 0x88, 0x29, 0xd1, 0x1a, 0xdd, 0xf6, 0x3a, 0xdd, 0xf1, 0xe6, 0xdd, 0xfd, 0x80, - 0x7c, 0xbe, 0x1f, 0x90, 0x2f, 0xf7, 0x03, 0xf2, 0xe1, 0xdb, 0xe0, 0x97, 0x8b, 0x16, 0xfe, 0x0e, - 0xff, 0xfb, 0x1e, 0x00, 0x00, 0xff, 0xff, 0x1c, 0xa1, 0x99, 0xa7, 0x1e, 0x07, 0x00, 0x00, + // 771 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6e, 0xd3, 0x4c, + 0x14, 0xfe, 0x27, 0x76, 0x12, 0xe7, 0xa4, 0xc9, 0x5f, 0x0d, 0xa5, 0x58, 0xa8, 0x0a, 0x96, 0x85, + 0x90, 0x57, 0xa9, 0x14, 0x24, 0x24, 0x36, 0x20, 0xd2, 0x0b, 0x8a, 0x0a, 0x15, 0x4c, 0x4b, 0x11, + 0x4b, 0xb7, 0x19, 0xb5, 0x96, 0x1c, 0x8f, 0xf1, 0x45, 0x69, 0x9e, 0xa3, 0x1b, 0x1e, 0x81, 0x05, + 0x0f, 0xd2, 0x25, 0x8f, 0x00, 0xe5, 0x45, 0xd0, 0x9c, 0xf1, 0xd8, 0x4e, 0x8a, 0x2a, 0x16, 0xec, + 0xe6, 0xfb, 0xce, 0x9c, 0xf1, 0x77, 0xae, 0x86, 0xb5, 0x38, 0x3f, 0x0d, 0x83, 0xb3, 0x61, 0x9c, + 0x88, 0x4c, 0x50, 0x2b, 0x88, 0x32, 0x9e, 0x44, 0x7e, 0xe8, 0x7e, 0x02, 0x83, 0x89, 0x39, 0xb5, + 0xa1, 0xbd, 0x23, 0xc2, 0x7c, 0x16, 0xa5, 0x36, 0x71, 0x0c, 0xcf, 0x64, 0x1a, 0xd2, 0xc7, 0xd0, + 0x7c, 0x95, 0x65, 0x49, 0x6a, 0x37, 0x1c, 0xc3, 0xeb, 0x8e, 0xfa, 0x43, 0xed, 0x3a, 0x94, 0x34, + 0x53, 0x46, 0x4a, 0xc1, 0x3c, 0xe0, 0x8b, 0xd4, 0x36, 0x1c, 0xc3, 0xeb, 0x30, 0x3c, 0xbb, 0x2f, + 0xc0, 0x7c, 0xe7, 0x07, 0x09, 0xed, 0x43, 0x63, 0xb2, 0x6b, 0x13, 0x87, 0x78, 0x26, 0x6b, 0x4c, + 0x76, 0xe9, 0x06, 0x34, 0x77, 0x44, 0x1e, 0x65, 0x76, 0x03, 0x29, 0x05, 0xe8, 0x3a, 0x18, 0x07, + 0x7c, 0x61, 0x1b, 0x0e, 0xf1, 0x3a, 0x4c, 0x1e, 0xdd, 0x67, 0x60, 0xed, 0x07, 0x3c, 0x9c, 0x4a, + 0x7d, 0x1b, 0xd0, 0xc4, 0x33, 0x3e, 0xd3, 0x61, 0x0a, 0x48, 0x96, 0x89, 0xf9, 0x64, 0x57, 0xbf, + 0x84, 0xc0, 0x3d, 0x80, 0xce, 0xeb, 0x44, 0xe4, 0xf1, 0x9b, 0x20, 0xe2, 0xd4, 0x83, 0x26, 0x02, + 0x0c, 0xab, 0x3b, 0xa2, 0x95, 0x7c, 0xfd, 0x36, 0x53, 0x17, 0xfe, 0x2c, 0xcb, 0x1d, 0x81, 0x75, + 0xe2, 0x87, 0xa5, 0xc4, 0x13, 0x3f, 0x44, 0x09, 0x06, 0x93, 0xc7, 0x65, 0x1f, 0x43, 0xfb, 0x7c, + 0x00, 0x63, 0x1c, 0x64, 0x95, 0x3a, 0x52, 0x53, 0x47, 0x1f, 0x82, 0xa5, 0x52, 0x5b, 0xca, 0x2e, + 0x31, 0xdd, 0x82, 0xce, 0x71, 0x30, 0xe3, 0x69, 0xe6, 0xcf, 0x62, 0xcc, 0x84, 0xc1, 0x2a, 0xc2, + 0xfd, 0x08, 0x3d, 0x75, 0x53, 0xa6, 0xfc, 0x88, 0x67, 0xb7, 0x12, 0xfb, 0x77, 0xa5, 0xba, 0x9d, + 0xe8, 0xaf, 0x04, 0x4c, 0x69, 0xd3, 0x26, 0x52, 0x9a, 0x64, 0x5d, 0x8f, 0x17, 0x31, 0x2f, 0x94, + 0xe2, 0x99, 0x3a, 0xd0, 0x3d, 0xca, 0x92, 0x20, 0x3a, 0x3f, 0xf1, 0xc3, 0x9c, 0x17, 0x0f, 0xd5, + 0x29, 0x19, 0xe3, 0x24, 0xca, 0x94, 0xd9, 0xc4, 0x30, 0x4a, 0x2c, 0x63, 0x1c, 0x0b, 0x11, 0x2a, + 0x63, 0xd3, 0x21, 0x9e, 0xc5, 0x2a, 0x82, 0x0e, 0x00, 0xf6, 0x43, 0xe1, 0x17, 0xbe, 0x2d, 0x87, + 0x78, 0x84, 0xd5, 0x18, 0x77, 0x1b, 0xda, 0x52, 0xe9, 0x5b, 0x3f, 0xae, 0xa2, 0x25, 0x77, 0x44, + 0xeb, 0x5e, 0x13, 0x58, 0x7b, 0x9f, 0xf3, 0x64, 0xc1, 0xf8, 0xe7, 0x9c, 0xa7, 0x58, 0x15, 0xc4, + 0xba, 0x93, 0x10, 0xd0, 0x4d, 0x68, 0x1d, 0x5d, 0xf8, 0xc9, 0x54, 0xe5, 0xce, 0x64, 0x05, 0x92, + 0xb1, 0x56, 0x39, 0x4f, 0x31, 0x56, 0x8b, 0xd5, 0x29, 0xe9, 0xc9, 0xf8, 0x4c, 0x64, 0x3a, 0x98, + 0x02, 0x51, 0x0f, 0xfe, 0xdf, 0xbb, 0x3c, 0x0b, 0xf3, 0x29, 0x67, 0x62, 0xae, 0xbc, 0x5b, 0x78, + 0x61, 0x95, 0xa6, 0x4f, 0xa0, 0x5f, 0x50, 0x7a, 0x04, 0xdb, 0x78, 0x71, 0x85, 0x75, 0xaf, 0x08, + 0xf4, 0x8a, 0x50, 0xd2, 0x58, 0x44, 0x29, 0x97, 0xf5, 0xda, 0x4b, 0x12, 0x5d, 0xaf, 0xbd, 0x24, + 0xa1, 0xdb, 0xd0, 0x66, 0x3c, 0xcd, 0xc3, 0x4c, 0x37, 0xc1, 0xfd, 0x2a, 0x2d, 0xda, 0x37, 0x0f, + 0x33, 0xa6, 0x6f, 0xd1, 0x97, 0xd0, 0x5f, 0x6a, 0x2a, 0x35, 0xc2, 0xdd, 0xd1, 0x83, 0xca, 0x6f, + 0xc9, 0xce, 0x56, 0xae, 0xbb, 0x57, 0x0d, 0xe8, 0xd6, 0x5e, 0xa6, 0x8f, 0x70, 0xa1, 0xa0, 0xa6, + 0xee, 0xa8, 0x57, 0xbd, 0x22, 0x27, 0x0d, 0x57, 0xcd, 0x1a, 0x90, 0xc3, 0xa2, 0x9f, 0xc8, 0xa1, + 0xac, 0xa2, 0x5c, 0x12, 0xfa, 0xb3, 0xb5, 0x2a, 0x4a, 0x9a, 0x29, 0x23, 0xae, 0xa7, 0x0b, 0x3f, + 0x3a, 0xe7, 0x53, 0xec, 0x27, 0x8b, 0x69, 0x48, 0x87, 0xd5, 0x7c, 0x62, 0x01, 0x96, 0x46, 0x5c, + 0x5b, 0x58, 0x35, 0xc3, 0xba, 0xa1, 0x65, 0x2d, 0x7a, 0x45, 0x43, 0xcb, 0x12, 0xca, 0xd9, 0x94, + 0x89, 0xc7, 0xe2, 0x2b, 0x44, 0x9f, 0x43, 0x0f, 0x57, 0xc3, 0x78, 0x81, 0xbe, 0xa9, 0x6d, 0xa1, + 0xc6, 0x7b, 0xd5, 0x07, 0xca, 0x3d, 0xc3, 0x96, 0x6f, 0xba, 0x3f, 0x09, 0xf4, 0x26, 0xb3, 0x58, + 0x24, 0x59, 0xad, 0xef, 0x26, 0xd1, 0x94, 0x5f, 0xea, 0xbe, 0x43, 0x50, 0xed, 0xb5, 0xc6, 0xca, + 0x5e, 0xc3, 0xfe, 0xc3, 0x7e, 0x33, 0x99, 0x02, 0x35, 0x99, 0xe6, 0x92, 0xcc, 0x2d, 0xe8, 0xe8, + 0x0d, 0x92, 0xda, 0x4d, 0x34, 0x55, 0x84, 0x9c, 0xa8, 0x72, 0x85, 0xc8, 0x16, 0x34, 0x3c, 0x83, + 0xd5, 0x18, 0x99, 0x5a, 0x26, 0xe6, 0xb8, 0xbc, 0xdb, 0xb8, 0xbc, 0x35, 0x94, 0x9e, 0xea, 0x19, + 0x34, 0x5a, 0x68, 0xac, 0x31, 0xee, 0x37, 0x02, 0x54, 0xc5, 0x88, 0xb3, 0xf9, 0xef, 0x02, 0xbd, + 0x3b, 0xa0, 0x4d, 0x68, 0xe1, 0xf7, 0x74, 0x30, 0x05, 0x5a, 0x91, 0xdb, 0x5e, 0x95, 0x3b, 0x5e, + 0xbf, 0xbe, 0x19, 0x90, 0xef, 0x37, 0x03, 0xf2, 0xe3, 0x66, 0x40, 0xbe, 0xfc, 0x1a, 0xfc, 0x77, + 0xda, 0xc2, 0x9f, 0xe1, 0xd3, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x21, 0xc1, 0x72, 0xc6, 0x1c, + 0x07, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index a7a6a20f0..b207e3cae 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -20,8 +20,8 @@ message FieldRow{ } message GroupLine{ - repeated FieldRow Groups = 1; - uint64 Total = 2; + repeated FieldRow Group = 1; + uint64 Count = 2; } message ValCount { From b6d386ba53a6a4eca90580615bf7c31cc9caff13 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 30 Aug 2018 15:09:04 -0600 Subject: [PATCH 05/39] fix tests --- executor.go | 6 +++--- executor_test.go | 30 +++++++++++++++--------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/executor.go b/executor.go index 6759ba3ea..5f2cdc75f 100644 --- a/executor.go +++ b/executor.go @@ -822,13 +822,13 @@ type GroupByCounts []GroupLine func (gbc GroupByCounts) Merge(other GroupByCounts) GroupByCounts { m := make(map[string]struct { i int - total uint64 + count uint64 }) for i := range gbc { m[uniqueGroupString(gbc[i].Group)] = struct { i int - total uint64 - }{i, gbc[i].Total} + count uint64 + }{i, gbc[i].Count} } for i := range other { o, found := m[uniqueGroupString(other[i].Group)] diff --git a/executor_test.go b/executor_test.go index e3afda3ed..7261611db 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1234,10 +1234,10 @@ Set(4500001, fn=4) t.Fatalf("GroupBy querying: %v", err) } else { expected := pilosa.GroupByCounts{ - {Groups: []pilosa.FieldRow{{Field: "f", RowID: 10}}, Total: 4}, - {Groups: []pilosa.FieldRow{{Field: "f", RowID: 7}}, Total: 1}, + {Group: []pilosa.FieldRow{{Field: "f", RowID: 10}}, Count: 4}, + {Group: []pilosa.FieldRow{{Field: "f", RowID: 7}}, Count: 1}, } - results := res.Results[0].(pilosa.GroupByCounts) + results := res.Results[0].([]pilosa.GroupLine) checkGroupBy(t, expected, results) } }) @@ -1713,43 +1713,43 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { }) t.Run("Basic", func(t *testing.T) { expected := pilosa.GroupByCounts{ - {Groups: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Total: 1}, - {Groups: []pilosa.FieldRow{{Field: "general", RowID: 11}, {Field: "sub", RowID: 110}}, Total: 1}, - {Groups: []pilosa.FieldRow{{Field: "general", RowID: 12}, {Field: "sub", RowID: 110}}, Total: 1}, - {Groups: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Total: 3}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}, {Field: "sub", RowID: 110}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 12}, {Field: "sub", RowID: 110}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Count: 3}, } if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(fields=[general,sub])`}); err != nil { t.Fatal(err) } else { - results := res.Results[0].(pilosa.GroupByCounts) + results := res.Results[0].([]pilosa.GroupLine) checkGroupBy(t, expected, results) } }) t.Run("check field offset no limit", func(t *testing.T) { expected := pilosa.GroupByCounts{ - {Groups: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Total: 2}, - {Groups: []pilosa.FieldRow{{Field: "general", RowID: 12}}, Total: 2}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Count: 2}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 12}}, Count: 2}, } if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(fields=[general:11:])`}); err != nil { t.Fatal(err) } else { - results := res.Results[0].(pilosa.GroupByCounts) + results := res.Results[0].([]pilosa.GroupLine) checkGroupBy(t, expected, results) } }) t.Run("check field offset limit", func(t *testing.T) { expected := pilosa.GroupByCounts{ - {Groups: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Total: 2}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Count: 2}, } if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(fields=[general:11:1])`}); err != nil { t.Fatal(err) } else { - results := res.Results[0].(pilosa.GroupByCounts) + results := res.Results[0].([]pilosa.GroupLine) checkGroupBy(t, expected, results) } }) @@ -1758,8 +1758,8 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { func checkGroupBy(t *testing.T, expected, results pilosa.GroupByCounts) { notIn := func(item pilosa.GroupLine, expected pilosa.GroupByCounts) bool { for i := range expected { - if item.Total == expected[i].Total { - if reflect.DeepEqual(item.Groups, expected[i].Groups) { + if item.Count == expected[i].Count { + if reflect.DeepEqual(item.Group, expected[i].Group) { return false } } From f0666b2be01008292a3cc56c3da76388d4a1eff9 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 4 Sep 2018 15:56:26 -0500 Subject: [PATCH 06/39] change GroupByCounts to []GroupCount --- encoding/proto/proto.go | 26 +++---- executor.go | 59 ++++++++------- executor_test.go | 20 ++--- internal/public.pb.go | 163 ++++++++++++++++++++-------------------- internal/public.proto | 4 +- 5 files changed, 136 insertions(+), 136 deletions(-) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 4df552307..6273f2c7f 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -363,9 +363,9 @@ func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse { case pilosa.RowIDs: pb.Results[i].Type = queryResultTypeRowIDs pb.Results[i].RowIDs = result - case pilosa.GroupByCounts: - pb.Results[i].Type = queryResultTypeGroupByCounts - pb.Results[i].GroupByCounts = encodeGroupByCount(result) + case []pilosa.GroupCount: + pb.Results[i].Type = queryResultTypeGroupCounts + pb.Results[i].GroupCounts = encodeGroupCounts(result) case nil: pb.Results[i].Type = queryResultTypeNil } @@ -929,7 +929,7 @@ const ( queryResultTypeUint64 queryResultTypeBool queryResultTypeRowIDs - queryResultTypeGroupByCounts + queryResultTypeGroupCounts ) func decodeQueryResult(pb *internal.QueryResult) interface{} { @@ -946,8 +946,8 @@ func decodeQueryResult(pb *internal.QueryResult) interface{} { return pb.Changed case queryResultTypeNil: return nil - case queryResultTypeGroupByCounts: - return decodeGroupByCounts(pb.GroupByCounts) + case queryResultTypeGroupCounts: + return decodeGroupCounts(pb.GroupCounts) } panic(fmt.Sprintf("unknown type: %d", pb.Type)) } @@ -998,15 +998,15 @@ func decodeAttr(attr *internal.Attr) (key string, value interface{}) { } } -func decodeGroupByCounts(a []*internal.GroupLine) pilosa.GroupByCounts { - other := make([]pilosa.GroupLine, len(a)) +func decodeGroupCounts(a []*internal.GroupCount) []pilosa.GroupCount { + other := make([]pilosa.GroupCount, len(a)) for i := range a { - other[i] = pilosa.GroupLine{ + other[i] = pilosa.GroupCount{ decodeFieldRows(a[i].Group), a[i].Count, } } - return pilosa.GroupByCounts(other) + return other } func decodeFieldRows(a []*internal.FieldRow) []pilosa.FieldRow { @@ -1069,10 +1069,10 @@ func encodeRow(r *pilosa.Row) *internal.Row { } } -func encodeGroupByCount(counts pilosa.GroupByCounts) []*internal.GroupLine { - result := make([]*internal.GroupLine, len(counts)) +func encodeGroupCounts(counts []pilosa.GroupCount) []*internal.GroupCount { + result := make([]*internal.GroupCount, len(counts)) for i := range counts { - result[i] = &internal.GroupLine{ + result[i] = &internal.GroupCount{ Group: encodeFieldRows(counts[i].Group), Count: counts[i].Count, } diff --git a/executor.go b/executor.go index 5f2cdc75f..faa71cd3d 100644 --- a/executor.go +++ b/executor.go @@ -751,22 +751,24 @@ func (r RowIDs) Merge(other RowIDs) RowIDs { } return result } -func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (GroupByCounts, error) { + +func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) ([]GroupCount, error) { // Execute calls in bulk on each remote node and merge. mapFn := func(shard uint64) (interface{}, error) { return e.executeGroupByShard(ctx, index, c, shard) } // Merge returned results at coordinating node. reduceFn := func(prev, v interface{}) interface{} { - other, _ := prev.(GroupByCounts) - return other.Merge(v.(GroupByCounts)) + other, _ := prev.([]GroupCount) + return mergeGroupCounts(other, v.([]GroupCount)) } // Get full result set. other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { return nil, err } - results, _ := other.(GroupByCounts) + results, _ := other.([]GroupCount) + // Apply offset. if offset, hasOffset, err := c.UintArg("offset"); err != nil { return nil, err @@ -811,37 +813,35 @@ type gbi struct { row *Row fieldRow FieldRow } -type GroupLine struct { + +type GroupCount struct { Group []FieldRow `json:"group"` Count uint64 `json:"count"` } -// GroupByCounts is the return type for GroupBy queries. -type GroupByCounts []GroupLine - -func (gbc GroupByCounts) Merge(other GroupByCounts) GroupByCounts { +func mergeGroupCounts(gc, other []GroupCount) []GroupCount { m := make(map[string]struct { i int count uint64 }) - for i := range gbc { - m[uniqueGroupString(gbc[i].Group)] = struct { + for i := range gc { + m[uniqueGroupString(gc[i].Group)] = struct { i int count uint64 - }{i, gbc[i].Count} + }{i, gc[i].Count} } for i := range other { o, found := m[uniqueGroupString(other[i].Group)] if found { - gbc[o.i].Count += other[i].Count + gc[o.i].Count += other[i].Count } else { - gbc = append(gbc, other[i]) + gc = append(gc, other[i]) } } - return gbc + return gc } -func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql.Call, shard uint64) (GroupByCounts, error) { +func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql.Call, shard uint64) ([]GroupCount, error) { // Fetch index. idx := e.Holder.Index(index) if idx == nil { @@ -873,7 +873,8 @@ func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql return nil, errors.Wrap(ErrFieldNotFound, fmt.Sprintf("executeGroupBy: %s", fieldDirective.(string))) } } - results := make(GroupByCounts, 0) + + results := make([]GroupCount, 0) var work [][]gbi for _, fieldDirective := range fieldDirectives.([]interface{}) { fieldName := getFieldName(fieldDirective.(string)) @@ -901,9 +902,9 @@ func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql work = append(work, set) } for _, group := range product(work) { - group.gl.Count = group.row.Count() - if group.gl.Count > 0 { - results = append(results, group.gl) + group.gCnt.Count = group.row.Count() + if group.gCnt.Count > 0 { + results = append(results, group.gCnt) } } return results, nil @@ -911,8 +912,8 @@ func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql // ppi is a product process item. type ppi struct { - row *Row - gl GroupLine + row *Row + gCnt GroupCount } // product generates the cartesian product of the input @@ -920,7 +921,7 @@ type ppi struct { func product(input [][]gbi) []ppi { if len(input) == 0 { // base return empty list return []ppi{ - {gl: GroupLine{Group: make([]FieldRow, 0)}}, + {gCnt: GroupCount{Group: make([]FieldRow, 0)}}, } } @@ -929,9 +930,9 @@ func product(input [][]gbi) []ppi { tail := product(input[1:]) // invoke product on remaining element for h := range head { // for each head for t := range tail { // iterate over the tail - s := ppi{gl: GroupLine{Group: make([]FieldRow, 0)}} - s.gl.Group = append([]FieldRow{head[h].fieldRow}, tail[t].gl.Group...) // had to insert at the front to match input order - if tail[t].row != nil { // first time around nothing to intersect + s := ppi{gCnt: GroupCount{Group: make([]FieldRow, 0)}} + s.gCnt.Group = append([]FieldRow{head[h].fieldRow}, tail[t].gCnt.Group...) // had to insert at the front to match input order + if tail[t].row != nil { // first time around nothing to intersect s.row = head[h].row.Intersect(tail[t].row) } else { s.row = head[h].row @@ -2092,8 +2093,8 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res } } - case GroupByCounts: - other := make([]GroupLine, 0) + case []GroupCount: + other := make([]GroupCount, 0) for _, gl := range result { group := make([]FieldRow, len(gl.Group)) @@ -2114,7 +2115,7 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res } } - other = append(other, GroupLine{ + other = append(other, GroupCount{ Group: group, Count: gl.Count, }) diff --git a/executor_test.go b/executor_test.go index 7261611db..786b0ebfb 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1233,11 +1233,11 @@ Set(4500001, fn=4) }); err != nil { t.Fatalf("GroupBy querying: %v", err) } else { - expected := pilosa.GroupByCounts{ + expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "f", RowID: 10}}, Count: 4}, {Group: []pilosa.FieldRow{{Field: "f", RowID: 7}}, Count: 1}, } - results := res.Results[0].([]pilosa.GroupLine) + results := res.Results[0].([]pilosa.GroupCount) checkGroupBy(t, expected, results) } }) @@ -1712,7 +1712,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { } }) t.Run("Basic", func(t *testing.T) { - expected := pilosa.GroupByCounts{ + expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Count: 1}, {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}, {Field: "sub", RowID: 110}}, Count: 1}, {Group: []pilosa.FieldRow{{Field: "general", RowID: 12}, {Field: "sub", RowID: 110}}, Count: 1}, @@ -1722,13 +1722,13 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(fields=[general,sub])`}); err != nil { t.Fatal(err) } else { - results := res.Results[0].([]pilosa.GroupLine) + results := res.Results[0].([]pilosa.GroupCount) checkGroupBy(t, expected, results) } }) t.Run("check field offset no limit", func(t *testing.T) { - expected := pilosa.GroupByCounts{ + expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Count: 2}, {Group: []pilosa.FieldRow{{Field: "general", RowID: 12}}, Count: 2}, } @@ -1736,27 +1736,27 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(fields=[general:11:])`}); err != nil { t.Fatal(err) } else { - results := res.Results[0].([]pilosa.GroupLine) + results := res.Results[0].([]pilosa.GroupCount) checkGroupBy(t, expected, results) } }) t.Run("check field offset limit", func(t *testing.T) { - expected := pilosa.GroupByCounts{ + expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Count: 2}, } if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(fields=[general:11:1])`}); err != nil { t.Fatal(err) } else { - results := res.Results[0].([]pilosa.GroupLine) + results := res.Results[0].([]pilosa.GroupCount) checkGroupBy(t, expected, results) } }) } -func checkGroupBy(t *testing.T, expected, results pilosa.GroupByCounts) { - notIn := func(item pilosa.GroupLine, expected pilosa.GroupByCounts) bool { +func checkGroupBy(t *testing.T, expected, results []pilosa.GroupCount) { + notIn := func(item pilosa.GroupCount, expected []pilosa.GroupCount) bool { for i := range expected { if item.Count == expected[i].Count { if reflect.DeepEqual(item.Group, expected[i].Group) { diff --git a/internal/public.pb.go b/internal/public.pb.go index 29254543a..4f9aab2fc 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -11,7 +11,7 @@ Row Pair FieldRow - GroupLine + GroupCount ValCount Bit ColumnAttrSet @@ -132,24 +132,24 @@ func (m *FieldRow) GetRowID() uint64 { return 0 } -type GroupLine struct { +type GroupCount struct { Group []*FieldRow `protobuf:"bytes,1,rep,name=Group" json:"Group,omitempty"` Count uint64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` } -func (m *GroupLine) Reset() { *m = GroupLine{} } -func (m *GroupLine) String() string { return proto.CompactTextString(m) } -func (*GroupLine) ProtoMessage() {} -func (*GroupLine) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} } +func (m *GroupCount) Reset() { *m = GroupCount{} } +func (m *GroupCount) String() string { return proto.CompactTextString(m) } +func (*GroupCount) ProtoMessage() {} +func (*GroupCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} } -func (m *GroupLine) GetGroup() []*FieldRow { +func (m *GroupCount) GetGroup() []*FieldRow { if m != nil { return m.Group } return nil } -func (m *GroupLine) GetCount() uint64 { +func (m *GroupCount) GetCount() uint64 { if m != nil { return m.Count } @@ -405,14 +405,14 @@ func (m *QueryResponse) GetColumnAttrSets() []*ColumnAttrSet { } type QueryResult struct { - Type uint32 `protobuf:"varint,6,opt,name=Type,proto3" json:"Type,omitempty"` - Row *Row `protobuf:"bytes,1,opt,name=Row" json:"Row,omitempty"` - N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` - Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` - Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` - ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount" json:"ValCount,omitempty"` - RowIDs []uint64 `protobuf:"varint,7,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` - GroupByCounts []*GroupLine `protobuf:"bytes,8,rep,name=GroupByCounts" json:"GroupByCounts,omitempty"` + Type uint32 `protobuf:"varint,6,opt,name=Type,proto3" json:"Type,omitempty"` + Row *Row `protobuf:"bytes,1,opt,name=Row" json:"Row,omitempty"` + N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` + Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` + Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` + ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount" json:"ValCount,omitempty"` + RowIDs []uint64 `protobuf:"varint,7,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` + GroupCounts []*GroupCount `protobuf:"bytes,8,rep,name=GroupCounts" json:"GroupCounts,omitempty"` } func (m *QueryResult) Reset() { *m = QueryResult{} } @@ -469,9 +469,9 @@ func (m *QueryResult) GetRowIDs() []uint64 { return nil } -func (m *QueryResult) GetGroupByCounts() []*GroupLine { +func (m *QueryResult) GetGroupCounts() []*GroupCount { if m != nil { - return m.GroupByCounts + return m.GroupCounts } return nil } @@ -608,7 +608,7 @@ func init() { proto.RegisterType((*Row)(nil), "internal.Row") proto.RegisterType((*Pair)(nil), "internal.Pair") proto.RegisterType((*FieldRow)(nil), "internal.FieldRow") - proto.RegisterType((*GroupLine)(nil), "internal.GroupLine") + proto.RegisterType((*GroupCount)(nil), "internal.GroupCount") proto.RegisterType((*ValCount)(nil), "internal.ValCount") proto.RegisterType((*Bit)(nil), "internal.Bit") proto.RegisterType((*ColumnAttrSet)(nil), "internal.ColumnAttrSet") @@ -745,7 +745,7 @@ func (m *FieldRow) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *GroupLine) Marshal() (dAtA []byte, err error) { +func (m *GroupCount) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -755,7 +755,7 @@ func (m *GroupLine) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *GroupLine) MarshalTo(dAtA []byte) (int, error) { +func (m *GroupCount) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -1181,8 +1181,8 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(j7)) i += copy(dAtA[i:], dAtA8[:j7]) } - if len(m.GroupByCounts) > 0 { - for _, msg := range m.GroupByCounts { + if len(m.GroupCounts) > 0 { + for _, msg := range m.GroupCounts { dAtA[i] = 0x42 i++ i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) @@ -1461,7 +1461,7 @@ func (m *FieldRow) Size() (n int) { return n } -func (m *GroupLine) Size() (n int) { +func (m *GroupCount) Size() (n int) { var l int _ = l if len(m.Group) > 0 { @@ -1644,8 +1644,8 @@ func (m *QueryResult) Size() (n int) { } n += 1 + sovPublic(uint64(l)) + l } - if len(m.GroupByCounts) > 0 { - for _, e := range m.GroupByCounts { + if len(m.GroupCounts) > 0 { + for _, e := range m.GroupCounts { l = e.Size() n += 1 + l + sovPublic(uint64(l)) } @@ -2140,7 +2140,7 @@ func (m *FieldRow) Unmarshal(dAtA []byte) error { } return nil } -func (m *GroupLine) Unmarshal(dAtA []byte) error { +func (m *GroupCount) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -2163,10 +2163,10 @@ func (m *GroupLine) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: GroupLine: wiretype end group for non-group") + return fmt.Errorf("proto: GroupCount: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: GroupLine: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: GroupCount: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -3432,7 +3432,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } case 8: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field GroupByCounts", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field GroupCounts", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -3456,8 +3456,8 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.GroupByCounts = append(m.GroupByCounts, &GroupLine{}) - if err := m.GroupByCounts[len(m.GroupByCounts)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.GroupCounts = append(m.GroupCounts, &GroupCount{}) + if err := m.GroupCounts[len(m.GroupCounts)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -4241,54 +4241,53 @@ var ( func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ - // 771 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6e, 0xd3, 0x4c, - 0x14, 0xfe, 0x27, 0x76, 0x12, 0xe7, 0xa4, 0xc9, 0x5f, 0x0d, 0xa5, 0x58, 0xa8, 0x0a, 0x96, 0x85, - 0x90, 0x57, 0xa9, 0x14, 0x24, 0x24, 0x36, 0x20, 0xd2, 0x0b, 0x8a, 0x0a, 0x15, 0x4c, 0x4b, 0x11, - 0x4b, 0xb7, 0x19, 0xb5, 0x96, 0x1c, 0x8f, 0xf1, 0x45, 0x69, 0x9e, 0xa3, 0x1b, 0x1e, 0x81, 0x05, - 0x0f, 0xd2, 0x25, 0x8f, 0x00, 0xe5, 0x45, 0xd0, 0x9c, 0xf1, 0xd8, 0x4e, 0x8a, 0x2a, 0x16, 0xec, - 0xe6, 0xfb, 0xce, 0x9c, 0xf1, 0x77, 0xae, 0x86, 0xb5, 0x38, 0x3f, 0x0d, 0x83, 0xb3, 0x61, 0x9c, - 0x88, 0x4c, 0x50, 0x2b, 0x88, 0x32, 0x9e, 0x44, 0x7e, 0xe8, 0x7e, 0x02, 0x83, 0x89, 0x39, 0xb5, - 0xa1, 0xbd, 0x23, 0xc2, 0x7c, 0x16, 0xa5, 0x36, 0x71, 0x0c, 0xcf, 0x64, 0x1a, 0xd2, 0xc7, 0xd0, - 0x7c, 0x95, 0x65, 0x49, 0x6a, 0x37, 0x1c, 0xc3, 0xeb, 0x8e, 0xfa, 0x43, 0xed, 0x3a, 0x94, 0x34, - 0x53, 0x46, 0x4a, 0xc1, 0x3c, 0xe0, 0x8b, 0xd4, 0x36, 0x1c, 0xc3, 0xeb, 0x30, 0x3c, 0xbb, 0x2f, - 0xc0, 0x7c, 0xe7, 0x07, 0x09, 0xed, 0x43, 0x63, 0xb2, 0x6b, 0x13, 0x87, 0x78, 0x26, 0x6b, 0x4c, - 0x76, 0xe9, 0x06, 0x34, 0x77, 0x44, 0x1e, 0x65, 0x76, 0x03, 0x29, 0x05, 0xe8, 0x3a, 0x18, 0x07, - 0x7c, 0x61, 0x1b, 0x0e, 0xf1, 0x3a, 0x4c, 0x1e, 0xdd, 0x67, 0x60, 0xed, 0x07, 0x3c, 0x9c, 0x4a, - 0x7d, 0x1b, 0xd0, 0xc4, 0x33, 0x3e, 0xd3, 0x61, 0x0a, 0x48, 0x96, 0x89, 0xf9, 0x64, 0x57, 0xbf, - 0x84, 0xc0, 0x3d, 0x80, 0xce, 0xeb, 0x44, 0xe4, 0xf1, 0x9b, 0x20, 0xe2, 0xd4, 0x83, 0x26, 0x02, - 0x0c, 0xab, 0x3b, 0xa2, 0x95, 0x7c, 0xfd, 0x36, 0x53, 0x17, 0xfe, 0x2c, 0xcb, 0x1d, 0x81, 0x75, - 0xe2, 0x87, 0xa5, 0xc4, 0x13, 0x3f, 0x44, 0x09, 0x06, 0x93, 0xc7, 0x65, 0x1f, 0x43, 0xfb, 0x7c, - 0x00, 0x63, 0x1c, 0x64, 0x95, 0x3a, 0x52, 0x53, 0x47, 0x1f, 0x82, 0xa5, 0x52, 0x5b, 0xca, 0x2e, - 0x31, 0xdd, 0x82, 0xce, 0x71, 0x30, 0xe3, 0x69, 0xe6, 0xcf, 0x62, 0xcc, 0x84, 0xc1, 0x2a, 0xc2, - 0xfd, 0x08, 0x3d, 0x75, 0x53, 0xa6, 0xfc, 0x88, 0x67, 0xb7, 0x12, 0xfb, 0x77, 0xa5, 0xba, 0x9d, - 0xe8, 0xaf, 0x04, 0x4c, 0x69, 0xd3, 0x26, 0x52, 0x9a, 0x64, 0x5d, 0x8f, 0x17, 0x31, 0x2f, 0x94, - 0xe2, 0x99, 0x3a, 0xd0, 0x3d, 0xca, 0x92, 0x20, 0x3a, 0x3f, 0xf1, 0xc3, 0x9c, 0x17, 0x0f, 0xd5, - 0x29, 0x19, 0xe3, 0x24, 0xca, 0x94, 0xd9, 0xc4, 0x30, 0x4a, 0x2c, 0x63, 0x1c, 0x0b, 0x11, 0x2a, - 0x63, 0xd3, 0x21, 0x9e, 0xc5, 0x2a, 0x82, 0x0e, 0x00, 0xf6, 0x43, 0xe1, 0x17, 0xbe, 0x2d, 0x87, - 0x78, 0x84, 0xd5, 0x18, 0x77, 0x1b, 0xda, 0x52, 0xe9, 0x5b, 0x3f, 0xae, 0xa2, 0x25, 0x77, 0x44, - 0xeb, 0x5e, 0x13, 0x58, 0x7b, 0x9f, 0xf3, 0x64, 0xc1, 0xf8, 0xe7, 0x9c, 0xa7, 0x58, 0x15, 0xc4, - 0xba, 0x93, 0x10, 0xd0, 0x4d, 0x68, 0x1d, 0x5d, 0xf8, 0xc9, 0x54, 0xe5, 0xce, 0x64, 0x05, 0x92, - 0xb1, 0x56, 0x39, 0x4f, 0x31, 0x56, 0x8b, 0xd5, 0x29, 0xe9, 0xc9, 0xf8, 0x4c, 0x64, 0x3a, 0x98, - 0x02, 0x51, 0x0f, 0xfe, 0xdf, 0xbb, 0x3c, 0x0b, 0xf3, 0x29, 0x67, 0x62, 0xae, 0xbc, 0x5b, 0x78, - 0x61, 0x95, 0xa6, 0x4f, 0xa0, 0x5f, 0x50, 0x7a, 0x04, 0xdb, 0x78, 0x71, 0x85, 0x75, 0xaf, 0x08, - 0xf4, 0x8a, 0x50, 0xd2, 0x58, 0x44, 0x29, 0x97, 0xf5, 0xda, 0x4b, 0x12, 0x5d, 0xaf, 0xbd, 0x24, - 0xa1, 0xdb, 0xd0, 0x66, 0x3c, 0xcd, 0xc3, 0x4c, 0x37, 0xc1, 0xfd, 0x2a, 0x2d, 0xda, 0x37, 0x0f, - 0x33, 0xa6, 0x6f, 0xd1, 0x97, 0xd0, 0x5f, 0x6a, 0x2a, 0x35, 0xc2, 0xdd, 0xd1, 0x83, 0xca, 0x6f, - 0xc9, 0xce, 0x56, 0xae, 0xbb, 0x57, 0x0d, 0xe8, 0xd6, 0x5e, 0xa6, 0x8f, 0x70, 0xa1, 0xa0, 0xa6, - 0xee, 0xa8, 0x57, 0xbd, 0x22, 0x27, 0x0d, 0x57, 0xcd, 0x1a, 0x90, 0xc3, 0xa2, 0x9f, 0xc8, 0xa1, - 0xac, 0xa2, 0x5c, 0x12, 0xfa, 0xb3, 0xb5, 0x2a, 0x4a, 0x9a, 0x29, 0x23, 0xae, 0xa7, 0x0b, 0x3f, - 0x3a, 0xe7, 0x53, 0xec, 0x27, 0x8b, 0x69, 0x48, 0x87, 0xd5, 0x7c, 0x62, 0x01, 0x96, 0x46, 0x5c, - 0x5b, 0x58, 0x35, 0xc3, 0xba, 0xa1, 0x65, 0x2d, 0x7a, 0x45, 0x43, 0xcb, 0x12, 0xca, 0xd9, 0x94, - 0x89, 0xc7, 0xe2, 0x2b, 0x44, 0x9f, 0x43, 0x0f, 0x57, 0xc3, 0x78, 0x81, 0xbe, 0xa9, 0x6d, 0xa1, - 0xc6, 0x7b, 0xd5, 0x07, 0xca, 0x3d, 0xc3, 0x96, 0x6f, 0xba, 0x3f, 0x09, 0xf4, 0x26, 0xb3, 0x58, - 0x24, 0x59, 0xad, 0xef, 0x26, 0xd1, 0x94, 0x5f, 0xea, 0xbe, 0x43, 0x50, 0xed, 0xb5, 0xc6, 0xca, - 0x5e, 0xc3, 0xfe, 0xc3, 0x7e, 0x33, 0x99, 0x02, 0x35, 0x99, 0xe6, 0x92, 0xcc, 0x2d, 0xe8, 0xe8, - 0x0d, 0x92, 0xda, 0x4d, 0x34, 0x55, 0x84, 0x9c, 0xa8, 0x72, 0x85, 0xc8, 0x16, 0x34, 0x3c, 0x83, - 0xd5, 0x18, 0x99, 0x5a, 0x26, 0xe6, 0xb8, 0xbc, 0xdb, 0xb8, 0xbc, 0x35, 0x94, 0x9e, 0xea, 0x19, - 0x34, 0x5a, 0x68, 0xac, 0x31, 0xee, 0x37, 0x02, 0x54, 0xc5, 0x88, 0xb3, 0xf9, 0xef, 0x02, 0xbd, - 0x3b, 0xa0, 0x4d, 0x68, 0xe1, 0xf7, 0x74, 0x30, 0x05, 0x5a, 0x91, 0xdb, 0x5e, 0x95, 0x3b, 0x5e, - 0xbf, 0xbe, 0x19, 0x90, 0xef, 0x37, 0x03, 0xf2, 0xe3, 0x66, 0x40, 0xbe, 0xfc, 0x1a, 0xfc, 0x77, - 0xda, 0xc2, 0x9f, 0xe1, 0xd3, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x21, 0xc1, 0x72, 0xc6, 0x1c, - 0x07, 0x00, 0x00, + // 764 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd3, 0x4a, + 0x14, 0xbe, 0x13, 0x3b, 0x89, 0x73, 0xd2, 0xe4, 0x56, 0xa3, 0xde, 0x5e, 0x0b, 0x55, 0xc1, 0xb2, + 0x10, 0xf2, 0x2a, 0x95, 0x82, 0xd4, 0x25, 0x88, 0xfe, 0xa1, 0xa8, 0x50, 0xc1, 0xb4, 0x14, 0xb1, + 0x74, 0x9b, 0x51, 0x6b, 0xc9, 0xf1, 0x18, 0xff, 0x28, 0xcd, 0x5b, 0x20, 0xb1, 0xe1, 0x11, 0x58, + 0xf0, 0x20, 0x5d, 0xf2, 0x08, 0x50, 0x5e, 0x04, 0xcd, 0x19, 0x4f, 0xc6, 0x49, 0x51, 0xc5, 0x82, + 0x9d, 0xbf, 0xef, 0xcc, 0x39, 0xf9, 0xce, 0x6f, 0x60, 0x2d, 0x2d, 0xcf, 0xe3, 0xe8, 0x62, 0x98, + 0x66, 0xa2, 0x10, 0xd4, 0x89, 0x92, 0x82, 0x67, 0x49, 0x18, 0xfb, 0xef, 0xc1, 0x62, 0x62, 0x46, + 0x5d, 0x68, 0xef, 0x89, 0xb8, 0x9c, 0x26, 0xb9, 0x4b, 0x3c, 0x2b, 0xb0, 0x99, 0x86, 0xf4, 0x11, + 0x34, 0x9f, 0x17, 0x45, 0x96, 0xbb, 0x0d, 0xcf, 0x0a, 0xba, 0xa3, 0xfe, 0x50, 0xbb, 0x0e, 0x25, + 0xcd, 0x94, 0x91, 0x52, 0xb0, 0x8f, 0xf8, 0x3c, 0x77, 0x2d, 0xcf, 0x0a, 0x3a, 0x0c, 0xbf, 0xfd, + 0xa7, 0x60, 0xbf, 0x0e, 0xa3, 0x8c, 0xf6, 0xa1, 0x31, 0xde, 0x77, 0x89, 0x47, 0x02, 0x9b, 0x35, + 0xc6, 0xfb, 0x74, 0x03, 0x9a, 0x7b, 0xa2, 0x4c, 0x0a, 0xb7, 0x81, 0x94, 0x02, 0x74, 0x1d, 0xac, + 0x23, 0x3e, 0x77, 0x2d, 0x8f, 0x04, 0x1d, 0x26, 0x3f, 0xfd, 0x1d, 0x70, 0x0e, 0x23, 0x1e, 0x4f, + 0xa4, 0xbe, 0x0d, 0x68, 0xe2, 0x37, 0x86, 0xe9, 0x30, 0x05, 0x24, 0xcb, 0xc4, 0x6c, 0xbc, 0xaf, + 0x23, 0x21, 0xf0, 0x5f, 0x02, 0xbc, 0xc8, 0x44, 0x99, 0xaa, 0xb8, 0x01, 0x34, 0x11, 0x61, 0x5e, + 0xdd, 0x11, 0x35, 0xfa, 0x75, 0x70, 0xa6, 0x1e, 0xfc, 0x5e, 0x97, 0x3f, 0x02, 0xe7, 0x2c, 0x8c, + 0x17, 0x1a, 0xcf, 0xc2, 0x18, 0x35, 0x58, 0x4c, 0x7e, 0x2e, 0xfb, 0x58, 0xda, 0xe7, 0x2d, 0x58, + 0xbb, 0x51, 0x61, 0xe4, 0x91, 0x9a, 0x3c, 0xfa, 0x00, 0x1c, 0x55, 0xdb, 0x85, 0xee, 0x05, 0xa6, + 0x5b, 0xd0, 0x39, 0x8d, 0xa6, 0x3c, 0x2f, 0xc2, 0x69, 0x8a, 0xa5, 0xb0, 0x98, 0x21, 0xfc, 0x77, + 0xd0, 0x53, 0x2f, 0x65, 0xcd, 0x4f, 0x78, 0x71, 0xa7, 0xb2, 0x7f, 0xd6, 0xab, 0xbb, 0x95, 0xfe, + 0x42, 0xc0, 0x96, 0x36, 0x6d, 0x22, 0x0b, 0x93, 0x6c, 0xec, 0xe9, 0x3c, 0xe5, 0x95, 0x52, 0xfc, + 0xa6, 0x1e, 0x74, 0x4f, 0x8a, 0x2c, 0x4a, 0x2e, 0xcf, 0xc2, 0xb8, 0xe4, 0x55, 0xa0, 0x3a, 0x25, + 0x73, 0x1c, 0x27, 0x85, 0x32, 0xdb, 0x98, 0xc6, 0x02, 0xcb, 0x1c, 0x77, 0x85, 0x88, 0x95, 0xb1, + 0xe9, 0x91, 0xc0, 0x61, 0x86, 0xa0, 0x03, 0x80, 0xc3, 0x58, 0x84, 0x95, 0x6f, 0xcb, 0x23, 0x01, + 0x61, 0x35, 0xc6, 0xdf, 0x86, 0xb6, 0x54, 0xfa, 0x2a, 0x4c, 0x4d, 0xb6, 0xe4, 0x9e, 0x6c, 0xfd, + 0x1b, 0x02, 0x6b, 0x6f, 0x4a, 0x9e, 0xcd, 0x19, 0xff, 0x50, 0xf2, 0x1c, 0xbb, 0x82, 0x58, 0x8f, + 0x12, 0x02, 0xba, 0x09, 0xad, 0x93, 0xab, 0x30, 0x9b, 0xa8, 0xda, 0xd9, 0xac, 0x42, 0x32, 0x57, + 0x53, 0xf3, 0x1c, 0x73, 0x75, 0x58, 0x9d, 0x92, 0x9e, 0x8c, 0x4f, 0x45, 0xa1, 0x93, 0xa9, 0x10, + 0x0d, 0xe0, 0xdf, 0x83, 0xeb, 0x8b, 0xb8, 0x9c, 0x70, 0x26, 0x66, 0xca, 0xbb, 0x85, 0x0f, 0x56, + 0x69, 0xfa, 0x18, 0xfa, 0x15, 0xa5, 0x77, 0xb0, 0x8d, 0x0f, 0x57, 0x58, 0xff, 0x13, 0x81, 0x5e, + 0x95, 0x4a, 0x9e, 0x8a, 0x24, 0xe7, 0xb2, 0x5f, 0x07, 0x59, 0xa6, 0xfb, 0x75, 0x90, 0x65, 0x74, + 0x1b, 0xda, 0x8c, 0xe7, 0x65, 0x5c, 0xe8, 0x21, 0xf8, 0xcf, 0x94, 0x45, 0xfb, 0x96, 0x71, 0xc1, + 0xf4, 0x2b, 0xfa, 0x0c, 0xfa, 0x4b, 0x43, 0xa5, 0x76, 0xb8, 0x3b, 0xfa, 0xdf, 0xf8, 0x2d, 0xd9, + 0xd9, 0xca, 0x73, 0xff, 0x63, 0x03, 0xba, 0xb5, 0xc8, 0xf4, 0x21, 0x5e, 0x14, 0xd4, 0xd4, 0x1d, + 0xf5, 0x4c, 0x14, 0xb9, 0x69, 0x78, 0x6b, 0xd6, 0x80, 0x1c, 0x57, 0xf3, 0x44, 0x8e, 0x65, 0x17, + 0xe5, 0x95, 0xd0, 0x3f, 0x5b, 0xeb, 0xa2, 0xa4, 0x99, 0x32, 0xe2, 0x7d, 0xba, 0x0a, 0x93, 0x4b, + 0x3e, 0xc1, 0x79, 0x72, 0x98, 0x86, 0x74, 0x68, 0xf6, 0x13, 0x1b, 0xb0, 0xb4, 0xe2, 0xda, 0xc2, + 0xcc, 0x0e, 0xeb, 0x81, 0x96, 0xbd, 0xe8, 0x55, 0x03, 0x2d, 0x5b, 0x28, 0x77, 0x53, 0x16, 0x1e, + 0x9b, 0xaf, 0x10, 0xdd, 0x81, 0xae, 0xb9, 0x24, 0xb9, 0xeb, 0xa0, 0xc2, 0x0d, 0x13, 0xde, 0x18, + 0x59, 0xfd, 0xa1, 0xff, 0x83, 0x40, 0x6f, 0x3c, 0x4d, 0x45, 0x56, 0xd4, 0x86, 0x6e, 0x9c, 0x4c, + 0xf8, 0xb5, 0x1e, 0x3a, 0x04, 0xe6, 0xaa, 0x35, 0x56, 0xae, 0x1a, 0x0e, 0x1f, 0x0e, 0x9b, 0xcd, + 0x14, 0xa8, 0x69, 0xb4, 0x97, 0x34, 0x6e, 0x41, 0x47, 0x9f, 0x8f, 0xdc, 0x6d, 0xa2, 0xc9, 0x10, + 0x72, 0x9d, 0x16, 0xf7, 0x43, 0xce, 0x9f, 0x15, 0x58, 0xac, 0xc6, 0xc8, 0xba, 0x32, 0x31, 0xc3, + 0xd3, 0xdd, 0xc6, 0xd3, 0xad, 0xa1, 0xf4, 0x54, 0x61, 0xd0, 0xe8, 0xa0, 0xb1, 0xc6, 0xf8, 0x5f, + 0x09, 0x50, 0x95, 0x23, 0x2e, 0xe6, 0xdf, 0x4b, 0xf4, 0xfe, 0x84, 0x36, 0xa1, 0x85, 0xbf, 0xa7, + 0x93, 0xa9, 0xd0, 0x8a, 0xdc, 0xf6, 0xaa, 0xdc, 0xdd, 0xf5, 0x9b, 0xdb, 0x01, 0xf9, 0x76, 0x3b, + 0x20, 0xdf, 0x6f, 0x07, 0xe4, 0xf3, 0xcf, 0xc1, 0x3f, 0xe7, 0x2d, 0xfc, 0x2b, 0x7c, 0xf2, 0x2b, + 0x00, 0x00, 0xff, 0xff, 0x25, 0x1f, 0x0d, 0xa8, 0x1a, 0x07, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index b207e3cae..c455ab1e6 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -19,7 +19,7 @@ message FieldRow{ uint64 RowID = 2; } -message GroupLine{ +message GroupCount{ repeated FieldRow Group = 1; uint64 Count = 2; } @@ -77,7 +77,7 @@ message QueryResult { bool Changed = 4; ValCount ValCount = 5; repeated uint64 RowIDs = 7; - repeated GroupLine GroupByCounts = 8; + repeated GroupCount GroupCounts = 8; } message ImportRequest { From 71ad2974504e677c3bc35d49d624874383cb5a64 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 5 Sep 2018 10:55:00 -0500 Subject: [PATCH 07/39] change Rows() to RowIDs() and add RowIdentifiers return type to hold row keys --- encoding/proto/proto.go | 13 +- executor.go | 59 ++++- executor_test.go | 22 +- internal/public.pb.go | 554 ++++++++++++++++++++++++++++++---------- internal/public.proto | 7 + 5 files changed, 506 insertions(+), 149 deletions(-) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 6273f2c7f..b6d78799a 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -366,6 +366,9 @@ func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse { case []pilosa.GroupCount: pb.Results[i].Type = queryResultTypeGroupCounts pb.Results[i].GroupCounts = encodeGroupCounts(result) + case pilosa.RowIdentifiers: + pb.Results[i].Type = queryResultTypeRowIdentifiers + pb.Results[i].RowIdentifiers = encodeRowIdentifiers(result) case nil: pb.Results[i].Type = queryResultTypeNil } @@ -898,7 +901,6 @@ func decodeQueryResponse(pb *internal.QueryResponse, m *pilosa.QueryResponse) { } m.Results = make([]interface{}, len(pb.Results)) decodeQueryResults(pb.Results, m.Results) - } func decodeColumnAttrSets(pb []*internal.ColumnAttrSet, m []*pilosa.ColumnAttrSet) { @@ -930,6 +932,7 @@ const ( queryResultTypeBool queryResultTypeRowIDs queryResultTypeGroupCounts + queryResultTypeRowIdentifiers ) func decodeQueryResult(pb *internal.QueryResult) interface{} { @@ -1069,6 +1072,14 @@ func encodeRow(r *pilosa.Row) *internal.Row { } } +func encodeRowIdentifiers(r pilosa.RowIdentifiers) *internal.RowIdentifiers { + return &internal.RowIdentifiers{ + Rows: r.Rows, + Keys: r.Keys, + //Attrs: encodeAttrs(r.Attrs), + } +} + func encodeGroupCounts(counts []pilosa.GroupCount) []*internal.GroupCount { result := make([]*internal.GroupCount, len(counts)) for i := range counts { diff --git a/executor.go b/executor.go index faa71cd3d..0fb6f53a7 100644 --- a/executor.go +++ b/executor.go @@ -196,9 +196,9 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s case "TopN": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeTopN(ctx, index, c, shards, opt) - case "Rows": + case "RowIDs": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) - return e.executeRows(ctx, index, c, shards, opt) + return e.executeRowIDs(ctx, index, c, shards, opt) case "GroupBy": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeGroupBy(ctx, index, c, shards, opt) @@ -722,9 +722,23 @@ func (e *executor) executeDifferenceShard(ctx context.Context, index string, c * return other, nil } +// RowIdentifiers is a return type for a list of +// row ids or row keys. The names `Rows` and `Keys` +// are meant to follow the same convention as the +// Row query which returns `Columns` and `Keys`. +// TODO: Rename this to something better. Anything. +type RowIdentifiers struct { + Rows []uint64 `json:"rows"` + Keys []string `json:"keys,omitempty"` +} + +// RowIDs is a query return type for just uint64 row ids. +// It should only be used internally (since RowIdentifiers +// is the external return type), but it is exported because +// the proto package needs access to it. type RowIDs []uint64 -func (r RowIDs) Merge(other RowIDs) RowIDs { +func (r RowIDs) merge(other RowIDs) RowIDs { i, j := 0, 0 result := make(RowIDs, 0) for i < len(r) && j < len(other) { @@ -942,15 +956,16 @@ func product(input [][]gbi) []ppi { } return res } -func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (RowIDs, error) { + +func (e *executor) executeRowIDs(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (RowIDs, error) { // Execute calls in bulk on each remote node and merge. mapFn := func(shard uint64) (interface{}, error) { - return e.executeRowsShard(ctx, index, c, shard) + return e.executeRowIDsShard(ctx, index, c, shard) } // Merge returned results at coordinating node. reduceFn := func(prev, v interface{}) interface{} { other, _ := prev.(RowIDs) - return other.Merge(v.(RowIDs)) + return other.merge(v.(RowIDs)) } // Get full result set. other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) @@ -976,7 +991,8 @@ func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, s } return results, nil } -func (e *executor) executeRowsShard(ctx context.Context, index string, c *pql.Call, shard uint64) (RowIDs, error) { + +func (e *executor) executeRowIDsShard(ctx context.Context, index string, c *pql.Call, shard uint64) (RowIDs, error) { // Fetch index. idx := e.Holder.Index(index) if idx == nil { @@ -985,7 +1001,7 @@ func (e *executor) executeRowsShard(ctx context.Context, index string, c *pql.Ca // Fetch field name from argument. fieldName, ok := c.Args["field"].(string) if !ok { - return nil, errors.New("Rows() argument required: field") + return nil, errors.New("RowIDs() argument required: field") } // Fetch field. f := e.Holder.Field(index, fieldName) @@ -2121,6 +2137,31 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res }) } return other, nil + + case RowIDs: + other := RowIdentifiers{} + + fieldName := callArgString(call, "field") + if fieldName == "" { + return nil, ErrFieldNotFound + } + + if field := idx.Field(fieldName); field == nil { + return nil, ErrFieldNotFound + } else if field.keys() { + other.Keys = make([]string, len(result)) + for i, id := range result { + key, err := e.TranslateStore.TranslateRowToString(index, fieldName, id) + if err != nil { + return nil, err + } + other.Keys[i] = key + } + } else { + other.Rows = result + } + + return other, nil } return result, nil @@ -2171,7 +2212,7 @@ func needsShards(calls []*pql.Call) bool { switch call.Name { case "Clear", "Set", "SetRowAttrs", "SetColumnAttrs": continue - case "Count", "TopN", "Rows": + case "Count", "TopN", "RowIDs": return true // default catches Bitmap calls default: diff --git a/executor_test.go b/executor_test.go index 786b0ebfb..c6ae4d9ab 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1639,7 +1639,7 @@ func benchmarkExistence(nn bool, b *testing.B) { func BenchmarkExecutor_Existence_True(b *testing.B) { benchmarkExistence(true, b) } func BenchmarkExecutor_Existence_False(b *testing.B) { benchmarkExistence(false, b) } -func TestExecutor_Execute_Rows(t *testing.T) { +func TestExecutor_Execute_RowIDs(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() hldr := test.Holder{Holder: c[0].Server.Holder()} @@ -1649,24 +1649,28 @@ func TestExecutor_Execute_Rows(t *testing.T) { hldr.SetBit("i", "general", 11, ShardWidth+2) hldr.SetBit("i", "general", 12, 2) hldr.SetBit("i", "general", 12, ShardWidth+2) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=general)`}); err != nil { + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `RowIDs(field=general)`}); err != nil { t.Fatal(err) - } else if columns := res.Results[0].(pilosa.RowIDs); !reflect.DeepEqual(columns, pilosa.RowIDs{10, 11, 12}) { + } else if columns := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual(columns, pilosa.RowIdentifiers{Rows: []uint64{10, 11, 12}}) { t.Fatalf("unexpected columns: %+v", columns) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=general, limit=2)`}); err != nil { + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `RowIDs(field=general, limit=2)`}); err != nil { t.Fatal(err) - } else if columns := res.Results[0].(pilosa.RowIDs); !reflect.DeepEqual(columns, pilosa.RowIDs{10, 11}) { + } else if columns := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual(columns, pilosa.RowIdentifiers{Rows: []uint64{10, 11}}) { t.Fatalf("unexpected columns: %+v", columns) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=general, offset=1,limit=2)`}); err != nil { + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `RowIDs(field=general, offset=1,limit=2)`}); err != nil { t.Fatal(err) - } else if columns := res.Results[0].(pilosa.RowIDs); !reflect.DeepEqual(columns, pilosa.RowIDs{11, 12}) { + } else if columns := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual(columns, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) { t.Fatalf("unexpected columns: %+v", columns) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=general, column=2)`}); err != nil { + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `RowIDs(field=general, column=2)`}); err != nil { t.Fatal(err) - } else if columns := res.Results[0].(pilosa.RowIDs); !reflect.DeepEqual(columns, pilosa.RowIDs{11, 12}) { + } else if columns := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual(columns, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) { t.Fatalf("unexpected columns: %+v", columns) } } diff --git a/internal/public.pb.go b/internal/public.pb.go index 4f9aab2fc..bfa0b3be1 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -9,6 +9,7 @@ It has these top-level messages: Row + RowIdentifiers Pair FieldRow GroupCount @@ -76,6 +77,30 @@ func (m *Row) GetAttrs() []*Attr { return nil } +type RowIdentifiers struct { + Rows []uint64 `protobuf:"varint,1,rep,packed,name=Rows" json:"Rows,omitempty"` + Keys []string `protobuf:"bytes,2,rep,name=Keys" json:"Keys,omitempty"` +} + +func (m *RowIdentifiers) Reset() { *m = RowIdentifiers{} } +func (m *RowIdentifiers) String() string { return proto.CompactTextString(m) } +func (*RowIdentifiers) ProtoMessage() {} +func (*RowIdentifiers) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{1} } + +func (m *RowIdentifiers) GetRows() []uint64 { + if m != nil { + return m.Rows + } + return nil +} + +func (m *RowIdentifiers) GetKeys() []string { + if m != nil { + return m.Keys + } + return nil +} + type Pair struct { ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` Key string `protobuf:"bytes,3,opt,name=Key,proto3" json:"Key,omitempty"` @@ -85,7 +110,7 @@ type Pair struct { func (m *Pair) Reset() { *m = Pair{} } func (m *Pair) String() string { return proto.CompactTextString(m) } func (*Pair) ProtoMessage() {} -func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{1} } +func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} } func (m *Pair) GetID() uint64 { if m != nil { @@ -116,7 +141,7 @@ type FieldRow struct { func (m *FieldRow) Reset() { *m = FieldRow{} } func (m *FieldRow) String() string { return proto.CompactTextString(m) } func (*FieldRow) ProtoMessage() {} -func (*FieldRow) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} } +func (*FieldRow) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} } func (m *FieldRow) GetField() string { if m != nil { @@ -140,7 +165,7 @@ type GroupCount struct { func (m *GroupCount) Reset() { *m = GroupCount{} } func (m *GroupCount) String() string { return proto.CompactTextString(m) } func (*GroupCount) ProtoMessage() {} -func (*GroupCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} } +func (*GroupCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{4} } func (m *GroupCount) GetGroup() []*FieldRow { if m != nil { @@ -164,7 +189,7 @@ type ValCount struct { func (m *ValCount) Reset() { *m = ValCount{} } func (m *ValCount) String() string { return proto.CompactTextString(m) } func (*ValCount) ProtoMessage() {} -func (*ValCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{4} } +func (*ValCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{5} } func (m *ValCount) GetVal() int64 { if m != nil { @@ -189,7 +214,7 @@ type Bit struct { func (m *Bit) Reset() { *m = Bit{} } func (m *Bit) String() string { return proto.CompactTextString(m) } func (*Bit) ProtoMessage() {} -func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{5} } +func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{6} } func (m *Bit) GetRowID() uint64 { if m != nil { @@ -221,7 +246,7 @@ type ColumnAttrSet struct { func (m *ColumnAttrSet) Reset() { *m = ColumnAttrSet{} } func (m *ColumnAttrSet) String() string { return proto.CompactTextString(m) } func (*ColumnAttrSet) ProtoMessage() {} -func (*ColumnAttrSet) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{6} } +func (*ColumnAttrSet) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{7} } func (m *ColumnAttrSet) GetID() uint64 { if m != nil { @@ -256,7 +281,7 @@ type Attr struct { func (m *Attr) Reset() { *m = Attr{} } func (m *Attr) String() string { return proto.CompactTextString(m) } func (*Attr) ProtoMessage() {} -func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{7} } +func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{8} } func (m *Attr) GetKey() string { if m != nil { @@ -307,7 +332,7 @@ type AttrMap struct { func (m *AttrMap) Reset() { *m = AttrMap{} } func (m *AttrMap) String() string { return proto.CompactTextString(m) } func (*AttrMap) ProtoMessage() {} -func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{8} } +func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{9} } func (m *AttrMap) GetAttrs() []*Attr { if m != nil { @@ -328,7 +353,7 @@ type QueryRequest struct { func (m *QueryRequest) Reset() { *m = QueryRequest{} } func (m *QueryRequest) String() string { return proto.CompactTextString(m) } func (*QueryRequest) ProtoMessage() {} -func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{9} } +func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{10} } func (m *QueryRequest) GetQuery() string { if m != nil { @@ -381,7 +406,7 @@ type QueryResponse struct { func (m *QueryResponse) Reset() { *m = QueryResponse{} } func (m *QueryResponse) String() string { return proto.CompactTextString(m) } func (*QueryResponse) ProtoMessage() {} -func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{10} } +func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{11} } func (m *QueryResponse) GetErr() string { if m != nil { @@ -405,20 +430,21 @@ func (m *QueryResponse) GetColumnAttrSets() []*ColumnAttrSet { } type QueryResult struct { - Type uint32 `protobuf:"varint,6,opt,name=Type,proto3" json:"Type,omitempty"` - Row *Row `protobuf:"bytes,1,opt,name=Row" json:"Row,omitempty"` - N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` - Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` - Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` - ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount" json:"ValCount,omitempty"` - RowIDs []uint64 `protobuf:"varint,7,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` - GroupCounts []*GroupCount `protobuf:"bytes,8,rep,name=GroupCounts" json:"GroupCounts,omitempty"` + Type uint32 `protobuf:"varint,6,opt,name=Type,proto3" json:"Type,omitempty"` + Row *Row `protobuf:"bytes,1,opt,name=Row" json:"Row,omitempty"` + N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` + Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` + Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` + ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount" json:"ValCount,omitempty"` + RowIDs []uint64 `protobuf:"varint,7,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` + GroupCounts []*GroupCount `protobuf:"bytes,8,rep,name=GroupCounts" json:"GroupCounts,omitempty"` + RowIdentifiers *RowIdentifiers `protobuf:"bytes,9,opt,name=RowIdentifiers" json:"RowIdentifiers,omitempty"` } func (m *QueryResult) Reset() { *m = QueryResult{} } func (m *QueryResult) String() string { return proto.CompactTextString(m) } func (*QueryResult) ProtoMessage() {} -func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{11} } +func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{12} } func (m *QueryResult) GetType() uint32 { if m != nil { @@ -476,6 +502,13 @@ func (m *QueryResult) GetGroupCounts() []*GroupCount { return nil } +func (m *QueryResult) GetRowIdentifiers() *RowIdentifiers { + if m != nil { + return m.RowIdentifiers + } + return nil +} + type ImportRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` @@ -490,7 +523,7 @@ type ImportRequest struct { func (m *ImportRequest) Reset() { *m = ImportRequest{} } func (m *ImportRequest) String() string { return proto.CompactTextString(m) } func (*ImportRequest) ProtoMessage() {} -func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{12} } +func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{13} } func (m *ImportRequest) GetIndex() string { if m != nil { @@ -560,7 +593,7 @@ type ImportValueRequest struct { func (m *ImportValueRequest) Reset() { *m = ImportValueRequest{} } func (m *ImportValueRequest) String() string { return proto.CompactTextString(m) } func (*ImportValueRequest) ProtoMessage() {} -func (*ImportValueRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{13} } +func (*ImportValueRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{14} } func (m *ImportValueRequest) GetIndex() string { if m != nil { @@ -606,6 +639,7 @@ func (m *ImportValueRequest) GetValues() []int64 { func init() { proto.RegisterType((*Row)(nil), "internal.Row") + proto.RegisterType((*RowIdentifiers)(nil), "internal.RowIdentifiers") proto.RegisterType((*Pair)(nil), "internal.Pair") proto.RegisterType((*FieldRow)(nil), "internal.FieldRow") proto.RegisterType((*GroupCount)(nil), "internal.GroupCount") @@ -682,6 +716,56 @@ func (m *Row) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *RowIdentifiers) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RowIdentifiers) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Rows) > 0 { + dAtA4 := make([]byte, len(m.Rows)*10) + var j3 int + for _, num := range m.Rows { + for num >= 1<<7 { + dAtA4[j3] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j3++ + } + dAtA4[j3] = uint8(num) + j3++ + } + dAtA[i] = 0xa + i++ + i = encodeVarintPublic(dAtA, i, uint64(j3)) + i += copy(dAtA[i:], dAtA4[:j3]) + } + if len(m.Keys) > 0 { + for _, s := range m.Keys { + dAtA[i] = 0x12 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + return i, nil +} + func (m *Pair) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -990,21 +1074,21 @@ func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], m.Query) } if len(m.Shards) > 0 { - dAtA4 := make([]byte, len(m.Shards)*10) - var j3 int + dAtA6 := make([]byte, len(m.Shards)*10) + var j5 int for _, num := range m.Shards { for num >= 1<<7 { - dAtA4[j3] = uint8(uint64(num)&0x7f | 0x80) + dAtA6[j5] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j3++ + j5++ } - dAtA4[j3] = uint8(num) - j3++ + dAtA6[j5] = uint8(num) + j5++ } dAtA[i] = 0x12 i++ - i = encodeVarintPublic(dAtA, i, uint64(j3)) - i += copy(dAtA[i:], dAtA4[:j3]) + i = encodeVarintPublic(dAtA, i, uint64(j5)) + i += copy(dAtA[i:], dAtA6[:j5]) } if m.ColumnAttrs { dAtA[i] = 0x18 @@ -1116,11 +1200,11 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintPublic(dAtA, i, uint64(m.Row.Size())) - n5, err := m.Row.MarshalTo(dAtA[i:]) + n7, err := m.Row.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n5 + i += n7 } if m.N != 0 { dAtA[i] = 0x10 @@ -1153,11 +1237,11 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x2a i++ i = encodeVarintPublic(dAtA, i, uint64(m.ValCount.Size())) - n6, err := m.ValCount.MarshalTo(dAtA[i:]) + n8, err := m.ValCount.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n6 + i += n8 } if m.Type != 0 { dAtA[i] = 0x30 @@ -1165,21 +1249,21 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(m.Type)) } if len(m.RowIDs) > 0 { - dAtA8 := make([]byte, len(m.RowIDs)*10) - var j7 int + dAtA10 := make([]byte, len(m.RowIDs)*10) + var j9 int for _, num := range m.RowIDs { for num >= 1<<7 { - dAtA8[j7] = uint8(uint64(num)&0x7f | 0x80) + dAtA10[j9] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j7++ + j9++ } - dAtA8[j7] = uint8(num) - j7++ + dAtA10[j9] = uint8(num) + j9++ } dAtA[i] = 0x3a i++ - i = encodeVarintPublic(dAtA, i, uint64(j7)) - i += copy(dAtA[i:], dAtA8[:j7]) + i = encodeVarintPublic(dAtA, i, uint64(j9)) + i += copy(dAtA[i:], dAtA10[:j9]) } if len(m.GroupCounts) > 0 { for _, msg := range m.GroupCounts { @@ -1193,6 +1277,16 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { i += n } } + if m.RowIdentifiers != nil { + dAtA[i] = 0x4a + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.RowIdentifiers.Size())) + n11, err := m.RowIdentifiers.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n11 + } return i, nil } @@ -1229,56 +1323,56 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(m.Shard)) } if len(m.RowIDs) > 0 { - dAtA10 := make([]byte, len(m.RowIDs)*10) - var j9 int + dAtA13 := make([]byte, len(m.RowIDs)*10) + var j12 int for _, num := range m.RowIDs { for num >= 1<<7 { - dAtA10[j9] = uint8(uint64(num)&0x7f | 0x80) + dAtA13[j12] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j9++ + j12++ } - dAtA10[j9] = uint8(num) - j9++ + dAtA13[j12] = uint8(num) + j12++ } dAtA[i] = 0x22 i++ - i = encodeVarintPublic(dAtA, i, uint64(j9)) - i += copy(dAtA[i:], dAtA10[:j9]) + i = encodeVarintPublic(dAtA, i, uint64(j12)) + i += copy(dAtA[i:], dAtA13[:j12]) } if len(m.ColumnIDs) > 0 { - dAtA12 := make([]byte, len(m.ColumnIDs)*10) - var j11 int + dAtA15 := make([]byte, len(m.ColumnIDs)*10) + var j14 int for _, num := range m.ColumnIDs { for num >= 1<<7 { - dAtA12[j11] = uint8(uint64(num)&0x7f | 0x80) + dAtA15[j14] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j11++ + j14++ } - dAtA12[j11] = uint8(num) - j11++ + dAtA15[j14] = uint8(num) + j14++ } dAtA[i] = 0x2a i++ - i = encodeVarintPublic(dAtA, i, uint64(j11)) - i += copy(dAtA[i:], dAtA12[:j11]) + i = encodeVarintPublic(dAtA, i, uint64(j14)) + i += copy(dAtA[i:], dAtA15[:j14]) } if len(m.Timestamps) > 0 { - dAtA14 := make([]byte, len(m.Timestamps)*10) - var j13 int + dAtA17 := make([]byte, len(m.Timestamps)*10) + var j16 int for _, num1 := range m.Timestamps { num := uint64(num1) for num >= 1<<7 { - dAtA14[j13] = uint8(uint64(num)&0x7f | 0x80) + dAtA17[j16] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j13++ + j16++ } - dAtA14[j13] = uint8(num) - j13++ + dAtA17[j16] = uint8(num) + j16++ } dAtA[i] = 0x32 i++ - i = encodeVarintPublic(dAtA, i, uint64(j13)) - i += copy(dAtA[i:], dAtA14[:j13]) + i = encodeVarintPublic(dAtA, i, uint64(j16)) + i += copy(dAtA[i:], dAtA17[:j16]) } if len(m.RowKeys) > 0 { for _, s := range m.RowKeys { @@ -1346,39 +1440,39 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(m.Shard)) } if len(m.ColumnIDs) > 0 { - dAtA16 := make([]byte, len(m.ColumnIDs)*10) - var j15 int + dAtA19 := make([]byte, len(m.ColumnIDs)*10) + var j18 int for _, num := range m.ColumnIDs { for num >= 1<<7 { - dAtA16[j15] = uint8(uint64(num)&0x7f | 0x80) + dAtA19[j18] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j15++ + j18++ } - dAtA16[j15] = uint8(num) - j15++ + dAtA19[j18] = uint8(num) + j18++ } dAtA[i] = 0x2a i++ - i = encodeVarintPublic(dAtA, i, uint64(j15)) - i += copy(dAtA[i:], dAtA16[:j15]) + i = encodeVarintPublic(dAtA, i, uint64(j18)) + i += copy(dAtA[i:], dAtA19[:j18]) } if len(m.Values) > 0 { - dAtA18 := make([]byte, len(m.Values)*10) - var j17 int + dAtA21 := make([]byte, len(m.Values)*10) + var j20 int for _, num1 := range m.Values { num := uint64(num1) for num >= 1<<7 { - dAtA18[j17] = uint8(uint64(num)&0x7f | 0x80) + dAtA21[j20] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j17++ + j20++ } - dAtA18[j17] = uint8(num) - j17++ + dAtA21[j20] = uint8(num) + j20++ } dAtA[i] = 0x32 i++ - i = encodeVarintPublic(dAtA, i, uint64(j17)) - i += copy(dAtA[i:], dAtA18[:j17]) + i = encodeVarintPublic(dAtA, i, uint64(j20)) + i += copy(dAtA[i:], dAtA21[:j20]) } if len(m.ColumnKeys) > 0 { for _, s := range m.ColumnKeys { @@ -1432,6 +1526,25 @@ func (m *Row) Size() (n int) { return n } +func (m *RowIdentifiers) Size() (n int) { + var l int + _ = l + if len(m.Rows) > 0 { + l = 0 + for _, e := range m.Rows { + l += sovPublic(uint64(e)) + } + n += 1 + sovPublic(uint64(l)) + l + } + if len(m.Keys) > 0 { + for _, s := range m.Keys { + l = len(s) + n += 1 + l + sovPublic(uint64(l)) + } + } + return n +} + func (m *Pair) Size() (n int) { var l int _ = l @@ -1650,6 +1763,10 @@ func (m *QueryResult) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } + if m.RowIdentifiers != nil { + l = m.RowIdentifiers.Size() + n += 1 + l + sovPublic(uint64(l)) + } return n } @@ -1925,6 +2042,147 @@ func (m *Row) Unmarshal(dAtA []byte) error { } return nil } +func (m *RowIdentifiers) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RowIdentifiers: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RowIdentifiers: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Rows = append(m.Rows, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Rows = append(m.Rows, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field Rows", wireType) + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Keys", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Keys = append(m.Keys, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPublic(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPublic + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *Pair) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -3461,6 +3719,39 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 9: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RowIdentifiers", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.RowIdentifiers == nil { + m.RowIdentifiers = &RowIdentifiers{} + } + if err := m.RowIdentifiers.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -4241,53 +4532,56 @@ var ( func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ - // 764 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd3, 0x4a, - 0x14, 0xbe, 0x13, 0x3b, 0x89, 0x73, 0xd2, 0xe4, 0x56, 0xa3, 0xde, 0x5e, 0x0b, 0x55, 0xc1, 0xb2, - 0x10, 0xf2, 0x2a, 0x95, 0x82, 0xd4, 0x25, 0x88, 0xfe, 0xa1, 0xa8, 0x50, 0xc1, 0xb4, 0x14, 0xb1, - 0x74, 0x9b, 0x51, 0x6b, 0xc9, 0xf1, 0x18, 0xff, 0x28, 0xcd, 0x5b, 0x20, 0xb1, 0xe1, 0x11, 0x58, - 0xf0, 0x20, 0x5d, 0xf2, 0x08, 0x50, 0x5e, 0x04, 0xcd, 0x19, 0x4f, 0xc6, 0x49, 0x51, 0xc5, 0x82, - 0x9d, 0xbf, 0xef, 0xcc, 0x39, 0xf9, 0xce, 0x6f, 0x60, 0x2d, 0x2d, 0xcf, 0xe3, 0xe8, 0x62, 0x98, - 0x66, 0xa2, 0x10, 0xd4, 0x89, 0x92, 0x82, 0x67, 0x49, 0x18, 0xfb, 0xef, 0xc1, 0x62, 0x62, 0x46, - 0x5d, 0x68, 0xef, 0x89, 0xb8, 0x9c, 0x26, 0xb9, 0x4b, 0x3c, 0x2b, 0xb0, 0x99, 0x86, 0xf4, 0x11, - 0x34, 0x9f, 0x17, 0x45, 0x96, 0xbb, 0x0d, 0xcf, 0x0a, 0xba, 0xa3, 0xfe, 0x50, 0xbb, 0x0e, 0x25, - 0xcd, 0x94, 0x91, 0x52, 0xb0, 0x8f, 0xf8, 0x3c, 0x77, 0x2d, 0xcf, 0x0a, 0x3a, 0x0c, 0xbf, 0xfd, - 0xa7, 0x60, 0xbf, 0x0e, 0xa3, 0x8c, 0xf6, 0xa1, 0x31, 0xde, 0x77, 0x89, 0x47, 0x02, 0x9b, 0x35, - 0xc6, 0xfb, 0x74, 0x03, 0x9a, 0x7b, 0xa2, 0x4c, 0x0a, 0xb7, 0x81, 0x94, 0x02, 0x74, 0x1d, 0xac, - 0x23, 0x3e, 0x77, 0x2d, 0x8f, 0x04, 0x1d, 0x26, 0x3f, 0xfd, 0x1d, 0x70, 0x0e, 0x23, 0x1e, 0x4f, - 0xa4, 0xbe, 0x0d, 0x68, 0xe2, 0x37, 0x86, 0xe9, 0x30, 0x05, 0x24, 0xcb, 0xc4, 0x6c, 0xbc, 0xaf, - 0x23, 0x21, 0xf0, 0x5f, 0x02, 0xbc, 0xc8, 0x44, 0x99, 0xaa, 0xb8, 0x01, 0x34, 0x11, 0x61, 0x5e, - 0xdd, 0x11, 0x35, 0xfa, 0x75, 0x70, 0xa6, 0x1e, 0xfc, 0x5e, 0x97, 0x3f, 0x02, 0xe7, 0x2c, 0x8c, - 0x17, 0x1a, 0xcf, 0xc2, 0x18, 0x35, 0x58, 0x4c, 0x7e, 0x2e, 0xfb, 0x58, 0xda, 0xe7, 0x2d, 0x58, - 0xbb, 0x51, 0x61, 0xe4, 0x91, 0x9a, 0x3c, 0xfa, 0x00, 0x1c, 0x55, 0xdb, 0x85, 0xee, 0x05, 0xa6, - 0x5b, 0xd0, 0x39, 0x8d, 0xa6, 0x3c, 0x2f, 0xc2, 0x69, 0x8a, 0xa5, 0xb0, 0x98, 0x21, 0xfc, 0x77, - 0xd0, 0x53, 0x2f, 0x65, 0xcd, 0x4f, 0x78, 0x71, 0xa7, 0xb2, 0x7f, 0xd6, 0xab, 0xbb, 0x95, 0xfe, - 0x42, 0xc0, 0x96, 0x36, 0x6d, 0x22, 0x0b, 0x93, 0x6c, 0xec, 0xe9, 0x3c, 0xe5, 0x95, 0x52, 0xfc, - 0xa6, 0x1e, 0x74, 0x4f, 0x8a, 0x2c, 0x4a, 0x2e, 0xcf, 0xc2, 0xb8, 0xe4, 0x55, 0xa0, 0x3a, 0x25, - 0x73, 0x1c, 0x27, 0x85, 0x32, 0xdb, 0x98, 0xc6, 0x02, 0xcb, 0x1c, 0x77, 0x85, 0x88, 0x95, 0xb1, - 0xe9, 0x91, 0xc0, 0x61, 0x86, 0xa0, 0x03, 0x80, 0xc3, 0x58, 0x84, 0x95, 0x6f, 0xcb, 0x23, 0x01, - 0x61, 0x35, 0xc6, 0xdf, 0x86, 0xb6, 0x54, 0xfa, 0x2a, 0x4c, 0x4d, 0xb6, 0xe4, 0x9e, 0x6c, 0xfd, - 0x1b, 0x02, 0x6b, 0x6f, 0x4a, 0x9e, 0xcd, 0x19, 0xff, 0x50, 0xf2, 0x1c, 0xbb, 0x82, 0x58, 0x8f, - 0x12, 0x02, 0xba, 0x09, 0xad, 0x93, 0xab, 0x30, 0x9b, 0xa8, 0xda, 0xd9, 0xac, 0x42, 0x32, 0x57, - 0x53, 0xf3, 0x1c, 0x73, 0x75, 0x58, 0x9d, 0x92, 0x9e, 0x8c, 0x4f, 0x45, 0xa1, 0x93, 0xa9, 0x10, - 0x0d, 0xe0, 0xdf, 0x83, 0xeb, 0x8b, 0xb8, 0x9c, 0x70, 0x26, 0x66, 0xca, 0xbb, 0x85, 0x0f, 0x56, - 0x69, 0xfa, 0x18, 0xfa, 0x15, 0xa5, 0x77, 0xb0, 0x8d, 0x0f, 0x57, 0x58, 0xff, 0x13, 0x81, 0x5e, - 0x95, 0x4a, 0x9e, 0x8a, 0x24, 0xe7, 0xb2, 0x5f, 0x07, 0x59, 0xa6, 0xfb, 0x75, 0x90, 0x65, 0x74, - 0x1b, 0xda, 0x8c, 0xe7, 0x65, 0x5c, 0xe8, 0x21, 0xf8, 0xcf, 0x94, 0x45, 0xfb, 0x96, 0x71, 0xc1, - 0xf4, 0x2b, 0xfa, 0x0c, 0xfa, 0x4b, 0x43, 0xa5, 0x76, 0xb8, 0x3b, 0xfa, 0xdf, 0xf8, 0x2d, 0xd9, - 0xd9, 0xca, 0x73, 0xff, 0x63, 0x03, 0xba, 0xb5, 0xc8, 0xf4, 0x21, 0x5e, 0x14, 0xd4, 0xd4, 0x1d, - 0xf5, 0x4c, 0x14, 0xb9, 0x69, 0x78, 0x6b, 0xd6, 0x80, 0x1c, 0x57, 0xf3, 0x44, 0x8e, 0x65, 0x17, - 0xe5, 0x95, 0xd0, 0x3f, 0x5b, 0xeb, 0xa2, 0xa4, 0x99, 0x32, 0xe2, 0x7d, 0xba, 0x0a, 0x93, 0x4b, - 0x3e, 0xc1, 0x79, 0x72, 0x98, 0x86, 0x74, 0x68, 0xf6, 0x13, 0x1b, 0xb0, 0xb4, 0xe2, 0xda, 0xc2, - 0xcc, 0x0e, 0xeb, 0x81, 0x96, 0xbd, 0xe8, 0x55, 0x03, 0x2d, 0x5b, 0x28, 0x77, 0x53, 0x16, 0x1e, - 0x9b, 0xaf, 0x10, 0xdd, 0x81, 0xae, 0xb9, 0x24, 0xb9, 0xeb, 0xa0, 0xc2, 0x0d, 0x13, 0xde, 0x18, - 0x59, 0xfd, 0xa1, 0xff, 0x83, 0x40, 0x6f, 0x3c, 0x4d, 0x45, 0x56, 0xd4, 0x86, 0x6e, 0x9c, 0x4c, - 0xf8, 0xb5, 0x1e, 0x3a, 0x04, 0xe6, 0xaa, 0x35, 0x56, 0xae, 0x1a, 0x0e, 0x1f, 0x0e, 0x9b, 0xcd, - 0x14, 0xa8, 0x69, 0xb4, 0x97, 0x34, 0x6e, 0x41, 0x47, 0x9f, 0x8f, 0xdc, 0x6d, 0xa2, 0xc9, 0x10, - 0x72, 0x9d, 0x16, 0xf7, 0x43, 0xce, 0x9f, 0x15, 0x58, 0xac, 0xc6, 0xc8, 0xba, 0x32, 0x31, 0xc3, - 0xd3, 0xdd, 0xc6, 0xd3, 0xad, 0xa1, 0xf4, 0x54, 0x61, 0xd0, 0xe8, 0xa0, 0xb1, 0xc6, 0xf8, 0x5f, - 0x09, 0x50, 0x95, 0x23, 0x2e, 0xe6, 0xdf, 0x4b, 0xf4, 0xfe, 0x84, 0x36, 0xa1, 0x85, 0xbf, 0xa7, - 0x93, 0xa9, 0xd0, 0x8a, 0xdc, 0xf6, 0xaa, 0xdc, 0xdd, 0xf5, 0x9b, 0xdb, 0x01, 0xf9, 0x76, 0x3b, - 0x20, 0xdf, 0x6f, 0x07, 0xe4, 0xf3, 0xcf, 0xc1, 0x3f, 0xe7, 0x2d, 0xfc, 0x2b, 0x7c, 0xf2, 0x2b, - 0x00, 0x00, 0xff, 0xff, 0x25, 0x1f, 0x0d, 0xa8, 0x1a, 0x07, 0x00, 0x00, + // 804 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6e, 0xdb, 0x46, + 0x14, 0xed, 0x88, 0x94, 0x44, 0x5d, 0x59, 0xaa, 0x31, 0x70, 0x5d, 0xa2, 0x30, 0x54, 0x82, 0x28, + 0x0a, 0xae, 0x64, 0x40, 0x05, 0x8c, 0xae, 0xfa, 0xf0, 0xab, 0x10, 0xdc, 0x1a, 0xcd, 0xd8, 0x71, + 0x90, 0x25, 0x6d, 0x4d, 0x6c, 0x02, 0x14, 0x87, 0xe1, 0x03, 0xb2, 0xbe, 0x23, 0x9b, 0x7c, 0x42, + 0x16, 0xf9, 0x10, 0x2f, 0x83, 0x7c, 0x41, 0xe2, 0xfc, 0x48, 0x30, 0x77, 0x38, 0x1a, 0x8a, 0x0e, + 0x8c, 0x2c, 0xb2, 0x9b, 0x73, 0x5f, 0xbc, 0xe7, 0xbe, 0x08, 0x1b, 0x69, 0x79, 0x19, 0x47, 0x57, + 0xe3, 0x34, 0x13, 0x85, 0xa0, 0x4e, 0x94, 0x14, 0x3c, 0x4b, 0xc2, 0xd8, 0x7f, 0x0e, 0x16, 0x13, + 0x0b, 0xea, 0x42, 0xf7, 0x40, 0xc4, 0xe5, 0x3c, 0xc9, 0x5d, 0xe2, 0x59, 0x81, 0xcd, 0x34, 0xa4, + 0xbf, 0x40, 0xfb, 0xef, 0xa2, 0xc8, 0x72, 0xb7, 0xe5, 0x59, 0x41, 0x7f, 0x32, 0x1c, 0x6b, 0xd7, + 0xb1, 0x14, 0x33, 0xa5, 0xa4, 0x14, 0xec, 0x13, 0xbe, 0xcc, 0x5d, 0xcb, 0xb3, 0x82, 0x1e, 0xc3, + 0xb7, 0xff, 0x3b, 0x0c, 0x99, 0x58, 0x4c, 0x67, 0x3c, 0x29, 0xa2, 0x17, 0x11, 0x57, 0x56, 0x4c, + 0x2c, 0xf4, 0x27, 0xf0, 0xbd, 0xf2, 0x6c, 0xd5, 0x3c, 0xff, 0x00, 0xfb, 0xff, 0x30, 0xca, 0xe8, + 0x10, 0x5a, 0xd3, 0x43, 0x97, 0x78, 0x24, 0xb0, 0x59, 0x6b, 0x7a, 0x48, 0xb7, 0xa0, 0x7d, 0x20, + 0xca, 0xa4, 0x70, 0x5b, 0x28, 0x52, 0x80, 0x6e, 0x82, 0x75, 0xc2, 0x97, 0xae, 0xe5, 0x91, 0xa0, + 0xc7, 0xe4, 0xd3, 0xdf, 0x03, 0xe7, 0x38, 0xe2, 0xf1, 0x4c, 0x32, 0xdb, 0x82, 0x36, 0xbe, 0x31, + 0x4c, 0x8f, 0x29, 0x20, 0xa5, 0x32, 0xb7, 0x43, 0x1d, 0x09, 0x81, 0xff, 0x2f, 0xc0, 0x3f, 0x99, + 0x28, 0x53, 0x15, 0x37, 0x80, 0x36, 0x22, 0x4c, 0xb7, 0x3f, 0xa1, 0x86, 0xb9, 0x0e, 0xce, 0x94, + 0xc1, 0x97, 0xf3, 0xf2, 0x27, 0xe0, 0x5c, 0x84, 0xf1, 0x2a, 0xc7, 0x8b, 0x30, 0xc6, 0x1c, 0x2c, + 0x26, 0x9f, 0xeb, 0x3e, 0x96, 0xf6, 0x79, 0x0a, 0xd6, 0x7e, 0x54, 0x98, 0xf4, 0x48, 0x2d, 0x3d, + 0xfa, 0x13, 0x38, 0xaa, 0x2b, 0xab, 0xbc, 0x57, 0x98, 0xee, 0x40, 0xef, 0x3c, 0x9a, 0xf3, 0xbc, + 0x08, 0xe7, 0x29, 0x96, 0xc2, 0x62, 0x46, 0xe0, 0x3f, 0x83, 0x81, 0xb2, 0x94, 0xdd, 0x3a, 0xe3, + 0xc5, 0x83, 0xca, 0x7e, 0x5d, 0x97, 0x1f, 0x56, 0xfa, 0x0d, 0x01, 0x5b, 0xea, 0xb4, 0x8a, 0xac, + 0x54, 0xb2, 0xb1, 0xe7, 0xcb, 0x94, 0x57, 0x99, 0xe2, 0x9b, 0x7a, 0xd0, 0x3f, 0x2b, 0xb2, 0x28, + 0xb9, 0xbe, 0x08, 0xe3, 0x92, 0x57, 0x81, 0xea, 0x22, 0xc9, 0x71, 0x9a, 0x14, 0x4a, 0x6d, 0x23, + 0x8d, 0x15, 0x96, 0x1c, 0xf7, 0x85, 0x88, 0x95, 0xb2, 0xed, 0x91, 0xc0, 0x61, 0x46, 0x40, 0x47, + 0x00, 0xc7, 0xb1, 0x08, 0x2b, 0xdf, 0x8e, 0x47, 0x02, 0xc2, 0x6a, 0x12, 0x7f, 0x17, 0xba, 0x32, + 0xd3, 0xff, 0xc2, 0xd4, 0xb0, 0x25, 0x8f, 0xb0, 0xf5, 0xef, 0x08, 0x6c, 0x3c, 0x29, 0x79, 0xb6, + 0x64, 0xfc, 0x65, 0xc9, 0x73, 0xec, 0x0a, 0x62, 0x3d, 0x4a, 0x08, 0xe8, 0x36, 0x74, 0xce, 0x6e, + 0xc2, 0x6c, 0xa6, 0x6a, 0x67, 0xb3, 0x0a, 0x49, 0xae, 0xa6, 0xe6, 0x39, 0x72, 0x75, 0x58, 0x5d, + 0x24, 0x3d, 0x19, 0x9f, 0x8b, 0x42, 0x93, 0xa9, 0x10, 0x0d, 0xe0, 0xfb, 0xa3, 0xdb, 0xab, 0xb8, + 0x9c, 0x71, 0x26, 0x16, 0xca, 0xbb, 0x83, 0x06, 0x4d, 0x31, 0xfd, 0x15, 0x86, 0x95, 0x48, 0x6f, + 0x6f, 0x17, 0x0d, 0x1b, 0x52, 0xff, 0x15, 0x81, 0x41, 0x45, 0x25, 0x4f, 0x45, 0x92, 0x73, 0xd9, + 0xaf, 0xa3, 0x2c, 0xd3, 0xfd, 0x3a, 0xca, 0x32, 0xba, 0x0b, 0x5d, 0xc6, 0xf3, 0x32, 0x2e, 0xf4, + 0x10, 0xfc, 0x60, 0xca, 0xa2, 0x7d, 0xcb, 0xb8, 0x60, 0xda, 0x8a, 0xfe, 0x09, 0xc3, 0xb5, 0xa1, + 0x52, 0xdb, 0xdf, 0x9f, 0xfc, 0x68, 0xfc, 0xd6, 0xf4, 0xac, 0x61, 0xee, 0xbf, 0x6f, 0x41, 0xbf, + 0x16, 0x99, 0xfe, 0x8c, 0xb7, 0x08, 0x73, 0xea, 0x4f, 0x06, 0x26, 0x8a, 0xdc, 0x34, 0xbc, 0x52, + 0x1b, 0x40, 0x4e, 0xab, 0x79, 0x22, 0xa7, 0xb2, 0x8b, 0xf2, 0x4a, 0xe8, 0xcf, 0xd6, 0xba, 0x28, + 0xc5, 0x4c, 0x29, 0xf1, 0xb2, 0xdd, 0x84, 0xc9, 0x35, 0x9f, 0xe1, 0x3c, 0x39, 0x4c, 0x43, 0x3a, + 0x36, 0xfb, 0x89, 0x0d, 0x58, 0x5b, 0x71, 0xad, 0x61, 0x66, 0x87, 0xf5, 0x40, 0xcb, 0x5e, 0x0c, + 0xaa, 0x81, 0x96, 0x2d, 0x94, 0xbb, 0x29, 0x0b, 0x8f, 0xcd, 0x57, 0x88, 0xee, 0x41, 0xdf, 0x5c, + 0x92, 0xdc, 0x75, 0x30, 0xc3, 0x2d, 0x13, 0xde, 0x28, 0x59, 0xdd, 0x90, 0xfe, 0xd5, 0xbc, 0x99, + 0x6e, 0x0f, 0x33, 0x73, 0xd7, 0xaa, 0x51, 0xd3, 0xb3, 0x86, 0xbd, 0xff, 0x91, 0xc0, 0x60, 0x3a, + 0x4f, 0x45, 0x56, 0xd4, 0xc6, 0x76, 0x9a, 0xcc, 0xf8, 0xad, 0x1e, 0x5b, 0x04, 0xe6, 0x2e, 0xb6, + 0x1a, 0x77, 0x11, 0xc7, 0x17, 0xc7, 0xd5, 0x66, 0x0a, 0xd4, 0x58, 0xda, 0x6b, 0x2c, 0x77, 0xa0, + 0xa7, 0x0f, 0x50, 0xee, 0xb6, 0x51, 0x65, 0x04, 0x72, 0x21, 0x57, 0x17, 0x48, 0x4e, 0xb0, 0x15, + 0x58, 0xac, 0x26, 0x91, 0x9d, 0x61, 0x62, 0x81, 0xc7, 0xbf, 0x8b, 0xc7, 0x5f, 0x43, 0xe9, 0xa9, + 0xc2, 0xa0, 0xd2, 0x41, 0x65, 0x4d, 0xe2, 0xbf, 0x25, 0x40, 0x15, 0x47, 0x5c, 0xed, 0x6f, 0x47, + 0xf4, 0x71, 0x42, 0xdb, 0xd0, 0xc1, 0xef, 0x69, 0x32, 0x15, 0x6a, 0xa4, 0xdb, 0x6d, 0xa6, 0xbb, + 0xbf, 0x79, 0x77, 0x3f, 0x22, 0xef, 0xee, 0x47, 0xe4, 0xc3, 0xfd, 0x88, 0xbc, 0xfe, 0x34, 0xfa, + 0xee, 0xb2, 0x83, 0xbf, 0xe1, 0xdf, 0x3e, 0x07, 0x00, 0x00, 0xff, 0xff, 0xa6, 0x62, 0xa8, 0x25, + 0x96, 0x07, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index c455ab1e6..a102f1088 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -8,6 +8,12 @@ message Row { repeated Attr Attrs = 2; } +message RowIdentifiers { + repeated uint64 Rows = 1; + repeated string Keys = 2; + //repeated Attr Attrs = 3; +} + message Pair { uint64 ID = 1; string Key = 3; @@ -78,6 +84,7 @@ message QueryResult { ValCount ValCount = 5; repeated uint64 RowIDs = 7; repeated GroupCount GroupCounts = 8; + RowIdentifiers RowIdentifiers = 9; } message ImportRequest { From 0ab3e72520b5a98f8e962b02a2a721afde269010 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 18 Sep 2018 14:09:52 -0500 Subject: [PATCH 08/39] refactor mergeGroupCounts function --- executor.go | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/executor.go b/executor.go index 0fb6f53a7..858c88d9e 100644 --- a/executor.go +++ b/executor.go @@ -834,20 +834,13 @@ type GroupCount struct { } func mergeGroupCounts(gc, other []GroupCount) []GroupCount { - m := make(map[string]struct { - i int - count uint64 - }) + m := make(map[string]int) for i := range gc { - m[uniqueGroupString(gc[i].Group)] = struct { - i int - count uint64 - }{i, gc[i].Count} + m[uniqueGroupString(gc[i].Group)] = i } for i := range other { - o, found := m[uniqueGroupString(other[i].Group)] - if found { - gc[o.i].Count += other[i].Count + if idx, found := m[uniqueGroupString(other[i].Group)]; found { + gc[idx].Count += other[i].Count } else { gc = append(gc, other[i]) } From 4face45c2a0efc2c2800b5024550a198fd03cc4d Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 26 Sep 2018 16:03:26 -0500 Subject: [PATCH 09/39] rename RowIDs methods to Rows --- executor.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/executor.go b/executor.go index 858c88d9e..80d9b5a8b 100644 --- a/executor.go +++ b/executor.go @@ -198,7 +198,7 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s return e.executeTopN(ctx, index, c, shards, opt) case "RowIDs": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) - return e.executeRowIDs(ctx, index, c, shards, opt) + return e.executeRows(ctx, index, c, shards, opt) case "GroupBy": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeGroupBy(ctx, index, c, shards, opt) @@ -950,10 +950,10 @@ func product(input [][]gbi) []ppi { return res } -func (e *executor) executeRowIDs(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (RowIDs, error) { +func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (RowIDs, error) { // Execute calls in bulk on each remote node and merge. mapFn := func(shard uint64) (interface{}, error) { - return e.executeRowIDsShard(ctx, index, c, shard) + return e.executeRowsShard(ctx, index, c, shard) } // Merge returned results at coordinating node. reduceFn := func(prev, v interface{}) interface{} { @@ -985,7 +985,7 @@ func (e *executor) executeRowIDs(ctx context.Context, index string, c *pql.Call, return results, nil } -func (e *executor) executeRowIDsShard(ctx context.Context, index string, c *pql.Call, shard uint64) (RowIDs, error) { +func (e *executor) executeRowsShard(ctx context.Context, index string, c *pql.Call, shard uint64) (RowIDs, error) { // Fetch index. idx := e.Holder.Index(index) if idx == nil { From ac98fcb6d43a7f70150c06f7970b88c51e92ae66 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 26 Sep 2018 16:09:05 -0500 Subject: [PATCH 10/39] rename RowIDs PQL to Rows --- executor.go | 6 +++--- executor_test.go | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/executor.go b/executor.go index 80d9b5a8b..5de820a22 100644 --- a/executor.go +++ b/executor.go @@ -196,7 +196,7 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s case "TopN": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeTopN(ctx, index, c, shards, opt) - case "RowIDs": + case "Rows": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeRows(ctx, index, c, shards, opt) case "GroupBy": @@ -994,7 +994,7 @@ func (e *executor) executeRowsShard(ctx context.Context, index string, c *pql.Ca // Fetch field name from argument. fieldName, ok := c.Args["field"].(string) if !ok { - return nil, errors.New("RowIDs() argument required: field") + return nil, errors.New("Rows() argument required: field") } // Fetch field. f := e.Holder.Field(index, fieldName) @@ -2205,7 +2205,7 @@ func needsShards(calls []*pql.Call) bool { switch call.Name { case "Clear", "Set", "SetRowAttrs", "SetColumnAttrs": continue - case "Count", "TopN", "RowIDs": + case "Count", "TopN", "Rows": return true // default catches Bitmap calls default: diff --git a/executor_test.go b/executor_test.go index c6ae4d9ab..73af2e2f8 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1650,25 +1650,25 @@ func TestExecutor_Execute_RowIDs(t *testing.T) { hldr.SetBit("i", "general", 12, 2) hldr.SetBit("i", "general", 12, ShardWidth+2) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `RowIDs(field=general)`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=general)`}); err != nil { t.Fatal(err) } else if columns := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual(columns, pilosa.RowIdentifiers{Rows: []uint64{10, 11, 12}}) { t.Fatalf("unexpected columns: %+v", columns) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `RowIDs(field=general, limit=2)`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=general, limit=2)`}); err != nil { t.Fatal(err) } else if columns := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual(columns, pilosa.RowIdentifiers{Rows: []uint64{10, 11}}) { t.Fatalf("unexpected columns: %+v", columns) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `RowIDs(field=general, offset=1,limit=2)`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=general, offset=1,limit=2)`}); err != nil { t.Fatal(err) } else if columns := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual(columns, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) { t.Fatalf("unexpected columns: %+v", columns) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `RowIDs(field=general, column=2)`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=general, column=2)`}); err != nil { t.Fatal(err) } else if columns := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual(columns, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) { t.Fatalf("unexpected columns: %+v", columns) From 78ff75690d1afe6eca272c76dacb2fe6f6a13c23 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 27 Sep 2018 16:14:40 -0500 Subject: [PATCH 11/39] convert Rows to use previous/limit pass previous+1 directly to fragment.rows so that the iterator can seek directly to the start point. handle limit inside reduce so it can skip out early and avoid extra allocation. --- executor.go | 78 +++++++++++++++++++-------------------- executor_test.go | 4 +- fragment.go | 28 +++++++++----- fragment_internal_test.go | 6 +-- 4 files changed, 61 insertions(+), 55 deletions(-) diff --git a/executor.go b/executor.go index 5de820a22..d8d5fba2b 100644 --- a/executor.go +++ b/executor.go @@ -738,10 +738,10 @@ type RowIdentifiers struct { // the proto package needs access to it. type RowIDs []uint64 -func (r RowIDs) merge(other RowIDs) RowIDs { +func (r RowIDs) merge(other RowIDs, limit int) RowIDs { i, j := 0, 0 result := make(RowIDs, 0) - for i < len(r) && j < len(other) { + for i < len(r) && j < len(other) && len(result) < limit { av, bv := r[i], other[j] if av < bv { result = append(result, av) @@ -755,11 +755,11 @@ func (r RowIDs) merge(other RowIDs) RowIDs { j++ } } - for i < len(r) { + for i < len(r) && len(result) < limit { result = append(result, r[i]) i++ } - for j < len(other) { + for j < len(other) && len(result) < limit { result = append(result, other[j]) j++ } @@ -897,7 +897,7 @@ func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql } set := make([]gbi, 0) - for _, rowID := range frag.rowsWithFilter(filter) { + for _, rowID := range frag.rowsWithFilter(0, filter) { set = append(set, gbi{ row: frag.row(rowID), fieldRow: FieldRow{ @@ -955,10 +955,19 @@ func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, s mapFn := func(shard uint64) (interface{}, error) { return e.executeRowsShard(ctx, index, c, shard) } + + // Determine limit so we can use it when reducing. + limit := int(^uint(0) >> 1) + if lim, hasLimit, err := c.UintArg("limit"); err != nil { + return nil, err + } else if hasLimit { + limit = int(lim) + } + // Merge returned results at coordinating node. reduceFn := func(prev, v interface{}) interface{} { other, _ := prev.(RowIDs) - return other.merge(v.(RowIDs)) + return other.merge(v.(RowIDs), limit) } // Get full result set. other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) @@ -966,22 +975,6 @@ func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, s return nil, err } results, _ := other.(RowIDs) - // Apply offset. - if offset, hasOffset, err := c.UintArg("offset"); err != nil { - return nil, err - } else if hasOffset { - if int(offset) < len(results) { - results = results[offset:] - } - } - // Apply limit. - if limit, hasLimit, err := c.UintArg("limit"); err != nil { - return nil, err - } else if hasLimit { - if int(limit) < len(results) { - results = results[:limit] - } - } return results, nil } @@ -1005,14 +998,29 @@ func (e *executor) executeRowsShard(ctx context.Context, index string, c *pql.Ca if frag == nil { return make(RowIDs, 0), nil } + + start := uint64(0) + if previous, ok, err := c.UintArg("previous"); err != nil { + return nil, errors.Wrap(err, "getting previous") + } else if ok { + start = previous + 1 + } + fmt.Println("calculated start is", start) + + filter := noFilter + if limit, hasLimit, err := c.UintArg("limit"); err != nil { + return nil, errors.Wrap(err, "getting limit") + } else if hasLimit { + filter = (&filterWithLimit{limit: limit}).filter + } + if columnID, ok, err := c.UintArg("column"); err != nil { return nil, err } else if ok { - // TODO: it's possible that filters could be applied here, so this returns too early. - return frag.rowsForColumn(columnID), nil + return frag.rowsForColumnWithFilter(start, columnID, filter), nil + } else { + return frag.rowsWithFilter(start, filter), nil } - filter := getFilterFunction(c) - return frag.rowsWithFilter(filter), nil } // getGroupByFilterFunction returns a rowFilter based on the @@ -1061,21 +1069,6 @@ func getGroupByFilterFunction(fieldDirective string) (rowFilter, error) { f := filterWithOffset{offset: offset} return f.filter, nil } -func getFilterFunction(c *pql.Call) rowFilter { - offset, hasOffset, _ := c.UintArg("shardoffset") - limit, hasLimit, _ := c.UintArg("shardlimit") - if hasOffset && hasLimit { - f := filterWithOffsetLimit{offset: offset, limit: limit} - return f.filter - } else if hasOffset { - f := filterWithOffset{offset: offset} - return f.filter - } else if hasLimit { - f := filterWithLimit{limit: limit} - return f.filter - } - return noFilter -} func (e *executor) executeBitmapShard(_ context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { // Fetch index. @@ -2005,6 +1998,9 @@ func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error { // Positional args in new PQL syntax require special handling here. rowKey = "_" + rowLabel fieldName = callArgString(c, "_field") + } else if c.Name == "Rows" { + fieldName = callArgString(c, "field") + rowKey = "previous" } else { colKey = "col" fieldName = callArgString(c, "field") diff --git a/executor_test.go b/executor_test.go index 73af2e2f8..0de3016f1 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1639,7 +1639,7 @@ func benchmarkExistence(nn bool, b *testing.B) { func BenchmarkExecutor_Existence_True(b *testing.B) { benchmarkExistence(true, b) } func BenchmarkExecutor_Existence_False(b *testing.B) { benchmarkExistence(false, b) } -func TestExecutor_Execute_RowIDs(t *testing.T) { +func TestExecutor_Execute_Rows(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() hldr := test.Holder{Holder: c[0].Server.Holder()} @@ -1662,7 +1662,7 @@ func TestExecutor_Execute_RowIDs(t *testing.T) { t.Fatalf("unexpected columns: %+v", columns) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=general, offset=1,limit=2)`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=general, previous=10,limit=2)`}); err != nil { t.Fatal(err) } else if columns := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual(columns, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) { t.Fatalf("unexpected columns: %+v", columns) diff --git a/fragment.go b/fragment.go index bcd8e9b13..6b0a53fe3 100644 --- a/fragment.go +++ b/fragment.go @@ -58,6 +58,9 @@ const ( // exponent. shardVsContainerExponent = shardWidthExponent - 16 + // width of roaring containers is 2^16 + containerWidth = 1 << 16 + // snapshotExt is the file extension used for an in-process snapshot. snapshotExt = ".snapshotting" @@ -1772,12 +1775,13 @@ var noFilter = func(rowID uint64) (bool, bool) { return true, false } // rows returns all rows by calling rowsWithFilter() // with a completely unrestrictive filter. -func (f *fragment) rows() []uint64 { - return f.rowsWithFilter(noFilter) +func (f *fragment) rows(start uint64) []uint64 { + return f.rowsWithFilter(start, noFilter) } -func (f *fragment) rowsWithFilter(filter rowFilter) []uint64 { - i, _ := f.storage.Containers.Iterator(0) +func (f *fragment) rowsWithFilter(start uint64, filter rowFilter) []uint64 { + startKey := rowToKey(start) + i, _ := f.storage.Containers.Iterator(startKey) rows := make([]uint64, 0) var lastRow uint64 = math.MaxUint64 @@ -1806,14 +1810,13 @@ func (f *fragment) rowsWithFilter(filter rowFilter) []uint64 { } -// rowsForColumn is similar to the rows method, but isolated -// to a single column. func (f *fragment) rowsForColumn(columnID uint64) []uint64 { - return f.rowsForColumnWithFilter(columnID, noFilter) + return f.rowsForColumnWithFilter(0, columnID, noFilter) } -func (f *fragment) rowsForColumnWithFilter(columnID uint64, filter rowFilter) []uint64 { - i, _ := f.storage.Containers.Iterator(0) +func (f *fragment) rowsForColumnWithFilter(start, columnID uint64, filter rowFilter) []uint64 { + startKey := rowToKey(start) + i, _ := f.storage.Containers.Iterator(startKey) rows := make([]uint64, 0) colID := columnID % ShardWidth @@ -2136,3 +2139,10 @@ func (v *rowsVector) Get(colID uint64) (uint64, bool) { // Set is not used for rowsVector. func (v *rowsVector) Set(colID, rowID uint64) {} + +// rowToKey converts a Pilosa row ID to the key of the container which starts +// that row in the bitmap which represents this entire fragment. A fragment is +// all the rows within a shard within a field concatenated together. +func rowToKey(rowID uint64) (key uint64) { + return rowID * (ShardWidth / containerWidth) +} diff --git a/fragment_internal_test.go b/fragment_internal_test.go index dd2d00b41..70b6ad78e 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -1341,7 +1341,7 @@ func TestFragment_RowsIteration(t *testing.T) { } } - ids := f.rows() + ids := f.rows(0) if !reflect.DeepEqual(expectedAll, ids) { t.Fatalf("Do not match %v %v", expectedAll, ids) } @@ -1365,7 +1365,7 @@ func TestFragment_RowsIteration(t *testing.T) { t.Fatal(err) } - ids := f.rows() + ids := f.rows(0) if !reflect.DeepEqual(expected, ids) { t.Fatalf("Do not match %v %v", expected, ids) } @@ -1388,7 +1388,7 @@ func TestFragment_RowsIteration(t *testing.T) { t.Fatal(err) } - ids := f.rows() + ids := f.rows(0) if !reflect.DeepEqual(expectedRows, ids) { t.Fatalf("Do not match %v %v", expectedRows, ids) } From 7d24276a98dd73bc17eb9e828f2138446a304340 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 27 Sep 2018 16:18:37 -0500 Subject: [PATCH 12/39] rename "columns" to "rows" in Rows test so it makes sense --- executor_test.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/executor_test.go b/executor_test.go index 0de3016f1..a00a9ab93 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1652,28 +1652,29 @@ func TestExecutor_Execute_Rows(t *testing.T) { if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=general)`}); err != nil { t.Fatal(err) - } else if columns := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual(columns, pilosa.RowIdentifiers{Rows: []uint64{10, 11, 12}}) { - t.Fatalf("unexpected columns: %+v", columns) + } else if rows := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{10, 11, 12}}) { + t.Fatalf("unexpected rows: %+v", rows) } if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=general, limit=2)`}); err != nil { t.Fatal(err) - } else if columns := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual(columns, pilosa.RowIdentifiers{Rows: []uint64{10, 11}}) { - t.Fatalf("unexpected columns: %+v", columns) + } else if rows := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{10, 11}}) { + t.Fatalf("unexpected rows: %+v", rows) } if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=general, previous=10,limit=2)`}); err != nil { t.Fatal(err) - } else if columns := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual(columns, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) { - t.Fatalf("unexpected columns: %+v", columns) + } else if rows := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) { + t.Fatalf("unexpected rows: %+v", rows) } if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=general, column=2)`}); err != nil { t.Fatal(err) - } else if columns := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual(columns, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) { - t.Fatalf("unexpected columns: %+v", columns) + } else if rows := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) { + t.Fatalf("unexpected rows: %+v", rows) } } + func TestExecutor_Execute_GroupBy(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() From 93e9f242fe52d46a343e284f20f0197b1c45c799 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 27 Sep 2018 17:07:15 -0500 Subject: [PATCH 13/39] test Rows call with row keys, fix column id problem --- executor.go | 12 ++++- executor_test.go | 122 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 2 deletions(-) diff --git a/executor.go b/executor.go index d8d5fba2b..dd7ff5bb7 100644 --- a/executor.go +++ b/executor.go @@ -951,6 +951,12 @@ func product(input [][]gbi) []ppi { } func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (RowIDs, error) { + if columnID, ok, err := c.UintArg("column"); err != nil { + return nil, errors.Wrap(err, "getting column") + } else if ok { + shards = []uint64{columnID / ShardWidth} + } + // Execute calls in bulk on each remote node and merge. mapFn := func(shard uint64) (interface{}, error) { return e.executeRowsShard(ctx, index, c, shard) @@ -1005,7 +1011,6 @@ func (e *executor) executeRowsShard(ctx context.Context, index string, c *pql.Ca } else if ok { start = previous + 1 } - fmt.Println("calculated start is", start) filter := noFilter if limit, hasLimit, err := c.UintArg("limit"); err != nil { @@ -1017,7 +1022,10 @@ func (e *executor) executeRowsShard(ctx context.Context, index string, c *pql.Ca if columnID, ok, err := c.UintArg("column"); err != nil { return nil, err } else if ok { - return frag.rowsForColumnWithFilter(start, columnID, filter), nil + if columnID/ShardWidth == shard { + return frag.rowsForColumnWithFilter(start, columnID%ShardWidth, filter), nil + } + return RowIDs{}, nil } else { return frag.rowsWithFilter(start, filter), nil } diff --git a/executor_test.go b/executor_test.go index a00a9ab93..faa0acbf9 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1675,6 +1675,128 @@ func TestExecutor_Execute_Rows(t *testing.T) { } } +func TestExecutor_Execute_Rows_Keys(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + + _, err := c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + + _, err = c[0].API.CreateField(context.Background(), "i", "f", pilosa.OptFieldKeys()) + if err != nil { + t.Fatalf("creating field: %v", err) + } + + // setup some data. 10 bits in each of shards 0 through 9. starting at + // row/col shardNum and progressing to row/col shardNum+10. Also set the + // previous 2 for each bit if row >0. + query := strings.Builder{} + for shard := 0; shard < 10; shard++ { + for i := shard; i < shard+10; i++ { + for row := i; row >= 0 && row > i-3; row-- { + query.WriteString(fmt.Sprintf("Set(%d, f=\"%d\")", shard*pilosa.ShardWidth+i, row)) + + } + + } + } + _, err = c[0].API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "i", + Query: query.String(), + }) + if err != nil { + t.Fatalf("querying: %v", err) + } + + tests := []struct { + q string + exp []string + }{ + { + q: `Rows(field=f)`, + exp: []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18"}, + }, + { + q: `Rows(field=f, limit=2)`, + exp: []string{"0", "1"}, + }, + { + q: `Rows(field=f, previous="15")`, + exp: []string{"16", "17", "18"}, + }, + { + q: `Rows(field=f, previous="11", limit=2)`, + exp: []string{"12", "13"}, + }, + { + q: `Rows(field=f, previous="11", limit=2)`, + exp: []string{"12", "13"}, + }, + { + q: `Rows(field=f, previous="17", limit=5)`, + exp: []string{"18"}, + }, + { + q: `Rows(field=f, previous="18")`, + exp: []string{}, + }, + { + q: `Rows(field=f, previous="1", limit=0)`, + exp: []string{}, + }, + { + q: `Rows(field=f, column=1)`, + exp: []string{"0", "1"}, + }, + { + q: `Rows(field=f, column=2)`, + exp: []string{"0", "1", "2"}, + }, + { + q: `Rows(field=f, column=3)`, + exp: []string{"1", "2", "3"}, + }, + { + q: `Rows(field=f, limit=2, column=3)`, + exp: []string{"1", "2"}, + }, + { + q: fmt.Sprintf(`Rows(field=f, previous="15", column=%d)`, ShardWidth*9+17), + exp: []string{"16", "17"}, + }, + { + q: fmt.Sprintf(`Rows(field=f, previous="11", limit=2, column=%d)`, ShardWidth*5+14), + exp: []string{"12", "13"}, + }, + { + q: fmt.Sprintf(`Rows(field=f, previous="17", limit=5, column=%d)`, ShardWidth*9+18), + exp: []string{"18"}, + }, + { + q: `Rows(field=f, previous="18", column=19)`, + exp: []string{}, + }, + { + q: `Rows(field=f, previous="1", limit=0, column=0)`, + exp: []string{}, + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("#%d_%s", i, test.q), func(t *testing.T) { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.q}); err != nil { + t.Fatal(err) + } else if rows := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual( + rows, pilosa.RowIdentifiers{Keys: test.exp}) { + t.Fatalf("\ngot: %+v\nexp: %+v", rows, pilosa.RowIdentifiers{Keys: test.exp}) + } + }) + } + +} + func TestExecutor_Execute_GroupBy(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() From ed1b09a1cd55c40d8811d9f01c1d08801f479d05 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 28 Sep 2018 10:17:53 -0500 Subject: [PATCH 14/39] fix columnID<>shard checks in executor and fragment fragment panics if rowsForColumn is called with a column id not in the fragment's shard. The justification for this is that we're wasting resources if we're sending requests for a specific column to any shard other than the one which contains that column. --- executor.go | 2 +- fragment.go | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/executor.go b/executor.go index dd7ff5bb7..47b36d6f2 100644 --- a/executor.go +++ b/executor.go @@ -1023,7 +1023,7 @@ func (e *executor) executeRowsShard(ctx context.Context, index string, c *pql.Ca return nil, err } else if ok { if columnID/ShardWidth == shard { - return frag.rowsForColumnWithFilter(start, columnID%ShardWidth, filter), nil + return frag.rowsForColumnWithFilter(start, columnID, filter), nil } return RowIDs{}, nil } else { diff --git a/fragment.go b/fragment.go index 6b0a53fe3..a29f68109 100644 --- a/fragment.go +++ b/fragment.go @@ -1815,6 +1815,11 @@ func (f *fragment) rowsForColumn(columnID uint64) []uint64 { } func (f *fragment) rowsForColumnWithFilter(start, columnID uint64, filter rowFilter) []uint64 { + if columnID/ShardWidth != f.shard { + panic(fmt.Sprintln("fragment.rowsForColumn should never be called with a columnID which is not in the fragment's shard", + columnID, columnID/ShardWidth, f.shard)) + } + startKey := rowToKey(start) i, _ := f.storage.Containers.Iterator(startKey) rows := make([]uint64, 0) From 61089981a2767b2dd0a78dce6cfca33efaf78a0f Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 28 Sep 2018 10:20:48 -0500 Subject: [PATCH 15/39] remove check for column in shard in executeRowsShard the check happens in executeRows and frag.rowsForColumn will panic if given a column id not in its shard. --- executor.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/executor.go b/executor.go index 47b36d6f2..f83c0cbf3 100644 --- a/executor.go +++ b/executor.go @@ -1022,10 +1022,7 @@ func (e *executor) executeRowsShard(ctx context.Context, index string, c *pql.Ca if columnID, ok, err := c.UintArg("column"); err != nil { return nil, err } else if ok { - if columnID/ShardWidth == shard { - return frag.rowsForColumnWithFilter(start, columnID, filter), nil - } - return RowIDs{}, nil + return frag.rowsForColumnWithFilter(start, columnID, filter), nil } else { return frag.rowsWithFilter(start, filter), nil } From f94cd8ae7d9ea0a93fdeafcd77192edca053813a Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 28 Sep 2018 14:30:18 -0500 Subject: [PATCH 16/39] add translation code for GroupBy "previous" arg --- executor.go | 57 ++++++++++++++++++ executor_internal_test.go | 122 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 executor_internal_test.go diff --git a/executor.go b/executor.go index f83c0cbf3..7c23ac6e4 100644 --- a/executor.go +++ b/executor.go @@ -2006,6 +2006,8 @@ func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error { } else if c.Name == "Rows" { fieldName = callArgString(c, "field") rowKey = "previous" + } else if c.Name == "GroupBy" { + return errors.Wrap(e.translateGroupByCall(index, idx, c), "translating GroupBy") } else { colKey = "col" fieldName = callArgString(c, "field") @@ -2067,6 +2069,61 @@ func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error { return nil } +func (e *executor) translateGroupByCall(index string, idx *Index, c *pql.Call) error { + if c.Name != "GroupBy" { + panic("translateGroupByCall called with '" + c.Name + "'") + } + + for _, child := range c.Children { + if err := e.translateCall(index, idx, child); err != nil { + return errors.Wrapf(err, "translating %s", child) + } + } + + prev, ok := c.Args["previous"] + if !ok { + return nil // nothing else to be translated + } + previous, ok := prev.([]interface{}) + if !ok { + return errors.Errorf("'previous' argument must be list, but got %T", prev) + } + if len(c.Children) != len(previous) { + return errors.Errorf("mismatched lengths for previous: %d and children: %d in %s", len(previous), len(c.Children), c) + } + + fields := make([]*Field, len(c.Children)) + for i, child := range c.Children { + fieldname := callArgString(child, "field") + field := idx.Field(fieldname) + if field == nil { + return errors.Wrapf(ErrFieldNotFound, "getting field '%s' from '%s'", fieldname, child) + } + fields[i] = field + } + + for i, field := range fields { + prev := previous[i] + if field.keys() { + prevStr, ok := prev.(string) + if !ok { + return errors.New("prev value must be a string when field 'keys' option enabled") + } + ids, err := e.TranslateStore.TranslateRowsToUint64(index, field.Name(), []string{prevStr}) + if err != nil { + return errors.Wrapf(err, "translating row key '%s'", prevStr) + } + previous[i] = ids[0] + } else { + if prevStr, ok := prev.(string); ok { + return errors.Errorf("got string row val '%s' in 'previous' for field %s which doesn't use string keys", prevStr, field.Name()) + } + } + + } + return nil +} + func (e *executor) translateResult(index string, idx *Index, call *pql.Call, result interface{}) (interface{}, error) { switch result := result.(type) { case *Row: diff --git a/executor_internal_test.go b/executor_internal_test.go new file mode 100644 index 000000000..e90948561 --- /dev/null +++ b/executor_internal_test.go @@ -0,0 +1,122 @@ +package pilosa + +import ( + "fmt" + "io/ioutil" + "strings" + "testing" + + "github.com/pilosa/pilosa/pql" +) + +func TestExecutor_TranslateGroupByCall(t *testing.T) { + e := &executor{ + Holder: NewHolder(), + } + e.Holder.Path, _ = ioutil.TempDir("", "") + err := e.Holder.Open() + if err != nil { + t.Fatalf("opening holder: %v", err) + } + + e.TranslateStore = e.Holder.translateFile + tf, _ := ioutil.TempFile("", "") + e.Holder.translateFile.Path = tf.Name() + err = e.Holder.translateFile.Open() + if err != nil { + t.Fatalf("opening translateFile: %v", err) + } + + idx, err := e.Holder.CreateIndex("i", IndexOptions{}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + + _, erra := idx.CreateField("ak", OptFieldKeys()) + _, errb := idx.CreateField("b") + _, errc := idx.CreateField("ck", OptFieldKeys()) + if erra != nil || errb != nil || errc != nil { + t.Fatalf("creating fields %v, %v, %v", erra, errb, errc) + } + + _, erra = e.TranslateStore.TranslateRowsToUint64("i", "ak", []string{"la"}) + _, errb = e.TranslateStore.TranslateRowsToUint64("i", "ck", []string{"ha"}) + if erra != nil || errb != nil { + t.Fatalf("translating rows %v, %v", erra, errb) + } + + query, err := pql.ParseString(`GroupBy(Rows(field=ak), Rows(field=b), Rows(field=ck), previous=["la", 0, "ha"])`) + if err != nil { + t.Fatalf("parsing query: %v", err) + } + c := query.Calls[0] + err = e.translateGroupByCall("i", idx, c) + if err != nil { + t.Fatalf("translating call: %v", err) + } + if len(c.Args["previous"].([]interface{})) != 3 { + t.Fatalf("unexpected length for 'previous' arg %v", c.Args["previous"]) + } + for i, v := range c.Args["previous"].([]interface{}) { + if !isInt(v) { + t.Fatalf("expected all items in previous to be ints, but '%v' at index %d is %[1]T", v, i) + } + } + + errTests := []struct { + pql string + err string + }{ + { + pql: `GroupBy(Rows(field=notfound), previous=1)`, + err: "'previous' argument must be list", + }, + { + pql: `GroupBy(Rows(field=ak), previous=["la", 0])`, + err: "mismatched lengths", + }, + { + pql: `GroupBy(Rows(field=ak), previous=[1])`, + err: "prev value must be a string", + }, + { + pql: `GroupBy(Rows(field=notfound), previous=[1])`, + err: ErrFieldNotFound.Error(), + }, + // TODO: an unknown key will actually allocate an id. this is probably bad. + // { + // pql: `GroupBy(Rows(field=ak), previous=["zoop"])`, + // err: "translating row key '", + // }, + { + pql: `GroupBy(Rows(field=b), previous=["la"])`, + err: "which doesn't use string keys", + }, + } + + for i, test := range errTests { + t.Run(fmt.Sprintf("#%d_%s", i, test.err), func(t *testing.T) { + query, err := pql.ParseString(test.pql) + if err != nil { + t.Fatalf("parsing query: %v", err) + } + c := query.Calls[0] + err = e.translateGroupByCall("i", idx, c) + if err == nil { + t.Fatalf("expected error, but translated call is '%s", c) + } + if !strings.Contains(err.Error(), test.err) { + t.Fatalf("expected '%s', got '%v'", test.err, err) + } + }) + } +} + +func isInt(a interface{}) bool { + switch a.(type) { + case int, int64, uint, uint64: + return true + default: + return false + } +} From 4f3f2e1a490b0c1b250308a8fc2fabe568108191 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 2 Oct 2018 09:25:07 -0500 Subject: [PATCH 17/39] remove noFilter and filterWithOffsetLimit can use an empty list of filters and a list of offsetFilter followed by limit filter respectively --- executor.go | 53 +++++++++++++++++------------------------------------ fragment.go | 46 ++++++++++++++++++++++++++++++---------------- 2 files changed, 47 insertions(+), 52 deletions(-) diff --git a/executor.go b/executor.go index 7c23ac6e4..d6b9ed46e 100644 --- a/executor.go +++ b/executor.go @@ -891,13 +891,13 @@ func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql return results, nil } // Get filter based on the field directive. - filter, err := getGroupByFilterFunction(fieldDirective.(string)) + filters, err := getGroupByFilterFunction(fieldDirective.(string)) if err != nil { return nil, err } set := make([]gbi, 0) - for _, rowID := range frag.rowsWithFilter(0, filter) { + for _, rowID := range frag.rows(0, filters...) { set = append(set, gbi{ row: frag.row(rowID), fieldRow: FieldRow{ @@ -1012,31 +1012,30 @@ func (e *executor) executeRowsShard(ctx context.Context, index string, c *pql.Ca start = previous + 1 } - filter := noFilter + filters := []rowFilter{} if limit, hasLimit, err := c.UintArg("limit"); err != nil { return nil, errors.Wrap(err, "getting limit") } else if hasLimit { - filter = (&filterWithLimit{limit: limit}).filter + filters = append(filters, (&filterWithLimit{limit: limit}).filter) } if columnID, ok, err := c.UintArg("column"); err != nil { return nil, err } else if ok { - return frag.rowsForColumnWithFilter(start, columnID, filter), nil + return frag.rowsForColumnWithFilter(start, columnID, filters...), nil } else { - return frag.rowsWithFilter(start, filter), nil + return frag.rows(start, filters...), nil } } // getGroupByFilterFunction returns a rowFilter based on the // field directive provided. -func getGroupByFilterFunction(fieldDirective string) (rowFilter, error) { +func getGroupByFilterFunction(fieldDirective string) (ret []rowFilter, err error) { parts := strings.Split(fieldDirective, ":") hasLimit := false hasOffset := false limit := uint64(0) offset := uint64(0) - var err error // fieldDirective can have one of the following forms: // [fieldName] // [fieldName:limit] @@ -1046,33 +1045,31 @@ func getGroupByFilterFunction(fieldDirective string) (rowFilter, error) { // [fieldName:offset:limit:extra] will be treated as // [fieldName:offset:limit] (i.e. `extra` is ignored). if len(parts) == 1 { - return noFilter, nil + return ret, nil } else if len(parts) == 2 { hasLimit = true if limit, err = strconv.ParseUint(parts[1], 10, 64); err != nil { - return nil, errors.Wrap(err, "getting groupby field limit only value") + return ret, errors.Wrap(err, "getting groupby field limit only value") } } else { hasOffset = true if offset, err = strconv.ParseUint(parts[1], 10, 64); err != nil { - return nil, errors.Wrap(err, "getting groupby field offset value") + return ret, errors.Wrap(err, "getting groupby field offset value") } if parts[2] != "" { hasLimit = true if limit, err = strconv.ParseUint(parts[2], 10, 64); err != nil { - return nil, errors.Wrap(err, "getting groupby field limit value") + return ret, errors.Wrap(err, "getting groupby field limit value") } } } - if hasOffset && hasLimit { - f := filterWithOffsetLimit{offset: offset, limit: limit} - return f.filter, nil - } else if hasLimit { - f := filterWithLimit{limit: limit} - return f.filter, nil + if hasOffset { + ret = append(ret, (&filterWithOffset{offset: offset}).filter) } - f := filterWithOffset{offset: offset} - return f.filter, nil + if hasLimit { + ret = append(ret, (&filterWithLimit{limit: limit}).filter) + } + return ret, nil } func (e *executor) executeBitmapShard(_ context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { @@ -2322,22 +2319,6 @@ func isString(v interface{}) bool { return ok } -// Filters to be used with RowsWithFilter queries. -type filterWithOffsetLimit struct { - offset, limit uint64 -} - -func (fol *filterWithOffsetLimit) filter(rowID uint64) (bool, bool) { - if rowID >= fol.offset { - if fol.limit > 0 { - fol.limit-- - return true, false - } - return false, true - } - return false, false -} - type filterWithOffset struct { offset uint64 } diff --git a/fragment.go b/fragment.go index a29f68109..49850f800 100644 --- a/fragment.go +++ b/fragment.go @@ -1769,17 +1769,14 @@ func (f *fragment) readCacheFromArchive(r io.Reader) error { // processing. type rowFilter func(rowID uint64) (bool, bool) -// noFilter is a filter function which has no restrictions. -var noFilter = func(rowID uint64) (bool, bool) { return true, false } - // rows returns all rows by calling rowsWithFilter() // with a completely unrestrictive filter. -func (f *fragment) rows(start uint64) []uint64 { - return f.rowsWithFilter(start, noFilter) +func (f *fragment) rows(start uint64, filters ...rowFilter) []uint64 { + return f.rowsWithFilter(start, filters...) } -func (f *fragment) rowsWithFilter(start uint64, filter rowFilter) []uint64 { +func (f *fragment) rowsWithFilter(start uint64, filters ...rowFilter) []uint64 { startKey := rowToKey(start) i, _ := f.storage.Containers.Iterator(startKey) rows := make([]uint64, 0) @@ -1797,24 +1794,32 @@ func (f *fragment) rowsWithFilter(start uint64, filter rowFilter) []uint64 { continue } - // apply filter - if addRow, breakOut := filter(vRow); breakOut { - break - } else if addRow { + // apply filters + addRow := true + for _, filter := range filters { + add, done := filter(vRow) + if done { + return rows + } + addRow = add && addRow + if !addRow { + break + } + } + if addRow { rows = append(rows, vRow) } lastRow = vRow } return rows - } func (f *fragment) rowsForColumn(columnID uint64) []uint64 { - return f.rowsForColumnWithFilter(0, columnID, noFilter) + return f.rowsForColumnWithFilter(0, columnID) } -func (f *fragment) rowsForColumnWithFilter(start, columnID uint64, filter rowFilter) []uint64 { +func (f *fragment) rowsForColumnWithFilter(start, columnID uint64, filters ...rowFilter) []uint64 { if columnID/ShardWidth != f.shard { panic(fmt.Sprintln("fragment.rowsForColumn should never be called with a columnID which is not in the fragment's shard", columnID, columnID/ShardWidth, f.shard)) @@ -1845,9 +1850,18 @@ func (f *fragment) rowsForColumnWithFilter(start, columnID uint64, filter rowFil // apply filter if c.Contains(colVal) { - if addRow, breakOut := filter(vRow); breakOut { - break - } else if addRow { + addRow := true + for _, filter := range filters { + add, done := filter(vRow) + if done { + return rows + } + addRow = add && addRow + if !addRow { + break + } + } + if addRow { rows = append(rows, vRow) } } From 8cd82af2e76b0fb7899ec82e9bf8642a15e54086 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 2 Oct 2018 09:31:13 -0500 Subject: [PATCH 18/39] remove extraneous fragment.rows* methods variadic filters makes separate methods unnecessary --- executor.go | 2 +- fragment.go | 13 ++----------- fragment_internal_test.go | 6 +++--- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/executor.go b/executor.go index d6b9ed46e..349604139 100644 --- a/executor.go +++ b/executor.go @@ -1022,7 +1022,7 @@ func (e *executor) executeRowsShard(ctx context.Context, index string, c *pql.Ca if columnID, ok, err := c.UintArg("column"); err != nil { return nil, err } else if ok { - return frag.rowsForColumnWithFilter(start, columnID, filters...), nil + return frag.rowsForColumn(start, columnID, filters...), nil } else { return frag.rows(start, filters...), nil } diff --git a/fragment.go b/fragment.go index 49850f800..6f6b376a5 100644 --- a/fragment.go +++ b/fragment.go @@ -1771,12 +1771,7 @@ type rowFilter func(rowID uint64) (bool, bool) // rows returns all rows by calling rowsWithFilter() // with a completely unrestrictive filter. - func (f *fragment) rows(start uint64, filters ...rowFilter) []uint64 { - return f.rowsWithFilter(start, filters...) -} - -func (f *fragment) rowsWithFilter(start uint64, filters ...rowFilter) []uint64 { startKey := rowToKey(start) i, _ := f.storage.Containers.Iterator(startKey) rows := make([]uint64, 0) @@ -1815,11 +1810,7 @@ func (f *fragment) rowsWithFilter(start uint64, filters ...rowFilter) []uint64 { return rows } -func (f *fragment) rowsForColumn(columnID uint64) []uint64 { - return f.rowsForColumnWithFilter(0, columnID) -} - -func (f *fragment) rowsForColumnWithFilter(start, columnID uint64, filters ...rowFilter) []uint64 { +func (f *fragment) rowsForColumn(start, columnID uint64, filters ...rowFilter) []uint64 { if columnID/ShardWidth != f.shard { panic(fmt.Sprintln("fragment.rowsForColumn should never be called with a columnID which is not in the fragment's shard", columnID, columnID/ShardWidth, f.shard)) @@ -2149,7 +2140,7 @@ func newRowsVector(f *fragment) *rowsVector { // Additionally, it returns true if a value was found, // otherwise it returns false. func (v *rowsVector) Get(colID uint64) (uint64, bool) { - rows := v.f.rowsForColumn(colID) + rows := v.f.rowsForColumn(0, colID) if len(rows) == 1 { return rows[0], true } diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 70b6ad78e..f9d0bd999 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -1346,7 +1346,7 @@ func TestFragment_RowsIteration(t *testing.T) { t.Fatalf("Do not match %v %v", expectedAll, ids) } - ids = f.rowsForColumn(1) + ids = f.rowsForColumn(0, 1) if !reflect.DeepEqual(expectedOdd, ids) { t.Fatalf("Do not match %v %v", expectedOdd, ids) } @@ -1370,7 +1370,7 @@ func TestFragment_RowsIteration(t *testing.T) { t.Fatalf("Do not match %v %v", expected, ids) } - ids = f.rowsForColumn(66000) + ids = f.rowsForColumn(0, 66000) if !reflect.DeepEqual(expected, ids) { t.Fatalf("Do not match %v %v", expected, ids) } @@ -1392,7 +1392,7 @@ func TestFragment_RowsIteration(t *testing.T) { if !reflect.DeepEqual(expectedRows, ids) { t.Fatalf("Do not match %v %v", expectedRows, ids) } - ids = f.rowsForColumn(c) + ids = f.rowsForColumn(0, c) if !reflect.DeepEqual(expectedRows, ids) { t.Fatalf("Do not match %v %v", expectedRows, ids) } From c172ca0680a3e04aea20f49a06de7148b549008b Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 8 Oct 2018 16:16:39 -0500 Subject: [PATCH 19/39] combine fragment.rows and rowsForColumn with generalized filter use filter funcs with closures for state instead of methods on structs. seems a bit cleaner. --- executor.go | 65 ++++++++++++++++------------- executor_internal_test.go | 18 ++++++++ fragment.go | 87 +++++++++------------------------------ fragment_internal_test.go | 6 +-- 4 files changed, 78 insertions(+), 98 deletions(-) diff --git a/executor.go b/executor.go index 349604139..3d8618b9d 100644 --- a/executor.go +++ b/executor.go @@ -23,6 +23,7 @@ import ( "time" "github.com/pilosa/pilosa/pql" + "github.com/pilosa/pilosa/roaring" "github.com/pkg/errors" ) @@ -1013,19 +1014,18 @@ func (e *executor) executeRowsShard(ctx context.Context, index string, c *pql.Ca } filters := []rowFilter{} - if limit, hasLimit, err := c.UintArg("limit"); err != nil { - return nil, errors.Wrap(err, "getting limit") - } else if hasLimit { - filters = append(filters, (&filterWithLimit{limit: limit}).filter) - } - if columnID, ok, err := c.UintArg("column"); err != nil { return nil, err } else if ok { - return frag.rowsForColumn(start, columnID, filters...), nil - } else { - return frag.rows(start, filters...), nil + filters = append(filters, filterColumn(columnID)) } + if limit, hasLimit, err := c.UintArg("limit"); err != nil { + return nil, errors.Wrap(err, "getting limit") + } else if hasLimit { + filters = append(filters, filterWithLimit(limit)) + } + + return frag.rows(start, filters...), nil } // getGroupByFilterFunction returns a rowFilter based on the @@ -1064,10 +1064,10 @@ func getGroupByFilterFunction(fieldDirective string) (ret []rowFilter, err error } } if hasOffset { - ret = append(ret, (&filterWithOffset{offset: offset}).filter) + ret = append(ret, filterWithOffset(offset)) } if hasLimit { - ret = append(ret, (&filterWithLimit{limit: limit}).filter) + ret = append(ret, filterWithLimit(limit)) } return ret, nil } @@ -2319,22 +2319,31 @@ func isString(v interface{}) bool { return ok } -type filterWithOffset struct { - offset uint64 -} - -func (fo *filterWithOffset) filter(rowID uint64) (bool, bool) { - return rowID >= fo.offset, false -} - -type filterWithLimit struct { - limit uint64 -} - -func (fl *filterWithLimit) filter(rowID uint64) (bool, bool) { // nolint: unparam - if fl.limit > 0 { - fl.limit-- - return true, false +func filterWithOffset(offset uint64) rowFilter { + return func(rowID, key uint64, c *roaring.Container) (include, done bool) { + return rowID >= offset, false + } +} + +// filterWithLimit returns a filter which will only allow a limited number of +// rows to be returned. It should be applied last so that it is only called (and +// therefore only updates its internal state) if the row is being included by +// every other filter. +func filterWithLimit(limit uint64) rowFilter { + return func(rowID, key uint64, c *roaring.Container) (include, done bool) { + if limit > 0 { + limit-- + return true, false + } + return false, true + } +} + +func filterColumn(col uint64) rowFilter { + return func(rowID, key uint64, c *roaring.Container) (include, done bool) { + colID := col % ShardWidth + colKey := ((rowID * ShardWidth) + colID) >> 16 + colVal := uint16(colID & 0xFFFF) // columnID within the container + return colKey == key && c.Contains(colVal), false } - return false, true } diff --git a/executor_internal_test.go b/executor_internal_test.go index e90948561..990509128 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -120,3 +120,21 @@ func isInt(a interface{}) bool { return false } } + +func TestFilterWithLimit(t *testing.T) { + f := filterWithLimit(5) + + for i := uint64(0); i < 5; i++ { + include, done := f(i, i*(1<> shardVsContainerExponent @@ -1790,71 +1795,19 @@ func (f *fragment) rows(start uint64, filters ...rowFilter) []uint64 { } // apply filters - addRow := true + addRow, done := true, false for _, filter := range filters { - add, done := filter(vRow) + addRow, done = filter(vRow, key, c) if done { return rows } - addRow = add && addRow if !addRow { break } } if addRow { - rows = append(rows, vRow) - } - - lastRow = vRow - } - return rows -} - -func (f *fragment) rowsForColumn(start, columnID uint64, filters ...rowFilter) []uint64 { - if columnID/ShardWidth != f.shard { - panic(fmt.Sprintln("fragment.rowsForColumn should never be called with a columnID which is not in the fragment's shard", - columnID, columnID/ShardWidth, f.shard)) - } - - startKey := rowToKey(start) - i, _ := f.storage.Containers.Iterator(startKey) - rows := make([]uint64, 0) - - colID := columnID % ShardWidth - colVal := uint16(colID & 0xFFFF) // columnID within the container - - var colKey uint64 - - // Loop over the existing containers. - for i.Next() { - key, c := i.Value() - - // virtual row for the current container - vRow := key >> shardVsContainerExponent - - // column container key for virtual row - colKey = ((vRow * ShardWidth) + colID) >> 16 - - if colKey != key { - continue - } - - // apply filter - if c.Contains(colVal) { - addRow := true - for _, filter := range filters { - add, done := filter(vRow) - if done { - return rows - } - addRow = add && addRow - if !addRow { - break - } - } - if addRow { - rows = append(rows, vRow) - } + lastRow = vRow + rows = append(rows, key>>shardVsContainerExponent) } } return rows @@ -2140,7 +2093,7 @@ func newRowsVector(f *fragment) *rowsVector { // Additionally, it returns true if a value was found, // otherwise it returns false. func (v *rowsVector) Get(colID uint64) (uint64, bool) { - rows := v.f.rowsForColumn(0, colID) + rows := v.f.rows(0, filterColumn(colID)) if len(rows) == 1 { return rows[0], true } diff --git a/fragment_internal_test.go b/fragment_internal_test.go index f9d0bd999..36651fee0 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -1346,7 +1346,7 @@ func TestFragment_RowsIteration(t *testing.T) { t.Fatalf("Do not match %v %v", expectedAll, ids) } - ids = f.rowsForColumn(0, 1) + ids = f.rows(0, filterColumn(1)) if !reflect.DeepEqual(expectedOdd, ids) { t.Fatalf("Do not match %v %v", expectedOdd, ids) } @@ -1370,7 +1370,7 @@ func TestFragment_RowsIteration(t *testing.T) { t.Fatalf("Do not match %v %v", expected, ids) } - ids = f.rowsForColumn(0, 66000) + ids = f.rows(0, filterColumn(66000)) if !reflect.DeepEqual(expected, ids) { t.Fatalf("Do not match %v %v", expected, ids) } @@ -1392,7 +1392,7 @@ func TestFragment_RowsIteration(t *testing.T) { if !reflect.DeepEqual(expectedRows, ids) { t.Fatalf("Do not match %v %v", expectedRows, ids) } - ids = f.rowsForColumn(0, c) + ids = f.rows(0, filterColumn(c)) if !reflect.DeepEqual(expectedRows, ids) { t.Fatalf("Do not match %v %v", expectedRows, ids) } From cbd7e945b28dacc6fa90385eaf58e807c25c291e Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 8 Oct 2018 19:04:29 -0500 Subject: [PATCH 20/39] get GroupBy working with "Rows" child calls, remove fieldDirectives had to implement decoders for RowIDs and RowIdentifiers - a sign that we need better testing of remote Rows calls --- encoding/proto/proto.go | 11 +++ executor.go | 149 ++++++++++++++-------------------------- executor_test.go | 21 ++---- 3 files changed, 71 insertions(+), 110 deletions(-) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index b6d78799a..4ba2021bb 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -949,6 +949,10 @@ func decodeQueryResult(pb *internal.QueryResult) interface{} { return pb.Changed case queryResultTypeNil: return nil + case queryResultTypeRowIDs: + return pilosa.RowIDs(pb.RowIDs) + case queryResultTypeRowIdentifiers: + return decodeRowIdentifiers(pb.RowIdentifiers) case queryResultTypeGroupCounts: return decodeGroupCounts(pb.GroupCounts) } @@ -1001,6 +1005,13 @@ func decodeAttr(attr *internal.Attr) (key string, value interface{}) { } } +func decodeRowIdentifiers(a *internal.RowIdentifiers) *pilosa.RowIdentifiers { + return &pilosa.RowIdentifiers{ + Rows: a.Rows, + Keys: a.Keys, + } +} + func decodeGroupCounts(a []*internal.GroupCount) []pilosa.GroupCount { other := make([]pilosa.GroupCount, len(a)) for i := range a { diff --git a/executor.go b/executor.go index 3d8618b9d..3eb887b4e 100644 --- a/executor.go +++ b/executor.go @@ -18,7 +18,6 @@ import ( "context" "fmt" "sort" - "strconv" "strings" "time" @@ -768,9 +767,40 @@ func (r RowIDs) merge(other RowIDs, limit int) RowIDs { } func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) ([]GroupCount, error) { + // validate call + if len(c.Children) == 0 { + return nil, errors.New("need at least one child call") + } + // get limit + gbLimit := int(^uint(0) >> 1) // largest signed int + if limit, hasLimit, err := c.UintArg("limit"); err != nil { + return nil, err + } else if hasLimit { + gbLimit = int(limit) + } + // perform Rows queries - TODO, call async? run per shard in + // executeGroupByShard? (note: can only do this for Rows queries which do + // not include "column" arg) + childRows := make([]RowIDs, len(c.Children)) + for i, child := range c.Children { + if child.Name != "Rows" { + return nil, errors.Errorf("'%s' is not a valid child query for GroupBy, must be 'Rows'", c.Name) + } + if limit, hasLimit, err := child.UintArg("limit"); err != nil { + return nil, err + } else if hasLimit && int(limit) > gbLimit { + child.Args["limit"] = uint64(gbLimit) + } + var err error + childRows[i], err = e.executeRows(ctx, index, child, shards, opt) + if err != nil { + return nil, errors.Wrap(err, "getting rows for ") + } + } + // Execute calls in bulk on each remote node and merge. mapFn := func(shard uint64) (interface{}, error) { - return e.executeGroupByShard(ctx, index, c, shard) + return e.executeGroupByShard(ctx, index, c, shard, childRows) } // Merge returned results at coordinating node. reduceFn := func(prev, v interface{}) interface{} { @@ -849,66 +879,39 @@ func mergeGroupCounts(gc, other []GroupCount) []GroupCount { return gc } -func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql.Call, shard uint64) ([]GroupCount, error) { +func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql.Call, shard uint64, childRows []RowIDs) ([]GroupCount, error) { // Fetch index. idx := e.Holder.Index(index) if idx == nil { return nil, ErrIndexNotFound } - // fieldDirective is a combination of field - // instructions, represented as a string with - // the form [fieldName:offset:limit] or - // [fieldName:limit]. - fieldDirectives, ok := c.Args["fields"] - if !ok { - return nil, errors.Wrap(ErrFieldsArgumentRequired, "executeGroupBy") - } - // Ensure that fieldDirectives is a list. - if _, ok := fieldDirectives.([]interface{}); !ok { - return nil, errors.Wrap(ErrExpectedFieldListArgument, "executeGroupBy") - } - // getFieldName extracts the fieldName portion of the - // fieldDirective. - getFieldName := func(s string) string { - parts := strings.Split(s, ":") - return parts[0] - } - // Ensure that all of the fields exist. - for _, fieldDirective := range fieldDirectives.([]interface{}) { - fieldName := getFieldName(fieldDirective.(string)) - f := e.Holder.Field(index, fieldName) - if f == nil { - return nil, errors.Wrap(ErrFieldNotFound, fmt.Sprintf("executeGroupBy: %s", fieldDirective.(string))) - } - } - results := make([]GroupCount, 0) var work [][]gbi - for _, fieldDirective := range fieldDirectives.([]interface{}) { - fieldName := getFieldName(fieldDirective.(string)) + for i, rowIDs := range childRows { + fieldName := c.Children[i].Args["field"].(string) // this has already been validated by this point // Fetch fragment. frag := e.Holder.fragment(index, fieldName, viewStandard, shard) if frag == nil { // this means this whole shard doesn't have all it needs to continue - return results, nil - } - // Get filter based on the field directive. - filters, err := getGroupByFilterFunction(fieldDirective.(string)) - if err != nil { - return nil, err + return []GroupCount{}, nil } set := make([]gbi, 0) - for _, rowID := range frag.rows(0, filters...) { - set = append(set, gbi{ - row: frag.row(rowID), - fieldRow: FieldRow{ - Field: fieldName, - RowID: rowID, - }, - }) + for _, rowID := range rowIDs { + rs := frag.rows(rowID, filterWithLimit(1)) + if len(rs) > 0 && rs[0] == rowID { + set = append(set, gbi{ + row: frag.row(rowID), + fieldRow: FieldRow{ + Field: fieldName, + RowID: rowID, + }, + }) + } } work = append(work, set) } + + results := make([]GroupCount, 0) for _, group := range product(work) { group.gCnt.Count = group.row.Count() if group.gCnt.Count > 0 { @@ -1017,6 +1020,10 @@ func (e *executor) executeRowsShard(ctx context.Context, index string, c *pql.Ca if columnID, ok, err := c.UintArg("column"); err != nil { return nil, err } else if ok { + colShard := columnID >> shardWidthExponent + if colShard != shard { + return RowIDs{}, nil + } filters = append(filters, filterColumn(columnID)) } if limit, hasLimit, err := c.UintArg("limit"); err != nil { @@ -1028,50 +1035,6 @@ func (e *executor) executeRowsShard(ctx context.Context, index string, c *pql.Ca return frag.rows(start, filters...), nil } -// getGroupByFilterFunction returns a rowFilter based on the -// field directive provided. -func getGroupByFilterFunction(fieldDirective string) (ret []rowFilter, err error) { - parts := strings.Split(fieldDirective, ":") - hasLimit := false - hasOffset := false - limit := uint64(0) - offset := uint64(0) - // fieldDirective can have one of the following forms: - // [fieldName] - // [fieldName:limit] - // [fieldName:offset:limit] - // - // Note that a field directive with the form - // [fieldName:offset:limit:extra] will be treated as - // [fieldName:offset:limit] (i.e. `extra` is ignored). - if len(parts) == 1 { - return ret, nil - } else if len(parts) == 2 { - hasLimit = true - if limit, err = strconv.ParseUint(parts[1], 10, 64); err != nil { - return ret, errors.Wrap(err, "getting groupby field limit only value") - } - } else { - hasOffset = true - if offset, err = strconv.ParseUint(parts[1], 10, 64); err != nil { - return ret, errors.Wrap(err, "getting groupby field offset value") - } - if parts[2] != "" { - hasLimit = true - if limit, err = strconv.ParseUint(parts[2], 10, 64); err != nil { - return ret, errors.Wrap(err, "getting groupby field limit value") - } - } - } - if hasOffset { - ret = append(ret, filterWithOffset(offset)) - } - if hasLimit { - ret = append(ret, filterWithLimit(limit)) - } - return ret, nil -} - func (e *executor) executeBitmapShard(_ context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { // Fetch index. idx := e.Holder.Index(index) @@ -2319,12 +2282,6 @@ func isString(v interface{}) bool { return ok } -func filterWithOffset(offset uint64) rowFilter { - return func(rowID, key uint64, c *roaring.Container) (include, done bool) { - return rowID >= offset, false - } -} - // filterWithLimit returns a filter which will only allow a limited number of // rows to be returned. It should be applied last so that it is only called (and // therefore only updates its internal state) if the row is being included by diff --git a/executor_test.go b/executor_test.go index faa0acbf9..78e6d8790 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1229,7 +1229,7 @@ Set(4500001, fn=4) t.Run("remote groupBy", func(t *testing.T) { if res, err := c[1].API.Query(context.Background(), &pilosa.QueryRequest{ Index: "i", - Query: `GroupBy(fields=[f])`, + Query: `GroupBy(Rows(field=f))`, }); err != nil { t.Fatalf("GroupBy querying: %v", err) } else { @@ -1819,25 +1819,18 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { t.Run("No Field List Arguments", func(t *testing.T) { if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy()`}); err != nil { - if errors.Cause(err) != pilosa.ErrFieldsArgumentRequired { - t.Fatalf("unexpected error\n\"%s\" not returned instead \n\"%s\"", pilosa.ErrFieldsArgumentRequired, err) + if !strings.Contains(err.Error(), "need at least one child call") { + t.Fatalf("unexpected error: \"%v\"", err) } } }) t.Run("Unknown Field ", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(fields=[missing])`}); err != nil { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(Rows(field=missing))`}); err != nil { if errors.Cause(err) != pilosa.ErrFieldNotFound { t.Fatalf("unexpected error\n\"%s\" not returned instead \n\"%s\"", pilosa.ErrFieldNotFound, err) } } }) - t.Run("Bad Field Format", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(fields=missing)`}); err != nil { - if errors.Cause(err) != pilosa.ErrExpectedFieldListArgument { - t.Fatalf("unexpected error\n\"%s\" not returned instead \n\"%s\"", pilosa.ErrExpectedFieldListArgument, err) - } - } - }) t.Run("Basic", func(t *testing.T) { expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Count: 1}, @@ -1846,7 +1839,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Count: 3}, } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(fields=[general,sub])`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(Rows(field=general), Rows(field=sub))`}); err != nil { t.Fatal(err) } else { results := res.Results[0].([]pilosa.GroupCount) @@ -1860,7 +1853,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "general", RowID: 12}}, Count: 2}, } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(fields=[general:11:])`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(Rows(field=general, previous=10))`}); err != nil { t.Fatal(err) } else { results := res.Results[0].([]pilosa.GroupCount) @@ -1873,7 +1866,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Count: 2}, } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(fields=[general:11:1])`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(Rows(field=general, previous=10, limit=1))`}); err != nil { t.Fatal(err) } else { results := res.Results[0].([]pilosa.GroupCount) From e2bbcb28e5ab3f37cb50c20ad69146d14d528a2b Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 8 Oct 2018 19:10:16 -0500 Subject: [PATCH 21/39] fix linter issues --- encoding/proto/proto.go | 4 ++-- executor.go | 4 ++-- fragment.go | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 4ba2021bb..5aad595a5 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -1016,8 +1016,8 @@ func decodeGroupCounts(a []*internal.GroupCount) []pilosa.GroupCount { other := make([]pilosa.GroupCount, len(a)) for i := range a { other[i] = pilosa.GroupCount{ - decodeFieldRows(a[i].Group), - a[i].Count, + Group: decodeFieldRows(a[i].Group), + Count: a[i].Count, } } return other diff --git a/executor.go b/executor.go index 3eb887b4e..89892e5c5 100644 --- a/executor.go +++ b/executor.go @@ -879,7 +879,7 @@ func mergeGroupCounts(gc, other []GroupCount) []GroupCount { return gc } -func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql.Call, shard uint64, childRows []RowIDs) ([]GroupCount, error) { +func (e *executor) executeGroupByShard(_ context.Context, index string, c *pql.Call, shard uint64, childRows []RowIDs) ([]GroupCount, error) { // Fetch index. idx := e.Holder.Index(index) if idx == nil { @@ -988,7 +988,7 @@ func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, s return results, nil } -func (e *executor) executeRowsShard(ctx context.Context, index string, c *pql.Call, shard uint64) (RowIDs, error) { +func (e *executor) executeRowsShard(_ context.Context, index string, c *pql.Call, shard uint64) (RowIDs, error) { // Fetch index. idx := e.Holder.Index(index) if idx == nil { diff --git a/fragment.go b/fragment.go index a738b0829..9fdcb849a 100644 --- a/fragment.go +++ b/fragment.go @@ -1795,8 +1795,9 @@ func (f *fragment) rows(start uint64, filters ...rowFilter) []uint64 { } // apply filters - addRow, done := true, false + addRow := true for _, filter := range filters { + var done bool addRow, done = filter(vRow, key, c) if done { return rows From 1474884f5dd1b4cc7dd97a59f82b29d6e583b488 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 9 Oct 2018 10:27:57 -0500 Subject: [PATCH 22/39] use existing var instead of recalculating silly mistake - thanks todd --- fragment.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fragment.go b/fragment.go index 9fdcb849a..a22d2e1e4 100644 --- a/fragment.go +++ b/fragment.go @@ -1808,7 +1808,7 @@ func (f *fragment) rows(start uint64, filters ...rowFilter) []uint64 { } if addRow { lastRow = vRow - rows = append(rows, key>>shardVsContainerExponent) + rows = append(rows, vRow) } } return rows From 45fb6f0c0679cdb8ea3dd6f30f7e09bf2a7e0a3c Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 9 Oct 2018 18:44:48 -0500 Subject: [PATCH 23/39] add some new test utils and test Rows calls on cluster --- executor_test.go | 39 ++++++++++++++------------- test/pilosa.go | 68 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 20 deletions(-) diff --git a/executor_test.go b/executor_test.go index 366a044f9..f0eb4608c 100644 --- a/executor_test.go +++ b/executor_test.go @@ -2189,37 +2189,36 @@ func BenchmarkExecutor_Existence_True(b *testing.B) { benchmarkExistence(true, func BenchmarkExecutor_Existence_False(b *testing.B) { benchmarkExistence(false, b) } func TestExecutor_Execute_Rows(t *testing.T) { - c := test.MustRunCluster(t, 1) + c := test.MustRunCluster(t, 3) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - hldr.SetBit("i", "general", 10, 0) - hldr.SetBit("i", "general", 10, ShardWidth+1) - hldr.SetBit("i", "general", 11, 2) - hldr.SetBit("i", "general", 11, ShardWidth+2) - hldr.SetBit("i", "general", 12, 2) - hldr.SetBit("i", "general", 12, ShardWidth+2) + c.CreateField(t, "i", pilosa.IndexOptions{}, "general") + c.ImportBits(t, "i", "general", [][2]uint64{ + {10, 0}, + {10, ShardWidth + 1}, + {11, 2}, + {11, ShardWidth + 2}, + {12, 2}, + {12, ShardWidth + 2}, + {13, 3}, + }) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=general)`}); err != nil { - t.Fatal(err) - } else if rows := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{10, 11, 12}}) { + rows := c.Query(t, "i", `Rows(field=general)`).Results[0].(pilosa.RowIdentifiers) + if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{10, 11, 12, 13}}) { t.Fatalf("unexpected rows: %+v", rows) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=general, limit=2)`}); err != nil { - t.Fatal(err) - } else if rows := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{10, 11}}) { + rows = c.Query(t, "i", `Rows(field=general, limit=2)`).Results[0].(pilosa.RowIdentifiers) + if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{10, 11}}) { t.Fatalf("unexpected rows: %+v", rows) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=general, previous=10,limit=2)`}); err != nil { - t.Fatal(err) - } else if rows := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) { + rows = c.Query(t, "i", `Rows(field=general, previous=10,limit=2)`).Results[0].(pilosa.RowIdentifiers) + if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) { t.Fatalf("unexpected rows: %+v", rows) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=general, column=2)`}); err != nil { - t.Fatal(err) - } else if rows := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) { + rows = c.Query(t, "i", `Rows(field=general, column=2)`).Results[0].(pilosa.RowIdentifiers) + if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) { t.Fatalf("unexpected rows: %+v", rows) } } diff --git a/test/pilosa.go b/test/pilosa.go index 27331d53f..2e4564284 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -197,6 +197,74 @@ func (m *Command) RecalculateCaches() error { // Cluster represents a Pilosa cluster (multiple Command instances) type Cluster []*Command +// Query executes an API.Query through one of the cluster's node's API. It fails +// the test if there is an error. +func (c Cluster) Query(t testing.TB, index, query string) pilosa.QueryResponse { + if len(c) == 0 { + t.Fatal("must have at least one node in cluster to query") + } + + return c[0].MustQuery(t, &pilosa.QueryRequest{Index: index, Query: query}) +} + +func (c Cluster) ImportBits(t testing.TB, index, field string, rowcols [][2]uint64) { + byShard := make(map[uint64][][2]uint64) + for _, rowcol := range rowcols { + shard := rowcol[1] / pilosa.ShardWidth + byShard[shard] = append(byShard[shard], rowcol) + } + + for shard, bits := range byShard { + rowIDs := make([]uint64, len(bits)) + colIDs := make([]uint64, len(bits)) + for i, bit := range bits { + rowIDs[i] = bit[0] + colIDs[i] = bit[1] + } + nodes, err := c[0].API.ShardNodes(context.Background(), index, shard) + if err != nil { + t.Fatalf("getting shard nodes: %v", err) + } + // TODO won't be necessary to do all nodes once that works hits + for _, node := range nodes { + for _, com := range c { + if com.API.Node().ID != node.ID { + continue + } + err := com.API.Import(context.Background(), &pilosa.ImportRequest{ + Index: index, + Field: field, + Shard: shard, + RowIDs: rowIDs, + ColumnIDs: colIDs, + }) + if err != nil { + t.Fatalf("importing data: %v", err) + } + } + } + } +} + +// CreateField creates the index (if necessary) and field specified. +func (c Cluster) CreateField(t testing.TB, index string, iopts pilosa.IndexOptions, field string, fopts ...pilosa.FieldOption) *pilosa.Field { + idx, err := c[0].API.CreateIndex(context.Background(), index, iopts) + if err != nil && errors.Cause(err) != pilosa.ErrIndexExists { + t.Fatalf("creating index: %v", err) + } + if idx.Options() != iopts { + t.Logf("existing index options:\n%v\ndon't match given opts:\n%v\n in pilosa/test.Cluster.CreateField", idx.Options(), iopts) + } + + f, err := c[0].API.CreateField(context.Background(), index, field, fopts...) + // we'll assume the field doesn't exist because checking if the options + // match seems painful. + if err != nil { + t.Fatalf("creating field: %v", err) + } + return f +} + // Start runs a Cluster func (c Cluster) Start() error { var gossipSeeds = make([]string, len(c)) From 578c594755650a945ab2a4c7ab564431c2fa74c5 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 9 Oct 2018 19:13:12 -0500 Subject: [PATCH 24/39] convert GroupBy tests to use new utils; fix case where index exists --- executor_test.go | 61 ++++++++++++++++++++++-------------------------- test/pilosa.go | 7 +++++- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/executor_test.go b/executor_test.go index f0eb4608c..072dff0f4 100644 --- a/executor_test.go +++ b/executor_test.go @@ -2348,22 +2348,26 @@ func TestExecutor_Execute_Rows_Keys(t *testing.T) { func TestExecutor_Execute_GroupBy(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - hldr.SetBit("i", "general", 10, 0) - hldr.SetBit("i", "general", 10, 1) - hldr.SetBit("i", "general", 10, ShardWidth+1) - hldr.SetBit("i", "general", 11, 2) - hldr.SetBit("i", "general", 11, ShardWidth+2) - hldr.SetBit("i", "general", 12, 2) - hldr.SetBit("i", "general", 12, ShardWidth+2) + c.CreateField(t, "i", pilosa.IndexOptions{}, "general") + c.CreateField(t, "i", pilosa.IndexOptions{}, "sub") + c.ImportBits(t, "i", "general", [][2]uint64{ + {10, 0}, + {10, 1}, + {10, ShardWidth + 1}, + {11, 2}, + {11, ShardWidth + 2}, + {12, 2}, + {12, ShardWidth + 2}, + }) + c.ImportBits(t, "i", "sub", [][2]uint64{ + {100, 0}, + {100, 1}, + {100, 3}, + {100, ShardWidth + 1}, - hldr.SetBit("i", "sub", 100, 0) - hldr.SetBit("i", "sub", 100, 1) - hldr.SetBit("i", "sub", 100, 3) - hldr.SetBit("i", "sub", 100, ShardWidth+1) - - hldr.SetBit("i", "sub", 110, 2) - hldr.SetBit("i", "sub", 110, 0) + {110, 2}, + {110, 0}, + }) t.Run("No Field List Arguments", func(t *testing.T) { if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy()`}); err != nil { @@ -2372,6 +2376,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { } } }) + t.Run("Unknown Field ", func(t *testing.T) { if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(Rows(field=missing))`}); err != nil { if errors.Cause(err) != pilosa.ErrFieldNotFound { @@ -2379,6 +2384,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { } } }) + t.Run("Basic", func(t *testing.T) { expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Count: 1}, @@ -2387,12 +2393,8 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Count: 3}, } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(Rows(field=general), Rows(field=sub))`}); err != nil { - t.Fatal(err) - } else { - results := res.Results[0].([]pilosa.GroupCount) - checkGroupBy(t, expected, results) - } + results := c.Query(t, "i", `GroupBy(Rows(field=general), Rows(field=sub))`).Results[0].([]pilosa.GroupCount) + checkGroupBy(t, expected, results) }) t.Run("check field offset no limit", func(t *testing.T) { @@ -2401,12 +2403,8 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "general", RowID: 12}}, Count: 2}, } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(Rows(field=general, previous=10))`}); err != nil { - t.Fatal(err) - } else { - results := res.Results[0].([]pilosa.GroupCount) - checkGroupBy(t, expected, results) - } + results := c.Query(t, "i", `GroupBy(Rows(field=general, previous=10))`).Results[0].([]pilosa.GroupCount) + checkGroupBy(t, expected, results) }) t.Run("check field offset limit", func(t *testing.T) { @@ -2414,12 +2412,9 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Count: 2}, } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(Rows(field=general, previous=10, limit=1))`}); err != nil { - t.Fatal(err) - } else { - results := res.Results[0].([]pilosa.GroupCount) - checkGroupBy(t, expected, results) - } + results := c.Query(t, "i", `GroupBy(Rows(field=general, previous=10, limit=1))`).Results[0].([]pilosa.GroupCount) + checkGroupBy(t, expected, results) + }) } diff --git a/test/pilosa.go b/test/pilosa.go index 2e4564284..3bc43d593 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -249,8 +249,13 @@ func (c Cluster) ImportBits(t testing.TB, index, field string, rowcols [][2]uint // CreateField creates the index (if necessary) and field specified. func (c Cluster) CreateField(t testing.TB, index string, iopts pilosa.IndexOptions, field string, fopts ...pilosa.FieldOption) *pilosa.Field { idx, err := c[0].API.CreateIndex(context.Background(), index, iopts) - if err != nil && errors.Cause(err) != pilosa.ErrIndexExists { + if err != nil && !strings.Contains(err.Error(), "index already exists") { t.Fatalf("creating index: %v", err) + } else if err != nil { // index exists + idx, err = c[0].API.Index(context.Background(), index) + if err != nil { + t.Fatalf("getting index: %v", err) + } } if idx.Options() != iopts { t.Logf("existing index options:\n%v\ndon't match given opts:\n%v\n in pilosa/test.Cluster.CreateField", idx.Options(), iopts) From 603b0e5369eef89f53276cceecd32cb5e01ed739 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 9 Oct 2018 19:27:10 -0500 Subject: [PATCH 25/39] fix logic bug applying limit to group by rows check in failing test showing how applying the limit to each rows query can cause the query to falsely return no results --- executor.go | 2 +- executor_test.go | 22 +++++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/executor.go b/executor.go index 8cac5c034..1cd95eed4 100644 --- a/executor.go +++ b/executor.go @@ -851,7 +851,7 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call } if limit, hasLimit, err := child.UintArg("limit"); err != nil { return nil, err - } else if hasLimit && int(limit) > gbLimit { + } else if !hasLimit || int(limit) > gbLimit { child.Args["limit"] = uint64(gbLimit) } var err error diff --git a/executor_test.go b/executor_test.go index 072dff0f4..12bf0b1e8 100644 --- a/executor_test.go +++ b/executor_test.go @@ -2416,6 +2416,26 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { checkGroupBy(t, expected, results) }) + + c.CreateField(t, "i", pilosa.IndexOptions{}, "a") + c.CreateField(t, "i", pilosa.IndexOptions{}, "b") + c.ImportBits(t, "i", "a", [][2]uint64{ + {0, 1}, + {1, ShardWidth + 1}, + }) + c.ImportBits(t, "i", "b", [][2]uint64{ + {0, ShardWidth + 1}, + {1, 1}, + }) + + t.Run("tricky data", func(t *testing.T) { + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "a", RowID: 0}, {Field: "b", RowID: 1}}, Count: 1}, + } + + results := c.Query(t, "i", `GroupBy(Rows(field=a), Rows(field=b), limit=1)`).Results[0].([]pilosa.GroupCount) + checkGroupBy(t, expected, results) + }) } func checkGroupBy(t *testing.T, expected, results []pilosa.GroupCount) { @@ -2434,7 +2454,7 @@ func checkGroupBy(t *testing.T, expected, results []pilosa.GroupCount) { } for _, result := range results { if notIn(result, expected) { - t.Fatalf("unexpected grouping: \n%+v\n\n\n%+v\n", result, expected) + t.Fatalf("unexpected results: \n got:%+v\nwant:%+v\n", results, expected) } } } From 360623230fb7603abace67f57ee1925360b46976 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 10 Oct 2018 21:03:45 -0500 Subject: [PATCH 26/39] get a somewhat better groupBy working that passes new tests one test still fails due to reordering during merging --- executor.go | 261 ++++++++++++++++++++++++++++---------- executor_internal_test.go | 55 ++++++++ executor_test.go | 46 ++++++- 3 files changed, 290 insertions(+), 72 deletions(-) diff --git a/executor.go b/executor.go index 1cd95eed4..d9bdb54d2 100644 --- a/executor.go +++ b/executor.go @@ -834,13 +834,6 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call if len(c.Children) == 0 { return nil, errors.New("need at least one child call") } - // get limit - gbLimit := int(^uint(0) >> 1) // largest signed int - if limit, hasLimit, err := c.UintArg("limit"); err != nil { - return nil, err - } else if hasLimit { - gbLimit = int(limit) - } // perform Rows queries - TODO, call async? run per shard in // executeGroupByShard? (note: can only do this for Rows queries which do // not include "column" arg) @@ -849,15 +842,22 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call if child.Name != "Rows" { return nil, errors.Errorf("'%s' is not a valid child query for GroupBy, must be 'Rows'", c.Name) } - if limit, hasLimit, err := child.UintArg("limit"); err != nil { - return nil, err - } else if !hasLimit || int(limit) > gbLimit { - child.Args["limit"] = uint64(gbLimit) - } - var err error - childRows[i], err = e.executeRows(ctx, index, child, shards, opt) + _, hasLimit, err := child.UintArg("limit") if err != nil { - return nil, errors.Wrap(err, "getting rows for ") + return nil, errors.Wrap(err, "getting limit") + } + _, hasCol, err := child.UintArg("column") + if err != nil { + return nil, errors.Wrap(err, "getting column") + } + if hasLimit || hasCol { // we need to perform this query cluster-wide ahead of executeGroupByShard + childRows[i], err = e.executeRows(ctx, index, child, shards, opt) + if err != nil { + return nil, errors.Wrap(err, "getting rows for ") + } + if len(childRows[i]) == 0 { // there are no results because this field has no values. + return []GroupCount{}, nil + } } } @@ -943,44 +943,58 @@ func mergeGroupCounts(gc, other []GroupCount) []GroupCount { } func (e *executor) executeGroupByShard(_ context.Context, index string, c *pql.Call, shard uint64, childRows []RowIDs) ([]GroupCount, error) { - // Fetch index. - idx := e.Holder.Index(index) - if idx == nil { - return nil, ErrIndexNotFound + iter := newGroupByIterator(childRows, c.Children, index, shard, e.Holder) + if iter == nil { + return []GroupCount{}, nil } - var work [][]gbi - for i, rowIDs := range childRows { - fieldName := c.Children[i].Args["field"].(string) // this has already been validated by this point - // Fetch fragment. - frag := e.Holder.fragment(index, fieldName, viewStandard, shard) - if frag == nil { // this means this whole shard doesn't have all it needs to continue - return []GroupCount{}, nil - } + // var work [][]gbi + // for i, rowIDs := range childRows { + // fieldName := c.Children[i].Args["field"].(string) // this has already been validated by this point + // // Fetch fragment. + // frag := e.Holder.fragment(index, fieldName, viewStandard, shard) + // if frag == nil { // this means this whole shard doesn't have all it needs to continue + // return []GroupCount{}, nil + // } - set := make([]gbi, 0) - for _, rowID := range rowIDs { - rs := frag.rows(rowID, filterWithLimit(1)) - if len(rs) > 0 && rs[0] == rowID { - set = append(set, gbi{ - row: frag.row(rowID), - fieldRow: FieldRow{ - Field: fieldName, - RowID: rowID, - }, - }) - } - } - work = append(work, set) + // set := make([]gbi, 0) + // for _, rowID := range rowIDs { + // rs := frag.rows(rowID, filterWithLimit(1)) + // if len(rs) > 0 && rs[0] == rowID { + // set = append(set, gbi{ + // row: frag.row(rowID), + // fieldRow: FieldRow{ + // Field: fieldName, + // RowID: rowID, + // }, + // }) + // } + // } + // work = append(work, set) + // } + limit := int(^uint(0) >> 1) + if lim, hasLimit, err := c.UintArg("limit"); err != nil { + return nil, err + } else if hasLimit { + limit = int(lim) } results := make([]GroupCount, 0) - for _, group := range product(work) { - group.gCnt.Count = group.row.Count() - if group.gCnt.Count > 0 { - results = append(results, group.gCnt) + + num := 0 + for pp, done := iter.Next(); !done && num < limit; pp, done = iter.Next() { + if pp.gCnt.Count > 0 { + num++ + results = append(results, pp.gCnt) } } + + // for _, group := range product(work) { + // group.gCnt.Count = group.row.Count() + // if group.gCnt.Count > 0 { + // results = append(results, group.gCnt) + // } + // } return results, nil } @@ -990,32 +1004,32 @@ type ppi struct { gCnt GroupCount } -// product generates the cartesian product of the input -// using tail recursion. -func product(input [][]gbi) []ppi { - if len(input) == 0 { // base return empty list - return []ppi{ - {gCnt: GroupCount{Group: make([]FieldRow, 0)}}, - } - } +// // product generates the cartesian product of the input +// // using tail recursion. +// func product(input [][]gbi) []ppi { +// if len(input) == 0 { // base return empty list +// return []ppi{ +// {gCnt: GroupCount{Group: make([]FieldRow, 0)}}, +// } +// } - res := make([]ppi, 0) - head := input[0] // take first element of the list - tail := product(input[1:]) // invoke product on remaining element - for h := range head { // for each head - for t := range tail { // iterate over the tail - s := ppi{gCnt: GroupCount{Group: make([]FieldRow, 0)}} - s.gCnt.Group = append([]FieldRow{head[h].fieldRow}, tail[t].gCnt.Group...) // had to insert at the front to match input order - if tail[t].row != nil { // first time around nothing to intersect - s.row = head[h].row.Intersect(tail[t].row) - } else { - s.row = head[h].row - } - res = append(res, s) - } - } - return res -} +// res := make([]ppi, 0) +// head := input[0] // take first element of the list +// tail := product(input[1:]) // invoke product on remaining element +// for h := range head { // for each head +// for t := range tail { // iterate over the tail +// s := ppi{gCnt: GroupCount{Group: make([]FieldRow, 0)}} +// s.gCnt.Group = append([]FieldRow{head[h].fieldRow}, tail[t].gCnt.Group...) // had to insert at the front to match input order +// if tail[t].row != nil { // first time around nothing to intersect +// s.row = head[h].row.Intersect(tail[t].row) +// } else { +// s.row = head[h].row +// } +// res = append(res, s) +// } +// } +// return res +// } func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (RowIDs, error) { if columnID, ok, err := c.UintArg("column"); err != nil { @@ -2555,3 +2569,110 @@ func filterColumn(col uint64) rowFilter { return colKey == key && c.Contains(colVal), false } } + +// TODO: this works, but it would be more performant if the fragment could seek to the +// next row in the rows list rather than asking the filter for each container +// serially. +func filterWithRows(rows []uint64) rowFilter { + loc := 0 + return func(rowID, key uint64, c *roaring.Container) (include, done bool) { + if loc >= len(rows) { + return false, true + } + i := sort.Search(len(rows[loc:]), func(i int) bool { + return rows[loc+i] >= rowID + }) + loc += i + if loc >= len(rows) { + return false, true + } + if rows[loc] == rowID { + if loc == len(rows)-1 { + done = true + } + return true, done + } + return false, false + } +} + +type groupByIterator struct { + fragments []*fragment + current []uint64 + rows []RowIDs + fields []FieldRow +} + +func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, index string, shard uint64, holder *Holder) *groupByIterator { + gbi := &groupByIterator{ + fragments: make([]*fragment, len(rowIDs)), + current: make([]uint64, len(rowIDs)), + rows: rowIDs, + fields: make([]FieldRow, len(rowIDs)), + } + for i, call := range children { + fieldName := call.Args["field"].(string) // this has already been validated by this point + gbi.fields[i].Field = fieldName + // Fetch fragment. + frag := holder.fragment(index, fieldName, viewStandard, shard) + if frag == nil { // this means this whole shard doesn't have all it needs to continue + return nil + } + gbi.fragments[i] = frag + if prev, hasPrev, err := call.UintArg("previous"); err != nil { + panic("getting prev") + } else if hasPrev { + gbi.current[i] = prev + if i == len(children)-1 { + gbi.current[i] += 1 + } + } + } + return gbi +} + +func (gbi *groupByIterator) Next() (ret ppi, done bool) { + rows := make([]*Row, len(gbi.current)) + ret.gCnt.Group = make([]FieldRow, len(gbi.current)) + copy(ret.gCnt.Group, gbi.fields) + wrap := false + + // build rows slice + for i := len(gbi.current) - 1; i >= 0; i-- { + if wrap { + gbi.current[i] += 1 + wrap = false + } + frag := gbi.fragments[i] + filters := []rowFilter{} + if len(gbi.rows[i]) > 0 { + filters = append(filters, filterWithRows(gbi.rows[i])) + } + filters = append(filters, filterWithLimit(1)) + rowIDs := frag.rows(gbi.current[i], filters...) + if len(rowIDs) == 0 && i != 0 { // wrap around + wrap = true + gbi.current[i] = 0 + if len(gbi.rows[i]) > 0 { + gbi.current[i] = gbi.rows[i][0] + } + rowIDs = frag.rows(gbi.current[i], filters...) + } + if len(rowIDs) != 0 { + rows[i] = frag.row(rowIDs[0]) + gbi.current[i] = rowIDs[0] + ret.gCnt.Group[i].RowID = rowIDs[0] + } else { + return ppi{}, true + } + } + gbi.current[len(gbi.current)-1] += 1 + + // build ppi from rows + ret.row = rows[0] + for _, row := range rows[1:] { + ret.row = ret.row.Intersect(row) + } + ret.gCnt.Count = ret.row.Count() + return ret, false +} diff --git a/executor_internal_test.go b/executor_internal_test.go index 990509128..93ba86743 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -138,3 +138,58 @@ func TestFilterWithLimit(t *testing.T) { t.Fatalf("limit filter should have been done, but got inc: %v done: %v", inc, done) } } + +func TestFilterWithRows(t *testing.T) { + tests := []struct { + rows []uint64 + callWith []uint64 + expect [][2]bool + }{ + { + rows: []uint64{}, + callWith: []uint64{0}, + expect: [][2]bool{{false, true}}, + }, + { + rows: []uint64{0}, + callWith: []uint64{0}, + expect: [][2]bool{{true, true}}, + }, + { + rows: []uint64{1}, + callWith: []uint64{0, 2}, + expect: [][2]bool{{false, false}, {false, true}}, + }, + { + rows: []uint64{0}, + callWith: []uint64{1, 2}, + expect: [][2]bool{{false, true}, {false, true}}, + }, + { + rows: []uint64{3, 9}, + callWith: []uint64{1, 2, 3, 10}, + expect: [][2]bool{{false, false}, {false, false}, {true, false}, {false, true}}, + }, + { + rows: []uint64{0, 1, 2}, + callWith: []uint64{0, 1, 2}, + expect: [][2]bool{{true, false}, {true, false}, {true, true}}, + }, + } + + for num, test := range tests { + t.Run(fmt.Sprintf("%d_%v_with_%v", num, test.rows, test.callWith), func(t *testing.T) { + if len(test.callWith) != len(test.expect) { + t.Fatalf("Badly specified test - must expect the same number of values as calls.") + } + f := filterWithRows(test.rows) + for i, id := range test.callWith { + inc, done := f(id, 0, nil) + if inc != test.expect[i][0] || done != test.expect[i][1] { + t.Fatalf("Calling with %d\nexp: %v,%v\ngot: %v,%v", id, test.expect[i][0], test.expect[i][1], inc, done) + } + } + }) + } + +} diff --git a/executor_test.go b/executor_test.go index 12bf0b1e8..30a1f2890 100644 --- a/executor_test.go +++ b/executor_test.go @@ -2412,7 +2412,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Count: 2}, } - results := c.Query(t, "i", `GroupBy(Rows(field=general, previous=10, limit=1))`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(field=general, previous=10), limit=1)`).Results[0].([]pilosa.GroupCount) checkGroupBy(t, expected, results) }) @@ -2436,6 +2436,48 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { results := c.Query(t, "i", `GroupBy(Rows(field=a), Rows(field=b), limit=1)`).Results[0].([]pilosa.GroupCount) checkGroupBy(t, expected, results) }) + + // set the same bits in a single shard in three fields + c.CreateField(t, "i", pilosa.IndexOptions{}, "wa") + c.CreateField(t, "i", pilosa.IndexOptions{}, "wb") + c.CreateField(t, "i", pilosa.IndexOptions{}, "wc") + c.ImportBits(t, "i", "wa", [][2]uint64{ + {0, 0}, {0, 1}, {0, 2}, // all + {1, 1}, // odds + {2, 0}, {2, 2}, // evens + {3, 3}, // no overlap + }) + c.ImportBits(t, "i", "wb", [][2]uint64{ + {0, 0}, {0, 1}, {0, 2}, + {1, 1}, + {2, 0}, {2, 2}, + {3, 3}, + }) + c.ImportBits(t, "i", "wc", [][2]uint64{ + {0, 0}, {0, 1}, {0, 2}, + {1, 1}, + {2, 0}, {2, 2}, + {3, 3}, + }) + + t.Run("test wrapping with previous", func(t *testing.T) { + results := c.Query(t, "i", `GroupBy(Rows(field=wa), Rows(field=wb), Rows(field=wc, previous=1), limit=3)`).Results[0].([]pilosa.GroupCount) + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "wa", RowID: 0}, {Field: "wb", RowID: 0}, {Field: "wc", RowID: 2}}, Count: 2}, + {Group: []pilosa.FieldRow{{Field: "wa", RowID: 0}, {Field: "wb", RowID: 1}, {Field: "wc", RowID: 0}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "wa", RowID: 0}, {Field: "wb", RowID: 1}, {Field: "wc", RowID: 1}}, Count: 1}, + } + checkGroupBy(t, expected, results) + }) + + t.Run("test wrapping multiple", func(t *testing.T) { + results := c.Query(t, "i", `GroupBy(Rows(field=wa), Rows(field=wb, previous=2), Rows(field=wc, previous=2), limit=1)`).Results[0].([]pilosa.GroupCount) + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "wa", RowID: 1}, {Field: "wb", RowID: 0}, {Field: "wc", RowID: 0}}, Count: 1}, + } + checkGroupBy(t, expected, results) + }) + } func checkGroupBy(t *testing.T, expected, results []pilosa.GroupCount) { @@ -2450,7 +2492,7 @@ func checkGroupBy(t *testing.T, expected, results []pilosa.GroupCount) { return true } if len(results) != len(expected) { - t.Fatalf("number of groupings mismatch: \n%+v\n%+v\n", results, expected) + t.Fatalf("number of groupings mismatch:\n got:%+v\nwant:%+v\n", results, expected) } for _, result := range results { if notIn(result, expected) { From f9cb7f8fec6689c122fa48bfa513bf38a5bc8231 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 11 Oct 2018 12:08:17 -0500 Subject: [PATCH 27/39] add some group by benchmarks --- executor_test.go | 50 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/executor_test.go b/executor_test.go index 30a1f2890..f2d32ba8f 100644 --- a/executor_test.go +++ b/executor_test.go @@ -2480,6 +2480,56 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { } +func BenchmarkGroupBy(b *testing.B) { + c := test.MustRunCluster(b, 1) + defer c.Close() + c.CreateField(b, "i", pilosa.IndexOptions{}, "a") + c.CreateField(b, "i", pilosa.IndexOptions{}, "b") + c.CreateField(b, "i", pilosa.IndexOptions{}, "c") + // Set up identical representative data in 3 fields. In each row, we'll set + // a certain bit pattern for 100 bits, then skip 1000 up to ShardWidth. + bits := make([][2]uint64, 0) + for i := uint64(0); i < ShardWidth; i++ { + // row 0 has 100 bit runs + bits = append(bits, [2]uint64{0, i}) + if i%2 == 1 { + // row 1 has odd bits set + bits = append(bits, [2]uint64{1, i}) + } + if i%2 == 0 { + // row 2 has even bits set + bits = append(bits, [2]uint64{2, i}) + } + if i%27 == 0 { + // row 3 has every 27th bit set + bits = append(bits, [2]uint64{3, i}) + } + if i%100 == 99 { + i += 1000 + } + } + c.ImportBits(b, "i", "a", bits) + c.ImportBits(b, "i", "b", bits) + c.ImportBits(b, "i", "c", bits) + + b.Run("single shard group by", func(b *testing.B) { + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + c.Query(b, "i", `GroupBy(Rows(field=a), Rows(field=b), Rows(field=c))`) + } + }) + + b.Run("single shard with limit", func(b *testing.B) { + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + c.Query(b, "i", `GroupBy(Rows(field=a), Rows(field=b), Rows(field=c), limit=4)`) + } + }) + +} + func checkGroupBy(t *testing.T, expected, results []pilosa.GroupCount) { notIn := func(item pilosa.GroupCount, expected []pilosa.GroupCount) bool { for i := range expected { From 9d896c5d2fee6fdb160e08b8adbdfa7ebb017000 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 11 Oct 2018 18:01:44 -0500 Subject: [PATCH 28/39] implement alternate groupByIterator using fragment rowIterator doesn't re-intersect the same rows for every record --- executor.go | 126 +++++++++++++++++++++++++++++++++++--- fragment.go | 36 +++++++++++ fragment_internal_test.go | 119 +++++++++++++++++++++++++++++++++++ 3 files changed, 274 insertions(+), 7 deletions(-) diff --git a/executor.go b/executor.go index d9bdb54d2..33ef1070f 100644 --- a/executor.go +++ b/executor.go @@ -943,7 +943,7 @@ func mergeGroupCounts(gc, other []GroupCount) []GroupCount { } func (e *executor) executeGroupByShard(_ context.Context, index string, c *pql.Call, shard uint64, childRows []RowIDs) ([]GroupCount, error) { - iter := newGroupByIterator(childRows, c.Children, index, shard, e.Holder) + iter := newGroupByIterator2(childRows, c.Children, index, shard, e.Holder) if iter == nil { return []GroupCount{}, nil } @@ -2596,10 +2596,122 @@ func filterWithRows(rows []uint64) rowFilter { } } +type groupByIterator2 struct { + rowIters []*rowIterator + rows []struct { + row *Row + id uint64 + } + done bool + fields []FieldRow +} + +func newGroupByIterator2(rowIDs []RowIDs, children []*pql.Call, index string, shard uint64, holder *Holder) *groupByIterator2 { + gbi := &groupByIterator2{ + rowIters: make([]*rowIterator, len(children)), + rows: make([]struct { + row *Row + id uint64 + }, len(children)), + fields: make([]FieldRow, len(children)), + } + + ignorePrev := false + for i, call := range children { + fieldName := call.Args["field"].(string) // this has already been validated by this point + gbi.fields[i].Field = fieldName + // Fetch fragment. + frag := holder.fragment(index, fieldName, viewStandard, shard) + if frag == nil { // this means this whole shard doesn't have all it needs to continue + return nil + } + filters := []rowFilter{} + if len(rowIDs[i]) > 0 { + filters = append(filters, filterWithRows(rowIDs[i])) + } + gbi.rowIters[i] = frag.rowIterator(i != 0, filters...) + + prev, hasPrev, err := call.UintArg("previous") + if err != nil { + panic("getting prev") + } else if hasPrev && !ignorePrev { + if i == len(children)-1 { + prev += 1 + } + gbi.rowIters[i].Seek(prev) + } + nextRow, rowID, wrapped := gbi.rowIters[i].Next() + if nextRow == nil { + gbi.done = true + return gbi + } + gbi.rows[i].row = nextRow + gbi.rows[i].id = rowID + if hasPrev && rowID != prev { + ignorePrev = true + } + if wrapped { + for j := i - 1; j >= 0; j-- { + nextRow, rowID, wrapped := gbi.rowIters[j].Next() + if nextRow == nil { + gbi.done = true + return gbi + } + gbi.rows[j].row = nextRow + gbi.rows[j].id = rowID + if !wrapped { + break + } + } + } + } + + for i := 1; i < len(gbi.rows); i++ { + gbi.rows[i].row = gbi.rows[i].row.Intersect(gbi.rows[i-1].row) + } + + return gbi +} + +func (gbi *groupByIterator2) nextAtIdx(i int) { + nr, rowID, wrapped := gbi.rowIters[i].Next() + if nr == nil { + gbi.done = true + return + } + if wrapped && i != 0 { + gbi.nextAtIdx(i - 1) + } + if i != 0 { + gbi.rows[i].row = nr.Intersect(gbi.rows[i-1].row) + } else { + gbi.rows[i].row = nr + } + gbi.rows[i].id = rowID +} + +func (gbi *groupByIterator2) Next() (ret ppi, done bool) { + if gbi.done { + return ret, true + } + ret.row = gbi.rows[len(gbi.rows)-1].row + ret.gCnt.Count = ret.row.Count() + ret.gCnt.Group = make([]FieldRow, len(gbi.rows)) + copy(ret.gCnt.Group, gbi.fields) + for i, r := range gbi.rows { + ret.gCnt.Group[i].RowID = r.id + } + + // set up for next call + gbi.nextAtIdx(len(gbi.rows) - 1) + + return ret, false +} + type groupByIterator struct { fragments []*fragment current []uint64 - rows []RowIDs + rowIDs []RowIDs fields []FieldRow } @@ -2607,7 +2719,7 @@ func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, index string, sha gbi := &groupByIterator{ fragments: make([]*fragment, len(rowIDs)), current: make([]uint64, len(rowIDs)), - rows: rowIDs, + rowIDs: rowIDs, fields: make([]FieldRow, len(rowIDs)), } for i, call := range children { @@ -2645,16 +2757,16 @@ func (gbi *groupByIterator) Next() (ret ppi, done bool) { } frag := gbi.fragments[i] filters := []rowFilter{} - if len(gbi.rows[i]) > 0 { - filters = append(filters, filterWithRows(gbi.rows[i])) + if len(gbi.rowIDs[i]) > 0 { + filters = append(filters, filterWithRows(gbi.rowIDs[i])) } filters = append(filters, filterWithLimit(1)) rowIDs := frag.rows(gbi.current[i], filters...) if len(rowIDs) == 0 && i != 0 { // wrap around wrap = true gbi.current[i] = 0 - if len(gbi.rows[i]) > 0 { - gbi.current[i] = gbi.rows[i][0] + if len(gbi.rowIDs[i]) > 0 { + gbi.current[i] = gbi.rowIDs[i][0] } rowIDs = frag.rows(gbi.current[i], filters...) } diff --git a/fragment.go b/fragment.go index 9210866e7..621ecdfa5 100644 --- a/fragment.go +++ b/fragment.go @@ -2013,6 +2013,42 @@ func (f *fragment) rows(start uint64, filters ...rowFilter) []uint64 { return rows } +type rowIterator struct { + f *fragment + rowIDs []uint64 + cur int + wrap bool +} + +func (f *fragment) rowIterator(wrap bool, filters ...rowFilter) *rowIterator { + return &rowIterator{ + f: f, + rowIDs: f.rows(0, filters...), // TODO: this may be memory intensive in high cardinality cases + wrap: wrap, + } +} + +func (ri *rowIterator) Seek(rowID uint64) { + idx := sort.Search(len(ri.rowIDs), func(i int) bool { + return ri.rowIDs[i] >= rowID + }) + ri.cur = idx +} + +func (ri *rowIterator) Next() (r *Row, rowID uint64, wrapped bool) { + if ri.cur >= len(ri.rowIDs) { + if !ri.wrap || len(ri.rowIDs) == 0 { + return nil, 0, true + } + ri.Seek(0) + wrapped = true + } + rowID = ri.rowIDs[ri.cur] + r = ri.f.row(rowID) + ri.cur += 1 + return r, rowID, wrapped +} + // FragmentBlock represents info about a subsection of the rows in a block. // This is used for comparing data in remote blocks for active anti-entropy. type FragmentBlock struct { diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 59d15ef54..19e3c0f12 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -1819,3 +1819,122 @@ func calcExpected(inputs ...[]uint64) [][]uint64 { return ret } + +func TestFragmentRowIterator(t *testing.T) { + t.Run("basic", func(t *testing.T) { + f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) + f.mustSetBits(0, 0) + f.mustSetBits(1, 0) + f.mustSetBits(2, 0) + f.mustSetBits(3, 0) + + iter := f.rowIterator(false) + for i := uint64(0); i < 4; i++ { + row, id, wrapped := iter.Next() + if id != i { + t.Fatalf("expected row %d but got %d", i, id) + } + if wrapped != false { + t.Fatalf("shouldn't have wrapped") + } + if !reflect.DeepEqual(row.Columns(), []uint64{0}) { + t.Fatalf("got wrong columns back on iteration %d - should just be 0 but %v", i, row.Columns()) + } + } + row, id, wrapped := iter.Next() + if row != nil { + t.Fatalf("row should be nil after iterator is exhausted, got %v", row.Columns()) + } + if id != 0 { + t.Fatalf("id should be 0 after iterator is exhausted, got %d", id) + } + if wrapped != true { + t.Fatalf("wrapped should be true after iterator is exhausted") + } + f.Close() + }) + + t.Run("skipped rows", func(t *testing.T) { + f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) + f.mustSetBits(1, 0) + f.mustSetBits(3, 0) + f.mustSetBits(5, 0) + f.mustSetBits(7, 0) + + iter := f.rowIterator(false) + for i := uint64(1); i < 8; i += 2 { + row, id, wrapped := iter.Next() + if id != i { + t.Fatalf("expected row %d but got %d", i, id) + } + if wrapped != false { + t.Fatalf("shouldn't have wrapped") + } + if !reflect.DeepEqual(row.Columns(), []uint64{0}) { + t.Fatalf("got wrong columns back on iteration %d - should just be 0 but %v", i, row.Columns()) + } + } + row, id, wrapped := iter.Next() + if row != nil { + t.Fatalf("row should be nil after iterator is exhausted, got %v", row.Columns()) + } + if id != 0 { + t.Fatalf("id should be 0 after iterator is exhausted, got %d", id) + } + if wrapped != true { + t.Fatalf("wrapped should be true after iterator is exhausted") + } + f.Close() + }) + + t.Run("basic wrapped", func(t *testing.T) { + f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) + f.mustSetBits(0, 0) + f.mustSetBits(1, 0) + f.mustSetBits(2, 0) + f.mustSetBits(3, 0) + + iter := f.rowIterator(true) + for i := uint64(0); i < 5; i++ { + row, id, wrapped := iter.Next() + if id != i%4 { + t.Fatalf("expected row %d but got %d", i%4, id) + } + if wrapped && i < 4 { + t.Fatalf("shouldn't have wrapped") + } else if !wrapped && i >= 4 { + t.Fatalf("should have wrapped") + } + if !reflect.DeepEqual(row.Columns(), []uint64{0}) { + t.Fatalf("got wrong columns back on iteration %d - should just be 0 but %v", i, row.Columns()) + } + } + f.Close() + }) + + t.Run("skipped rows wrapped", func(t *testing.T) { + f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) + f.mustSetBits(1, 0) + f.mustSetBits(3, 0) + f.mustSetBits(5, 0) + f.mustSetBits(7, 0) + + iter := f.rowIterator(true) + for i := uint64(1); i < 10; i += 2 { + row, id, wrapped := iter.Next() + if id != i%8 { + t.Errorf("expected row %d but got %d", i%8, id) + } + if wrapped && i < 8 { + t.Errorf("shouldn't have wrapped") + } else if !wrapped && i >= 8 { + t.Errorf("should have wrapped") + } + if !reflect.DeepEqual(row.Columns(), []uint64{0}) { + t.Fatalf("got wrong columns back on iteration %d - should just be 0 but %v", i, row.Columns()) + } + } + f.Close() + }) + +} From 07d279a15538dc3f840b151069f9c280c2ba99ef Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 11 Oct 2018 19:06:46 -0500 Subject: [PATCH 29/39] implement mergeGroupCounts w/o map, remove dead code move rowFilters to fragment.go new mergeGroupCounts implementation takes limit into account while merging, exploits inherent order of group count results. --- executor.go | 277 +++++++++++----------------------------------------- fragment.go | 49 ++++++++++ 2 files changed, 104 insertions(+), 222 deletions(-) diff --git a/executor.go b/executor.go index 33ef1070f..ea43e7ffb 100644 --- a/executor.go +++ b/executor.go @@ -18,11 +18,9 @@ import ( "context" "fmt" "sort" - "strings" "time" "github.com/pilosa/pilosa/pql" - "github.com/pilosa/pilosa/roaring" "github.com/pkg/errors" ) @@ -834,6 +832,13 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call if len(c.Children) == 0 { return nil, errors.New("need at least one child call") } + limit := int(^uint(0) >> 1) + if lim, hasLimit, err := c.UintArg("limit"); err != nil { + return nil, err + } else if hasLimit { + limit = int(lim) + } + // perform Rows queries - TODO, call async? run per shard in // executeGroupByShard? (note: can only do this for Rows queries which do // not include "column" arg) @@ -868,7 +873,7 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call // Merge returned results at coordinating node. reduceFn := func(prev, v interface{}) interface{} { other, _ := prev.([]GroupCount) - return mergeGroupCounts(other, v.([]GroupCount)) + return mergeGroupCounts(other, v.([]GroupCount), limit) } // Get full result set. other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) @@ -907,71 +912,62 @@ func (fr FieldRow) String() string { return fmt.Sprintf("%s.%d", fr.Field, fr.RowID) } -// TODO: we shouldn't need to string this -func uniqueGroupString(fr []FieldRow) string { - s := []string{} - for _, f := range fr { - s = append(s, f.String()) - } - return strings.Join(s, "-") -} - -// gbi is a groupBy item. -type gbi struct { - row *Row - fieldRow FieldRow -} - type GroupCount struct { Group []FieldRow `json:"group"` Count uint64 `json:"count"` } -func mergeGroupCounts(gc, other []GroupCount) []GroupCount { - m := make(map[string]int) - for i := range gc { - m[uniqueGroupString(gc[i].Group)] = i +// mergeGroupCounts merges two slices of GroupCounts throwing away any that go +// beyond the limit. It assume that the two slices are sorted by the row ids in +// the fields of the group counts. It may modify its arguments. +func mergeGroupCounts(a, b []GroupCount, limit int) []GroupCount { + if limit > len(a)+len(b) { + limit = len(a) + len(b) } - for i := range other { - if idx, found := m[uniqueGroupString(other[i].Group)]; found { - gc[idx].Count += other[i].Count - } else { - gc = append(gc, other[i]) + ret := make([]GroupCount, 0, limit) + i, j := 0, 0 + for i < len(a) && j < len(b) && len(ret) < limit { + switch a[i].Compare(b[j]) { + case -1: + ret = append(ret, a[i]) + i++ + case 0: + a[i].Count += b[j].Count + ret = append(ret, a[i]) + i++ + j++ + case 1: + ret = append(ret, b[j]) + j++ } } - return gc + for ; i < len(a) && len(ret) < limit; i++ { + ret = append(ret, a[i]) + } + for ; j < len(b) && len(ret) < limit; j++ { + ret = append(ret, b[j]) + } + return ret +} + +func (g GroupCount) Compare(o GroupCount) int { + for i := range g.Group { + if g.Group[i].RowID < o.Group[i].RowID { + return -1 + } + if g.Group[i].RowID > o.Group[i].RowID { + return 1 + } + } + return 0 } func (e *executor) executeGroupByShard(_ context.Context, index string, c *pql.Call, shard uint64, childRows []RowIDs) ([]GroupCount, error) { - iter := newGroupByIterator2(childRows, c.Children, index, shard, e.Holder) + iter := newGroupByIterator(childRows, c.Children, index, shard, e.Holder) if iter == nil { return []GroupCount{}, nil } - // var work [][]gbi - // for i, rowIDs := range childRows { - // fieldName := c.Children[i].Args["field"].(string) // this has already been validated by this point - // // Fetch fragment. - // frag := e.Holder.fragment(index, fieldName, viewStandard, shard) - // if frag == nil { // this means this whole shard doesn't have all it needs to continue - // return []GroupCount{}, nil - // } - - // set := make([]gbi, 0) - // for _, rowID := range rowIDs { - // rs := frag.rows(rowID, filterWithLimit(1)) - // if len(rs) > 0 && rs[0] == rowID { - // set = append(set, gbi{ - // row: frag.row(rowID), - // fieldRow: FieldRow{ - // Field: fieldName, - // RowID: rowID, - // }, - // }) - // } - // } - // work = append(work, set) - // } limit := int(^uint(0) >> 1) if lim, hasLimit, err := c.UintArg("limit"); err != nil { return nil, err @@ -989,48 +985,15 @@ func (e *executor) executeGroupByShard(_ context.Context, index string, c *pql.C } } - // for _, group := range product(work) { - // group.gCnt.Count = group.row.Count() - // if group.gCnt.Count > 0 { - // results = append(results, group.gCnt) - // } - // } return results, nil } -// ppi is a product process item. +// ppi is a product process item. TODO: rename type ppi struct { row *Row gCnt GroupCount } -// // product generates the cartesian product of the input -// // using tail recursion. -// func product(input [][]gbi) []ppi { -// if len(input) == 0 { // base return empty list -// return []ppi{ -// {gCnt: GroupCount{Group: make([]FieldRow, 0)}}, -// } -// } - -// res := make([]ppi, 0) -// head := input[0] // take first element of the list -// tail := product(input[1:]) // invoke product on remaining element -// for h := range head { // for each head -// for t := range tail { // iterate over the tail -// s := ppi{gCnt: GroupCount{Group: make([]FieldRow, 0)}} -// s.gCnt.Group = append([]FieldRow{head[h].fieldRow}, tail[t].gCnt.Group...) // had to insert at the front to match input order -// if tail[t].row != nil { // first time around nothing to intersect -// s.row = head[h].row.Intersect(tail[t].row) -// } else { -// s.row = head[h].row -// } -// res = append(res, s) -// } -// } -// return res -// } - func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (RowIDs, error) { if columnID, ok, err := c.UintArg("column"); err != nil { return nil, errors.Wrap(err, "getting column") @@ -2547,56 +2510,7 @@ func isString(v interface{}) bool { return ok } -// filterWithLimit returns a filter which will only allow a limited number of -// rows to be returned. It should be applied last so that it is only called (and -// therefore only updates its internal state) if the row is being included by -// every other filter. -func filterWithLimit(limit uint64) rowFilter { - return func(rowID, key uint64, c *roaring.Container) (include, done bool) { - if limit > 0 { - limit-- - return true, false - } - return false, true - } -} - -func filterColumn(col uint64) rowFilter { - return func(rowID, key uint64, c *roaring.Container) (include, done bool) { - colID := col % ShardWidth - colKey := ((rowID * ShardWidth) + colID) >> 16 - colVal := uint16(colID & 0xFFFF) // columnID within the container - return colKey == key && c.Contains(colVal), false - } -} - -// TODO: this works, but it would be more performant if the fragment could seek to the -// next row in the rows list rather than asking the filter for each container -// serially. -func filterWithRows(rows []uint64) rowFilter { - loc := 0 - return func(rowID, key uint64, c *roaring.Container) (include, done bool) { - if loc >= len(rows) { - return false, true - } - i := sort.Search(len(rows[loc:]), func(i int) bool { - return rows[loc+i] >= rowID - }) - loc += i - if loc >= len(rows) { - return false, true - } - if rows[loc] == rowID { - if loc == len(rows)-1 { - done = true - } - return true, done - } - return false, false - } -} - -type groupByIterator2 struct { +type groupByIterator struct { rowIters []*rowIterator rows []struct { row *Row @@ -2606,8 +2520,8 @@ type groupByIterator2 struct { fields []FieldRow } -func newGroupByIterator2(rowIDs []RowIDs, children []*pql.Call, index string, shard uint64, holder *Holder) *groupByIterator2 { - gbi := &groupByIterator2{ +func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, index string, shard uint64, holder *Holder) *groupByIterator { + gbi := &groupByIterator{ rowIters: make([]*rowIterator, len(children)), rows: make([]struct { row *Row @@ -2673,7 +2587,7 @@ func newGroupByIterator2(rowIDs []RowIDs, children []*pql.Call, index string, sh return gbi } -func (gbi *groupByIterator2) nextAtIdx(i int) { +func (gbi *groupByIterator) nextAtIdx(i int) { nr, rowID, wrapped := gbi.rowIters[i].Next() if nr == nil { gbi.done = true @@ -2690,7 +2604,7 @@ func (gbi *groupByIterator2) nextAtIdx(i int) { gbi.rows[i].id = rowID } -func (gbi *groupByIterator2) Next() (ret ppi, done bool) { +func (gbi *groupByIterator) Next() (ret ppi, done bool) { if gbi.done { return ret, true } @@ -2707,84 +2621,3 @@ func (gbi *groupByIterator2) Next() (ret ppi, done bool) { return ret, false } - -type groupByIterator struct { - fragments []*fragment - current []uint64 - rowIDs []RowIDs - fields []FieldRow -} - -func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, index string, shard uint64, holder *Holder) *groupByIterator { - gbi := &groupByIterator{ - fragments: make([]*fragment, len(rowIDs)), - current: make([]uint64, len(rowIDs)), - rowIDs: rowIDs, - fields: make([]FieldRow, len(rowIDs)), - } - for i, call := range children { - fieldName := call.Args["field"].(string) // this has already been validated by this point - gbi.fields[i].Field = fieldName - // Fetch fragment. - frag := holder.fragment(index, fieldName, viewStandard, shard) - if frag == nil { // this means this whole shard doesn't have all it needs to continue - return nil - } - gbi.fragments[i] = frag - if prev, hasPrev, err := call.UintArg("previous"); err != nil { - panic("getting prev") - } else if hasPrev { - gbi.current[i] = prev - if i == len(children)-1 { - gbi.current[i] += 1 - } - } - } - return gbi -} - -func (gbi *groupByIterator) Next() (ret ppi, done bool) { - rows := make([]*Row, len(gbi.current)) - ret.gCnt.Group = make([]FieldRow, len(gbi.current)) - copy(ret.gCnt.Group, gbi.fields) - wrap := false - - // build rows slice - for i := len(gbi.current) - 1; i >= 0; i-- { - if wrap { - gbi.current[i] += 1 - wrap = false - } - frag := gbi.fragments[i] - filters := []rowFilter{} - if len(gbi.rowIDs[i]) > 0 { - filters = append(filters, filterWithRows(gbi.rowIDs[i])) - } - filters = append(filters, filterWithLimit(1)) - rowIDs := frag.rows(gbi.current[i], filters...) - if len(rowIDs) == 0 && i != 0 { // wrap around - wrap = true - gbi.current[i] = 0 - if len(gbi.rowIDs[i]) > 0 { - gbi.current[i] = gbi.rowIDs[i][0] - } - rowIDs = frag.rows(gbi.current[i], filters...) - } - if len(rowIDs) != 0 { - rows[i] = frag.row(rowIDs[0]) - gbi.current[i] = rowIDs[0] - ret.gCnt.Group[i].RowID = rowIDs[0] - } else { - return ppi{}, true - } - } - gbi.current[len(gbi.current)-1] += 1 - - // build ppi from rows - ret.row = rows[0] - for _, row := range rows[1:] { - ret.row = ret.row.Intersect(row) - } - ret.gCnt.Count = ret.row.Count() - return ret, false -} diff --git a/fragment.go b/fragment.go index 621ecdfa5..cf38972ab 100644 --- a/fragment.go +++ b/fragment.go @@ -1967,6 +1967,55 @@ func (f *fragment) readCacheFromArchive(r io.Reader) error { // continue. type rowFilter func(rowID, key uint64, c *roaring.Container) (include, done bool) +// filterWithLimit returns a filter which will only allow a limited number of +// rows to be returned. It should be applied last so that it is only called (and +// therefore only updates its internal state) if the row is being included by +// every other filter. +func filterWithLimit(limit uint64) rowFilter { + return func(rowID, key uint64, c *roaring.Container) (include, done bool) { + if limit > 0 { + limit-- + return true, false + } + return false, true + } +} + +func filterColumn(col uint64) rowFilter { + return func(rowID, key uint64, c *roaring.Container) (include, done bool) { + colID := col % ShardWidth + colKey := ((rowID * ShardWidth) + colID) >> 16 + colVal := uint16(colID & 0xFFFF) // columnID within the container + return colKey == key && c.Contains(colVal), false + } +} + +// TODO: this works, but it would be more performant if the fragment could seek to the +// next row in the rows list rather than asking the filter for each container +// serially. +func filterWithRows(rows []uint64) rowFilter { + loc := 0 + return func(rowID, key uint64, c *roaring.Container) (include, done bool) { + if loc >= len(rows) { + return false, true + } + i := sort.Search(len(rows[loc:]), func(i int) bool { + return rows[loc+i] >= rowID + }) + loc += i + if loc >= len(rows) { + return false, true + } + if rows[loc] == rowID { + if loc == len(rows)-1 { + done = true + } + return true, done + } + return false, false + } +} + // rows returns all rows starting from 'start'. Filters will be applied in // order. All filters must return true to include the row. Once a row is // included, further containers in that row will be skipped. So, for a row to be From a4edd397154f738aaec4c6d6673d712294713ded Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 12 Oct 2018 15:30:25 -0500 Subject: [PATCH 30/39] use intersectionCounts for final row of groupBy record since we only need the counts and not the data, this optimization actually provides enormous speedup (2x?) and massive decrease in allocations. also in this commit (unfortunately), a bunch of renaming and documentation, returning a GroupCount from the GroupBy iterator instead of a ppi (ppi is now gone). also added TODOs for tests and benchmarks --- executor.go | 52 ++++++++++++++++++++++++++++++------------------ executor_test.go | 13 ++++++++++++ 2 files changed, 46 insertions(+), 19 deletions(-) diff --git a/executor.go b/executor.go index ea43e7ffb..8bb413f09 100644 --- a/executor.go +++ b/executor.go @@ -978,22 +978,16 @@ func (e *executor) executeGroupByShard(_ context.Context, index string, c *pql.C results := make([]GroupCount, 0) num := 0 - for pp, done := iter.Next(); !done && num < limit; pp, done = iter.Next() { - if pp.gCnt.Count > 0 { + for gc, done := iter.Next(); !done && num < limit; gc, done = iter.Next() { + if gc.Count > 0 { num++ - results = append(results, pp.gCnt) + results = append(results, gc) } } return results, nil } -// ppi is a product process item. TODO: rename -type ppi struct { - row *Row - gCnt GroupCount -} - func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (RowIDs, error) { if columnID, ok, err := c.UintArg("column"); err != nil { return nil, errors.Wrap(err, "getting column") @@ -2510,16 +2504,27 @@ func isString(v interface{}) bool { return ok } +// groupByIterator contains several slices. Each slice contains a number of +// elements equal to the number of fields in the group by (the number of Rows +// calls). type groupByIterator struct { + // rowIters contains a rowIterator for each of the fields in the Group By. rowIters []*rowIterator - rows []struct { + // rows contains the current row data for each of the fields in the Group + // By. Each row is the intersection of itself and the rows of the fields + // with an index lower than its own. This is a performance optimization so + // that the expected common case of getting the next row in the furthest + // field to the right require only a single intersect with the row of the + // previous field to determine the count of the new group. + rows []struct { row *Row id uint64 } - done bool fields []FieldRow + done bool } +// newGroupByIterator initializes a new groupByIterator. func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, index string, shard uint64, holder *Holder) *groupByIterator { gbi := &groupByIterator{ rowIters: make([]*rowIterator, len(children)), @@ -2580,13 +2585,15 @@ func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, index string, sha } } - for i := 1; i < len(gbi.rows); i++ { + for i := 1; i < len(gbi.rows)-1; i++ { gbi.rows[i].row = gbi.rows[i].row.Intersect(gbi.rows[i-1].row) } return gbi } +// nextAtIdx is a recursive helper method for getting the next row for the field +// at index i, and then updating the rows in the "higher" fields if it wraps. func (gbi *groupByIterator) nextAtIdx(i int) { nr, rowID, wrapped := gbi.rowIters[i].Next() if nr == nil { @@ -2596,24 +2603,31 @@ func (gbi *groupByIterator) nextAtIdx(i int) { if wrapped && i != 0 { gbi.nextAtIdx(i - 1) } - if i != 0 { + if i != 0 && i != len(gbi.rows)-1 { gbi.rows[i].row = nr.Intersect(gbi.rows[i-1].row) + } else if i == len(gbi.rows)-1 { + gbi.rows[i].row = nr } else { gbi.rows[i].row = nr } gbi.rows[i].id = rowID } -func (gbi *groupByIterator) Next() (ret ppi, done bool) { +// Next returns a ppi representing the next group by record. When there are no +// more records it will return an empty ppi and done==true. +func (gbi *groupByIterator) Next() (ret GroupCount, done bool) { if gbi.done { return ret, true } - ret.row = gbi.rows[len(gbi.rows)-1].row - ret.gCnt.Count = ret.row.Count() - ret.gCnt.Group = make([]FieldRow, len(gbi.rows)) - copy(ret.gCnt.Group, gbi.fields) + if len(gbi.rows) == 1 { + ret.Count = gbi.rows[len(gbi.rows)-1].row.Count() + } else { + ret.Count = gbi.rows[len(gbi.rows)-1].row.intersectionCount(gbi.rows[len(gbi.rows)-2].row) + } + ret.Group = make([]FieldRow, len(gbi.rows)) + copy(ret.Group, gbi.fields) for i, r := range gbi.rows { - ret.gCnt.Group[i].RowID = r.id + ret.Group[i].RowID = r.id } // set up for next call diff --git a/executor_test.go b/executor_test.go index f2d32ba8f..28b89aa92 100644 --- a/executor_test.go +++ b/executor_test.go @@ -2478,6 +2478,15 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { checkGroupBy(t, expected, results) }) + // TODO test multiple shards with distinct results (different rows) and same + // rows to ensure ordering, limit behavior and correctness + + // TODO test column queries to row call (also with multiple shards) + + // TODO test limit query to Rows calls + + // TODO test paging over results using previous. + } func BenchmarkGroupBy(b *testing.B) { @@ -2528,6 +2537,10 @@ func BenchmarkGroupBy(b *testing.B) { } }) + // TODO benchmark over multiple shards + + // TODO benchmark paging over large numbers of rows + } func checkGroupBy(t *testing.T, expected, results []pilosa.GroupCount) { From 431185110bb87a4e23ef8bcba04b7e77d636f487 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 12 Oct 2018 15:58:28 -0500 Subject: [PATCH 31/39] add different shard test and simplify checking logic we now guarantee result order --- executor_test.go | 45 +++++++++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/executor_test.go b/executor_test.go index 28b89aa92..666eb104d 100644 --- a/executor_test.go +++ b/executor_test.go @@ -2387,10 +2387,10 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { t.Run("Basic", func(t *testing.T) { expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Count: 3}, {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Count: 1}, {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}, {Field: "sub", RowID: 110}}, Count: 1}, {Group: []pilosa.FieldRow{{Field: "general", RowID: 12}, {Field: "sub", RowID: 110}}, Count: 1}, - {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Count: 3}, } results := c.Query(t, "i", `GroupBy(Rows(field=general), Rows(field=sub))`).Results[0].([]pilosa.GroupCount) @@ -2480,6 +2480,33 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { // TODO test multiple shards with distinct results (different rows) and same // rows to ensure ordering, limit behavior and correctness + c.CreateField(t, "i", pilosa.IndexOptions{}, "ma") + c.CreateField(t, "i", pilosa.IndexOptions{}, "mb") + c.CreateField(t, "i", pilosa.IndexOptions{}, "mc") + c.ImportBits(t, "i", "ma", [][2]uint64{ + {0, 0}, + {1, ShardWidth}, + {2, 0}, + {3, ShardWidth}, + }) + c.ImportBits(t, "i", "mb", [][2]uint64{ + {0, 0}, + {1, ShardWidth}, + {2, 0}, + {3, ShardWidth}, + }) + t.Run("distinct rows in different shards", func(t *testing.T) { + results := c.Query(t, "i", `GroupBy(Rows(field=ma), Rows(field=mb), limit=5)`).Results[0].([]pilosa.GroupCount) + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "ma", RowID: 0}, {Field: "mb", RowID: 0}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "ma", RowID: 0}, {Field: "mb", RowID: 2}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 1}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 3}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "ma", RowID: 2}, {Field: "mb", RowID: 0}}, Count: 1}, + } + checkGroupBy(t, expected, results) + + }) // TODO test column queries to row call (also with multiple shards) @@ -2544,22 +2571,12 @@ func BenchmarkGroupBy(b *testing.B) { } func checkGroupBy(t *testing.T, expected, results []pilosa.GroupCount) { - notIn := func(item pilosa.GroupCount, expected []pilosa.GroupCount) bool { - for i := range expected { - if item.Count == expected[i].Count { - if reflect.DeepEqual(item.Group, expected[i].Group) { - return false - } - } - } - return true - } if len(results) != len(expected) { t.Fatalf("number of groupings mismatch:\n got:%+v\nwant:%+v\n", results, expected) } - for _, result := range results { - if notIn(result, expected) { - t.Fatalf("unexpected results: \n got:%+v\nwant:%+v\n", results, expected) + for i, result := range results { + if !reflect.DeepEqual(expected[i], result) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) } } } From fb706ab883e87ecf257740bcc3aea26bb2939cb8 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 12 Oct 2018 18:47:41 -0500 Subject: [PATCH 32/39] add GroupBy Rows(limit) test and fix bug run all group by tests on two cluster sizes --- executor_test.go | 372 +++++++++++++++++++++++++++++------------------ fragment.go | 18 ++- 2 files changed, 237 insertions(+), 153 deletions(-) diff --git a/executor_test.go b/executor_test.go index 666eb104d..6b903acc2 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1358,8 +1358,8 @@ Set(4500001, fn=4) t.Fatalf("GroupBy querying: %v", err) } else { expected := []pilosa.GroupCount{ - {Group: []pilosa.FieldRow{{Field: "f", RowID: 10}}, Count: 4}, {Group: []pilosa.FieldRow{{Field: "f", RowID: 7}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "f", RowID: 10}}, Count: 4}, } results := res.Results[0].([]pilosa.GroupCount) checkGroupBy(t, expected, results) @@ -2346,174 +2346,256 @@ func TestExecutor_Execute_Rows_Keys(t *testing.T) { } func TestExecutor_Execute_GroupBy(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - c.CreateField(t, "i", pilosa.IndexOptions{}, "general") - c.CreateField(t, "i", pilosa.IndexOptions{}, "sub") - c.ImportBits(t, "i", "general", [][2]uint64{ - {10, 0}, - {10, 1}, - {10, ShardWidth + 1}, - {11, 2}, - {11, ShardWidth + 2}, - {12, 2}, - {12, ShardWidth + 2}, - }) - c.ImportBits(t, "i", "sub", [][2]uint64{ - {100, 0}, - {100, 1}, - {100, 3}, - {100, ShardWidth + 1}, + groupByTest := func(t *testing.T, clusterSize int) { + c := test.MustRunCluster(t, 1) + defer c.Close() + c.CreateField(t, "i", pilosa.IndexOptions{}, "general") + c.CreateField(t, "i", pilosa.IndexOptions{}, "sub") + c.ImportBits(t, "i", "general", [][2]uint64{ + {10, 0}, + {10, 1}, + {10, ShardWidth + 1}, + {11, 2}, + {11, ShardWidth + 2}, + {12, 2}, + {12, ShardWidth + 2}, + }) + c.ImportBits(t, "i", "sub", [][2]uint64{ + {100, 0}, + {100, 1}, + {100, 3}, + {100, ShardWidth + 1}, - {110, 2}, - {110, 0}, - }) + {110, 2}, + {110, 0}, + }) - t.Run("No Field List Arguments", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy()`}); err != nil { - if !strings.Contains(err.Error(), "need at least one child call") { - t.Fatalf("unexpected error: \"%v\"", err) + t.Run("No Field List Arguments", func(t *testing.T) { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy()`}); err != nil { + if !strings.Contains(err.Error(), "need at least one child call") { + t.Fatalf("unexpected error: \"%v\"", err) + } } - } - }) + }) - t.Run("Unknown Field ", func(t *testing.T) { - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(Rows(field=missing))`}); err != nil { - if errors.Cause(err) != pilosa.ErrFieldNotFound { - t.Fatalf("unexpected error\n\"%s\" not returned instead \n\"%s\"", pilosa.ErrFieldNotFound, err) + t.Run("Unknown Field ", func(t *testing.T) { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(Rows(field=missing))`}); err != nil { + if errors.Cause(err) != pilosa.ErrFieldNotFound { + t.Fatalf("unexpected error\n\"%s\" not returned instead \n\"%s\"", pilosa.ErrFieldNotFound, err) + } } - } - }) + }) - t.Run("Basic", func(t *testing.T) { - expected := []pilosa.GroupCount{ - {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Count: 3}, - {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Count: 1}, - {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}, {Field: "sub", RowID: 110}}, Count: 1}, - {Group: []pilosa.FieldRow{{Field: "general", RowID: 12}, {Field: "sub", RowID: 110}}, Count: 1}, - } + t.Run("Basic", func(t *testing.T) { + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Count: 3}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}, {Field: "sub", RowID: 110}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 12}, {Field: "sub", RowID: 110}}, Count: 1}, + } - results := c.Query(t, "i", `GroupBy(Rows(field=general), Rows(field=sub))`).Results[0].([]pilosa.GroupCount) - checkGroupBy(t, expected, results) - }) + results := c.Query(t, "i", `GroupBy(Rows(field=general), Rows(field=sub))`).Results[0].([]pilosa.GroupCount) + checkGroupBy(t, expected, results) + }) - t.Run("check field offset no limit", func(t *testing.T) { - expected := []pilosa.GroupCount{ - {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Count: 2}, - {Group: []pilosa.FieldRow{{Field: "general", RowID: 12}}, Count: 2}, - } + t.Run("check field offset no limit", func(t *testing.T) { + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Count: 2}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 12}}, Count: 2}, + } - results := c.Query(t, "i", `GroupBy(Rows(field=general, previous=10))`).Results[0].([]pilosa.GroupCount) - checkGroupBy(t, expected, results) - }) + results := c.Query(t, "i", `GroupBy(Rows(field=general, previous=10))`).Results[0].([]pilosa.GroupCount) + checkGroupBy(t, expected, results) + }) - t.Run("check field offset limit", func(t *testing.T) { - expected := []pilosa.GroupCount{ - {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Count: 2}, - } + t.Run("check field offset limit", func(t *testing.T) { + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Count: 2}, + } - results := c.Query(t, "i", `GroupBy(Rows(field=general, previous=10), limit=1)`).Results[0].([]pilosa.GroupCount) - checkGroupBy(t, expected, results) + results := c.Query(t, "i", `GroupBy(Rows(field=general, previous=10), limit=1)`).Results[0].([]pilosa.GroupCount) + checkGroupBy(t, expected, results) - }) + }) - c.CreateField(t, "i", pilosa.IndexOptions{}, "a") - c.CreateField(t, "i", pilosa.IndexOptions{}, "b") - c.ImportBits(t, "i", "a", [][2]uint64{ - {0, 1}, - {1, ShardWidth + 1}, - }) - c.ImportBits(t, "i", "b", [][2]uint64{ - {0, ShardWidth + 1}, - {1, 1}, - }) + c.CreateField(t, "i", pilosa.IndexOptions{}, "a") + c.CreateField(t, "i", pilosa.IndexOptions{}, "b") + c.ImportBits(t, "i", "a", [][2]uint64{ + {0, 1}, + {1, ShardWidth + 1}, + }) + c.ImportBits(t, "i", "b", [][2]uint64{ + {0, ShardWidth + 1}, + {1, 1}, + }) - t.Run("tricky data", func(t *testing.T) { - expected := []pilosa.GroupCount{ - {Group: []pilosa.FieldRow{{Field: "a", RowID: 0}, {Field: "b", RowID: 1}}, Count: 1}, - } + t.Run("tricky data", func(t *testing.T) { + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "a", RowID: 0}, {Field: "b", RowID: 1}}, Count: 1}, + } - results := c.Query(t, "i", `GroupBy(Rows(field=a), Rows(field=b), limit=1)`).Results[0].([]pilosa.GroupCount) - checkGroupBy(t, expected, results) - }) + results := c.Query(t, "i", `GroupBy(Rows(field=a), Rows(field=b), limit=1)`).Results[0].([]pilosa.GroupCount) + checkGroupBy(t, expected, results) + }) - // set the same bits in a single shard in three fields - c.CreateField(t, "i", pilosa.IndexOptions{}, "wa") - c.CreateField(t, "i", pilosa.IndexOptions{}, "wb") - c.CreateField(t, "i", pilosa.IndexOptions{}, "wc") - c.ImportBits(t, "i", "wa", [][2]uint64{ - {0, 0}, {0, 1}, {0, 2}, // all - {1, 1}, // odds - {2, 0}, {2, 2}, // evens - {3, 3}, // no overlap - }) - c.ImportBits(t, "i", "wb", [][2]uint64{ - {0, 0}, {0, 1}, {0, 2}, - {1, 1}, - {2, 0}, {2, 2}, - {3, 3}, - }) - c.ImportBits(t, "i", "wc", [][2]uint64{ - {0, 0}, {0, 1}, {0, 2}, - {1, 1}, - {2, 0}, {2, 2}, - {3, 3}, - }) + // set the same bits in a single shard in three fields + c.CreateField(t, "i", pilosa.IndexOptions{}, "wa") + c.CreateField(t, "i", pilosa.IndexOptions{}, "wb") + c.CreateField(t, "i", pilosa.IndexOptions{}, "wc") + c.ImportBits(t, "i", "wa", [][2]uint64{ + {0, 0}, {0, 1}, {0, 2}, // all + {1, 1}, // odds + {2, 0}, {2, 2}, // evens + {3, 3}, // no overlap + }) + c.ImportBits(t, "i", "wb", [][2]uint64{ + {0, 0}, {0, 1}, {0, 2}, + {1, 1}, + {2, 0}, {2, 2}, + {3, 3}, + }) + c.ImportBits(t, "i", "wc", [][2]uint64{ + {0, 0}, {0, 1}, {0, 2}, + {1, 1}, + {2, 0}, {2, 2}, + {3, 3}, + }) - t.Run("test wrapping with previous", func(t *testing.T) { - results := c.Query(t, "i", `GroupBy(Rows(field=wa), Rows(field=wb), Rows(field=wc, previous=1), limit=3)`).Results[0].([]pilosa.GroupCount) - expected := []pilosa.GroupCount{ - {Group: []pilosa.FieldRow{{Field: "wa", RowID: 0}, {Field: "wb", RowID: 0}, {Field: "wc", RowID: 2}}, Count: 2}, - {Group: []pilosa.FieldRow{{Field: "wa", RowID: 0}, {Field: "wb", RowID: 1}, {Field: "wc", RowID: 0}}, Count: 1}, - {Group: []pilosa.FieldRow{{Field: "wa", RowID: 0}, {Field: "wb", RowID: 1}, {Field: "wc", RowID: 1}}, Count: 1}, - } - checkGroupBy(t, expected, results) - }) + t.Run("test wrapping with previous", func(t *testing.T) { + results := c.Query(t, "i", `GroupBy(Rows(field=wa), Rows(field=wb), Rows(field=wc, previous=1), limit=3)`).Results[0].([]pilosa.GroupCount) + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "wa", RowID: 0}, {Field: "wb", RowID: 0}, {Field: "wc", RowID: 2}}, Count: 2}, + {Group: []pilosa.FieldRow{{Field: "wa", RowID: 0}, {Field: "wb", RowID: 1}, {Field: "wc", RowID: 0}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "wa", RowID: 0}, {Field: "wb", RowID: 1}, {Field: "wc", RowID: 1}}, Count: 1}, + } + checkGroupBy(t, expected, results) + }) - t.Run("test wrapping multiple", func(t *testing.T) { - results := c.Query(t, "i", `GroupBy(Rows(field=wa), Rows(field=wb, previous=2), Rows(field=wc, previous=2), limit=1)`).Results[0].([]pilosa.GroupCount) - expected := []pilosa.GroupCount{ - {Group: []pilosa.FieldRow{{Field: "wa", RowID: 1}, {Field: "wb", RowID: 0}, {Field: "wc", RowID: 0}}, Count: 1}, - } - checkGroupBy(t, expected, results) - }) + t.Run("test wrapping multiple", func(t *testing.T) { + results := c.Query(t, "i", `GroupBy(Rows(field=wa), Rows(field=wb, previous=2), Rows(field=wc, previous=2), limit=1)`).Results[0].([]pilosa.GroupCount) + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "wa", RowID: 1}, {Field: "wb", RowID: 0}, {Field: "wc", RowID: 0}}, Count: 1}, + } + checkGroupBy(t, expected, results) + }) - // TODO test multiple shards with distinct results (different rows) and same - // rows to ensure ordering, limit behavior and correctness - c.CreateField(t, "i", pilosa.IndexOptions{}, "ma") - c.CreateField(t, "i", pilosa.IndexOptions{}, "mb") - c.CreateField(t, "i", pilosa.IndexOptions{}, "mc") - c.ImportBits(t, "i", "ma", [][2]uint64{ - {0, 0}, - {1, ShardWidth}, - {2, 0}, - {3, ShardWidth}, - }) - c.ImportBits(t, "i", "mb", [][2]uint64{ - {0, 0}, - {1, ShardWidth}, - {2, 0}, - {3, ShardWidth}, - }) - t.Run("distinct rows in different shards", func(t *testing.T) { - results := c.Query(t, "i", `GroupBy(Rows(field=ma), Rows(field=mb), limit=5)`).Results[0].([]pilosa.GroupCount) - expected := []pilosa.GroupCount{ - {Group: []pilosa.FieldRow{{Field: "ma", RowID: 0}, {Field: "mb", RowID: 0}}, Count: 1}, - {Group: []pilosa.FieldRow{{Field: "ma", RowID: 0}, {Field: "mb", RowID: 2}}, Count: 1}, - {Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 1}}, Count: 1}, - {Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 3}}, Count: 1}, - {Group: []pilosa.FieldRow{{Field: "ma", RowID: 2}, {Field: "mb", RowID: 0}}, Count: 1}, - } - checkGroupBy(t, expected, results) + // test multiple shards with distinct results (different rows) and same + // rows to ensure ordering, limit behavior and correctness + c.CreateField(t, "i", pilosa.IndexOptions{}, "ma") + c.CreateField(t, "i", pilosa.IndexOptions{}, "mb") + c.ImportBits(t, "i", "ma", [][2]uint64{ + {0, 0}, + {1, ShardWidth}, + {2, 0}, + {3, ShardWidth}, + }) + c.ImportBits(t, "i", "mb", [][2]uint64{ + {0, 0}, + {1, ShardWidth}, + {2, 0}, + {3, ShardWidth}, + }) + t.Run("distinct rows in different shards", func(t *testing.T) { + results := c.Query(t, "i", `GroupBy(Rows(field=ma), Rows(field=mb), limit=5)`).Results[0].([]pilosa.GroupCount) + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "ma", RowID: 0}, {Field: "mb", RowID: 0}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "ma", RowID: 0}, {Field: "mb", RowID: 2}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 1}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 3}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "ma", RowID: 2}, {Field: "mb", RowID: 0}}, Count: 1}, + } + checkGroupBy(t, expected, results) + }) - }) + t.Run("distinct rows in different shards with row limit", func(t *testing.T) { + results := c.Query(t, "i", `GroupBy(Rows(field=ma), Rows(field=mb, limit=2), limit=5)`).Results[0].([]pilosa.GroupCount) + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "ma", RowID: 0}, {Field: "mb", RowID: 0}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 1}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "ma", RowID: 2}, {Field: "mb", RowID: 0}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "ma", RowID: 3}, {Field: "mb", RowID: 1}}, Count: 1}, + } + checkGroupBy(t, expected, results) + }) - // TODO test column queries to row call (also with multiple shards) + c.CreateField(t, "i", pilosa.IndexOptions{}, "na") + c.CreateField(t, "i", pilosa.IndexOptions{}, "nb") + c.ImportBits(t, "i", "na", [][2]uint64{ + {0, 0}, + {0, ShardWidth}, + {1, 0}, + {1, ShardWidth}, + }) + c.ImportBits(t, "i", "nb", [][2]uint64{ + {0, 0}, + {0, ShardWidth}, + {1, 0}, + {1, ShardWidth}, + }) + t.Run("same rows in different shards", func(t *testing.T) { + results := c.Query(t, "i", `GroupBy(Rows(field=na), Rows(field=nb))`).Results[0].([]pilosa.GroupCount) + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "na", RowID: 0}, {Field: "nb", RowID: 0}}, Count: 2}, + {Group: []pilosa.FieldRow{{Field: "na", RowID: 0}, {Field: "nb", RowID: 1}}, Count: 2}, + {Group: []pilosa.FieldRow{{Field: "na", RowID: 1}, {Field: "nb", RowID: 0}}, Count: 2}, + {Group: []pilosa.FieldRow{{Field: "na", RowID: 1}, {Field: "nb", RowID: 1}}, Count: 2}, + } + checkGroupBy(t, expected, results) - // TODO test limit query to Rows calls + }) - // TODO test paging over results using previous. + // TODO test column queries to row call (also with multiple shards) + // test paging over results using previous. set the same bits in three + // fields + c.CreateField(t, "i", pilosa.IndexOptions{}, "ppa") + c.CreateField(t, "i", pilosa.IndexOptions{}, "ppb") + c.CreateField(t, "i", pilosa.IndexOptions{}, "ppc") + c.ImportBits(t, "i", "ppa", [][2]uint64{ + {0, 0}, + {1, 0}, + {2, 0}, + {3, 0}, {3, 91000}, {3, ShardWidth}, {3, ShardWidth * 2}, {3, ShardWidth * 3}, + }) + c.ImportBits(t, "i", "ppb", [][2]uint64{ + {0, 0}, + {1, 0}, + {2, 0}, + {3, 0}, {3, 91000}, {3, ShardWidth}, {3, ShardWidth * 2}, {3, ShardWidth * 3}, + }) + c.ImportBits(t, "i", "ppc", [][2]uint64{ + {0, 0}, + {1, 0}, + {2, 0}, + {3, 0}, {3, 91000}, {3, ShardWidth}, {3, ShardWidth * 2}, {3, ShardWidth * 3}, + }) + + t.Run("test wrapping with previous", func(t *testing.T) { + totalResults := make([]pilosa.GroupCount, 0) + var results []pilosa.GroupCount + results = c.Query(t, "i", `GroupBy(Rows(field=ppa), Rows(field=ppb), Rows(field=ppc), limit=3)`).Results[0].([]pilosa.GroupCount) + totalResults = append(totalResults, results...) + for len(totalResults) < 64 { + lastGroup := results[len(results)-1].Group + query := fmt.Sprintf("GroupBy(Rows(field=ppa, previous=%d), Rows(field=ppb, previous=%d), Rows(field=ppc, previous=%d), limit=3)", lastGroup[0].RowID, lastGroup[1].RowID, lastGroup[2].RowID) + results = c.Query(t, "i", query).Results[0].([]pilosa.GroupCount) + totalResults = append(totalResults, results...) + } + + expected := make([]pilosa.GroupCount, 64) + for i := 0; i < 64; i++ { + expected[i] = pilosa.GroupCount{Group: []pilosa.FieldRow{{Field: "ppa", RowID: uint64(i / 16)}, {Field: "ppb", RowID: uint64((i % 16) / 4)}, {Field: "ppc", RowID: uint64(i % 4)}}, Count: 1} + } + expected[63].Count = 5 + + checkGroupBy(t, expected, totalResults) + }) + } + for size := range []int{1, 3} { + t.Run(fmt.Sprintf("%d_nodes", size), func(t *testing.T) { + groupByTest(t, size) + }) + } } func BenchmarkGroupBy(b *testing.B) { diff --git a/fragment.go b/fragment.go index cf38972ab..f70ddf482 100644 --- a/fragment.go +++ b/fragment.go @@ -2022,8 +2022,9 @@ func filterWithRows(rows []uint64) rowFilter { // included, there must be one container in that row where all filters return // true. For a row to be skipped, at least one filter must return false for each // container in that row (it need not be the same filter for each). Any filter -// returning done == true will cause processing to stop and the rows accumulated -// so far will be returned. +// returning done == true will cause processing to stop after all filters for +// this container have been processed. The rows accumulated up to this point +// (including this row if all filters passed) will be returned. func (f *fragment) rows(start uint64, filters ...rowFilter) []uint64 { startKey := rowToKey(start) i, _ := f.storage.Containers.Iterator(startKey) @@ -2043,13 +2044,11 @@ func (f *fragment) rows(start uint64, filters ...rowFilter) []uint64 { } // apply filters - addRow := true + addRow, done := true, false for _, filter := range filters { - var done bool - addRow, done = filter(vRow, key, c) - if done { - return rows - } + var d bool + addRow, d = filter(vRow, key, c) + done = done || d if !addRow { break } @@ -2058,6 +2057,9 @@ func (f *fragment) rows(start uint64, filters ...rowFilter) []uint64 { lastRow = vRow rows = append(rows, vRow) } + if done { + return rows + } } return rows } From 38f459a6d431c219c6829df33f234a1e4a2f75f7 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 12 Oct 2018 18:58:27 -0500 Subject: [PATCH 33/39] add GroupBy(Rows(column)) test and fix comments --- executor.go | 7 ++++--- executor_test.go | 13 +++++++++++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/executor.go b/executor.go index 8bb413f09..40589f6e2 100644 --- a/executor.go +++ b/executor.go @@ -839,9 +839,10 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call limit = int(lim) } - // perform Rows queries - TODO, call async? run per shard in - // executeGroupByShard? (note: can only do this for Rows queries which do - // not include "column" arg) + // perform necessary Rows queries (any that have limit or columns args) - + // TODO, call async? would only help if multiple Rows queries had a column + // or limit arg. + // TODO support TopN in here would be really cool - and pretty easy I think. childRows := make([]RowIDs, len(c.Children)) for i, child := range c.Children { if child.Name != "Rows" { diff --git a/executor_test.go b/executor_test.go index 6b903acc2..a5e7b984a 100644 --- a/executor_test.go +++ b/executor_test.go @@ -2518,6 +2518,17 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { checkGroupBy(t, expected, results) }) + t.Run("distinct rows in different shards with column arg", func(t *testing.T) { + results := c.Query(t, "i", fmt.Sprintf(`GroupBy(Rows(field=ma), Rows(field=mb, column=%d), limit=5)`, ShardWidth)).Results[0].([]pilosa.GroupCount) + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 1}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 3}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "ma", RowID: 3}, {Field: "mb", RowID: 1}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "ma", RowID: 3}, {Field: "mb", RowID: 3}}, Count: 1}, + } + checkGroupBy(t, expected, results) + }) + c.CreateField(t, "i", pilosa.IndexOptions{}, "na") c.CreateField(t, "i", pilosa.IndexOptions{}, "nb") c.ImportBits(t, "i", "na", [][2]uint64{ @@ -2544,8 +2555,6 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { }) - // TODO test column queries to row call (also with multiple shards) - // test paging over results using previous. set the same bits in three // fields c.CreateField(t, "i", pilosa.IndexOptions{}, "ppa") From b6a953cade031acde377f147cef699c71c394942 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 16 Oct 2018 19:56:54 -0500 Subject: [PATCH 34/39] cleanup groupby - more comments, remove panic, remove dup test --- executor.go | 28 ++++++++++++++++++++-------- executor_test.go | 4 ---- fragment.go | 8 +++++--- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/executor.go b/executor.go index 40589f6e2..32f914f1b 100644 --- a/executor.go +++ b/executor.go @@ -964,7 +964,10 @@ func (g GroupCount) Compare(o GroupCount) int { } func (e *executor) executeGroupByShard(_ context.Context, index string, c *pql.Call, shard uint64, childRows []RowIDs) ([]GroupCount, error) { - iter := newGroupByIterator(childRows, c.Children, index, shard, e.Holder) + iter, err := newGroupByIterator(childRows, c.Children, index, shard, e.Holder) + if err != nil { + return errors.Wrapf(err, "getting group by iterator for shard %d", shard) + } if iter == nil { return []GroupCount{}, nil } @@ -2521,12 +2524,16 @@ type groupByIterator struct { row *Row id uint64 } + + // fields helps with the construction of GroupCount results by holding all + // the field names that are being grouped by. Each results makes a copy of + // fields and then sets the row ids. fields []FieldRow done bool } // newGroupByIterator initializes a new groupByIterator. -func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, index string, shard uint64, holder *Holder) *groupByIterator { +func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, index string, shard uint64, holder *Holder) (*groupByIterator, error) { gbi := &groupByIterator{ rowIters: make([]*rowIterator, len(children)), rows: make([]struct { @@ -2543,7 +2550,7 @@ func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, index string, sha // Fetch fragment. frag := holder.fragment(index, fieldName, viewStandard, shard) if frag == nil { // this means this whole shard doesn't have all it needs to continue - return nil + return nil, nil } filters := []rowFilter{} if len(rowIDs[i]) > 0 { @@ -2553,7 +2560,7 @@ func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, index string, sha prev, hasPrev, err := call.UintArg("previous") if err != nil { - panic("getting prev") + return nil, errors.Wrap(err, "getting previous") } else if hasPrev && !ignorePrev { if i == len(children)-1 { prev += 1 @@ -2563,19 +2570,25 @@ func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, index string, sha nextRow, rowID, wrapped := gbi.rowIters[i].Next() if nextRow == nil { gbi.done = true - return gbi + return gbi, nil } gbi.rows[i].row = nextRow gbi.rows[i].id = rowID if hasPrev && rowID != prev { + // ignorePrev signals that we didn't find a previous row, so all + // Rows queries "deeper" than it need to ignore the previous + // argument and start at the beginning. ignorePrev = true } if wrapped { + // if a field has wrapped, we need to get the next row for the + // previous field, and if that one wraps we need to keep going + // backward. for j := i - 1; j >= 0; j-- { nextRow, rowID, wrapped := gbi.rowIters[j].Next() if nextRow == nil { gbi.done = true - return gbi + return gbi, nil } gbi.rows[j].row = nextRow gbi.rows[j].id = rowID @@ -2606,8 +2619,6 @@ func (gbi *groupByIterator) nextAtIdx(i int) { } if i != 0 && i != len(gbi.rows)-1 { gbi.rows[i].row = nr.Intersect(gbi.rows[i-1].row) - } else if i == len(gbi.rows)-1 { - gbi.rows[i].row = nr } else { gbi.rows[i].row = nr } @@ -2625,6 +2636,7 @@ func (gbi *groupByIterator) Next() (ret GroupCount, done bool) { } else { ret.Count = gbi.rows[len(gbi.rows)-1].row.intersectionCount(gbi.rows[len(gbi.rows)-2].row) } + ret.Group = make([]FieldRow, len(gbi.rows)) copy(ret.Group, gbi.fields) for i, r := range gbi.rows { diff --git a/executor_test.go b/executor_test.go index 85669cffc..311745c56 100644 --- a/executor_test.go +++ b/executor_test.go @@ -2729,10 +2729,6 @@ func TestExecutor_Execute_Rows_Keys(t *testing.T) { q: `Rows(field=f, previous="11", limit=2)`, exp: []string{"12", "13"}, }, - { - q: `Rows(field=f, previous="11", limit=2)`, - exp: []string{"12", "13"}, - }, { q: `Rows(field=f, previous="17", limit=5)`, exp: []string{"18"}, diff --git a/fragment.go b/fragment.go index f70ddf482..a68f57b5f 100644 --- a/fragment.go +++ b/fragment.go @@ -1990,9 +1990,11 @@ func filterColumn(col uint64) rowFilter { } } -// TODO: this works, but it would be more performant if the fragment could seek to the -// next row in the rows list rather than asking the filter for each container -// serially. +// TODO: this works, but it would be more performant if the fragment could seek +// to the next row in the rows list rather than asking the filter for each +// container serially. The container iterator would need to expose a seek +// method, and the rowFilter would need some way of communicating to +// fragment.rows what the next rowID to seek to is. func filterWithRows(rows []uint64) rowFilter { loc := 0 return func(rowID, key uint64, c *roaring.Container) (include, done bool) { From f9dbacf3322b6ca282525456e0fa71344ab5c225 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 18 Oct 2018 14:36:44 -0500 Subject: [PATCH 35/39] fix compile errors --- executor.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/executor.go b/executor.go index 32f914f1b..c57f80618 100644 --- a/executor.go +++ b/executor.go @@ -966,7 +966,7 @@ func (g GroupCount) Compare(o GroupCount) int { func (e *executor) executeGroupByShard(_ context.Context, index string, c *pql.Call, shard uint64, childRows []RowIDs) ([]GroupCount, error) { iter, err := newGroupByIterator(childRows, c.Children, index, shard, e.Holder) if err != nil { - return errors.Wrapf(err, "getting group by iterator for shard %d", shard) + return nil, errors.Wrapf(err, "getting group by iterator for shard %d", shard) } if iter == nil { return []GroupCount{}, nil @@ -2603,7 +2603,7 @@ func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, index string, sha gbi.rows[i].row = gbi.rows[i].row.Intersect(gbi.rows[i-1].row) } - return gbi + return gbi, nil } // nextAtIdx is a recursive helper method for getting the next row for the field From 9fee746e76bf3a908c79d0a99a121e2aa6f4ecdf Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 22 Oct 2018 17:41:59 -0500 Subject: [PATCH 36/39] translate column argument in Rows() query --- executor.go | 1 + executor_test.go | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/executor.go b/executor.go index c57f80618..82d2135ab 100644 --- a/executor.go +++ b/executor.go @@ -2166,6 +2166,7 @@ func (e *executor) translateCall(index string, idx *Index, c *pql.Call) error { case "Rows": fieldName = callArgString(c, "field") rowKey = "previous" + colKey = "column" case "GroupBy": return errors.Wrap(e.translateGroupByCall(index, idx, c), "translating GroupBy") default: diff --git a/executor_test.go b/executor_test.go index 311745c56..03b659f33 100644 --- a/executor_test.go +++ b/executor_test.go @@ -2678,7 +2678,7 @@ func TestExecutor_Execute_Rows_Keys(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() - _, err := c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) + _, err := c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{Keys: true}) if err != nil { t.Fatalf("creating index: %v", err) } @@ -2695,7 +2695,7 @@ func TestExecutor_Execute_Rows_Keys(t *testing.T) { for shard := 0; shard < 10; shard++ { for i := shard; i < shard+10; i++ { for row := i; row >= 0 && row > i-3; row-- { - query.WriteString(fmt.Sprintf("Set(%d, f=\"%d\")", shard*pilosa.ShardWidth+i, row)) + query.WriteString(fmt.Sprintf("Set(\"%d\", f=\"%d\")", shard*pilosa.ShardWidth+i, row)) } @@ -2742,39 +2742,39 @@ func TestExecutor_Execute_Rows_Keys(t *testing.T) { exp: []string{}, }, { - q: `Rows(field=f, column=1)`, + q: `Rows(field=f, column="1")`, exp: []string{"0", "1"}, }, { - q: `Rows(field=f, column=2)`, + q: `Rows(field=f, column="2")`, exp: []string{"0", "1", "2"}, }, { - q: `Rows(field=f, column=3)`, + q: `Rows(field=f, column="3")`, exp: []string{"1", "2", "3"}, }, { - q: `Rows(field=f, limit=2, column=3)`, + q: `Rows(field=f, limit=2, column="3")`, exp: []string{"1", "2"}, }, { - q: fmt.Sprintf(`Rows(field=f, previous="15", column=%d)`, ShardWidth*9+17), + q: fmt.Sprintf(`Rows(field=f, previous="15", column="%d")`, ShardWidth*9+17), exp: []string{"16", "17"}, }, { - q: fmt.Sprintf(`Rows(field=f, previous="11", limit=2, column=%d)`, ShardWidth*5+14), + q: fmt.Sprintf(`Rows(field=f, previous="11", limit=2, column="%d")`, ShardWidth*5+14), exp: []string{"12", "13"}, }, { - q: fmt.Sprintf(`Rows(field=f, previous="17", limit=5, column=%d)`, ShardWidth*9+18), + q: fmt.Sprintf(`Rows(field=f, previous="17", limit=5, column="%d")`, ShardWidth*9+18), exp: []string{"18"}, }, { - q: `Rows(field=f, previous="18", column=19)`, + q: `Rows(field=f, previous="18", column="19")`, exp: []string{}, }, { - q: `Rows(field=f, previous="1", limit=0, column=0)`, + q: `Rows(field=f, previous="1", limit=0, column="0")`, exp: []string{}, }, } From 671420f31d2ee61b71d02e5d19c0fab1ee8da024 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 24 Oct 2018 17:11:45 -0500 Subject: [PATCH 37/39] add custom json marshal for FieldRow --- executor.go | 20 ++++++++++++++++++++ executor_internal_test.go | 29 +++++++++++++++++++++++++++++ internal/public.proto | 3 +-- 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/executor.go b/executor.go index 82d2135ab..7d71088ba 100644 --- a/executor.go +++ b/executor.go @@ -16,6 +16,7 @@ package pilosa import ( "context" + "encoding/json" "fmt" "sort" "time" @@ -909,6 +910,25 @@ type FieldRow struct { RowKey string `json:"rowKey,omitempty"` } +func (fr FieldRow) MarshalJSON() ([]byte, error) { + if fr.RowKey != "" { + return json.Marshal(struct { + Field string `json:"field"` + RowKey string `json:"rowKey"` + }{ + Field: fr.Field, + RowKey: fr.RowKey, + }) + } + return json.Marshal(struct { + Field string `json:"field"` + RowID uint64 `json:"rowID"` + }{ + Field: fr.Field, + RowID: fr.RowID, + }) +} + func (fr FieldRow) String() string { return fmt.Sprintf("%s.%d", fr.Field, fr.RowID) } diff --git a/executor_internal_test.go b/executor_internal_test.go index 93ba86743..629d704db 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -1,6 +1,7 @@ package pilosa import ( + "encoding/json" "fmt" "io/ioutil" "strings" @@ -193,3 +194,31 @@ func TestFilterWithRows(t *testing.T) { } } + +func TestFieldRowMarshalJSON(t *testing.T) { + fr := FieldRow{ + Field: "blah", + RowID: 0, + RowKey: "ha", + } + b, err := json.Marshal(fr) + if err != nil { + t.Fatalf("marshalling fieldrow: %v", err) + } + if string(b) != `{"field":"blah","rowKey":"ha"}` { + t.Fatalf("unexpected json: %s", b) + } + + fr = FieldRow{ + Field: "blah", + RowID: 2, + RowKey: "", + } + b, err = json.Marshal(fr) + if err != nil { + t.Fatalf("marshalling fieldrow: %v", err) + } + if string(b) != `{"field":"blah","rowID":2}` { + t.Fatalf("unexpected json: %s", b) + } +} diff --git a/internal/public.proto b/internal/public.proto index a102f1088..229fcfadd 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -11,7 +11,6 @@ message Row { message RowIdentifiers { repeated uint64 Rows = 1; repeated string Keys = 2; - //repeated Attr Attrs = 3; } message Pair { @@ -81,7 +80,7 @@ message QueryResult { uint64 N = 2; repeated Pair Pairs = 3; bool Changed = 4; - ValCount ValCount = 5; + ValCount ValCount = 5; repeated uint64 RowIDs = 7; repeated GroupCount GroupCounts = 8; RowIdentifiers RowIdentifiers = 9; From 21c35e6861d575281fe474b6a4746339bd1d87c8 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 24 Oct 2018 17:18:39 -0500 Subject: [PATCH 38/39] wrap errors, fix comment, add test --- executor.go | 8 ++++---- executor_test.go | 7 +++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/executor.go b/executor.go index 7d71088ba..c4153ab61 100644 --- a/executor.go +++ b/executor.go @@ -2371,7 +2371,7 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res if field.keys() { key, err := e.TranslateStore.TranslateRowToString(index, g.Field, g.RowID) if err != nil { - return nil, err + return nil, errors.Wrap(err, "translating row ID in Group") } group[i].RowKey = key } @@ -2399,7 +2399,7 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res for i, id := range result { key, err := e.TranslateStore.TranslateRowToString(index, fieldName, id) if err != nil { - return nil, err + return nil, errors.Wrap(err, "translating row ID") } other.Keys[i] = key } @@ -2646,8 +2646,8 @@ func (gbi *groupByIterator) nextAtIdx(i int) { gbi.rows[i].id = rowID } -// Next returns a ppi representing the next group by record. When there are no -// more records it will return an empty ppi and done==true. +// Next returns a GroupCount representing the next group by record. When there +// are no more records it will return an empty GroupCount and done==true. func (gbi *groupByIterator) Next() (ret GroupCount, done bool) { if gbi.done { return ret, true diff --git a/executor_test.go b/executor_test.go index 03b659f33..4ac1d20ea 100644 --- a/executor_test.go +++ b/executor_test.go @@ -2918,6 +2918,13 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { checkGroupBy(t, expected, results) }) + t.Run("test previous is last result", func(t *testing.T) { + results := c.Query(t, "i", `GroupBy(Rows(field=wa, previous=3), Rows(field=wb, previous=3), Rows(field=wc, previous=3), limit=3)`).Results[0].([]pilosa.GroupCount) + if len(results) > 0 { + t.Fatalf("expected no results because previous specified last result") + } + }) + t.Run("test wrapping multiple", func(t *testing.T) { results := c.Query(t, "i", `GroupBy(Rows(field=wa), Rows(field=wb, previous=2), Rows(field=wc, previous=2), limit=1)`).Results[0].([]pilosa.GroupCount) expected := []pilosa.GroupCount{ From a0f4522b6a31f210f322fe9975897be3b18838a2 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 25 Oct 2018 08:35:09 -0500 Subject: [PATCH 39/39] fix up linter issues in new groupby/rows tests --- executor_test.go | 3 +-- fragment_internal_test.go | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/executor_test.go b/executor_test.go index 4ac1d20ea..87b72689c 100644 --- a/executor_test.go +++ b/executor_test.go @@ -3035,8 +3035,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { t.Run("test wrapping with previous", func(t *testing.T) { totalResults := make([]pilosa.GroupCount, 0) - var results []pilosa.GroupCount - results = c.Query(t, "i", `GroupBy(Rows(field=ppa), Rows(field=ppb), Rows(field=ppc), limit=3)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(field=ppa), Rows(field=ppb), Rows(field=ppc), limit=3)`).Results[0].([]pilosa.GroupCount) totalResults = append(totalResults, results...) for len(totalResults) < 64 { lastGroup := results[len(results)-1].Group diff --git a/fragment_internal_test.go b/fragment_internal_test.go index eaf28acfe..4e3007957 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -2080,7 +2080,7 @@ func TestFragmentRowIterator(t *testing.T) { if id != i { t.Fatalf("expected row %d but got %d", i, id) } - if wrapped != false { + if wrapped { t.Fatalf("shouldn't have wrapped") } if !reflect.DeepEqual(row.Columns(), []uint64{0}) { @@ -2094,7 +2094,7 @@ func TestFragmentRowIterator(t *testing.T) { if id != 0 { t.Fatalf("id should be 0 after iterator is exhausted, got %d", id) } - if wrapped != true { + if !wrapped { t.Fatalf("wrapped should be true after iterator is exhausted") } f.Close() @@ -2113,7 +2113,7 @@ func TestFragmentRowIterator(t *testing.T) { if id != i { t.Fatalf("expected row %d but got %d", i, id) } - if wrapped != false { + if wrapped { t.Fatalf("shouldn't have wrapped") } if !reflect.DeepEqual(row.Columns(), []uint64{0}) { @@ -2127,7 +2127,7 @@ func TestFragmentRowIterator(t *testing.T) { if id != 0 { t.Fatalf("id should be 0 after iterator is exhausted, got %d", id) } - if wrapped != true { + if !wrapped { t.Fatalf("wrapped should be true after iterator is exhausted") } f.Close()