From e1b938e52c7187d3bc266f24c1ece678a7e24a05 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 22 Aug 2018 12:18:27 -0500 Subject: [PATCH 1/7] add FieldRow struct to replace the groupBy string key --- encoding/proto/proto.go | 44 +++++- executor.go | 102 +++++++------ executor_test.go | 60 +++++--- internal/public.pb.go | 326 ++++++++++++++++++++++++++++++---------- internal/public.proto | 9 +- 5 files changed, 381 insertions(+), 160 deletions(-) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index e133de7fc..4f886acce 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -999,11 +999,29 @@ func decodeAttr(attr *internal.Attr) (key string, value interface{}) { } func decodeGroupByCounts(a []*internal.GroupLine) pilosa.GroupByCounts { - gbc := make(pilosa.GroupByCounts, 0) + other := make([]pilosa.GroupLine, len(a)) for i := range a { - gbc = append(gbc, pilosa.GroupLine{a[i].Groups, a[i].Total}) + other[i] = pilosa.GroupLine{ + decodeFieldRows(a[i].Groups), + a[i].Total, + } + } + return pilosa.GroupByCounts(other) +} + +func decodeFieldRows(a []*internal.FieldRow) []pilosa.FieldRow { + other := make([]pilosa.FieldRow, len(a)) + for i := range a { + other[i] = decodeFieldRow(a[i]) + } + return other +} + +func decodeFieldRow(pb *internal.FieldRow) pilosa.FieldRow { + return pilosa.FieldRow{ + Field: pb.Field, + RowID: pb.RowID, } - return gbc } func decodePairs(a []*internal.Pair) []pilosa.Pair { @@ -1060,11 +1078,29 @@ func encodeRow(r *pilosa.Row) *internal.Row { func encodeGroupByCount(counts pilosa.GroupByCounts) []*internal.GroupLine { result := make([]*internal.GroupLine, len(counts)) for i := range counts { - result[i] = &internal.GroupLine{Groups: counts[i].Groups, Total: counts[i].Total} + result[i] = &internal.GroupLine{ + Groups: encodeFieldRows(counts[i].Groups), + Total: counts[i].Total, + } } return result } +func encodeFieldRows(a []pilosa.FieldRow) []*internal.FieldRow { + other := make([]*internal.FieldRow, len(a)) + for i := range a { + other[i] = encodeFieldRow(a[i]) + } + return other +} + +func encodeFieldRow(p pilosa.FieldRow) *internal.FieldRow { + return &internal.FieldRow{ + Field: p.Field, + RowID: p.RowID, + } +} + func encodePairs(a pilosa.Pairs) []*internal.Pair { other := make([]*internal.Pair, len(a)) for i := range a { diff --git a/executor.go b/executor.go index 56ab05269..35740866b 100644 --- a/executor.go +++ b/executor.go @@ -786,16 +786,37 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call return results, nil } +// FieldRow is used to distinguish rows in a group by result. +type FieldRow struct { + Field string + RowID uint64 + RowKey string +} + +func (fr FieldRow) String() string { + return fmt.Sprintf("%s.%d", fr.Field, fr.RowID) +} + +// TODO: we shouldn't need to string this +func uniqueGroupString(fr []FieldRow) string { + s := []string{} + for _, f := range fr { + s = append(s, f.String()) + } + return strings.Join(s, "-") +} + // gbi is a groupBy item. type gbi struct { row *Row - fieldKey string - rowID uint64 + fieldRow FieldRow } type GroupLine struct { - Groups []string + Groups []FieldRow Total uint64 } + +// GroupByCounts is the return type for GroupBy queries. type GroupByCounts []GroupLine func (gbc GroupByCounts) Merge(other GroupByCounts) GroupByCounts { @@ -804,14 +825,13 @@ func (gbc GroupByCounts) Merge(other GroupByCounts) GroupByCounts { total uint64 }) for i := range gbc { - m[strings.Join(gbc[i].Groups, "-")] = struct { + m[uniqueGroupString(gbc[i].Groups)] = struct { i int total uint64 - }{total: gbc[i].Total, i: i} + }{i, gbc[i].Total} } for i := range other { - key := strings.Join(other[i].Groups, "-") - o, found := m[key] + o, found := m[uniqueGroupString(other[i].Groups)] if found { gbc[o.i].Total += other[i].Total } else { @@ -820,20 +840,7 @@ func (gbc GroupByCounts) Merge(other GroupByCounts) GroupByCounts { } return gbc } -func makeGroup(parts []gbi) GroupLine { - var other *Row - line := GroupLine{} - for i, o := range parts { - if i == 0 { - other = o.row - } else { - other = other.Intersect(o.row) - } - line.Groups = append(line.Groups, o.fieldKey) - } - line.Total = other.Count() - return line -} + func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql.Call, shard uint64) (GroupByCounts, error) { // Fetch index. idx := e.Holder.Index(index) @@ -867,7 +874,7 @@ func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql } } results := make(GroupByCounts, 0) - var work listOfGBILists + var work [][]gbi for _, fieldDirective := range fieldDirectives.([]interface{}) { fieldName := getFieldName(fieldDirective.(string)) // Fetch fragment. @@ -880,12 +887,15 @@ func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql if err != nil { return nil, err } - set := make(gbiList, 0) + + set := make([]gbi, 0) for _, rowID := range frag.rowsWithFilter(filter) { set = append(set, gbi{ - row: frag.row(rowID), - rowID: rowID, - fieldKey: fmt.Sprintf("%s.%d", fieldName, rowID), + row: frag.row(rowID), + fieldRow: FieldRow{ + Field: fieldName, + RowID: rowID, + }, }) } work = append(work, set) @@ -899,35 +909,29 @@ func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql return results, nil } -type gbiList []gbi -type listOfGBILists []gbiList - -// pi is a product process item. -type pi struct { +// ppi is a product process item. +type ppi struct { row *Row gl GroupLine } -type piList []pi -// product generates the cartiesian product of the input -// using tail recursion -func product(input listOfGBILists) piList { - res := make(piList, 0) - if len(input) == 0 { //base return empty list - res = append(res, pi{gl: GroupLine{Groups: make([]string, 0)}}) - } else { - res = productHelper(input, res) +// product generates the cartesian product of the input +// using tail recursion. +func product(input [][]gbi) []ppi { + if len(input) == 0 { // base return empty list + return []ppi{ + {gl: GroupLine{Groups: make([]FieldRow, 0)}}, + } } - return res -} -func productHelper(lists listOfGBILists, res piList) piList { - head := lists[0] //take first element of the list - tail := product(lists[1:]) //invoke product on remaining element + + res := make([]ppi, 0) + head := input[0] // take first element of the list + tail := product(input[1:]) // invoke product on remaining element for h := range head { // for each head - for t := range tail { //iterate over the tail - s := pi{gl: GroupLine{Groups: make([]string, 0)}} - s.gl.Groups = append([]string{head[h].fieldKey}, tail[t].gl.Groups...) //had to insert at the front to match input order - if tail[t].row != nil { //first time around nothing to intersect + for t := range tail { // iterate over the tail + s := ppi{gl: GroupLine{Groups: make([]FieldRow, 0)}} + s.gl.Groups = append([]FieldRow{head[h].fieldRow}, tail[t].gl.Groups...) // had to insert at the front to match input order + if tail[t].row != nil { // first time around nothing to intersect s.row = head[h].row.Intersect(tail[t].row) } else { s.row = head[h].row diff --git a/executor_test.go b/executor_test.go index 84e00936d..e3afda3ed 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1234,11 +1234,11 @@ Set(4500001, fn=4) t.Fatalf("GroupBy querying: %v", err) } else { expected := pilosa.GroupByCounts{ - {Groups: []string{"f.10"}, Total: 4}, - {Groups: []string{"f.7"}, Total: 1}, + {Groups: []pilosa.FieldRow{{Field: "f", RowID: 10}}, Total: 4}, + {Groups: []pilosa.FieldRow{{Field: "f", RowID: 7}}, Total: 1}, } results := res.Results[0].(pilosa.GroupByCounts) - checkGroupBy(expected, results, t) + checkGroupBy(t, expected, results) } }) } @@ -1681,17 +1681,15 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { hldr.SetBit("i", "general", 11, ShardWidth+2) hldr.SetBit("i", "general", 12, 2) hldr.SetBit("i", "general", 12, ShardWidth+2) - hldr.SetBit("i", "sub", 10, 0) - hldr.SetBit("i", "sub", 10, 1) - hldr.SetBit("i", "sub", 10, 3) - hldr.SetBit("i", "sub", 11, 2) - hldr.SetBit("i", "sub", 11, 0) - expected := pilosa.GroupByCounts{ - {Groups: []string{"general.10", "sub.11"}, Total: 1}, - {Groups: []string{"general.11", "sub.11"}, Total: 1}, - {Groups: []string{"general.12", "sub.11"}, Total: 1}, - {Groups: []string{"general.10", "sub.10"}, Total: 2}, - } + + hldr.SetBit("i", "sub", 100, 0) + hldr.SetBit("i", "sub", 100, 1) + hldr.SetBit("i", "sub", 100, 3) + hldr.SetBit("i", "sub", 100, ShardWidth+1) + + hldr.SetBit("i", "sub", 110, 2) + hldr.SetBit("i", "sub", 110, 0) + t.Run("No Field List Arguments", func(t *testing.T) { if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy()`}); err != nil { if errors.Cause(err) != pilosa.ErrFieldsArgumentRequired { @@ -1714,38 +1712,50 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { } }) t.Run("Basic", func(t *testing.T) { + expected := pilosa.GroupByCounts{ + {Groups: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Total: 1}, + {Groups: []pilosa.FieldRow{{Field: "general", RowID: 11}, {Field: "sub", RowID: 110}}, Total: 1}, + {Groups: []pilosa.FieldRow{{Field: "general", RowID: 12}, {Field: "sub", RowID: 110}}, Total: 1}, + {Groups: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Total: 3}, + } + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(fields=[general,sub])`}); err != nil { t.Fatal(err) } else { results := res.Results[0].(pilosa.GroupByCounts) - checkGroupBy(expected, results, t) + checkGroupBy(t, expected, results) } }) - expected = pilosa.GroupByCounts{ - {Groups: []string{"general.11"}, Total: 2}, - {Groups: []string{"general.12"}, Total: 2}, - } + t.Run("check field offset no limit", func(t *testing.T) { + expected := pilosa.GroupByCounts{ + {Groups: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Total: 2}, + {Groups: []pilosa.FieldRow{{Field: "general", RowID: 12}}, Total: 2}, + } + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(fields=[general:11:])`}); err != nil { t.Fatal(err) } else { results := res.Results[0].(pilosa.GroupByCounts) - checkGroupBy(expected, results, t) + checkGroupBy(t, expected, results) } }) - expected = pilosa.GroupByCounts{ - {Groups: []string{"general.11"}, Total: 2}, - } + t.Run("check field offset limit", func(t *testing.T) { + expected := pilosa.GroupByCounts{ + {Groups: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Total: 2}, + } + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(fields=[general:11:1])`}); err != nil { t.Fatal(err) } else { results := res.Results[0].(pilosa.GroupByCounts) - checkGroupBy(expected, results, t) + checkGroupBy(t, expected, results) } }) } -func checkGroupBy(expected, results pilosa.GroupByCounts, t *testing.T) { + +func checkGroupBy(t *testing.T, expected, results pilosa.GroupByCounts) { notIn := func(item pilosa.GroupLine, expected pilosa.GroupByCounts) bool { for i := range expected { if item.Total == expected[i].Total { diff --git a/internal/public.pb.go b/internal/public.pb.go index 8139d3a28..2203c87d6 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -10,6 +10,7 @@ It has these top-level messages: Row Pair + FieldRow GroupLine ValCount Bit @@ -107,17 +108,41 @@ func (m *Pair) GetCount() uint64 { return 0 } +type FieldRow struct { + Field string `protobuf:"bytes,1,opt,name=Field,proto3" json:"Field,omitempty"` + RowID uint64 `protobuf:"varint,2,opt,name=RowID,proto3" json:"RowID,omitempty"` +} + +func (m *FieldRow) Reset() { *m = FieldRow{} } +func (m *FieldRow) String() string { return proto.CompactTextString(m) } +func (*FieldRow) ProtoMessage() {} +func (*FieldRow) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} } + +func (m *FieldRow) GetField() string { + if m != nil { + return m.Field + } + return "" +} + +func (m *FieldRow) GetRowID() uint64 { + if m != nil { + return m.RowID + } + return 0 +} + type GroupLine struct { - Groups []string `protobuf:"bytes,1,rep,name=Groups" json:"Groups,omitempty"` - Total uint64 `protobuf:"varint,2,opt,name=Total,proto3" json:"Total,omitempty"` + Groups []*FieldRow `protobuf:"bytes,1,rep,name=Groups" json:"Groups,omitempty"` + Total uint64 `protobuf:"varint,2,opt,name=Total,proto3" json:"Total,omitempty"` } func (m *GroupLine) Reset() { *m = GroupLine{} } func (m *GroupLine) String() string { return proto.CompactTextString(m) } func (*GroupLine) ProtoMessage() {} -func (*GroupLine) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} } +func (*GroupLine) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} } -func (m *GroupLine) GetGroups() []string { +func (m *GroupLine) GetGroups() []*FieldRow { if m != nil { return m.Groups } @@ -139,7 +164,7 @@ type ValCount struct { func (m *ValCount) Reset() { *m = ValCount{} } func (m *ValCount) String() string { return proto.CompactTextString(m) } func (*ValCount) ProtoMessage() {} -func (*ValCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} } +func (*ValCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{4} } func (m *ValCount) GetVal() int64 { if m != nil { @@ -164,7 +189,7 @@ type Bit struct { func (m *Bit) Reset() { *m = Bit{} } func (m *Bit) String() string { return proto.CompactTextString(m) } func (*Bit) ProtoMessage() {} -func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{4} } +func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{5} } func (m *Bit) GetRowID() uint64 { if m != nil { @@ -196,7 +221,7 @@ type ColumnAttrSet struct { func (m *ColumnAttrSet) Reset() { *m = ColumnAttrSet{} } func (m *ColumnAttrSet) String() string { return proto.CompactTextString(m) } func (*ColumnAttrSet) ProtoMessage() {} -func (*ColumnAttrSet) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{5} } +func (*ColumnAttrSet) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{6} } func (m *ColumnAttrSet) GetID() uint64 { if m != nil { @@ -231,7 +256,7 @@ type Attr struct { func (m *Attr) Reset() { *m = Attr{} } func (m *Attr) String() string { return proto.CompactTextString(m) } func (*Attr) ProtoMessage() {} -func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{6} } +func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{7} } func (m *Attr) GetKey() string { if m != nil { @@ -282,7 +307,7 @@ type AttrMap struct { func (m *AttrMap) Reset() { *m = AttrMap{} } func (m *AttrMap) String() string { return proto.CompactTextString(m) } func (*AttrMap) ProtoMessage() {} -func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{7} } +func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{8} } func (m *AttrMap) GetAttrs() []*Attr { if m != nil { @@ -303,7 +328,7 @@ type QueryRequest struct { func (m *QueryRequest) Reset() { *m = QueryRequest{} } func (m *QueryRequest) String() string { return proto.CompactTextString(m) } func (*QueryRequest) ProtoMessage() {} -func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{8} } +func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{9} } func (m *QueryRequest) GetQuery() string { if m != nil { @@ -356,7 +381,7 @@ type QueryResponse struct { func (m *QueryResponse) Reset() { *m = QueryResponse{} } func (m *QueryResponse) String() string { return proto.CompactTextString(m) } func (*QueryResponse) ProtoMessage() {} -func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{9} } +func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{10} } func (m *QueryResponse) GetErr() string { if m != nil { @@ -393,7 +418,7 @@ type QueryResult struct { func (m *QueryResult) Reset() { *m = QueryResult{} } func (m *QueryResult) String() string { return proto.CompactTextString(m) } func (*QueryResult) ProtoMessage() {} -func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{10} } +func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{11} } func (m *QueryResult) GetType() uint32 { if m != nil { @@ -465,7 +490,7 @@ type ImportRequest struct { func (m *ImportRequest) Reset() { *m = ImportRequest{} } func (m *ImportRequest) String() string { return proto.CompactTextString(m) } func (*ImportRequest) ProtoMessage() {} -func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{11} } +func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{12} } func (m *ImportRequest) GetIndex() string { if m != nil { @@ -535,7 +560,7 @@ type ImportValueRequest struct { func (m *ImportValueRequest) Reset() { *m = ImportValueRequest{} } func (m *ImportValueRequest) String() string { return proto.CompactTextString(m) } func (*ImportValueRequest) ProtoMessage() {} -func (*ImportValueRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{12} } +func (*ImportValueRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{13} } func (m *ImportValueRequest) GetIndex() string { if m != nil { @@ -582,6 +607,7 @@ func (m *ImportValueRequest) GetValues() []int64 { func init() { proto.RegisterType((*Row)(nil), "internal.Row") proto.RegisterType((*Pair)(nil), "internal.Pair") + proto.RegisterType((*FieldRow)(nil), "internal.FieldRow") proto.RegisterType((*GroupLine)(nil), "internal.GroupLine") proto.RegisterType((*ValCount)(nil), "internal.ValCount") proto.RegisterType((*Bit)(nil), "internal.Bit") @@ -690,6 +716,35 @@ func (m *Pair) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *FieldRow) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FieldRow) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Field) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) + } + if m.RowID != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.RowID)) + } + return i, nil +} + func (m *GroupLine) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -706,18 +761,15 @@ func (m *GroupLine) MarshalTo(dAtA []byte) (int, error) { var l int _ = l if len(m.Groups) > 0 { - for _, s := range m.Groups { + for _, msg := range m.Groups { dAtA[i] = 0xa i++ - l = len(s) - for l >= 1<<7 { - dAtA[i] = uint8(uint64(l)&0x7f | 0x80) - l >>= 7 - i++ + i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err } - dAtA[i] = uint8(l) - i++ - i += copy(dAtA[i:], s) + i += n } } if m.Total != 0 { @@ -1396,12 +1448,25 @@ func (m *Pair) Size() (n int) { return n } +func (m *FieldRow) Size() (n int) { + var l int + _ = l + l = len(m.Field) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } + if m.RowID != 0 { + n += 1 + sovPublic(uint64(m.RowID)) + } + return n +} + func (m *GroupLine) Size() (n int) { var l int _ = l if len(m.Groups) > 0 { - for _, s := range m.Groups { - l = len(s) + for _, e := range m.Groups { + l = e.Size() n += 1 + l + sovPublic(uint64(l)) } } @@ -1977,6 +2042,104 @@ func (m *Pair) Unmarshal(dAtA []byte) error { } return nil } +func (m *FieldRow) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FieldRow: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FieldRow: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Field = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field RowID", wireType) + } + m.RowID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.RowID |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipPublic(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPublic + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *GroupLine) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -2010,7 +2173,7 @@ func (m *GroupLine) Unmarshal(dAtA []byte) error { if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Groups", wireType) } - var stringLen uint64 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPublic @@ -2020,20 +2183,22 @@ func (m *GroupLine) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + msglen |= (int(b) & 0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { + if msglen < 0 { return ErrInvalidLengthPublic } - postIndex := iNdEx + intStringLen + postIndex := iNdEx + msglen if postIndex > l { return io.ErrUnexpectedEOF } - m.Groups = append(m.Groups, string(dAtA[iNdEx:postIndex])) + m.Groups = append(m.Groups, &FieldRow{}) + if err := m.Groups[len(m.Groups)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } iNdEx = postIndex case 2: if wireType != 0 { @@ -4076,53 +4241,54 @@ var ( func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ - // 760 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd3, 0x4a, - 0x14, 0xbe, 0x13, 0x3b, 0x89, 0x73, 0xd2, 0xe4, 0x56, 0x73, 0xef, 0xed, 0xb5, 0x50, 0x15, 0x2c, - 0x0b, 0x21, 0xaf, 0x52, 0x29, 0xac, 0xba, 0x01, 0x91, 0xfe, 0xa0, 0xa8, 0x50, 0xc1, 0xb4, 0x14, - 0xb1, 0x74, 0x9b, 0x51, 0x6b, 0xc9, 0xf1, 0x18, 0x7b, 0xac, 0x34, 0xcf, 0xd1, 0x0d, 0x8f, 0xc0, - 0x82, 0x07, 0xe9, 0x92, 0x47, 0x80, 0xf2, 0x22, 0x68, 0xce, 0x78, 0x62, 0x27, 0x95, 0x2a, 0x16, - 0xec, 0xfc, 0x7d, 0x67, 0xe6, 0xcc, 0xf9, 0xce, 0x9f, 0x61, 0x23, 0x2d, 0xce, 0xe3, 0xe8, 0x62, - 0x98, 0x66, 0x42, 0x0a, 0xea, 0x44, 0x89, 0xe4, 0x59, 0x12, 0xc6, 0xfe, 0x47, 0xb0, 0x98, 0x98, - 0x53, 0x17, 0xda, 0x7b, 0x22, 0x2e, 0x66, 0x49, 0xee, 0x12, 0xcf, 0x0a, 0x6c, 0x66, 0x20, 0x7d, - 0x02, 0xcd, 0x97, 0x52, 0x66, 0xb9, 0xdb, 0xf0, 0xac, 0xa0, 0x3b, 0xea, 0x0f, 0xcd, 0xd5, 0xa1, - 0xa2, 0x99, 0x36, 0x52, 0x0a, 0xf6, 0x11, 0x5f, 0xe4, 0xae, 0xe5, 0x59, 0x41, 0x87, 0xe1, 0xb7, - 0xff, 0x1c, 0xec, 0xb7, 0x61, 0x94, 0xd1, 0x3e, 0x34, 0x26, 0xfb, 0x2e, 0xf1, 0x48, 0x60, 0xb3, - 0xc6, 0x64, 0x9f, 0xfe, 0x0b, 0xcd, 0x3d, 0x51, 0x24, 0xd2, 0x6d, 0x20, 0xa5, 0x01, 0xdd, 0x04, - 0xeb, 0x88, 0x2f, 0x5c, 0xcb, 0x23, 0x41, 0x87, 0xa9, 0x4f, 0x7f, 0x17, 0x3a, 0xaf, 0x32, 0x51, - 0xa4, 0xaf, 0xa3, 0x84, 0xd3, 0x2d, 0x68, 0x21, 0xd0, 0xf1, 0x75, 0x58, 0x89, 0x94, 0xb3, 0x53, - 0x21, 0xc3, 0xd8, 0x38, 0x43, 0xe0, 0x8f, 0xc0, 0x39, 0x0b, 0xe3, 0xa5, 0xe3, 0xb3, 0x30, 0xc6, - 0xf7, 0x2d, 0xa6, 0x3e, 0x57, 0x03, 0xb0, 0xca, 0x00, 0xfc, 0xf7, 0x60, 0x8d, 0x23, 0xa9, 0x8c, - 0x4c, 0xcc, 0x97, 0x01, 0x6b, 0x40, 0x1f, 0x81, 0xa3, 0x13, 0x32, 0xd9, 0x2f, 0x5f, 0x5a, 0x62, - 0xba, 0x0d, 0x9d, 0xd3, 0x68, 0xc6, 0x73, 0x19, 0xce, 0x52, 0x8c, 0xdf, 0x62, 0x15, 0xe1, 0x7f, - 0x80, 0x9e, 0x3e, 0xa9, 0x12, 0x75, 0xc2, 0xe5, 0xbd, 0x74, 0xfc, 0x5e, 0x82, 0xef, 0xa7, 0xe7, - 0x0b, 0x01, 0x5b, 0xd9, 0x8c, 0x89, 0x2c, 0x4d, 0xaa, 0x1a, 0xa7, 0x8b, 0x94, 0x97, 0x91, 0xe2, - 0x37, 0xf5, 0xa0, 0x7b, 0x22, 0xb3, 0x28, 0xb9, 0x3c, 0x0b, 0xe3, 0x82, 0x97, 0x8e, 0xea, 0x94, - 0xd2, 0x38, 0x49, 0xa4, 0x36, 0xdb, 0x28, 0x63, 0x89, 0x95, 0xc6, 0xb1, 0x10, 0xb1, 0x36, 0x36, - 0x3d, 0x12, 0x38, 0xac, 0x22, 0xe8, 0x00, 0xe0, 0x30, 0x16, 0x61, 0x79, 0xb7, 0xe5, 0x91, 0x80, - 0xb0, 0x1a, 0xe3, 0xef, 0x40, 0x5b, 0x45, 0xfa, 0x26, 0x4c, 0x2b, 0xb5, 0xe4, 0x01, 0xb5, 0xfe, - 0x2d, 0x81, 0x8d, 0x77, 0x05, 0xcf, 0x16, 0x8c, 0x7f, 0x2a, 0x78, 0x8e, 0x55, 0x41, 0x5c, 0xaa, - 0xd4, 0x40, 0x35, 0xc5, 0xc9, 0x55, 0x98, 0x4d, 0x75, 0xee, 0x6c, 0x56, 0x22, 0xa5, 0xb5, 0xca, - 0x79, 0x8e, 0x5a, 0x1d, 0x56, 0xa7, 0xd4, 0x4d, 0xc6, 0x67, 0x42, 0x1a, 0x31, 0x25, 0xa2, 0x01, - 0xfc, 0x7d, 0x70, 0x7d, 0x11, 0x17, 0x53, 0xce, 0xc4, 0x5c, 0xdf, 0x6e, 0xe1, 0x81, 0x75, 0x9a, - 0x3e, 0x85, 0x7e, 0x49, 0x99, 0xc1, 0x69, 0xe3, 0xc1, 0x35, 0xd6, 0xbf, 0x21, 0xd0, 0x2b, 0xa5, - 0xe4, 0xa9, 0x48, 0x72, 0xae, 0xea, 0x75, 0x90, 0x65, 0xa6, 0x5e, 0x07, 0x59, 0x46, 0x77, 0xa0, - 0xcd, 0x78, 0x5e, 0xc4, 0xd2, 0x34, 0xc1, 0x7f, 0x55, 0x5a, 0xcc, 0xdd, 0x22, 0x96, 0xcc, 0x9c, - 0xa2, 0x2f, 0xa0, 0xbf, 0xd2, 0x54, 0x7a, 0xf0, 0xba, 0xa3, 0xff, 0xab, 0x7b, 0x2b, 0x76, 0xb6, - 0x76, 0xdc, 0xbf, 0x69, 0x40, 0xb7, 0xe6, 0x99, 0x3e, 0xc6, 0x35, 0x80, 0x31, 0x75, 0x47, 0xbd, - 0xca, 0x0b, 0x13, 0x73, 0x86, 0x0b, 0x62, 0x03, 0xc8, 0x71, 0xd9, 0x4f, 0xe4, 0x58, 0x55, 0x51, - 0x8d, 0xb6, 0x79, 0xb6, 0x56, 0x45, 0x45, 0x33, 0x6d, 0xc4, 0xa5, 0x72, 0x15, 0x26, 0x97, 0x7c, - 0x8a, 0xfd, 0xe4, 0x30, 0x03, 0xe9, 0xb0, 0x9a, 0x4f, 0x2c, 0x40, 0x77, 0x44, 0x2b, 0x17, 0xc6, - 0xc2, 0xaa, 0x19, 0x36, 0x0d, 0xad, 0x6a, 0xd1, 0x2b, 0x1b, 0x5a, 0x95, 0x50, 0xcd, 0xa6, 0x4a, - 0x3c, 0x16, 0x5f, 0x23, 0xba, 0x0b, 0x3d, 0xdc, 0x0d, 0xe3, 0x05, 0xde, 0xcd, 0x5d, 0x07, 0x63, - 0xfc, 0xa7, 0x7a, 0x60, 0xb9, 0x55, 0xd8, 0xea, 0x49, 0xff, 0x07, 0x81, 0xde, 0x64, 0x96, 0x8a, - 0x4c, 0xd6, 0xfa, 0x6e, 0x92, 0x4c, 0xf9, 0xb5, 0xe9, 0x3b, 0x04, 0x8a, 0x3d, 0x8c, 0x78, 0x3c, - 0xc5, 0x84, 0x74, 0x98, 0x06, 0x8a, 0xc5, 0xfe, 0xc3, 0x7e, 0xb3, 0x99, 0x06, 0xb5, 0x30, 0xed, - 0x95, 0x30, 0xb7, 0xa1, 0x63, 0x36, 0x48, 0xee, 0x36, 0xd1, 0x54, 0x11, 0x6a, 0xa2, 0x96, 0x2b, - 0x44, 0xb5, 0xa0, 0x15, 0x58, 0xac, 0xc6, 0xa8, 0xd4, 0x32, 0x31, 0xc7, 0x95, 0xdb, 0xc6, 0x7d, - 0x68, 0xa0, 0xba, 0xa9, 0xdd, 0xa0, 0xd1, 0x41, 0x63, 0x8d, 0xf1, 0xbf, 0x12, 0xa0, 0x5a, 0x23, - 0xce, 0xe6, 0x9f, 0x13, 0xfa, 0xb0, 0xa0, 0x2d, 0x68, 0xe1, 0x7b, 0x46, 0x4c, 0x89, 0xd6, 0xc2, - 0x6d, 0xaf, 0x87, 0x3b, 0xde, 0xbc, 0xbd, 0x1b, 0x90, 0x6f, 0x77, 0x03, 0xf2, 0xfd, 0x6e, 0x40, - 0x3e, 0xff, 0x1c, 0xfc, 0x75, 0xde, 0xc2, 0x5f, 0xd8, 0xb3, 0x5f, 0x01, 0x00, 0x00, 0xff, 0xff, - 0x22, 0x44, 0x8c, 0x98, 0xd2, 0x06, 0x00, 0x00, + // 783 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xdd, 0x6a, 0xdb, 0x48, + 0x14, 0xde, 0xb1, 0x64, 0x5b, 0x3e, 0x8e, 0xbd, 0x61, 0x36, 0x9b, 0x15, 0x4b, 0xf0, 0x0a, 0xb1, + 0x2c, 0x62, 0x2f, 0x1c, 0xf0, 0xc2, 0x42, 0x6f, 0x5a, 0xea, 0xfc, 0x14, 0x93, 0x26, 0xb4, 0x93, + 0x34, 0xa5, 0x97, 0x4a, 0x3c, 0x24, 0x02, 0x59, 0xa3, 0x4a, 0x23, 0x1c, 0x3f, 0x47, 0x6e, 0xfa, + 0x08, 0xbd, 0xe8, 0x83, 0xe4, 0xb2, 0x8f, 0xd0, 0xa6, 0x2f, 0x52, 0xe6, 0x8c, 0xc6, 0x92, 0x1d, + 0x08, 0xbd, 0xe8, 0xdd, 0x7c, 0xe7, 0xcc, 0x1c, 0x7d, 0xdf, 0xf9, 0x13, 0x6c, 0xa4, 0xc5, 0x45, + 0x1c, 0x5d, 0x0e, 0xd3, 0x4c, 0x48, 0x41, 0x9d, 0x28, 0x91, 0x3c, 0x4b, 0xc2, 0xd8, 0x7f, 0x07, + 0x16, 0x13, 0x73, 0xea, 0x42, 0x7b, 0x4f, 0xc4, 0xc5, 0x2c, 0xc9, 0x5d, 0xe2, 0x59, 0x81, 0xcd, + 0x0c, 0xa4, 0x7f, 0x43, 0xf3, 0xb9, 0x94, 0x59, 0xee, 0x36, 0x3c, 0x2b, 0xe8, 0x8e, 0xfa, 0x43, + 0xf3, 0x74, 0xa8, 0xcc, 0x4c, 0x3b, 0x29, 0x05, 0xfb, 0x88, 0x2f, 0x72, 0xd7, 0xf2, 0xac, 0xa0, + 0xc3, 0xf0, 0xec, 0x3f, 0x05, 0xfb, 0x55, 0x18, 0x65, 0xb4, 0x0f, 0x8d, 0xc9, 0xbe, 0x4b, 0x3c, + 0x12, 0xd8, 0xac, 0x31, 0xd9, 0xa7, 0x5b, 0xd0, 0xdc, 0x13, 0x45, 0x22, 0xdd, 0x06, 0x9a, 0x34, + 0xa0, 0x9b, 0x60, 0x1d, 0xf1, 0x85, 0x6b, 0x79, 0x24, 0xe8, 0x30, 0x75, 0xf4, 0xff, 0x07, 0xe7, + 0x30, 0xe2, 0xf1, 0x54, 0xf1, 0xdb, 0x82, 0x26, 0x9e, 0x31, 0x4c, 0x87, 0x69, 0xa0, 0xac, 0x4c, + 0xcc, 0x27, 0xfb, 0x26, 0x12, 0x02, 0xff, 0x18, 0x3a, 0x2f, 0x32, 0x51, 0xa4, 0x2f, 0xa3, 0x84, + 0xd3, 0x7f, 0xa1, 0x85, 0x40, 0xeb, 0xea, 0x8e, 0x68, 0xc5, 0xdf, 0x04, 0x67, 0xe5, 0x0d, 0x15, + 0xee, 0x4c, 0xc8, 0x30, 0x36, 0xe1, 0x10, 0xf8, 0x23, 0x70, 0xce, 0xc3, 0x78, 0x49, 0xf2, 0x3c, + 0x8c, 0x91, 0x84, 0xc5, 0xd4, 0x71, 0x55, 0x8c, 0x55, 0x8a, 0xf1, 0xdf, 0x80, 0x35, 0x8e, 0x64, + 0xc5, 0x8f, 0xd4, 0xf8, 0xd1, 0x3f, 0xc1, 0xd1, 0xc9, 0x5d, 0x12, 0x5f, 0x62, 0xba, 0x03, 0x9d, + 0xb3, 0x68, 0xc6, 0x73, 0x19, 0xce, 0x52, 0xcc, 0x85, 0xc5, 0x2a, 0x83, 0xff, 0x16, 0x7a, 0xfa, + 0xa6, 0x4a, 0xfa, 0x29, 0x97, 0x0f, 0x52, 0xfb, 0x63, 0xc5, 0x7a, 0x98, 0xea, 0x8f, 0x04, 0x6c, + 0xe5, 0x33, 0x2e, 0xb2, 0x74, 0xa9, 0xca, 0x9e, 0x2d, 0x52, 0x5e, 0x32, 0xc5, 0x33, 0xf5, 0xa0, + 0x7b, 0x2a, 0xb3, 0x28, 0xb9, 0x3a, 0x0f, 0xe3, 0x82, 0x97, 0x81, 0xea, 0x26, 0xa5, 0x71, 0x92, + 0x48, 0xed, 0xb6, 0x51, 0xc6, 0x12, 0x2b, 0x8d, 0x63, 0x21, 0x62, 0xed, 0x6c, 0x7a, 0x24, 0x70, + 0x58, 0x65, 0xa0, 0x03, 0x80, 0xc3, 0x58, 0x84, 0xe5, 0xdb, 0x96, 0x47, 0x02, 0xc2, 0x6a, 0x16, + 0x7f, 0x17, 0xda, 0x8a, 0xe9, 0x71, 0x98, 0x56, 0x6a, 0xc9, 0x23, 0x6a, 0xfd, 0x3b, 0x02, 0x1b, + 0xaf, 0x0b, 0x9e, 0x2d, 0x18, 0x7f, 0x5f, 0xf0, 0x1c, 0xab, 0x82, 0xd8, 0xf4, 0x12, 0x02, 0xba, + 0x0d, 0xad, 0xd3, 0xeb, 0x30, 0x9b, 0xea, 0xdc, 0xd9, 0xac, 0x44, 0x4a, 0x6b, 0x95, 0xf3, 0x1c, + 0xb5, 0x3a, 0xac, 0x6e, 0x52, 0x2f, 0x19, 0x9f, 0x09, 0x69, 0xc4, 0x94, 0x88, 0x06, 0xf0, 0xeb, + 0xc1, 0xcd, 0x65, 0x5c, 0x4c, 0x39, 0x13, 0x73, 0xfd, 0xba, 0x85, 0x17, 0xd6, 0xcd, 0xf4, 0x1f, + 0xe8, 0x97, 0x26, 0x33, 0x84, 0x6d, 0xbc, 0xb8, 0x66, 0xf5, 0x6f, 0x09, 0xf4, 0x4a, 0x29, 0x79, + 0x2a, 0x92, 0x9c, 0xab, 0x7a, 0x1d, 0x64, 0x99, 0xa9, 0xd7, 0x41, 0x96, 0xd1, 0x5d, 0x68, 0x33, + 0x9e, 0x17, 0xb1, 0x34, 0x4d, 0xf0, 0x7b, 0x95, 0x16, 0xf3, 0xb6, 0x88, 0x25, 0x33, 0xb7, 0xe8, + 0x33, 0xe8, 0xaf, 0x34, 0x95, 0x1e, 0xe2, 0xee, 0xe8, 0x8f, 0xea, 0xdd, 0x8a, 0x9f, 0xad, 0x5d, + 0xf7, 0x6f, 0x1b, 0xd0, 0xad, 0x45, 0xa6, 0x7f, 0xe1, 0x4a, 0x41, 0x4e, 0xdd, 0x51, 0xaf, 0x8a, + 0xa2, 0x46, 0x0d, 0x97, 0xcd, 0x06, 0x90, 0x93, 0xb2, 0x9f, 0xc8, 0x89, 0xaa, 0xa2, 0x5a, 0x13, + 0xe6, 0xb3, 0xb5, 0x2a, 0x2a, 0x33, 0xd3, 0x4e, 0x5c, 0x50, 0xd7, 0x61, 0x72, 0xc5, 0xa7, 0xd8, + 0x4f, 0x0e, 0x33, 0x90, 0x0e, 0xab, 0xf9, 0xc4, 0x02, 0xac, 0xcc, 0xb8, 0xf1, 0xb0, 0x6a, 0x86, + 0x4d, 0x43, 0xab, 0x5a, 0xf4, 0xca, 0x86, 0x56, 0x25, 0x54, 0xb3, 0xa9, 0x12, 0x8f, 0xc5, 0xd7, + 0x88, 0x3e, 0x81, 0x1e, 0xee, 0x86, 0xf1, 0x02, 0xdf, 0xe6, 0xae, 0x83, 0x1c, 0x7f, 0xab, 0x3e, + 0xb0, 0xdc, 0x34, 0x6c, 0xf5, 0xa6, 0xff, 0x95, 0x40, 0x6f, 0x32, 0x4b, 0x45, 0x26, 0x6b, 0x7d, + 0x37, 0x49, 0xa6, 0xfc, 0xc6, 0xf4, 0x1d, 0x82, 0x6a, 0xb3, 0x35, 0xd6, 0x36, 0x1b, 0xf6, 0x1f, + 0xf6, 0x9b, 0xcd, 0x34, 0xa8, 0xd1, 0xb4, 0x57, 0x68, 0xee, 0x40, 0xc7, 0x6c, 0x90, 0xdc, 0x6d, + 0xa2, 0xab, 0x32, 0xa8, 0x89, 0x5a, 0xae, 0x10, 0xd5, 0x82, 0x56, 0x60, 0xb1, 0x9a, 0x45, 0xa5, + 0x96, 0x89, 0x39, 0xae, 0xef, 0x36, 0xae, 0x6f, 0x03, 0xd5, 0x4b, 0x1d, 0x06, 0x9d, 0x0e, 0x3a, + 0x6b, 0x16, 0xff, 0x13, 0x01, 0xaa, 0x35, 0xe2, 0x6c, 0xfe, 0x3c, 0xa1, 0x8f, 0x0b, 0xda, 0x86, + 0x16, 0x7e, 0xcf, 0x88, 0x29, 0xd1, 0x1a, 0xdd, 0xf6, 0x3a, 0xdd, 0xf1, 0xe6, 0xdd, 0xfd, 0x80, + 0x7c, 0xbe, 0x1f, 0x90, 0x2f, 0xf7, 0x03, 0xf2, 0xe1, 0xdb, 0xe0, 0x97, 0x8b, 0x16, 0xfe, 0x0e, + 0xff, 0xfb, 0x1e, 0x00, 0x00, 0xff, 0xff, 0x1c, 0xa1, 0x99, 0xa7, 0x1e, 0x07, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index f3f362fd1..a7a6a20f0 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -14,9 +14,14 @@ message Pair { uint64 Count = 2; } +message FieldRow{ + string Field = 1; + uint64 RowID = 2; +} + message GroupLine{ - repeated string Groups = 1; - uint64 Total=2; + repeated FieldRow Groups = 1; + uint64 Total = 2; } message ValCount { From de071d548a59f9f7c10651e85b88beb185fc0056 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 24 Aug 2018 15:28:18 -0500 Subject: [PATCH 2/7] remove additional decodeFieldRow (and hopefully allocation) --- encoding/proto/proto.go | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 4f886acce..ba89a1b95 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -1012,18 +1012,12 @@ func decodeGroupByCounts(a []*internal.GroupLine) pilosa.GroupByCounts { func decodeFieldRows(a []*internal.FieldRow) []pilosa.FieldRow { other := make([]pilosa.FieldRow, len(a)) for i := range a { - other[i] = decodeFieldRow(a[i]) + other[i].Field = a[i].Field + other[i].RowID = a[i].RowID } return other } -func decodeFieldRow(pb *internal.FieldRow) pilosa.FieldRow { - return pilosa.FieldRow{ - Field: pb.Field, - RowID: pb.RowID, - } -} - func decodePairs(a []*internal.Pair) []pilosa.Pair { other := make([]pilosa.Pair, len(a)) for i := range a { @@ -1089,18 +1083,14 @@ func encodeGroupByCount(counts pilosa.GroupByCounts) []*internal.GroupLine { func encodeFieldRows(a []pilosa.FieldRow) []*internal.FieldRow { other := make([]*internal.FieldRow, len(a)) for i := range a { - other[i] = encodeFieldRow(a[i]) + other[i] = &internal.FieldRow{ + Field: a[i].Field, + RowID: a[i].RowID, + } } return other } -func encodeFieldRow(p pilosa.FieldRow) *internal.FieldRow { - return &internal.FieldRow{ - Field: p.Field, - RowID: p.RowID, - } -} - func encodePairs(a pilosa.Pairs) []*internal.Pair { other := make([]*internal.Pair, len(a)) for i := range a { From 96b408636065691998eaca077293de5a076383b4 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 22 Aug 2018 14:58:50 -0500 Subject: [PATCH 3/7] plug in in translation. adjust output format. --- encoding/proto/proto.go | 8 +-- executor.go | 58 +++++++++++++---- internal/public.pb.go | 140 ++++++++++++++++++++-------------------- internal/public.proto | 4 +- 4 files changed, 120 insertions(+), 90 deletions(-) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index ba89a1b95..4df552307 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -1002,8 +1002,8 @@ func decodeGroupByCounts(a []*internal.GroupLine) pilosa.GroupByCounts { other := make([]pilosa.GroupLine, len(a)) for i := range a { other[i] = pilosa.GroupLine{ - decodeFieldRows(a[i].Groups), - a[i].Total, + decodeFieldRows(a[i].Group), + a[i].Count, } } return pilosa.GroupByCounts(other) @@ -1073,8 +1073,8 @@ func encodeGroupByCount(counts pilosa.GroupByCounts) []*internal.GroupLine { result := make([]*internal.GroupLine, len(counts)) for i := range counts { result[i] = &internal.GroupLine{ - Groups: encodeFieldRows(counts[i].Groups), - Total: counts[i].Total, + Group: encodeFieldRows(counts[i].Group), + Count: counts[i].Count, } } return result diff --git a/executor.go b/executor.go index 35740866b..6759ba3ea 100644 --- a/executor.go +++ b/executor.go @@ -788,9 +788,9 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call // FieldRow is used to distinguish rows in a group by result. type FieldRow struct { - Field string - RowID uint64 - RowKey string + Field string `json:"field"` + RowID uint64 `json:"rowID"` + RowKey string `json:"rowKey,omitempty"` } func (fr FieldRow) String() string { @@ -812,8 +812,8 @@ type gbi struct { fieldRow FieldRow } type GroupLine struct { - Groups []FieldRow - Total uint64 + Group []FieldRow `json:"group"` + Count uint64 `json:"count"` } // GroupByCounts is the return type for GroupBy queries. @@ -825,15 +825,15 @@ func (gbc GroupByCounts) Merge(other GroupByCounts) GroupByCounts { total uint64 }) for i := range gbc { - m[uniqueGroupString(gbc[i].Groups)] = struct { + m[uniqueGroupString(gbc[i].Group)] = struct { i int total uint64 }{i, gbc[i].Total} } for i := range other { - o, found := m[uniqueGroupString(other[i].Groups)] + o, found := m[uniqueGroupString(other[i].Group)] if found { - gbc[o.i].Total += other[i].Total + gbc[o.i].Count += other[i].Count } else { gbc = append(gbc, other[i]) } @@ -901,8 +901,8 @@ func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql work = append(work, set) } for _, group := range product(work) { - group.gl.Total = group.row.Count() - if group.gl.Total > 0 { + group.gl.Count = group.row.Count() + if group.gl.Count > 0 { results = append(results, group.gl) } } @@ -920,7 +920,7 @@ type ppi struct { func product(input [][]gbi) []ppi { if len(input) == 0 { // base return empty list return []ppi{ - {gl: GroupLine{Groups: make([]FieldRow, 0)}}, + {gl: GroupLine{Group: make([]FieldRow, 0)}}, } } @@ -929,9 +929,9 @@ func product(input [][]gbi) []ppi { tail := product(input[1:]) // invoke product on remaining element for h := range head { // for each head for t := range tail { // iterate over the tail - s := ppi{gl: GroupLine{Groups: make([]FieldRow, 0)}} - s.gl.Groups = append([]FieldRow{head[h].fieldRow}, tail[t].gl.Groups...) // had to insert at the front to match input order - if tail[t].row != nil { // first time around nothing to intersect + s := ppi{gl: GroupLine{Group: make([]FieldRow, 0)}} + s.gl.Group = append([]FieldRow{head[h].fieldRow}, tail[t].gl.Group...) // had to insert at the front to match input order + if tail[t].row != nil { // first time around nothing to intersect s.row = head[h].row.Intersect(tail[t].row) } else { s.row = head[h].row @@ -2091,7 +2091,37 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res return other, nil } } + + case GroupByCounts: + other := make([]GroupLine, 0) + for _, gl := range result { + + group := make([]FieldRow, len(gl.Group)) + for i, g := range gl.Group { + group[i] = g + + // TODO: It may be useful to cache this field lookup. + field := idx.Field(g.Field) + if field == nil { + return nil, ErrFieldNotFound + } + if field.keys() { + key, err := e.TranslateStore.TranslateRowToString(index, g.Field, g.RowID) + if err != nil { + return nil, err + } + group[i].RowKey = key + } + } + + other = append(other, GroupLine{ + Group: group, + Count: gl.Count, + }) + } + return other, nil } + return result, nil } diff --git a/internal/public.pb.go b/internal/public.pb.go index 2203c87d6..29254543a 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -133,8 +133,8 @@ func (m *FieldRow) GetRowID() uint64 { } type GroupLine struct { - Groups []*FieldRow `protobuf:"bytes,1,rep,name=Groups" json:"Groups,omitempty"` - Total uint64 `protobuf:"varint,2,opt,name=Total,proto3" json:"Total,omitempty"` + Group []*FieldRow `protobuf:"bytes,1,rep,name=Group" json:"Group,omitempty"` + Count uint64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` } func (m *GroupLine) Reset() { *m = GroupLine{} } @@ -142,16 +142,16 @@ func (m *GroupLine) String() string { return proto.CompactTextString( func (*GroupLine) ProtoMessage() {} func (*GroupLine) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} } -func (m *GroupLine) GetGroups() []*FieldRow { +func (m *GroupLine) GetGroup() []*FieldRow { if m != nil { - return m.Groups + return m.Group } return nil } -func (m *GroupLine) GetTotal() uint64 { +func (m *GroupLine) GetCount() uint64 { if m != nil { - return m.Total + return m.Count } return 0 } @@ -760,8 +760,8 @@ func (m *GroupLine) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.Groups) > 0 { - for _, msg := range m.Groups { + if len(m.Group) > 0 { + for _, msg := range m.Group { dAtA[i] = 0xa i++ i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) @@ -772,10 +772,10 @@ func (m *GroupLine) MarshalTo(dAtA []byte) (int, error) { i += n } } - if m.Total != 0 { + if m.Count != 0 { dAtA[i] = 0x10 i++ - i = encodeVarintPublic(dAtA, i, uint64(m.Total)) + i = encodeVarintPublic(dAtA, i, uint64(m.Count)) } return i, nil } @@ -1464,14 +1464,14 @@ func (m *FieldRow) Size() (n int) { func (m *GroupLine) Size() (n int) { var l int _ = l - if len(m.Groups) > 0 { - for _, e := range m.Groups { + if len(m.Group) > 0 { + for _, e := range m.Group { l = e.Size() n += 1 + l + sovPublic(uint64(l)) } } - if m.Total != 0 { - n += 1 + sovPublic(uint64(m.Total)) + if m.Count != 0 { + n += 1 + sovPublic(uint64(m.Count)) } return n } @@ -2171,7 +2171,7 @@ func (m *GroupLine) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Groups", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Group", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -2195,16 +2195,16 @@ func (m *GroupLine) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Groups = append(m.Groups, &FieldRow{}) - if err := m.Groups[len(m.Groups)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.Group = append(m.Group, &FieldRow{}) + if err := m.Group[len(m.Group)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Total", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Count", wireType) } - m.Total = 0 + m.Count = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPublic @@ -2214,7 +2214,7 @@ func (m *GroupLine) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Total |= (uint64(b) & 0x7F) << shift + m.Count |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -4241,54 +4241,54 @@ var ( func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ - // 783 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xdd, 0x6a, 0xdb, 0x48, - 0x14, 0xde, 0xb1, 0x64, 0x5b, 0x3e, 0x8e, 0xbd, 0x61, 0x36, 0x9b, 0x15, 0x4b, 0xf0, 0x0a, 0xb1, - 0x2c, 0x62, 0x2f, 0x1c, 0xf0, 0xc2, 0x42, 0x6f, 0x5a, 0xea, 0xfc, 0x14, 0x93, 0x26, 0xb4, 0x93, - 0x34, 0xa5, 0x97, 0x4a, 0x3c, 0x24, 0x02, 0x59, 0xa3, 0x4a, 0x23, 0x1c, 0x3f, 0x47, 0x6e, 0xfa, - 0x08, 0xbd, 0xe8, 0x83, 0xe4, 0xb2, 0x8f, 0xd0, 0xa6, 0x2f, 0x52, 0xe6, 0x8c, 0xc6, 0x92, 0x1d, - 0x08, 0xbd, 0xe8, 0xdd, 0x7c, 0xe7, 0xcc, 0x1c, 0x7d, 0xdf, 0xf9, 0x13, 0x6c, 0xa4, 0xc5, 0x45, - 0x1c, 0x5d, 0x0e, 0xd3, 0x4c, 0x48, 0x41, 0x9d, 0x28, 0x91, 0x3c, 0x4b, 0xc2, 0xd8, 0x7f, 0x07, - 0x16, 0x13, 0x73, 0xea, 0x42, 0x7b, 0x4f, 0xc4, 0xc5, 0x2c, 0xc9, 0x5d, 0xe2, 0x59, 0x81, 0xcd, - 0x0c, 0xa4, 0x7f, 0x43, 0xf3, 0xb9, 0x94, 0x59, 0xee, 0x36, 0x3c, 0x2b, 0xe8, 0x8e, 0xfa, 0x43, - 0xf3, 0x74, 0xa8, 0xcc, 0x4c, 0x3b, 0x29, 0x05, 0xfb, 0x88, 0x2f, 0x72, 0xd7, 0xf2, 0xac, 0xa0, - 0xc3, 0xf0, 0xec, 0x3f, 0x05, 0xfb, 0x55, 0x18, 0x65, 0xb4, 0x0f, 0x8d, 0xc9, 0xbe, 0x4b, 0x3c, - 0x12, 0xd8, 0xac, 0x31, 0xd9, 0xa7, 0x5b, 0xd0, 0xdc, 0x13, 0x45, 0x22, 0xdd, 0x06, 0x9a, 0x34, - 0xa0, 0x9b, 0x60, 0x1d, 0xf1, 0x85, 0x6b, 0x79, 0x24, 0xe8, 0x30, 0x75, 0xf4, 0xff, 0x07, 0xe7, - 0x30, 0xe2, 0xf1, 0x54, 0xf1, 0xdb, 0x82, 0x26, 0x9e, 0x31, 0x4c, 0x87, 0x69, 0xa0, 0xac, 0x4c, - 0xcc, 0x27, 0xfb, 0x26, 0x12, 0x02, 0xff, 0x18, 0x3a, 0x2f, 0x32, 0x51, 0xa4, 0x2f, 0xa3, 0x84, - 0xd3, 0x7f, 0xa1, 0x85, 0x40, 0xeb, 0xea, 0x8e, 0x68, 0xc5, 0xdf, 0x04, 0x67, 0xe5, 0x0d, 0x15, - 0xee, 0x4c, 0xc8, 0x30, 0x36, 0xe1, 0x10, 0xf8, 0x23, 0x70, 0xce, 0xc3, 0x78, 0x49, 0xf2, 0x3c, - 0x8c, 0x91, 0x84, 0xc5, 0xd4, 0x71, 0x55, 0x8c, 0x55, 0x8a, 0xf1, 0xdf, 0x80, 0x35, 0x8e, 0x64, - 0xc5, 0x8f, 0xd4, 0xf8, 0xd1, 0x3f, 0xc1, 0xd1, 0xc9, 0x5d, 0x12, 0x5f, 0x62, 0xba, 0x03, 0x9d, - 0xb3, 0x68, 0xc6, 0x73, 0x19, 0xce, 0x52, 0xcc, 0x85, 0xc5, 0x2a, 0x83, 0xff, 0x16, 0x7a, 0xfa, - 0xa6, 0x4a, 0xfa, 0x29, 0x97, 0x0f, 0x52, 0xfb, 0x63, 0xc5, 0x7a, 0x98, 0xea, 0x8f, 0x04, 0x6c, - 0xe5, 0x33, 0x2e, 0xb2, 0x74, 0xa9, 0xca, 0x9e, 0x2d, 0x52, 0x5e, 0x32, 0xc5, 0x33, 0xf5, 0xa0, - 0x7b, 0x2a, 0xb3, 0x28, 0xb9, 0x3a, 0x0f, 0xe3, 0x82, 0x97, 0x81, 0xea, 0x26, 0xa5, 0x71, 0x92, - 0x48, 0xed, 0xb6, 0x51, 0xc6, 0x12, 0x2b, 0x8d, 0x63, 0x21, 0x62, 0xed, 0x6c, 0x7a, 0x24, 0x70, - 0x58, 0x65, 0xa0, 0x03, 0x80, 0xc3, 0x58, 0x84, 0xe5, 0xdb, 0x96, 0x47, 0x02, 0xc2, 0x6a, 0x16, - 0x7f, 0x17, 0xda, 0x8a, 0xe9, 0x71, 0x98, 0x56, 0x6a, 0xc9, 0x23, 0x6a, 0xfd, 0x3b, 0x02, 0x1b, - 0xaf, 0x0b, 0x9e, 0x2d, 0x18, 0x7f, 0x5f, 0xf0, 0x1c, 0xab, 0x82, 0xd8, 0xf4, 0x12, 0x02, 0xba, - 0x0d, 0xad, 0xd3, 0xeb, 0x30, 0x9b, 0xea, 0xdc, 0xd9, 0xac, 0x44, 0x4a, 0x6b, 0x95, 0xf3, 0x1c, - 0xb5, 0x3a, 0xac, 0x6e, 0x52, 0x2f, 0x19, 0x9f, 0x09, 0x69, 0xc4, 0x94, 0x88, 0x06, 0xf0, 0xeb, - 0xc1, 0xcd, 0x65, 0x5c, 0x4c, 0x39, 0x13, 0x73, 0xfd, 0xba, 0x85, 0x17, 0xd6, 0xcd, 0xf4, 0x1f, - 0xe8, 0x97, 0x26, 0x33, 0x84, 0x6d, 0xbc, 0xb8, 0x66, 0xf5, 0x6f, 0x09, 0xf4, 0x4a, 0x29, 0x79, - 0x2a, 0x92, 0x9c, 0xab, 0x7a, 0x1d, 0x64, 0x99, 0xa9, 0xd7, 0x41, 0x96, 0xd1, 0x5d, 0x68, 0x33, - 0x9e, 0x17, 0xb1, 0x34, 0x4d, 0xf0, 0x7b, 0x95, 0x16, 0xf3, 0xb6, 0x88, 0x25, 0x33, 0xb7, 0xe8, - 0x33, 0xe8, 0xaf, 0x34, 0x95, 0x1e, 0xe2, 0xee, 0xe8, 0x8f, 0xea, 0xdd, 0x8a, 0x9f, 0xad, 0x5d, - 0xf7, 0x6f, 0x1b, 0xd0, 0xad, 0x45, 0xa6, 0x7f, 0xe1, 0x4a, 0x41, 0x4e, 0xdd, 0x51, 0xaf, 0x8a, - 0xa2, 0x46, 0x0d, 0x97, 0xcd, 0x06, 0x90, 0x93, 0xb2, 0x9f, 0xc8, 0x89, 0xaa, 0xa2, 0x5a, 0x13, - 0xe6, 0xb3, 0xb5, 0x2a, 0x2a, 0x33, 0xd3, 0x4e, 0x5c, 0x50, 0xd7, 0x61, 0x72, 0xc5, 0xa7, 0xd8, - 0x4f, 0x0e, 0x33, 0x90, 0x0e, 0xab, 0xf9, 0xc4, 0x02, 0xac, 0xcc, 0xb8, 0xf1, 0xb0, 0x6a, 0x86, - 0x4d, 0x43, 0xab, 0x5a, 0xf4, 0xca, 0x86, 0x56, 0x25, 0x54, 0xb3, 0xa9, 0x12, 0x8f, 0xc5, 0xd7, - 0x88, 0x3e, 0x81, 0x1e, 0xee, 0x86, 0xf1, 0x02, 0xdf, 0xe6, 0xae, 0x83, 0x1c, 0x7f, 0xab, 0x3e, - 0xb0, 0xdc, 0x34, 0x6c, 0xf5, 0xa6, 0xff, 0x95, 0x40, 0x6f, 0x32, 0x4b, 0x45, 0x26, 0x6b, 0x7d, - 0x37, 0x49, 0xa6, 0xfc, 0xc6, 0xf4, 0x1d, 0x82, 0x6a, 0xb3, 0x35, 0xd6, 0x36, 0x1b, 0xf6, 0x1f, - 0xf6, 0x9b, 0xcd, 0x34, 0xa8, 0xd1, 0xb4, 0x57, 0x68, 0xee, 0x40, 0xc7, 0x6c, 0x90, 0xdc, 0x6d, - 0xa2, 0xab, 0x32, 0xa8, 0x89, 0x5a, 0xae, 0x10, 0xd5, 0x82, 0x56, 0x60, 0xb1, 0x9a, 0x45, 0xa5, - 0x96, 0x89, 0x39, 0xae, 0xef, 0x36, 0xae, 0x6f, 0x03, 0xd5, 0x4b, 0x1d, 0x06, 0x9d, 0x0e, 0x3a, - 0x6b, 0x16, 0xff, 0x13, 0x01, 0xaa, 0x35, 0xe2, 0x6c, 0xfe, 0x3c, 0xa1, 0x8f, 0x0b, 0xda, 0x86, - 0x16, 0x7e, 0xcf, 0x88, 0x29, 0xd1, 0x1a, 0xdd, 0xf6, 0x3a, 0xdd, 0xf1, 0xe6, 0xdd, 0xfd, 0x80, - 0x7c, 0xbe, 0x1f, 0x90, 0x2f, 0xf7, 0x03, 0xf2, 0xe1, 0xdb, 0xe0, 0x97, 0x8b, 0x16, 0xfe, 0x0e, - 0xff, 0xfb, 0x1e, 0x00, 0x00, 0xff, 0xff, 0x1c, 0xa1, 0x99, 0xa7, 0x1e, 0x07, 0x00, 0x00, + // 771 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6e, 0xd3, 0x4c, + 0x14, 0xfe, 0x27, 0x76, 0x12, 0xe7, 0xa4, 0xc9, 0x5f, 0x0d, 0xa5, 0x58, 0xa8, 0x0a, 0x96, 0x85, + 0x90, 0x57, 0xa9, 0x14, 0x24, 0x24, 0x36, 0x20, 0xd2, 0x0b, 0x8a, 0x0a, 0x15, 0x4c, 0x4b, 0x11, + 0x4b, 0xb7, 0x19, 0xb5, 0x96, 0x1c, 0x8f, 0xf1, 0x45, 0x69, 0x9e, 0xa3, 0x1b, 0x1e, 0x81, 0x05, + 0x0f, 0xd2, 0x25, 0x8f, 0x00, 0xe5, 0x45, 0xd0, 0x9c, 0xf1, 0xd8, 0x4e, 0x8a, 0x2a, 0x16, 0xec, + 0xe6, 0xfb, 0xce, 0x9c, 0xf1, 0x77, 0xae, 0x86, 0xb5, 0x38, 0x3f, 0x0d, 0x83, 0xb3, 0x61, 0x9c, + 0x88, 0x4c, 0x50, 0x2b, 0x88, 0x32, 0x9e, 0x44, 0x7e, 0xe8, 0x7e, 0x02, 0x83, 0x89, 0x39, 0xb5, + 0xa1, 0xbd, 0x23, 0xc2, 0x7c, 0x16, 0xa5, 0x36, 0x71, 0x0c, 0xcf, 0x64, 0x1a, 0xd2, 0xc7, 0xd0, + 0x7c, 0x95, 0x65, 0x49, 0x6a, 0x37, 0x1c, 0xc3, 0xeb, 0x8e, 0xfa, 0x43, 0xed, 0x3a, 0x94, 0x34, + 0x53, 0x46, 0x4a, 0xc1, 0x3c, 0xe0, 0x8b, 0xd4, 0x36, 0x1c, 0xc3, 0xeb, 0x30, 0x3c, 0xbb, 0x2f, + 0xc0, 0x7c, 0xe7, 0x07, 0x09, 0xed, 0x43, 0x63, 0xb2, 0x6b, 0x13, 0x87, 0x78, 0x26, 0x6b, 0x4c, + 0x76, 0xe9, 0x06, 0x34, 0x77, 0x44, 0x1e, 0x65, 0x76, 0x03, 0x29, 0x05, 0xe8, 0x3a, 0x18, 0x07, + 0x7c, 0x61, 0x1b, 0x0e, 0xf1, 0x3a, 0x4c, 0x1e, 0xdd, 0x67, 0x60, 0xed, 0x07, 0x3c, 0x9c, 0x4a, + 0x7d, 0x1b, 0xd0, 0xc4, 0x33, 0x3e, 0xd3, 0x61, 0x0a, 0x48, 0x96, 0x89, 0xf9, 0x64, 0x57, 0xbf, + 0x84, 0xc0, 0x3d, 0x80, 0xce, 0xeb, 0x44, 0xe4, 0xf1, 0x9b, 0x20, 0xe2, 0xd4, 0x83, 0x26, 0x02, + 0x0c, 0xab, 0x3b, 0xa2, 0x95, 0x7c, 0xfd, 0x36, 0x53, 0x17, 0xfe, 0x2c, 0xcb, 0x1d, 0x81, 0x75, + 0xe2, 0x87, 0xa5, 0xc4, 0x13, 0x3f, 0x44, 0x09, 0x06, 0x93, 0xc7, 0x65, 0x1f, 0x43, 0xfb, 0x7c, + 0x00, 0x63, 0x1c, 0x64, 0x95, 0x3a, 0x52, 0x53, 0x47, 0x1f, 0x82, 0xa5, 0x52, 0x5b, 0xca, 0x2e, + 0x31, 0xdd, 0x82, 0xce, 0x71, 0x30, 0xe3, 0x69, 0xe6, 0xcf, 0x62, 0xcc, 0x84, 0xc1, 0x2a, 0xc2, + 0xfd, 0x08, 0x3d, 0x75, 0x53, 0xa6, 0xfc, 0x88, 0x67, 0xb7, 0x12, 0xfb, 0x77, 0xa5, 0xba, 0x9d, + 0xe8, 0xaf, 0x04, 0x4c, 0x69, 0xd3, 0x26, 0x52, 0x9a, 0x64, 0x5d, 0x8f, 0x17, 0x31, 0x2f, 0x94, + 0xe2, 0x99, 0x3a, 0xd0, 0x3d, 0xca, 0x92, 0x20, 0x3a, 0x3f, 0xf1, 0xc3, 0x9c, 0x17, 0x0f, 0xd5, + 0x29, 0x19, 0xe3, 0x24, 0xca, 0x94, 0xd9, 0xc4, 0x30, 0x4a, 0x2c, 0x63, 0x1c, 0x0b, 0x11, 0x2a, + 0x63, 0xd3, 0x21, 0x9e, 0xc5, 0x2a, 0x82, 0x0e, 0x00, 0xf6, 0x43, 0xe1, 0x17, 0xbe, 0x2d, 0x87, + 0x78, 0x84, 0xd5, 0x18, 0x77, 0x1b, 0xda, 0x52, 0xe9, 0x5b, 0x3f, 0xae, 0xa2, 0x25, 0x77, 0x44, + 0xeb, 0x5e, 0x13, 0x58, 0x7b, 0x9f, 0xf3, 0x64, 0xc1, 0xf8, 0xe7, 0x9c, 0xa7, 0x58, 0x15, 0xc4, + 0xba, 0x93, 0x10, 0xd0, 0x4d, 0x68, 0x1d, 0x5d, 0xf8, 0xc9, 0x54, 0xe5, 0xce, 0x64, 0x05, 0x92, + 0xb1, 0x56, 0x39, 0x4f, 0x31, 0x56, 0x8b, 0xd5, 0x29, 0xe9, 0xc9, 0xf8, 0x4c, 0x64, 0x3a, 0x98, + 0x02, 0x51, 0x0f, 0xfe, 0xdf, 0xbb, 0x3c, 0x0b, 0xf3, 0x29, 0x67, 0x62, 0xae, 0xbc, 0x5b, 0x78, + 0x61, 0x95, 0xa6, 0x4f, 0xa0, 0x5f, 0x50, 0x7a, 0x04, 0xdb, 0x78, 0x71, 0x85, 0x75, 0xaf, 0x08, + 0xf4, 0x8a, 0x50, 0xd2, 0x58, 0x44, 0x29, 0x97, 0xf5, 0xda, 0x4b, 0x12, 0x5d, 0xaf, 0xbd, 0x24, + 0xa1, 0xdb, 0xd0, 0x66, 0x3c, 0xcd, 0xc3, 0x4c, 0x37, 0xc1, 0xfd, 0x2a, 0x2d, 0xda, 0x37, 0x0f, + 0x33, 0xa6, 0x6f, 0xd1, 0x97, 0xd0, 0x5f, 0x6a, 0x2a, 0x35, 0xc2, 0xdd, 0xd1, 0x83, 0xca, 0x6f, + 0xc9, 0xce, 0x56, 0xae, 0xbb, 0x57, 0x0d, 0xe8, 0xd6, 0x5e, 0xa6, 0x8f, 0x70, 0xa1, 0xa0, 0xa6, + 0xee, 0xa8, 0x57, 0xbd, 0x22, 0x27, 0x0d, 0x57, 0xcd, 0x1a, 0x90, 0xc3, 0xa2, 0x9f, 0xc8, 0xa1, + 0xac, 0xa2, 0x5c, 0x12, 0xfa, 0xb3, 0xb5, 0x2a, 0x4a, 0x9a, 0x29, 0x23, 0xae, 0xa7, 0x0b, 0x3f, + 0x3a, 0xe7, 0x53, 0xec, 0x27, 0x8b, 0x69, 0x48, 0x87, 0xd5, 0x7c, 0x62, 0x01, 0x96, 0x46, 0x5c, + 0x5b, 0x58, 0x35, 0xc3, 0xba, 0xa1, 0x65, 0x2d, 0x7a, 0x45, 0x43, 0xcb, 0x12, 0xca, 0xd9, 0x94, + 0x89, 0xc7, 0xe2, 0x2b, 0x44, 0x9f, 0x43, 0x0f, 0x57, 0xc3, 0x78, 0x81, 0xbe, 0xa9, 0x6d, 0xa1, + 0xc6, 0x7b, 0xd5, 0x07, 0xca, 0x3d, 0xc3, 0x96, 0x6f, 0xba, 0x3f, 0x09, 0xf4, 0x26, 0xb3, 0x58, + 0x24, 0x59, 0xad, 0xef, 0x26, 0xd1, 0x94, 0x5f, 0xea, 0xbe, 0x43, 0x50, 0xed, 0xb5, 0xc6, 0xca, + 0x5e, 0xc3, 0xfe, 0xc3, 0x7e, 0x33, 0x99, 0x02, 0x35, 0x99, 0xe6, 0x92, 0xcc, 0x2d, 0xe8, 0xe8, + 0x0d, 0x92, 0xda, 0x4d, 0x34, 0x55, 0x84, 0x9c, 0xa8, 0x72, 0x85, 0xc8, 0x16, 0x34, 0x3c, 0x83, + 0xd5, 0x18, 0x99, 0x5a, 0x26, 0xe6, 0xb8, 0xbc, 0xdb, 0xb8, 0xbc, 0x35, 0x94, 0x9e, 0xea, 0x19, + 0x34, 0x5a, 0x68, 0xac, 0x31, 0xee, 0x37, 0x02, 0x54, 0xc5, 0x88, 0xb3, 0xf9, 0xef, 0x02, 0xbd, + 0x3b, 0xa0, 0x4d, 0x68, 0xe1, 0xf7, 0x74, 0x30, 0x05, 0x5a, 0x91, 0xdb, 0x5e, 0x95, 0x3b, 0x5e, + 0xbf, 0xbe, 0x19, 0x90, 0xef, 0x37, 0x03, 0xf2, 0xe3, 0x66, 0x40, 0xbe, 0xfc, 0x1a, 0xfc, 0x77, + 0xda, 0xc2, 0x9f, 0xe1, 0xd3, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x21, 0xc1, 0x72, 0xc6, 0x1c, + 0x07, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index a7a6a20f0..b207e3cae 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -20,8 +20,8 @@ message FieldRow{ } message GroupLine{ - repeated FieldRow Groups = 1; - uint64 Total = 2; + repeated FieldRow Group = 1; + uint64 Count = 2; } message ValCount { From b6d386ba53a6a4eca90580615bf7c31cc9caff13 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 30 Aug 2018 15:09:04 -0600 Subject: [PATCH 4/7] fix tests --- executor.go | 6 +++--- executor_test.go | 30 +++++++++++++++--------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/executor.go b/executor.go index 6759ba3ea..5f2cdc75f 100644 --- a/executor.go +++ b/executor.go @@ -822,13 +822,13 @@ type GroupByCounts []GroupLine func (gbc GroupByCounts) Merge(other GroupByCounts) GroupByCounts { m := make(map[string]struct { i int - total uint64 + count uint64 }) for i := range gbc { m[uniqueGroupString(gbc[i].Group)] = struct { i int - total uint64 - }{i, gbc[i].Total} + count uint64 + }{i, gbc[i].Count} } for i := range other { o, found := m[uniqueGroupString(other[i].Group)] diff --git a/executor_test.go b/executor_test.go index e3afda3ed..7261611db 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1234,10 +1234,10 @@ Set(4500001, fn=4) t.Fatalf("GroupBy querying: %v", err) } else { expected := pilosa.GroupByCounts{ - {Groups: []pilosa.FieldRow{{Field: "f", RowID: 10}}, Total: 4}, - {Groups: []pilosa.FieldRow{{Field: "f", RowID: 7}}, Total: 1}, + {Group: []pilosa.FieldRow{{Field: "f", RowID: 10}}, Count: 4}, + {Group: []pilosa.FieldRow{{Field: "f", RowID: 7}}, Count: 1}, } - results := res.Results[0].(pilosa.GroupByCounts) + results := res.Results[0].([]pilosa.GroupLine) checkGroupBy(t, expected, results) } }) @@ -1713,43 +1713,43 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { }) t.Run("Basic", func(t *testing.T) { expected := pilosa.GroupByCounts{ - {Groups: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Total: 1}, - {Groups: []pilosa.FieldRow{{Field: "general", RowID: 11}, {Field: "sub", RowID: 110}}, Total: 1}, - {Groups: []pilosa.FieldRow{{Field: "general", RowID: 12}, {Field: "sub", RowID: 110}}, Total: 1}, - {Groups: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Total: 3}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}, {Field: "sub", RowID: 110}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 12}, {Field: "sub", RowID: 110}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Count: 3}, } if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(fields=[general,sub])`}); err != nil { t.Fatal(err) } else { - results := res.Results[0].(pilosa.GroupByCounts) + results := res.Results[0].([]pilosa.GroupLine) checkGroupBy(t, expected, results) } }) t.Run("check field offset no limit", func(t *testing.T) { expected := pilosa.GroupByCounts{ - {Groups: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Total: 2}, - {Groups: []pilosa.FieldRow{{Field: "general", RowID: 12}}, Total: 2}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Count: 2}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 12}}, Count: 2}, } if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(fields=[general:11:])`}); err != nil { t.Fatal(err) } else { - results := res.Results[0].(pilosa.GroupByCounts) + results := res.Results[0].([]pilosa.GroupLine) checkGroupBy(t, expected, results) } }) t.Run("check field offset limit", func(t *testing.T) { expected := pilosa.GroupByCounts{ - {Groups: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Total: 2}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Count: 2}, } if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(fields=[general:11:1])`}); err != nil { t.Fatal(err) } else { - results := res.Results[0].(pilosa.GroupByCounts) + results := res.Results[0].([]pilosa.GroupLine) checkGroupBy(t, expected, results) } }) @@ -1758,8 +1758,8 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { func checkGroupBy(t *testing.T, expected, results pilosa.GroupByCounts) { notIn := func(item pilosa.GroupLine, expected pilosa.GroupByCounts) bool { for i := range expected { - if item.Total == expected[i].Total { - if reflect.DeepEqual(item.Groups, expected[i].Groups) { + if item.Count == expected[i].Count { + if reflect.DeepEqual(item.Group, expected[i].Group) { return false } } From f0666b2be01008292a3cc56c3da76388d4a1eff9 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 4 Sep 2018 15:56:26 -0500 Subject: [PATCH 5/7] change GroupByCounts to []GroupCount --- encoding/proto/proto.go | 26 +++---- executor.go | 59 ++++++++------- executor_test.go | 20 ++--- internal/public.pb.go | 163 ++++++++++++++++++++-------------------- internal/public.proto | 4 +- 5 files changed, 136 insertions(+), 136 deletions(-) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 4df552307..6273f2c7f 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -363,9 +363,9 @@ func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse { case pilosa.RowIDs: pb.Results[i].Type = queryResultTypeRowIDs pb.Results[i].RowIDs = result - case pilosa.GroupByCounts: - pb.Results[i].Type = queryResultTypeGroupByCounts - pb.Results[i].GroupByCounts = encodeGroupByCount(result) + case []pilosa.GroupCount: + pb.Results[i].Type = queryResultTypeGroupCounts + pb.Results[i].GroupCounts = encodeGroupCounts(result) case nil: pb.Results[i].Type = queryResultTypeNil } @@ -929,7 +929,7 @@ const ( queryResultTypeUint64 queryResultTypeBool queryResultTypeRowIDs - queryResultTypeGroupByCounts + queryResultTypeGroupCounts ) func decodeQueryResult(pb *internal.QueryResult) interface{} { @@ -946,8 +946,8 @@ func decodeQueryResult(pb *internal.QueryResult) interface{} { return pb.Changed case queryResultTypeNil: return nil - case queryResultTypeGroupByCounts: - return decodeGroupByCounts(pb.GroupByCounts) + case queryResultTypeGroupCounts: + return decodeGroupCounts(pb.GroupCounts) } panic(fmt.Sprintf("unknown type: %d", pb.Type)) } @@ -998,15 +998,15 @@ func decodeAttr(attr *internal.Attr) (key string, value interface{}) { } } -func decodeGroupByCounts(a []*internal.GroupLine) pilosa.GroupByCounts { - other := make([]pilosa.GroupLine, len(a)) +func decodeGroupCounts(a []*internal.GroupCount) []pilosa.GroupCount { + other := make([]pilosa.GroupCount, len(a)) for i := range a { - other[i] = pilosa.GroupLine{ + other[i] = pilosa.GroupCount{ decodeFieldRows(a[i].Group), a[i].Count, } } - return pilosa.GroupByCounts(other) + return other } func decodeFieldRows(a []*internal.FieldRow) []pilosa.FieldRow { @@ -1069,10 +1069,10 @@ func encodeRow(r *pilosa.Row) *internal.Row { } } -func encodeGroupByCount(counts pilosa.GroupByCounts) []*internal.GroupLine { - result := make([]*internal.GroupLine, len(counts)) +func encodeGroupCounts(counts []pilosa.GroupCount) []*internal.GroupCount { + result := make([]*internal.GroupCount, len(counts)) for i := range counts { - result[i] = &internal.GroupLine{ + result[i] = &internal.GroupCount{ Group: encodeFieldRows(counts[i].Group), Count: counts[i].Count, } diff --git a/executor.go b/executor.go index 5f2cdc75f..faa71cd3d 100644 --- a/executor.go +++ b/executor.go @@ -751,22 +751,24 @@ func (r RowIDs) Merge(other RowIDs) RowIDs { } return result } -func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (GroupByCounts, error) { + +func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) ([]GroupCount, error) { // Execute calls in bulk on each remote node and merge. mapFn := func(shard uint64) (interface{}, error) { return e.executeGroupByShard(ctx, index, c, shard) } // Merge returned results at coordinating node. reduceFn := func(prev, v interface{}) interface{} { - other, _ := prev.(GroupByCounts) - return other.Merge(v.(GroupByCounts)) + other, _ := prev.([]GroupCount) + return mergeGroupCounts(other, v.([]GroupCount)) } // Get full result set. other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { return nil, err } - results, _ := other.(GroupByCounts) + results, _ := other.([]GroupCount) + // Apply offset. if offset, hasOffset, err := c.UintArg("offset"); err != nil { return nil, err @@ -811,37 +813,35 @@ type gbi struct { row *Row fieldRow FieldRow } -type GroupLine struct { + +type GroupCount struct { Group []FieldRow `json:"group"` Count uint64 `json:"count"` } -// GroupByCounts is the return type for GroupBy queries. -type GroupByCounts []GroupLine - -func (gbc GroupByCounts) Merge(other GroupByCounts) GroupByCounts { +func mergeGroupCounts(gc, other []GroupCount) []GroupCount { m := make(map[string]struct { i int count uint64 }) - for i := range gbc { - m[uniqueGroupString(gbc[i].Group)] = struct { + for i := range gc { + m[uniqueGroupString(gc[i].Group)] = struct { i int count uint64 - }{i, gbc[i].Count} + }{i, gc[i].Count} } for i := range other { o, found := m[uniqueGroupString(other[i].Group)] if found { - gbc[o.i].Count += other[i].Count + gc[o.i].Count += other[i].Count } else { - gbc = append(gbc, other[i]) + gc = append(gc, other[i]) } } - return gbc + return gc } -func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql.Call, shard uint64) (GroupByCounts, error) { +func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql.Call, shard uint64) ([]GroupCount, error) { // Fetch index. idx := e.Holder.Index(index) if idx == nil { @@ -873,7 +873,8 @@ func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql return nil, errors.Wrap(ErrFieldNotFound, fmt.Sprintf("executeGroupBy: %s", fieldDirective.(string))) } } - results := make(GroupByCounts, 0) + + results := make([]GroupCount, 0) var work [][]gbi for _, fieldDirective := range fieldDirectives.([]interface{}) { fieldName := getFieldName(fieldDirective.(string)) @@ -901,9 +902,9 @@ func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql work = append(work, set) } for _, group := range product(work) { - group.gl.Count = group.row.Count() - if group.gl.Count > 0 { - results = append(results, group.gl) + group.gCnt.Count = group.row.Count() + if group.gCnt.Count > 0 { + results = append(results, group.gCnt) } } return results, nil @@ -911,8 +912,8 @@ func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql // ppi is a product process item. type ppi struct { - row *Row - gl GroupLine + row *Row + gCnt GroupCount } // product generates the cartesian product of the input @@ -920,7 +921,7 @@ type ppi struct { func product(input [][]gbi) []ppi { if len(input) == 0 { // base return empty list return []ppi{ - {gl: GroupLine{Group: make([]FieldRow, 0)}}, + {gCnt: GroupCount{Group: make([]FieldRow, 0)}}, } } @@ -929,9 +930,9 @@ func product(input [][]gbi) []ppi { tail := product(input[1:]) // invoke product on remaining element for h := range head { // for each head for t := range tail { // iterate over the tail - s := ppi{gl: GroupLine{Group: make([]FieldRow, 0)}} - s.gl.Group = append([]FieldRow{head[h].fieldRow}, tail[t].gl.Group...) // had to insert at the front to match input order - if tail[t].row != nil { // first time around nothing to intersect + s := ppi{gCnt: GroupCount{Group: make([]FieldRow, 0)}} + s.gCnt.Group = append([]FieldRow{head[h].fieldRow}, tail[t].gCnt.Group...) // had to insert at the front to match input order + if tail[t].row != nil { // first time around nothing to intersect s.row = head[h].row.Intersect(tail[t].row) } else { s.row = head[h].row @@ -2092,8 +2093,8 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res } } - case GroupByCounts: - other := make([]GroupLine, 0) + case []GroupCount: + other := make([]GroupCount, 0) for _, gl := range result { group := make([]FieldRow, len(gl.Group)) @@ -2114,7 +2115,7 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res } } - other = append(other, GroupLine{ + other = append(other, GroupCount{ Group: group, Count: gl.Count, }) diff --git a/executor_test.go b/executor_test.go index 7261611db..786b0ebfb 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1233,11 +1233,11 @@ Set(4500001, fn=4) }); err != nil { t.Fatalf("GroupBy querying: %v", err) } else { - expected := pilosa.GroupByCounts{ + expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "f", RowID: 10}}, Count: 4}, {Group: []pilosa.FieldRow{{Field: "f", RowID: 7}}, Count: 1}, } - results := res.Results[0].([]pilosa.GroupLine) + results := res.Results[0].([]pilosa.GroupCount) checkGroupBy(t, expected, results) } }) @@ -1712,7 +1712,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { } }) t.Run("Basic", func(t *testing.T) { - expected := pilosa.GroupByCounts{ + expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Count: 1}, {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}, {Field: "sub", RowID: 110}}, Count: 1}, {Group: []pilosa.FieldRow{{Field: "general", RowID: 12}, {Field: "sub", RowID: 110}}, Count: 1}, @@ -1722,13 +1722,13 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(fields=[general,sub])`}); err != nil { t.Fatal(err) } else { - results := res.Results[0].([]pilosa.GroupLine) + results := res.Results[0].([]pilosa.GroupCount) checkGroupBy(t, expected, results) } }) t.Run("check field offset no limit", func(t *testing.T) { - expected := pilosa.GroupByCounts{ + expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Count: 2}, {Group: []pilosa.FieldRow{{Field: "general", RowID: 12}}, Count: 2}, } @@ -1736,27 +1736,27 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(fields=[general:11:])`}); err != nil { t.Fatal(err) } else { - results := res.Results[0].([]pilosa.GroupLine) + results := res.Results[0].([]pilosa.GroupCount) checkGroupBy(t, expected, results) } }) t.Run("check field offset limit", func(t *testing.T) { - expected := pilosa.GroupByCounts{ + expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Count: 2}, } if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `GroupBy(fields=[general:11:1])`}); err != nil { t.Fatal(err) } else { - results := res.Results[0].([]pilosa.GroupLine) + results := res.Results[0].([]pilosa.GroupCount) checkGroupBy(t, expected, results) } }) } -func checkGroupBy(t *testing.T, expected, results pilosa.GroupByCounts) { - notIn := func(item pilosa.GroupLine, expected pilosa.GroupByCounts) bool { +func checkGroupBy(t *testing.T, expected, results []pilosa.GroupCount) { + notIn := func(item pilosa.GroupCount, expected []pilosa.GroupCount) bool { for i := range expected { if item.Count == expected[i].Count { if reflect.DeepEqual(item.Group, expected[i].Group) { diff --git a/internal/public.pb.go b/internal/public.pb.go index 29254543a..4f9aab2fc 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -11,7 +11,7 @@ Row Pair FieldRow - GroupLine + GroupCount ValCount Bit ColumnAttrSet @@ -132,24 +132,24 @@ func (m *FieldRow) GetRowID() uint64 { return 0 } -type GroupLine struct { +type GroupCount struct { Group []*FieldRow `protobuf:"bytes,1,rep,name=Group" json:"Group,omitempty"` Count uint64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` } -func (m *GroupLine) Reset() { *m = GroupLine{} } -func (m *GroupLine) String() string { return proto.CompactTextString(m) } -func (*GroupLine) ProtoMessage() {} -func (*GroupLine) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} } +func (m *GroupCount) Reset() { *m = GroupCount{} } +func (m *GroupCount) String() string { return proto.CompactTextString(m) } +func (*GroupCount) ProtoMessage() {} +func (*GroupCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} } -func (m *GroupLine) GetGroup() []*FieldRow { +func (m *GroupCount) GetGroup() []*FieldRow { if m != nil { return m.Group } return nil } -func (m *GroupLine) GetCount() uint64 { +func (m *GroupCount) GetCount() uint64 { if m != nil { return m.Count } @@ -405,14 +405,14 @@ func (m *QueryResponse) GetColumnAttrSets() []*ColumnAttrSet { } type QueryResult struct { - Type uint32 `protobuf:"varint,6,opt,name=Type,proto3" json:"Type,omitempty"` - Row *Row `protobuf:"bytes,1,opt,name=Row" json:"Row,omitempty"` - N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` - Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` - Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` - ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount" json:"ValCount,omitempty"` - RowIDs []uint64 `protobuf:"varint,7,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` - GroupByCounts []*GroupLine `protobuf:"bytes,8,rep,name=GroupByCounts" json:"GroupByCounts,omitempty"` + Type uint32 `protobuf:"varint,6,opt,name=Type,proto3" json:"Type,omitempty"` + Row *Row `protobuf:"bytes,1,opt,name=Row" json:"Row,omitempty"` + N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` + Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` + Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` + ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount" json:"ValCount,omitempty"` + RowIDs []uint64 `protobuf:"varint,7,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` + GroupCounts []*GroupCount `protobuf:"bytes,8,rep,name=GroupCounts" json:"GroupCounts,omitempty"` } func (m *QueryResult) Reset() { *m = QueryResult{} } @@ -469,9 +469,9 @@ func (m *QueryResult) GetRowIDs() []uint64 { return nil } -func (m *QueryResult) GetGroupByCounts() []*GroupLine { +func (m *QueryResult) GetGroupCounts() []*GroupCount { if m != nil { - return m.GroupByCounts + return m.GroupCounts } return nil } @@ -608,7 +608,7 @@ func init() { proto.RegisterType((*Row)(nil), "internal.Row") proto.RegisterType((*Pair)(nil), "internal.Pair") proto.RegisterType((*FieldRow)(nil), "internal.FieldRow") - proto.RegisterType((*GroupLine)(nil), "internal.GroupLine") + proto.RegisterType((*GroupCount)(nil), "internal.GroupCount") proto.RegisterType((*ValCount)(nil), "internal.ValCount") proto.RegisterType((*Bit)(nil), "internal.Bit") proto.RegisterType((*ColumnAttrSet)(nil), "internal.ColumnAttrSet") @@ -745,7 +745,7 @@ func (m *FieldRow) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *GroupLine) Marshal() (dAtA []byte, err error) { +func (m *GroupCount) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -755,7 +755,7 @@ func (m *GroupLine) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *GroupLine) MarshalTo(dAtA []byte) (int, error) { +func (m *GroupCount) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -1181,8 +1181,8 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(j7)) i += copy(dAtA[i:], dAtA8[:j7]) } - if len(m.GroupByCounts) > 0 { - for _, msg := range m.GroupByCounts { + if len(m.GroupCounts) > 0 { + for _, msg := range m.GroupCounts { dAtA[i] = 0x42 i++ i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) @@ -1461,7 +1461,7 @@ func (m *FieldRow) Size() (n int) { return n } -func (m *GroupLine) Size() (n int) { +func (m *GroupCount) Size() (n int) { var l int _ = l if len(m.Group) > 0 { @@ -1644,8 +1644,8 @@ func (m *QueryResult) Size() (n int) { } n += 1 + sovPublic(uint64(l)) + l } - if len(m.GroupByCounts) > 0 { - for _, e := range m.GroupByCounts { + if len(m.GroupCounts) > 0 { + for _, e := range m.GroupCounts { l = e.Size() n += 1 + l + sovPublic(uint64(l)) } @@ -2140,7 +2140,7 @@ func (m *FieldRow) Unmarshal(dAtA []byte) error { } return nil } -func (m *GroupLine) Unmarshal(dAtA []byte) error { +func (m *GroupCount) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -2163,10 +2163,10 @@ func (m *GroupLine) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: GroupLine: wiretype end group for non-group") + return fmt.Errorf("proto: GroupCount: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: GroupLine: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: GroupCount: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -3432,7 +3432,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } case 8: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field GroupByCounts", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field GroupCounts", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -3456,8 +3456,8 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.GroupByCounts = append(m.GroupByCounts, &GroupLine{}) - if err := m.GroupByCounts[len(m.GroupByCounts)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.GroupCounts = append(m.GroupCounts, &GroupCount{}) + if err := m.GroupCounts[len(m.GroupCounts)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -4241,54 +4241,53 @@ var ( func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ - // 771 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6e, 0xd3, 0x4c, - 0x14, 0xfe, 0x27, 0x76, 0x12, 0xe7, 0xa4, 0xc9, 0x5f, 0x0d, 0xa5, 0x58, 0xa8, 0x0a, 0x96, 0x85, - 0x90, 0x57, 0xa9, 0x14, 0x24, 0x24, 0x36, 0x20, 0xd2, 0x0b, 0x8a, 0x0a, 0x15, 0x4c, 0x4b, 0x11, - 0x4b, 0xb7, 0x19, 0xb5, 0x96, 0x1c, 0x8f, 0xf1, 0x45, 0x69, 0x9e, 0xa3, 0x1b, 0x1e, 0x81, 0x05, - 0x0f, 0xd2, 0x25, 0x8f, 0x00, 0xe5, 0x45, 0xd0, 0x9c, 0xf1, 0xd8, 0x4e, 0x8a, 0x2a, 0x16, 0xec, - 0xe6, 0xfb, 0xce, 0x9c, 0xf1, 0x77, 0xae, 0x86, 0xb5, 0x38, 0x3f, 0x0d, 0x83, 0xb3, 0x61, 0x9c, - 0x88, 0x4c, 0x50, 0x2b, 0x88, 0x32, 0x9e, 0x44, 0x7e, 0xe8, 0x7e, 0x02, 0x83, 0x89, 0x39, 0xb5, - 0xa1, 0xbd, 0x23, 0xc2, 0x7c, 0x16, 0xa5, 0x36, 0x71, 0x0c, 0xcf, 0x64, 0x1a, 0xd2, 0xc7, 0xd0, - 0x7c, 0x95, 0x65, 0x49, 0x6a, 0x37, 0x1c, 0xc3, 0xeb, 0x8e, 0xfa, 0x43, 0xed, 0x3a, 0x94, 0x34, - 0x53, 0x46, 0x4a, 0xc1, 0x3c, 0xe0, 0x8b, 0xd4, 0x36, 0x1c, 0xc3, 0xeb, 0x30, 0x3c, 0xbb, 0x2f, - 0xc0, 0x7c, 0xe7, 0x07, 0x09, 0xed, 0x43, 0x63, 0xb2, 0x6b, 0x13, 0x87, 0x78, 0x26, 0x6b, 0x4c, - 0x76, 0xe9, 0x06, 0x34, 0x77, 0x44, 0x1e, 0x65, 0x76, 0x03, 0x29, 0x05, 0xe8, 0x3a, 0x18, 0x07, - 0x7c, 0x61, 0x1b, 0x0e, 0xf1, 0x3a, 0x4c, 0x1e, 0xdd, 0x67, 0x60, 0xed, 0x07, 0x3c, 0x9c, 0x4a, - 0x7d, 0x1b, 0xd0, 0xc4, 0x33, 0x3e, 0xd3, 0x61, 0x0a, 0x48, 0x96, 0x89, 0xf9, 0x64, 0x57, 0xbf, - 0x84, 0xc0, 0x3d, 0x80, 0xce, 0xeb, 0x44, 0xe4, 0xf1, 0x9b, 0x20, 0xe2, 0xd4, 0x83, 0x26, 0x02, - 0x0c, 0xab, 0x3b, 0xa2, 0x95, 0x7c, 0xfd, 0x36, 0x53, 0x17, 0xfe, 0x2c, 0xcb, 0x1d, 0x81, 0x75, - 0xe2, 0x87, 0xa5, 0xc4, 0x13, 0x3f, 0x44, 0x09, 0x06, 0x93, 0xc7, 0x65, 0x1f, 0x43, 0xfb, 0x7c, - 0x00, 0x63, 0x1c, 0x64, 0x95, 0x3a, 0x52, 0x53, 0x47, 0x1f, 0x82, 0xa5, 0x52, 0x5b, 0xca, 0x2e, - 0x31, 0xdd, 0x82, 0xce, 0x71, 0x30, 0xe3, 0x69, 0xe6, 0xcf, 0x62, 0xcc, 0x84, 0xc1, 0x2a, 0xc2, - 0xfd, 0x08, 0x3d, 0x75, 0x53, 0xa6, 0xfc, 0x88, 0x67, 0xb7, 0x12, 0xfb, 0x77, 0xa5, 0xba, 0x9d, - 0xe8, 0xaf, 0x04, 0x4c, 0x69, 0xd3, 0x26, 0x52, 0x9a, 0x64, 0x5d, 0x8f, 0x17, 0x31, 0x2f, 0x94, - 0xe2, 0x99, 0x3a, 0xd0, 0x3d, 0xca, 0x92, 0x20, 0x3a, 0x3f, 0xf1, 0xc3, 0x9c, 0x17, 0x0f, 0xd5, - 0x29, 0x19, 0xe3, 0x24, 0xca, 0x94, 0xd9, 0xc4, 0x30, 0x4a, 0x2c, 0x63, 0x1c, 0x0b, 0x11, 0x2a, - 0x63, 0xd3, 0x21, 0x9e, 0xc5, 0x2a, 0x82, 0x0e, 0x00, 0xf6, 0x43, 0xe1, 0x17, 0xbe, 0x2d, 0x87, - 0x78, 0x84, 0xd5, 0x18, 0x77, 0x1b, 0xda, 0x52, 0xe9, 0x5b, 0x3f, 0xae, 0xa2, 0x25, 0x77, 0x44, - 0xeb, 0x5e, 0x13, 0x58, 0x7b, 0x9f, 0xf3, 0x64, 0xc1, 0xf8, 0xe7, 0x9c, 0xa7, 0x58, 0x15, 0xc4, - 0xba, 0x93, 0x10, 0xd0, 0x4d, 0x68, 0x1d, 0x5d, 0xf8, 0xc9, 0x54, 0xe5, 0xce, 0x64, 0x05, 0x92, - 0xb1, 0x56, 0x39, 0x4f, 0x31, 0x56, 0x8b, 0xd5, 0x29, 0xe9, 0xc9, 0xf8, 0x4c, 0x64, 0x3a, 0x98, - 0x02, 0x51, 0x0f, 0xfe, 0xdf, 0xbb, 0x3c, 0x0b, 0xf3, 0x29, 0x67, 0x62, 0xae, 0xbc, 0x5b, 0x78, - 0x61, 0x95, 0xa6, 0x4f, 0xa0, 0x5f, 0x50, 0x7a, 0x04, 0xdb, 0x78, 0x71, 0x85, 0x75, 0xaf, 0x08, - 0xf4, 0x8a, 0x50, 0xd2, 0x58, 0x44, 0x29, 0x97, 0xf5, 0xda, 0x4b, 0x12, 0x5d, 0xaf, 0xbd, 0x24, - 0xa1, 0xdb, 0xd0, 0x66, 0x3c, 0xcd, 0xc3, 0x4c, 0x37, 0xc1, 0xfd, 0x2a, 0x2d, 0xda, 0x37, 0x0f, - 0x33, 0xa6, 0x6f, 0xd1, 0x97, 0xd0, 0x5f, 0x6a, 0x2a, 0x35, 0xc2, 0xdd, 0xd1, 0x83, 0xca, 0x6f, - 0xc9, 0xce, 0x56, 0xae, 0xbb, 0x57, 0x0d, 0xe8, 0xd6, 0x5e, 0xa6, 0x8f, 0x70, 0xa1, 0xa0, 0xa6, - 0xee, 0xa8, 0x57, 0xbd, 0x22, 0x27, 0x0d, 0x57, 0xcd, 0x1a, 0x90, 0xc3, 0xa2, 0x9f, 0xc8, 0xa1, - 0xac, 0xa2, 0x5c, 0x12, 0xfa, 0xb3, 0xb5, 0x2a, 0x4a, 0x9a, 0x29, 0x23, 0xae, 0xa7, 0x0b, 0x3f, - 0x3a, 0xe7, 0x53, 0xec, 0x27, 0x8b, 0x69, 0x48, 0x87, 0xd5, 0x7c, 0x62, 0x01, 0x96, 0x46, 0x5c, - 0x5b, 0x58, 0x35, 0xc3, 0xba, 0xa1, 0x65, 0x2d, 0x7a, 0x45, 0x43, 0xcb, 0x12, 0xca, 0xd9, 0x94, - 0x89, 0xc7, 0xe2, 0x2b, 0x44, 0x9f, 0x43, 0x0f, 0x57, 0xc3, 0x78, 0x81, 0xbe, 0xa9, 0x6d, 0xa1, - 0xc6, 0x7b, 0xd5, 0x07, 0xca, 0x3d, 0xc3, 0x96, 0x6f, 0xba, 0x3f, 0x09, 0xf4, 0x26, 0xb3, 0x58, - 0x24, 0x59, 0xad, 0xef, 0x26, 0xd1, 0x94, 0x5f, 0xea, 0xbe, 0x43, 0x50, 0xed, 0xb5, 0xc6, 0xca, - 0x5e, 0xc3, 0xfe, 0xc3, 0x7e, 0x33, 0x99, 0x02, 0x35, 0x99, 0xe6, 0x92, 0xcc, 0x2d, 0xe8, 0xe8, - 0x0d, 0x92, 0xda, 0x4d, 0x34, 0x55, 0x84, 0x9c, 0xa8, 0x72, 0x85, 0xc8, 0x16, 0x34, 0x3c, 0x83, - 0xd5, 0x18, 0x99, 0x5a, 0x26, 0xe6, 0xb8, 0xbc, 0xdb, 0xb8, 0xbc, 0x35, 0x94, 0x9e, 0xea, 0x19, - 0x34, 0x5a, 0x68, 0xac, 0x31, 0xee, 0x37, 0x02, 0x54, 0xc5, 0x88, 0xb3, 0xf9, 0xef, 0x02, 0xbd, - 0x3b, 0xa0, 0x4d, 0x68, 0xe1, 0xf7, 0x74, 0x30, 0x05, 0x5a, 0x91, 0xdb, 0x5e, 0x95, 0x3b, 0x5e, - 0xbf, 0xbe, 0x19, 0x90, 0xef, 0x37, 0x03, 0xf2, 0xe3, 0x66, 0x40, 0xbe, 0xfc, 0x1a, 0xfc, 0x77, - 0xda, 0xc2, 0x9f, 0xe1, 0xd3, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x21, 0xc1, 0x72, 0xc6, 0x1c, - 0x07, 0x00, 0x00, + // 764 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd3, 0x4a, + 0x14, 0xbe, 0x13, 0x3b, 0x89, 0x73, 0xd2, 0xe4, 0x56, 0xa3, 0xde, 0x5e, 0x0b, 0x55, 0xc1, 0xb2, + 0x10, 0xf2, 0x2a, 0x95, 0x82, 0xd4, 0x25, 0x88, 0xfe, 0xa1, 0xa8, 0x50, 0xc1, 0xb4, 0x14, 0xb1, + 0x74, 0x9b, 0x51, 0x6b, 0xc9, 0xf1, 0x18, 0xff, 0x28, 0xcd, 0x5b, 0x20, 0xb1, 0xe1, 0x11, 0x58, + 0xf0, 0x20, 0x5d, 0xf2, 0x08, 0x50, 0x5e, 0x04, 0xcd, 0x19, 0x4f, 0xc6, 0x49, 0x51, 0xc5, 0x82, + 0x9d, 0xbf, 0xef, 0xcc, 0x39, 0xf9, 0xce, 0x6f, 0x60, 0x2d, 0x2d, 0xcf, 0xe3, 0xe8, 0x62, 0x98, + 0x66, 0xa2, 0x10, 0xd4, 0x89, 0x92, 0x82, 0x67, 0x49, 0x18, 0xfb, 0xef, 0xc1, 0x62, 0x62, 0x46, + 0x5d, 0x68, 0xef, 0x89, 0xb8, 0x9c, 0x26, 0xb9, 0x4b, 0x3c, 0x2b, 0xb0, 0x99, 0x86, 0xf4, 0x11, + 0x34, 0x9f, 0x17, 0x45, 0x96, 0xbb, 0x0d, 0xcf, 0x0a, 0xba, 0xa3, 0xfe, 0x50, 0xbb, 0x0e, 0x25, + 0xcd, 0x94, 0x91, 0x52, 0xb0, 0x8f, 0xf8, 0x3c, 0x77, 0x2d, 0xcf, 0x0a, 0x3a, 0x0c, 0xbf, 0xfd, + 0xa7, 0x60, 0xbf, 0x0e, 0xa3, 0x8c, 0xf6, 0xa1, 0x31, 0xde, 0x77, 0x89, 0x47, 0x02, 0x9b, 0x35, + 0xc6, 0xfb, 0x74, 0x03, 0x9a, 0x7b, 0xa2, 0x4c, 0x0a, 0xb7, 0x81, 0x94, 0x02, 0x74, 0x1d, 0xac, + 0x23, 0x3e, 0x77, 0x2d, 0x8f, 0x04, 0x1d, 0x26, 0x3f, 0xfd, 0x1d, 0x70, 0x0e, 0x23, 0x1e, 0x4f, + 0xa4, 0xbe, 0x0d, 0x68, 0xe2, 0x37, 0x86, 0xe9, 0x30, 0x05, 0x24, 0xcb, 0xc4, 0x6c, 0xbc, 0xaf, + 0x23, 0x21, 0xf0, 0x5f, 0x02, 0xbc, 0xc8, 0x44, 0x99, 0xaa, 0xb8, 0x01, 0x34, 0x11, 0x61, 0x5e, + 0xdd, 0x11, 0x35, 0xfa, 0x75, 0x70, 0xa6, 0x1e, 0xfc, 0x5e, 0x97, 0x3f, 0x02, 0xe7, 0x2c, 0x8c, + 0x17, 0x1a, 0xcf, 0xc2, 0x18, 0x35, 0x58, 0x4c, 0x7e, 0x2e, 0xfb, 0x58, 0xda, 0xe7, 0x2d, 0x58, + 0xbb, 0x51, 0x61, 0xe4, 0x91, 0x9a, 0x3c, 0xfa, 0x00, 0x1c, 0x55, 0xdb, 0x85, 0xee, 0x05, 0xa6, + 0x5b, 0xd0, 0x39, 0x8d, 0xa6, 0x3c, 0x2f, 0xc2, 0x69, 0x8a, 0xa5, 0xb0, 0x98, 0x21, 0xfc, 0x77, + 0xd0, 0x53, 0x2f, 0x65, 0xcd, 0x4f, 0x78, 0x71, 0xa7, 0xb2, 0x7f, 0xd6, 0xab, 0xbb, 0x95, 0xfe, + 0x42, 0xc0, 0x96, 0x36, 0x6d, 0x22, 0x0b, 0x93, 0x6c, 0xec, 0xe9, 0x3c, 0xe5, 0x95, 0x52, 0xfc, + 0xa6, 0x1e, 0x74, 0x4f, 0x8a, 0x2c, 0x4a, 0x2e, 0xcf, 0xc2, 0xb8, 0xe4, 0x55, 0xa0, 0x3a, 0x25, + 0x73, 0x1c, 0x27, 0x85, 0x32, 0xdb, 0x98, 0xc6, 0x02, 0xcb, 0x1c, 0x77, 0x85, 0x88, 0x95, 0xb1, + 0xe9, 0x91, 0xc0, 0x61, 0x86, 0xa0, 0x03, 0x80, 0xc3, 0x58, 0x84, 0x95, 0x6f, 0xcb, 0x23, 0x01, + 0x61, 0x35, 0xc6, 0xdf, 0x86, 0xb6, 0x54, 0xfa, 0x2a, 0x4c, 0x4d, 0xb6, 0xe4, 0x9e, 0x6c, 0xfd, + 0x1b, 0x02, 0x6b, 0x6f, 0x4a, 0x9e, 0xcd, 0x19, 0xff, 0x50, 0xf2, 0x1c, 0xbb, 0x82, 0x58, 0x8f, + 0x12, 0x02, 0xba, 0x09, 0xad, 0x93, 0xab, 0x30, 0x9b, 0xa8, 0xda, 0xd9, 0xac, 0x42, 0x32, 0x57, + 0x53, 0xf3, 0x1c, 0x73, 0x75, 0x58, 0x9d, 0x92, 0x9e, 0x8c, 0x4f, 0x45, 0xa1, 0x93, 0xa9, 0x10, + 0x0d, 0xe0, 0xdf, 0x83, 0xeb, 0x8b, 0xb8, 0x9c, 0x70, 0x26, 0x66, 0xca, 0xbb, 0x85, 0x0f, 0x56, + 0x69, 0xfa, 0x18, 0xfa, 0x15, 0xa5, 0x77, 0xb0, 0x8d, 0x0f, 0x57, 0x58, 0xff, 0x13, 0x81, 0x5e, + 0x95, 0x4a, 0x9e, 0x8a, 0x24, 0xe7, 0xb2, 0x5f, 0x07, 0x59, 0xa6, 0xfb, 0x75, 0x90, 0x65, 0x74, + 0x1b, 0xda, 0x8c, 0xe7, 0x65, 0x5c, 0xe8, 0x21, 0xf8, 0xcf, 0x94, 0x45, 0xfb, 0x96, 0x71, 0xc1, + 0xf4, 0x2b, 0xfa, 0x0c, 0xfa, 0x4b, 0x43, 0xa5, 0x76, 0xb8, 0x3b, 0xfa, 0xdf, 0xf8, 0x2d, 0xd9, + 0xd9, 0xca, 0x73, 0xff, 0x63, 0x03, 0xba, 0xb5, 0xc8, 0xf4, 0x21, 0x5e, 0x14, 0xd4, 0xd4, 0x1d, + 0xf5, 0x4c, 0x14, 0xb9, 0x69, 0x78, 0x6b, 0xd6, 0x80, 0x1c, 0x57, 0xf3, 0x44, 0x8e, 0x65, 0x17, + 0xe5, 0x95, 0xd0, 0x3f, 0x5b, 0xeb, 0xa2, 0xa4, 0x99, 0x32, 0xe2, 0x7d, 0xba, 0x0a, 0x93, 0x4b, + 0x3e, 0xc1, 0x79, 0x72, 0x98, 0x86, 0x74, 0x68, 0xf6, 0x13, 0x1b, 0xb0, 0xb4, 0xe2, 0xda, 0xc2, + 0xcc, 0x0e, 0xeb, 0x81, 0x96, 0xbd, 0xe8, 0x55, 0x03, 0x2d, 0x5b, 0x28, 0x77, 0x53, 0x16, 0x1e, + 0x9b, 0xaf, 0x10, 0xdd, 0x81, 0xae, 0xb9, 0x24, 0xb9, 0xeb, 0xa0, 0xc2, 0x0d, 0x13, 0xde, 0x18, + 0x59, 0xfd, 0xa1, 0xff, 0x83, 0x40, 0x6f, 0x3c, 0x4d, 0x45, 0x56, 0xd4, 0x86, 0x6e, 0x9c, 0x4c, + 0xf8, 0xb5, 0x1e, 0x3a, 0x04, 0xe6, 0xaa, 0x35, 0x56, 0xae, 0x1a, 0x0e, 0x1f, 0x0e, 0x9b, 0xcd, + 0x14, 0xa8, 0x69, 0xb4, 0x97, 0x34, 0x6e, 0x41, 0x47, 0x9f, 0x8f, 0xdc, 0x6d, 0xa2, 0xc9, 0x10, + 0x72, 0x9d, 0x16, 0xf7, 0x43, 0xce, 0x9f, 0x15, 0x58, 0xac, 0xc6, 0xc8, 0xba, 0x32, 0x31, 0xc3, + 0xd3, 0xdd, 0xc6, 0xd3, 0xad, 0xa1, 0xf4, 0x54, 0x61, 0xd0, 0xe8, 0xa0, 0xb1, 0xc6, 0xf8, 0x5f, + 0x09, 0x50, 0x95, 0x23, 0x2e, 0xe6, 0xdf, 0x4b, 0xf4, 0xfe, 0x84, 0x36, 0xa1, 0x85, 0xbf, 0xa7, + 0x93, 0xa9, 0xd0, 0x8a, 0xdc, 0xf6, 0xaa, 0xdc, 0xdd, 0xf5, 0x9b, 0xdb, 0x01, 0xf9, 0x76, 0x3b, + 0x20, 0xdf, 0x6f, 0x07, 0xe4, 0xf3, 0xcf, 0xc1, 0x3f, 0xe7, 0x2d, 0xfc, 0x2b, 0x7c, 0xf2, 0x2b, + 0x00, 0x00, 0xff, 0xff, 0x25, 0x1f, 0x0d, 0xa8, 0x1a, 0x07, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index b207e3cae..c455ab1e6 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -19,7 +19,7 @@ message FieldRow{ uint64 RowID = 2; } -message GroupLine{ +message GroupCount{ repeated FieldRow Group = 1; uint64 Count = 2; } @@ -77,7 +77,7 @@ message QueryResult { bool Changed = 4; ValCount ValCount = 5; repeated uint64 RowIDs = 7; - repeated GroupLine GroupByCounts = 8; + repeated GroupCount GroupCounts = 8; } message ImportRequest { From 71ad2974504e677c3bc35d49d624874383cb5a64 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 5 Sep 2018 10:55:00 -0500 Subject: [PATCH 6/7] change Rows() to RowIDs() and add RowIdentifiers return type to hold row keys --- encoding/proto/proto.go | 13 +- executor.go | 59 ++++- executor_test.go | 22 +- internal/public.pb.go | 554 ++++++++++++++++++++++++++++++---------- internal/public.proto | 7 + 5 files changed, 506 insertions(+), 149 deletions(-) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 6273f2c7f..b6d78799a 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -366,6 +366,9 @@ func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse { case []pilosa.GroupCount: pb.Results[i].Type = queryResultTypeGroupCounts pb.Results[i].GroupCounts = encodeGroupCounts(result) + case pilosa.RowIdentifiers: + pb.Results[i].Type = queryResultTypeRowIdentifiers + pb.Results[i].RowIdentifiers = encodeRowIdentifiers(result) case nil: pb.Results[i].Type = queryResultTypeNil } @@ -898,7 +901,6 @@ func decodeQueryResponse(pb *internal.QueryResponse, m *pilosa.QueryResponse) { } m.Results = make([]interface{}, len(pb.Results)) decodeQueryResults(pb.Results, m.Results) - } func decodeColumnAttrSets(pb []*internal.ColumnAttrSet, m []*pilosa.ColumnAttrSet) { @@ -930,6 +932,7 @@ const ( queryResultTypeBool queryResultTypeRowIDs queryResultTypeGroupCounts + queryResultTypeRowIdentifiers ) func decodeQueryResult(pb *internal.QueryResult) interface{} { @@ -1069,6 +1072,14 @@ func encodeRow(r *pilosa.Row) *internal.Row { } } +func encodeRowIdentifiers(r pilosa.RowIdentifiers) *internal.RowIdentifiers { + return &internal.RowIdentifiers{ + Rows: r.Rows, + Keys: r.Keys, + //Attrs: encodeAttrs(r.Attrs), + } +} + func encodeGroupCounts(counts []pilosa.GroupCount) []*internal.GroupCount { result := make([]*internal.GroupCount, len(counts)) for i := range counts { diff --git a/executor.go b/executor.go index faa71cd3d..0fb6f53a7 100644 --- a/executor.go +++ b/executor.go @@ -196,9 +196,9 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s case "TopN": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeTopN(ctx, index, c, shards, opt) - case "Rows": + case "RowIDs": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) - return e.executeRows(ctx, index, c, shards, opt) + return e.executeRowIDs(ctx, index, c, shards, opt) case "GroupBy": e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag}) return e.executeGroupBy(ctx, index, c, shards, opt) @@ -722,9 +722,23 @@ func (e *executor) executeDifferenceShard(ctx context.Context, index string, c * return other, nil } +// RowIdentifiers is a return type for a list of +// row ids or row keys. The names `Rows` and `Keys` +// are meant to follow the same convention as the +// Row query which returns `Columns` and `Keys`. +// TODO: Rename this to something better. Anything. +type RowIdentifiers struct { + Rows []uint64 `json:"rows"` + Keys []string `json:"keys,omitempty"` +} + +// RowIDs is a query return type for just uint64 row ids. +// It should only be used internally (since RowIdentifiers +// is the external return type), but it is exported because +// the proto package needs access to it. type RowIDs []uint64 -func (r RowIDs) Merge(other RowIDs) RowIDs { +func (r RowIDs) merge(other RowIDs) RowIDs { i, j := 0, 0 result := make(RowIDs, 0) for i < len(r) && j < len(other) { @@ -942,15 +956,16 @@ func product(input [][]gbi) []ppi { } return res } -func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (RowIDs, error) { + +func (e *executor) executeRowIDs(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (RowIDs, error) { // Execute calls in bulk on each remote node and merge. mapFn := func(shard uint64) (interface{}, error) { - return e.executeRowsShard(ctx, index, c, shard) + return e.executeRowIDsShard(ctx, index, c, shard) } // Merge returned results at coordinating node. reduceFn := func(prev, v interface{}) interface{} { other, _ := prev.(RowIDs) - return other.Merge(v.(RowIDs)) + return other.merge(v.(RowIDs)) } // Get full result set. other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) @@ -976,7 +991,8 @@ func (e *executor) executeRows(ctx context.Context, index string, c *pql.Call, s } return results, nil } -func (e *executor) executeRowsShard(ctx context.Context, index string, c *pql.Call, shard uint64) (RowIDs, error) { + +func (e *executor) executeRowIDsShard(ctx context.Context, index string, c *pql.Call, shard uint64) (RowIDs, error) { // Fetch index. idx := e.Holder.Index(index) if idx == nil { @@ -985,7 +1001,7 @@ func (e *executor) executeRowsShard(ctx context.Context, index string, c *pql.Ca // Fetch field name from argument. fieldName, ok := c.Args["field"].(string) if !ok { - return nil, errors.New("Rows() argument required: field") + return nil, errors.New("RowIDs() argument required: field") } // Fetch field. f := e.Holder.Field(index, fieldName) @@ -2121,6 +2137,31 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res }) } return other, nil + + case RowIDs: + other := RowIdentifiers{} + + fieldName := callArgString(call, "field") + if fieldName == "" { + return nil, ErrFieldNotFound + } + + if field := idx.Field(fieldName); field == nil { + return nil, ErrFieldNotFound + } else if field.keys() { + other.Keys = make([]string, len(result)) + for i, id := range result { + key, err := e.TranslateStore.TranslateRowToString(index, fieldName, id) + if err != nil { + return nil, err + } + other.Keys[i] = key + } + } else { + other.Rows = result + } + + return other, nil } return result, nil @@ -2171,7 +2212,7 @@ func needsShards(calls []*pql.Call) bool { switch call.Name { case "Clear", "Set", "SetRowAttrs", "SetColumnAttrs": continue - case "Count", "TopN", "Rows": + case "Count", "TopN", "RowIDs": return true // default catches Bitmap calls default: diff --git a/executor_test.go b/executor_test.go index 786b0ebfb..c6ae4d9ab 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1639,7 +1639,7 @@ func benchmarkExistence(nn bool, b *testing.B) { func BenchmarkExecutor_Existence_True(b *testing.B) { benchmarkExistence(true, b) } func BenchmarkExecutor_Existence_False(b *testing.B) { benchmarkExistence(false, b) } -func TestExecutor_Execute_Rows(t *testing.T) { +func TestExecutor_Execute_RowIDs(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() hldr := test.Holder{Holder: c[0].Server.Holder()} @@ -1649,24 +1649,28 @@ func TestExecutor_Execute_Rows(t *testing.T) { hldr.SetBit("i", "general", 11, ShardWidth+2) hldr.SetBit("i", "general", 12, 2) hldr.SetBit("i", "general", 12, ShardWidth+2) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=general)`}); err != nil { + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `RowIDs(field=general)`}); err != nil { t.Fatal(err) - } else if columns := res.Results[0].(pilosa.RowIDs); !reflect.DeepEqual(columns, pilosa.RowIDs{10, 11, 12}) { + } else if columns := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual(columns, pilosa.RowIdentifiers{Rows: []uint64{10, 11, 12}}) { t.Fatalf("unexpected columns: %+v", columns) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=general, limit=2)`}); err != nil { + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `RowIDs(field=general, limit=2)`}); err != nil { t.Fatal(err) - } else if columns := res.Results[0].(pilosa.RowIDs); !reflect.DeepEqual(columns, pilosa.RowIDs{10, 11}) { + } else if columns := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual(columns, pilosa.RowIdentifiers{Rows: []uint64{10, 11}}) { t.Fatalf("unexpected columns: %+v", columns) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=general, offset=1,limit=2)`}); err != nil { + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `RowIDs(field=general, offset=1,limit=2)`}); err != nil { t.Fatal(err) - } else if columns := res.Results[0].(pilosa.RowIDs); !reflect.DeepEqual(columns, pilosa.RowIDs{11, 12}) { + } else if columns := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual(columns, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) { t.Fatalf("unexpected columns: %+v", columns) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Rows(field=general, column=2)`}); err != nil { + + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `RowIDs(field=general, column=2)`}); err != nil { t.Fatal(err) - } else if columns := res.Results[0].(pilosa.RowIDs); !reflect.DeepEqual(columns, pilosa.RowIDs{11, 12}) { + } else if columns := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual(columns, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) { t.Fatalf("unexpected columns: %+v", columns) } } diff --git a/internal/public.pb.go b/internal/public.pb.go index 4f9aab2fc..bfa0b3be1 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -9,6 +9,7 @@ It has these top-level messages: Row + RowIdentifiers Pair FieldRow GroupCount @@ -76,6 +77,30 @@ func (m *Row) GetAttrs() []*Attr { return nil } +type RowIdentifiers struct { + Rows []uint64 `protobuf:"varint,1,rep,packed,name=Rows" json:"Rows,omitempty"` + Keys []string `protobuf:"bytes,2,rep,name=Keys" json:"Keys,omitempty"` +} + +func (m *RowIdentifiers) Reset() { *m = RowIdentifiers{} } +func (m *RowIdentifiers) String() string { return proto.CompactTextString(m) } +func (*RowIdentifiers) ProtoMessage() {} +func (*RowIdentifiers) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{1} } + +func (m *RowIdentifiers) GetRows() []uint64 { + if m != nil { + return m.Rows + } + return nil +} + +func (m *RowIdentifiers) GetKeys() []string { + if m != nil { + return m.Keys + } + return nil +} + type Pair struct { ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` Key string `protobuf:"bytes,3,opt,name=Key,proto3" json:"Key,omitempty"` @@ -85,7 +110,7 @@ type Pair struct { func (m *Pair) Reset() { *m = Pair{} } func (m *Pair) String() string { return proto.CompactTextString(m) } func (*Pair) ProtoMessage() {} -func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{1} } +func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} } func (m *Pair) GetID() uint64 { if m != nil { @@ -116,7 +141,7 @@ type FieldRow struct { func (m *FieldRow) Reset() { *m = FieldRow{} } func (m *FieldRow) String() string { return proto.CompactTextString(m) } func (*FieldRow) ProtoMessage() {} -func (*FieldRow) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} } +func (*FieldRow) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} } func (m *FieldRow) GetField() string { if m != nil { @@ -140,7 +165,7 @@ type GroupCount struct { func (m *GroupCount) Reset() { *m = GroupCount{} } func (m *GroupCount) String() string { return proto.CompactTextString(m) } func (*GroupCount) ProtoMessage() {} -func (*GroupCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} } +func (*GroupCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{4} } func (m *GroupCount) GetGroup() []*FieldRow { if m != nil { @@ -164,7 +189,7 @@ type ValCount struct { func (m *ValCount) Reset() { *m = ValCount{} } func (m *ValCount) String() string { return proto.CompactTextString(m) } func (*ValCount) ProtoMessage() {} -func (*ValCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{4} } +func (*ValCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{5} } func (m *ValCount) GetVal() int64 { if m != nil { @@ -189,7 +214,7 @@ type Bit struct { func (m *Bit) Reset() { *m = Bit{} } func (m *Bit) String() string { return proto.CompactTextString(m) } func (*Bit) ProtoMessage() {} -func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{5} } +func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{6} } func (m *Bit) GetRowID() uint64 { if m != nil { @@ -221,7 +246,7 @@ type ColumnAttrSet struct { func (m *ColumnAttrSet) Reset() { *m = ColumnAttrSet{} } func (m *ColumnAttrSet) String() string { return proto.CompactTextString(m) } func (*ColumnAttrSet) ProtoMessage() {} -func (*ColumnAttrSet) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{6} } +func (*ColumnAttrSet) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{7} } func (m *ColumnAttrSet) GetID() uint64 { if m != nil { @@ -256,7 +281,7 @@ type Attr struct { func (m *Attr) Reset() { *m = Attr{} } func (m *Attr) String() string { return proto.CompactTextString(m) } func (*Attr) ProtoMessage() {} -func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{7} } +func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{8} } func (m *Attr) GetKey() string { if m != nil { @@ -307,7 +332,7 @@ type AttrMap struct { func (m *AttrMap) Reset() { *m = AttrMap{} } func (m *AttrMap) String() string { return proto.CompactTextString(m) } func (*AttrMap) ProtoMessage() {} -func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{8} } +func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{9} } func (m *AttrMap) GetAttrs() []*Attr { if m != nil { @@ -328,7 +353,7 @@ type QueryRequest struct { func (m *QueryRequest) Reset() { *m = QueryRequest{} } func (m *QueryRequest) String() string { return proto.CompactTextString(m) } func (*QueryRequest) ProtoMessage() {} -func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{9} } +func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{10} } func (m *QueryRequest) GetQuery() string { if m != nil { @@ -381,7 +406,7 @@ type QueryResponse struct { func (m *QueryResponse) Reset() { *m = QueryResponse{} } func (m *QueryResponse) String() string { return proto.CompactTextString(m) } func (*QueryResponse) ProtoMessage() {} -func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{10} } +func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{11} } func (m *QueryResponse) GetErr() string { if m != nil { @@ -405,20 +430,21 @@ func (m *QueryResponse) GetColumnAttrSets() []*ColumnAttrSet { } type QueryResult struct { - Type uint32 `protobuf:"varint,6,opt,name=Type,proto3" json:"Type,omitempty"` - Row *Row `protobuf:"bytes,1,opt,name=Row" json:"Row,omitempty"` - N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` - Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` - Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` - ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount" json:"ValCount,omitempty"` - RowIDs []uint64 `protobuf:"varint,7,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` - GroupCounts []*GroupCount `protobuf:"bytes,8,rep,name=GroupCounts" json:"GroupCounts,omitempty"` + Type uint32 `protobuf:"varint,6,opt,name=Type,proto3" json:"Type,omitempty"` + Row *Row `protobuf:"bytes,1,opt,name=Row" json:"Row,omitempty"` + N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` + Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` + Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` + ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount" json:"ValCount,omitempty"` + RowIDs []uint64 `protobuf:"varint,7,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` + GroupCounts []*GroupCount `protobuf:"bytes,8,rep,name=GroupCounts" json:"GroupCounts,omitempty"` + RowIdentifiers *RowIdentifiers `protobuf:"bytes,9,opt,name=RowIdentifiers" json:"RowIdentifiers,omitempty"` } func (m *QueryResult) Reset() { *m = QueryResult{} } func (m *QueryResult) String() string { return proto.CompactTextString(m) } func (*QueryResult) ProtoMessage() {} -func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{11} } +func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{12} } func (m *QueryResult) GetType() uint32 { if m != nil { @@ -476,6 +502,13 @@ func (m *QueryResult) GetGroupCounts() []*GroupCount { return nil } +func (m *QueryResult) GetRowIdentifiers() *RowIdentifiers { + if m != nil { + return m.RowIdentifiers + } + return nil +} + type ImportRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` @@ -490,7 +523,7 @@ type ImportRequest struct { func (m *ImportRequest) Reset() { *m = ImportRequest{} } func (m *ImportRequest) String() string { return proto.CompactTextString(m) } func (*ImportRequest) ProtoMessage() {} -func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{12} } +func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{13} } func (m *ImportRequest) GetIndex() string { if m != nil { @@ -560,7 +593,7 @@ type ImportValueRequest struct { func (m *ImportValueRequest) Reset() { *m = ImportValueRequest{} } func (m *ImportValueRequest) String() string { return proto.CompactTextString(m) } func (*ImportValueRequest) ProtoMessage() {} -func (*ImportValueRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{13} } +func (*ImportValueRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{14} } func (m *ImportValueRequest) GetIndex() string { if m != nil { @@ -606,6 +639,7 @@ func (m *ImportValueRequest) GetValues() []int64 { func init() { proto.RegisterType((*Row)(nil), "internal.Row") + proto.RegisterType((*RowIdentifiers)(nil), "internal.RowIdentifiers") proto.RegisterType((*Pair)(nil), "internal.Pair") proto.RegisterType((*FieldRow)(nil), "internal.FieldRow") proto.RegisterType((*GroupCount)(nil), "internal.GroupCount") @@ -682,6 +716,56 @@ func (m *Row) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *RowIdentifiers) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RowIdentifiers) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Rows) > 0 { + dAtA4 := make([]byte, len(m.Rows)*10) + var j3 int + for _, num := range m.Rows { + for num >= 1<<7 { + dAtA4[j3] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j3++ + } + dAtA4[j3] = uint8(num) + j3++ + } + dAtA[i] = 0xa + i++ + i = encodeVarintPublic(dAtA, i, uint64(j3)) + i += copy(dAtA[i:], dAtA4[:j3]) + } + if len(m.Keys) > 0 { + for _, s := range m.Keys { + dAtA[i] = 0x12 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + return i, nil +} + func (m *Pair) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -990,21 +1074,21 @@ func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], m.Query) } if len(m.Shards) > 0 { - dAtA4 := make([]byte, len(m.Shards)*10) - var j3 int + dAtA6 := make([]byte, len(m.Shards)*10) + var j5 int for _, num := range m.Shards { for num >= 1<<7 { - dAtA4[j3] = uint8(uint64(num)&0x7f | 0x80) + dAtA6[j5] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j3++ + j5++ } - dAtA4[j3] = uint8(num) - j3++ + dAtA6[j5] = uint8(num) + j5++ } dAtA[i] = 0x12 i++ - i = encodeVarintPublic(dAtA, i, uint64(j3)) - i += copy(dAtA[i:], dAtA4[:j3]) + i = encodeVarintPublic(dAtA, i, uint64(j5)) + i += copy(dAtA[i:], dAtA6[:j5]) } if m.ColumnAttrs { dAtA[i] = 0x18 @@ -1116,11 +1200,11 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintPublic(dAtA, i, uint64(m.Row.Size())) - n5, err := m.Row.MarshalTo(dAtA[i:]) + n7, err := m.Row.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n5 + i += n7 } if m.N != 0 { dAtA[i] = 0x10 @@ -1153,11 +1237,11 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x2a i++ i = encodeVarintPublic(dAtA, i, uint64(m.ValCount.Size())) - n6, err := m.ValCount.MarshalTo(dAtA[i:]) + n8, err := m.ValCount.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n6 + i += n8 } if m.Type != 0 { dAtA[i] = 0x30 @@ -1165,21 +1249,21 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(m.Type)) } if len(m.RowIDs) > 0 { - dAtA8 := make([]byte, len(m.RowIDs)*10) - var j7 int + dAtA10 := make([]byte, len(m.RowIDs)*10) + var j9 int for _, num := range m.RowIDs { for num >= 1<<7 { - dAtA8[j7] = uint8(uint64(num)&0x7f | 0x80) + dAtA10[j9] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j7++ + j9++ } - dAtA8[j7] = uint8(num) - j7++ + dAtA10[j9] = uint8(num) + j9++ } dAtA[i] = 0x3a i++ - i = encodeVarintPublic(dAtA, i, uint64(j7)) - i += copy(dAtA[i:], dAtA8[:j7]) + i = encodeVarintPublic(dAtA, i, uint64(j9)) + i += copy(dAtA[i:], dAtA10[:j9]) } if len(m.GroupCounts) > 0 { for _, msg := range m.GroupCounts { @@ -1193,6 +1277,16 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { i += n } } + if m.RowIdentifiers != nil { + dAtA[i] = 0x4a + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.RowIdentifiers.Size())) + n11, err := m.RowIdentifiers.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n11 + } return i, nil } @@ -1229,56 +1323,56 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(m.Shard)) } if len(m.RowIDs) > 0 { - dAtA10 := make([]byte, len(m.RowIDs)*10) - var j9 int + dAtA13 := make([]byte, len(m.RowIDs)*10) + var j12 int for _, num := range m.RowIDs { for num >= 1<<7 { - dAtA10[j9] = uint8(uint64(num)&0x7f | 0x80) + dAtA13[j12] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j9++ + j12++ } - dAtA10[j9] = uint8(num) - j9++ + dAtA13[j12] = uint8(num) + j12++ } dAtA[i] = 0x22 i++ - i = encodeVarintPublic(dAtA, i, uint64(j9)) - i += copy(dAtA[i:], dAtA10[:j9]) + i = encodeVarintPublic(dAtA, i, uint64(j12)) + i += copy(dAtA[i:], dAtA13[:j12]) } if len(m.ColumnIDs) > 0 { - dAtA12 := make([]byte, len(m.ColumnIDs)*10) - var j11 int + dAtA15 := make([]byte, len(m.ColumnIDs)*10) + var j14 int for _, num := range m.ColumnIDs { for num >= 1<<7 { - dAtA12[j11] = uint8(uint64(num)&0x7f | 0x80) + dAtA15[j14] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j11++ + j14++ } - dAtA12[j11] = uint8(num) - j11++ + dAtA15[j14] = uint8(num) + j14++ } dAtA[i] = 0x2a i++ - i = encodeVarintPublic(dAtA, i, uint64(j11)) - i += copy(dAtA[i:], dAtA12[:j11]) + i = encodeVarintPublic(dAtA, i, uint64(j14)) + i += copy(dAtA[i:], dAtA15[:j14]) } if len(m.Timestamps) > 0 { - dAtA14 := make([]byte, len(m.Timestamps)*10) - var j13 int + dAtA17 := make([]byte, len(m.Timestamps)*10) + var j16 int for _, num1 := range m.Timestamps { num := uint64(num1) for num >= 1<<7 { - dAtA14[j13] = uint8(uint64(num)&0x7f | 0x80) + dAtA17[j16] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j13++ + j16++ } - dAtA14[j13] = uint8(num) - j13++ + dAtA17[j16] = uint8(num) + j16++ } dAtA[i] = 0x32 i++ - i = encodeVarintPublic(dAtA, i, uint64(j13)) - i += copy(dAtA[i:], dAtA14[:j13]) + i = encodeVarintPublic(dAtA, i, uint64(j16)) + i += copy(dAtA[i:], dAtA17[:j16]) } if len(m.RowKeys) > 0 { for _, s := range m.RowKeys { @@ -1346,39 +1440,39 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(m.Shard)) } if len(m.ColumnIDs) > 0 { - dAtA16 := make([]byte, len(m.ColumnIDs)*10) - var j15 int + dAtA19 := make([]byte, len(m.ColumnIDs)*10) + var j18 int for _, num := range m.ColumnIDs { for num >= 1<<7 { - dAtA16[j15] = uint8(uint64(num)&0x7f | 0x80) + dAtA19[j18] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j15++ + j18++ } - dAtA16[j15] = uint8(num) - j15++ + dAtA19[j18] = uint8(num) + j18++ } dAtA[i] = 0x2a i++ - i = encodeVarintPublic(dAtA, i, uint64(j15)) - i += copy(dAtA[i:], dAtA16[:j15]) + i = encodeVarintPublic(dAtA, i, uint64(j18)) + i += copy(dAtA[i:], dAtA19[:j18]) } if len(m.Values) > 0 { - dAtA18 := make([]byte, len(m.Values)*10) - var j17 int + dAtA21 := make([]byte, len(m.Values)*10) + var j20 int for _, num1 := range m.Values { num := uint64(num1) for num >= 1<<7 { - dAtA18[j17] = uint8(uint64(num)&0x7f | 0x80) + dAtA21[j20] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j17++ + j20++ } - dAtA18[j17] = uint8(num) - j17++ + dAtA21[j20] = uint8(num) + j20++ } dAtA[i] = 0x32 i++ - i = encodeVarintPublic(dAtA, i, uint64(j17)) - i += copy(dAtA[i:], dAtA18[:j17]) + i = encodeVarintPublic(dAtA, i, uint64(j20)) + i += copy(dAtA[i:], dAtA21[:j20]) } if len(m.ColumnKeys) > 0 { for _, s := range m.ColumnKeys { @@ -1432,6 +1526,25 @@ func (m *Row) Size() (n int) { return n } +func (m *RowIdentifiers) Size() (n int) { + var l int + _ = l + if len(m.Rows) > 0 { + l = 0 + for _, e := range m.Rows { + l += sovPublic(uint64(e)) + } + n += 1 + sovPublic(uint64(l)) + l + } + if len(m.Keys) > 0 { + for _, s := range m.Keys { + l = len(s) + n += 1 + l + sovPublic(uint64(l)) + } + } + return n +} + func (m *Pair) Size() (n int) { var l int _ = l @@ -1650,6 +1763,10 @@ func (m *QueryResult) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } + if m.RowIdentifiers != nil { + l = m.RowIdentifiers.Size() + n += 1 + l + sovPublic(uint64(l)) + } return n } @@ -1925,6 +2042,147 @@ func (m *Row) Unmarshal(dAtA []byte) error { } return nil } +func (m *RowIdentifiers) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RowIdentifiers: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RowIdentifiers: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Rows = append(m.Rows, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Rows = append(m.Rows, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field Rows", wireType) + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Keys", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Keys = append(m.Keys, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPublic(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPublic + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *Pair) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -3461,6 +3719,39 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 9: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RowIdentifiers", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.RowIdentifiers == nil { + m.RowIdentifiers = &RowIdentifiers{} + } + if err := m.RowIdentifiers.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -4241,53 +4532,56 @@ var ( func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ - // 764 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd3, 0x4a, - 0x14, 0xbe, 0x13, 0x3b, 0x89, 0x73, 0xd2, 0xe4, 0x56, 0xa3, 0xde, 0x5e, 0x0b, 0x55, 0xc1, 0xb2, - 0x10, 0xf2, 0x2a, 0x95, 0x82, 0xd4, 0x25, 0x88, 0xfe, 0xa1, 0xa8, 0x50, 0xc1, 0xb4, 0x14, 0xb1, - 0x74, 0x9b, 0x51, 0x6b, 0xc9, 0xf1, 0x18, 0xff, 0x28, 0xcd, 0x5b, 0x20, 0xb1, 0xe1, 0x11, 0x58, - 0xf0, 0x20, 0x5d, 0xf2, 0x08, 0x50, 0x5e, 0x04, 0xcd, 0x19, 0x4f, 0xc6, 0x49, 0x51, 0xc5, 0x82, - 0x9d, 0xbf, 0xef, 0xcc, 0x39, 0xf9, 0xce, 0x6f, 0x60, 0x2d, 0x2d, 0xcf, 0xe3, 0xe8, 0x62, 0x98, - 0x66, 0xa2, 0x10, 0xd4, 0x89, 0x92, 0x82, 0x67, 0x49, 0x18, 0xfb, 0xef, 0xc1, 0x62, 0x62, 0x46, - 0x5d, 0x68, 0xef, 0x89, 0xb8, 0x9c, 0x26, 0xb9, 0x4b, 0x3c, 0x2b, 0xb0, 0x99, 0x86, 0xf4, 0x11, - 0x34, 0x9f, 0x17, 0x45, 0x96, 0xbb, 0x0d, 0xcf, 0x0a, 0xba, 0xa3, 0xfe, 0x50, 0xbb, 0x0e, 0x25, - 0xcd, 0x94, 0x91, 0x52, 0xb0, 0x8f, 0xf8, 0x3c, 0x77, 0x2d, 0xcf, 0x0a, 0x3a, 0x0c, 0xbf, 0xfd, - 0xa7, 0x60, 0xbf, 0x0e, 0xa3, 0x8c, 0xf6, 0xa1, 0x31, 0xde, 0x77, 0x89, 0x47, 0x02, 0x9b, 0x35, - 0xc6, 0xfb, 0x74, 0x03, 0x9a, 0x7b, 0xa2, 0x4c, 0x0a, 0xb7, 0x81, 0x94, 0x02, 0x74, 0x1d, 0xac, - 0x23, 0x3e, 0x77, 0x2d, 0x8f, 0x04, 0x1d, 0x26, 0x3f, 0xfd, 0x1d, 0x70, 0x0e, 0x23, 0x1e, 0x4f, - 0xa4, 0xbe, 0x0d, 0x68, 0xe2, 0x37, 0x86, 0xe9, 0x30, 0x05, 0x24, 0xcb, 0xc4, 0x6c, 0xbc, 0xaf, - 0x23, 0x21, 0xf0, 0x5f, 0x02, 0xbc, 0xc8, 0x44, 0x99, 0xaa, 0xb8, 0x01, 0x34, 0x11, 0x61, 0x5e, - 0xdd, 0x11, 0x35, 0xfa, 0x75, 0x70, 0xa6, 0x1e, 0xfc, 0x5e, 0x97, 0x3f, 0x02, 0xe7, 0x2c, 0x8c, - 0x17, 0x1a, 0xcf, 0xc2, 0x18, 0x35, 0x58, 0x4c, 0x7e, 0x2e, 0xfb, 0x58, 0xda, 0xe7, 0x2d, 0x58, - 0xbb, 0x51, 0x61, 0xe4, 0x91, 0x9a, 0x3c, 0xfa, 0x00, 0x1c, 0x55, 0xdb, 0x85, 0xee, 0x05, 0xa6, - 0x5b, 0xd0, 0x39, 0x8d, 0xa6, 0x3c, 0x2f, 0xc2, 0x69, 0x8a, 0xa5, 0xb0, 0x98, 0x21, 0xfc, 0x77, - 0xd0, 0x53, 0x2f, 0x65, 0xcd, 0x4f, 0x78, 0x71, 0xa7, 0xb2, 0x7f, 0xd6, 0xab, 0xbb, 0x95, 0xfe, - 0x42, 0xc0, 0x96, 0x36, 0x6d, 0x22, 0x0b, 0x93, 0x6c, 0xec, 0xe9, 0x3c, 0xe5, 0x95, 0x52, 0xfc, - 0xa6, 0x1e, 0x74, 0x4f, 0x8a, 0x2c, 0x4a, 0x2e, 0xcf, 0xc2, 0xb8, 0xe4, 0x55, 0xa0, 0x3a, 0x25, - 0x73, 0x1c, 0x27, 0x85, 0x32, 0xdb, 0x98, 0xc6, 0x02, 0xcb, 0x1c, 0x77, 0x85, 0x88, 0x95, 0xb1, - 0xe9, 0x91, 0xc0, 0x61, 0x86, 0xa0, 0x03, 0x80, 0xc3, 0x58, 0x84, 0x95, 0x6f, 0xcb, 0x23, 0x01, - 0x61, 0x35, 0xc6, 0xdf, 0x86, 0xb6, 0x54, 0xfa, 0x2a, 0x4c, 0x4d, 0xb6, 0xe4, 0x9e, 0x6c, 0xfd, - 0x1b, 0x02, 0x6b, 0x6f, 0x4a, 0x9e, 0xcd, 0x19, 0xff, 0x50, 0xf2, 0x1c, 0xbb, 0x82, 0x58, 0x8f, - 0x12, 0x02, 0xba, 0x09, 0xad, 0x93, 0xab, 0x30, 0x9b, 0xa8, 0xda, 0xd9, 0xac, 0x42, 0x32, 0x57, - 0x53, 0xf3, 0x1c, 0x73, 0x75, 0x58, 0x9d, 0x92, 0x9e, 0x8c, 0x4f, 0x45, 0xa1, 0x93, 0xa9, 0x10, - 0x0d, 0xe0, 0xdf, 0x83, 0xeb, 0x8b, 0xb8, 0x9c, 0x70, 0x26, 0x66, 0xca, 0xbb, 0x85, 0x0f, 0x56, - 0x69, 0xfa, 0x18, 0xfa, 0x15, 0xa5, 0x77, 0xb0, 0x8d, 0x0f, 0x57, 0x58, 0xff, 0x13, 0x81, 0x5e, - 0x95, 0x4a, 0x9e, 0x8a, 0x24, 0xe7, 0xb2, 0x5f, 0x07, 0x59, 0xa6, 0xfb, 0x75, 0x90, 0x65, 0x74, - 0x1b, 0xda, 0x8c, 0xe7, 0x65, 0x5c, 0xe8, 0x21, 0xf8, 0xcf, 0x94, 0x45, 0xfb, 0x96, 0x71, 0xc1, - 0xf4, 0x2b, 0xfa, 0x0c, 0xfa, 0x4b, 0x43, 0xa5, 0x76, 0xb8, 0x3b, 0xfa, 0xdf, 0xf8, 0x2d, 0xd9, - 0xd9, 0xca, 0x73, 0xff, 0x63, 0x03, 0xba, 0xb5, 0xc8, 0xf4, 0x21, 0x5e, 0x14, 0xd4, 0xd4, 0x1d, - 0xf5, 0x4c, 0x14, 0xb9, 0x69, 0x78, 0x6b, 0xd6, 0x80, 0x1c, 0x57, 0xf3, 0x44, 0x8e, 0x65, 0x17, - 0xe5, 0x95, 0xd0, 0x3f, 0x5b, 0xeb, 0xa2, 0xa4, 0x99, 0x32, 0xe2, 0x7d, 0xba, 0x0a, 0x93, 0x4b, - 0x3e, 0xc1, 0x79, 0x72, 0x98, 0x86, 0x74, 0x68, 0xf6, 0x13, 0x1b, 0xb0, 0xb4, 0xe2, 0xda, 0xc2, - 0xcc, 0x0e, 0xeb, 0x81, 0x96, 0xbd, 0xe8, 0x55, 0x03, 0x2d, 0x5b, 0x28, 0x77, 0x53, 0x16, 0x1e, - 0x9b, 0xaf, 0x10, 0xdd, 0x81, 0xae, 0xb9, 0x24, 0xb9, 0xeb, 0xa0, 0xc2, 0x0d, 0x13, 0xde, 0x18, - 0x59, 0xfd, 0xa1, 0xff, 0x83, 0x40, 0x6f, 0x3c, 0x4d, 0x45, 0x56, 0xd4, 0x86, 0x6e, 0x9c, 0x4c, - 0xf8, 0xb5, 0x1e, 0x3a, 0x04, 0xe6, 0xaa, 0x35, 0x56, 0xae, 0x1a, 0x0e, 0x1f, 0x0e, 0x9b, 0xcd, - 0x14, 0xa8, 0x69, 0xb4, 0x97, 0x34, 0x6e, 0x41, 0x47, 0x9f, 0x8f, 0xdc, 0x6d, 0xa2, 0xc9, 0x10, - 0x72, 0x9d, 0x16, 0xf7, 0x43, 0xce, 0x9f, 0x15, 0x58, 0xac, 0xc6, 0xc8, 0xba, 0x32, 0x31, 0xc3, - 0xd3, 0xdd, 0xc6, 0xd3, 0xad, 0xa1, 0xf4, 0x54, 0x61, 0xd0, 0xe8, 0xa0, 0xb1, 0xc6, 0xf8, 0x5f, - 0x09, 0x50, 0x95, 0x23, 0x2e, 0xe6, 0xdf, 0x4b, 0xf4, 0xfe, 0x84, 0x36, 0xa1, 0x85, 0xbf, 0xa7, - 0x93, 0xa9, 0xd0, 0x8a, 0xdc, 0xf6, 0xaa, 0xdc, 0xdd, 0xf5, 0x9b, 0xdb, 0x01, 0xf9, 0x76, 0x3b, - 0x20, 0xdf, 0x6f, 0x07, 0xe4, 0xf3, 0xcf, 0xc1, 0x3f, 0xe7, 0x2d, 0xfc, 0x2b, 0x7c, 0xf2, 0x2b, - 0x00, 0x00, 0xff, 0xff, 0x25, 0x1f, 0x0d, 0xa8, 0x1a, 0x07, 0x00, 0x00, + // 804 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6e, 0xdb, 0x46, + 0x14, 0xed, 0x88, 0x94, 0x44, 0x5d, 0x59, 0xaa, 0x31, 0x70, 0x5d, 0xa2, 0x30, 0x54, 0x82, 0x28, + 0x0a, 0xae, 0x64, 0x40, 0x05, 0x8c, 0xae, 0xfa, 0xf0, 0xab, 0x10, 0xdc, 0x1a, 0xcd, 0xd8, 0x71, + 0x90, 0x25, 0x6d, 0x4d, 0x6c, 0x02, 0x14, 0x87, 0xe1, 0x03, 0xb2, 0xbe, 0x23, 0x9b, 0x7c, 0x42, + 0x16, 0xf9, 0x10, 0x2f, 0x83, 0x7c, 0x41, 0xe2, 0xfc, 0x48, 0x30, 0x77, 0x38, 0x1a, 0x8a, 0x0e, + 0x8c, 0x2c, 0xb2, 0x9b, 0x73, 0x5f, 0xbc, 0xe7, 0xbe, 0x08, 0x1b, 0x69, 0x79, 0x19, 0x47, 0x57, + 0xe3, 0x34, 0x13, 0x85, 0xa0, 0x4e, 0x94, 0x14, 0x3c, 0x4b, 0xc2, 0xd8, 0x7f, 0x0e, 0x16, 0x13, + 0x0b, 0xea, 0x42, 0xf7, 0x40, 0xc4, 0xe5, 0x3c, 0xc9, 0x5d, 0xe2, 0x59, 0x81, 0xcd, 0x34, 0xa4, + 0xbf, 0x40, 0xfb, 0xef, 0xa2, 0xc8, 0x72, 0xb7, 0xe5, 0x59, 0x41, 0x7f, 0x32, 0x1c, 0x6b, 0xd7, + 0xb1, 0x14, 0x33, 0xa5, 0xa4, 0x14, 0xec, 0x13, 0xbe, 0xcc, 0x5d, 0xcb, 0xb3, 0x82, 0x1e, 0xc3, + 0xb7, 0xff, 0x3b, 0x0c, 0x99, 0x58, 0x4c, 0x67, 0x3c, 0x29, 0xa2, 0x17, 0x11, 0x57, 0x56, 0x4c, + 0x2c, 0xf4, 0x27, 0xf0, 0xbd, 0xf2, 0x6c, 0xd5, 0x3c, 0xff, 0x00, 0xfb, 0xff, 0x30, 0xca, 0xe8, + 0x10, 0x5a, 0xd3, 0x43, 0x97, 0x78, 0x24, 0xb0, 0x59, 0x6b, 0x7a, 0x48, 0xb7, 0xa0, 0x7d, 0x20, + 0xca, 0xa4, 0x70, 0x5b, 0x28, 0x52, 0x80, 0x6e, 0x82, 0x75, 0xc2, 0x97, 0xae, 0xe5, 0x91, 0xa0, + 0xc7, 0xe4, 0xd3, 0xdf, 0x03, 0xe7, 0x38, 0xe2, 0xf1, 0x4c, 0x32, 0xdb, 0x82, 0x36, 0xbe, 0x31, + 0x4c, 0x8f, 0x29, 0x20, 0xa5, 0x32, 0xb7, 0x43, 0x1d, 0x09, 0x81, 0xff, 0x2f, 0xc0, 0x3f, 0x99, + 0x28, 0x53, 0x15, 0x37, 0x80, 0x36, 0x22, 0x4c, 0xb7, 0x3f, 0xa1, 0x86, 0xb9, 0x0e, 0xce, 0x94, + 0xc1, 0x97, 0xf3, 0xf2, 0x27, 0xe0, 0x5c, 0x84, 0xf1, 0x2a, 0xc7, 0x8b, 0x30, 0xc6, 0x1c, 0x2c, + 0x26, 0x9f, 0xeb, 0x3e, 0x96, 0xf6, 0x79, 0x0a, 0xd6, 0x7e, 0x54, 0x98, 0xf4, 0x48, 0x2d, 0x3d, + 0xfa, 0x13, 0x38, 0xaa, 0x2b, 0xab, 0xbc, 0x57, 0x98, 0xee, 0x40, 0xef, 0x3c, 0x9a, 0xf3, 0xbc, + 0x08, 0xe7, 0x29, 0x96, 0xc2, 0x62, 0x46, 0xe0, 0x3f, 0x83, 0x81, 0xb2, 0x94, 0xdd, 0x3a, 0xe3, + 0xc5, 0x83, 0xca, 0x7e, 0x5d, 0x97, 0x1f, 0x56, 0xfa, 0x0d, 0x01, 0x5b, 0xea, 0xb4, 0x8a, 0xac, + 0x54, 0xb2, 0xb1, 0xe7, 0xcb, 0x94, 0x57, 0x99, 0xe2, 0x9b, 0x7a, 0xd0, 0x3f, 0x2b, 0xb2, 0x28, + 0xb9, 0xbe, 0x08, 0xe3, 0x92, 0x57, 0x81, 0xea, 0x22, 0xc9, 0x71, 0x9a, 0x14, 0x4a, 0x6d, 0x23, + 0x8d, 0x15, 0x96, 0x1c, 0xf7, 0x85, 0x88, 0x95, 0xb2, 0xed, 0x91, 0xc0, 0x61, 0x46, 0x40, 0x47, + 0x00, 0xc7, 0xb1, 0x08, 0x2b, 0xdf, 0x8e, 0x47, 0x02, 0xc2, 0x6a, 0x12, 0x7f, 0x17, 0xba, 0x32, + 0xd3, 0xff, 0xc2, 0xd4, 0xb0, 0x25, 0x8f, 0xb0, 0xf5, 0xef, 0x08, 0x6c, 0x3c, 0x29, 0x79, 0xb6, + 0x64, 0xfc, 0x65, 0xc9, 0x73, 0xec, 0x0a, 0x62, 0x3d, 0x4a, 0x08, 0xe8, 0x36, 0x74, 0xce, 0x6e, + 0xc2, 0x6c, 0xa6, 0x6a, 0x67, 0xb3, 0x0a, 0x49, 0xae, 0xa6, 0xe6, 0x39, 0x72, 0x75, 0x58, 0x5d, + 0x24, 0x3d, 0x19, 0x9f, 0x8b, 0x42, 0x93, 0xa9, 0x10, 0x0d, 0xe0, 0xfb, 0xa3, 0xdb, 0xab, 0xb8, + 0x9c, 0x71, 0x26, 0x16, 0xca, 0xbb, 0x83, 0x06, 0x4d, 0x31, 0xfd, 0x15, 0x86, 0x95, 0x48, 0x6f, + 0x6f, 0x17, 0x0d, 0x1b, 0x52, 0xff, 0x15, 0x81, 0x41, 0x45, 0x25, 0x4f, 0x45, 0x92, 0x73, 0xd9, + 0xaf, 0xa3, 0x2c, 0xd3, 0xfd, 0x3a, 0xca, 0x32, 0xba, 0x0b, 0x5d, 0xc6, 0xf3, 0x32, 0x2e, 0xf4, + 0x10, 0xfc, 0x60, 0xca, 0xa2, 0x7d, 0xcb, 0xb8, 0x60, 0xda, 0x8a, 0xfe, 0x09, 0xc3, 0xb5, 0xa1, + 0x52, 0xdb, 0xdf, 0x9f, 0xfc, 0x68, 0xfc, 0xd6, 0xf4, 0xac, 0x61, 0xee, 0xbf, 0x6f, 0x41, 0xbf, + 0x16, 0x99, 0xfe, 0x8c, 0xb7, 0x08, 0x73, 0xea, 0x4f, 0x06, 0x26, 0x8a, 0xdc, 0x34, 0xbc, 0x52, + 0x1b, 0x40, 0x4e, 0xab, 0x79, 0x22, 0xa7, 0xb2, 0x8b, 0xf2, 0x4a, 0xe8, 0xcf, 0xd6, 0xba, 0x28, + 0xc5, 0x4c, 0x29, 0xf1, 0xb2, 0xdd, 0x84, 0xc9, 0x35, 0x9f, 0xe1, 0x3c, 0x39, 0x4c, 0x43, 0x3a, + 0x36, 0xfb, 0x89, 0x0d, 0x58, 0x5b, 0x71, 0xad, 0x61, 0x66, 0x87, 0xf5, 0x40, 0xcb, 0x5e, 0x0c, + 0xaa, 0x81, 0x96, 0x2d, 0x94, 0xbb, 0x29, 0x0b, 0x8f, 0xcd, 0x57, 0x88, 0xee, 0x41, 0xdf, 0x5c, + 0x92, 0xdc, 0x75, 0x30, 0xc3, 0x2d, 0x13, 0xde, 0x28, 0x59, 0xdd, 0x90, 0xfe, 0xd5, 0xbc, 0x99, + 0x6e, 0x0f, 0x33, 0x73, 0xd7, 0xaa, 0x51, 0xd3, 0xb3, 0x86, 0xbd, 0xff, 0x91, 0xc0, 0x60, 0x3a, + 0x4f, 0x45, 0x56, 0xd4, 0xc6, 0x76, 0x9a, 0xcc, 0xf8, 0xad, 0x1e, 0x5b, 0x04, 0xe6, 0x2e, 0xb6, + 0x1a, 0x77, 0x11, 0xc7, 0x17, 0xc7, 0xd5, 0x66, 0x0a, 0xd4, 0x58, 0xda, 0x6b, 0x2c, 0x77, 0xa0, + 0xa7, 0x0f, 0x50, 0xee, 0xb6, 0x51, 0x65, 0x04, 0x72, 0x21, 0x57, 0x17, 0x48, 0x4e, 0xb0, 0x15, + 0x58, 0xac, 0x26, 0x91, 0x9d, 0x61, 0x62, 0x81, 0xc7, 0xbf, 0x8b, 0xc7, 0x5f, 0x43, 0xe9, 0xa9, + 0xc2, 0xa0, 0xd2, 0x41, 0x65, 0x4d, 0xe2, 0xbf, 0x25, 0x40, 0x15, 0x47, 0x5c, 0xed, 0x6f, 0x47, + 0xf4, 0x71, 0x42, 0xdb, 0xd0, 0xc1, 0xef, 0x69, 0x32, 0x15, 0x6a, 0xa4, 0xdb, 0x6d, 0xa6, 0xbb, + 0xbf, 0x79, 0x77, 0x3f, 0x22, 0xef, 0xee, 0x47, 0xe4, 0xc3, 0xfd, 0x88, 0xbc, 0xfe, 0x34, 0xfa, + 0xee, 0xb2, 0x83, 0xbf, 0xe1, 0xdf, 0x3e, 0x07, 0x00, 0x00, 0xff, 0xff, 0xa6, 0x62, 0xa8, 0x25, + 0x96, 0x07, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index c455ab1e6..a102f1088 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -8,6 +8,12 @@ message Row { repeated Attr Attrs = 2; } +message RowIdentifiers { + repeated uint64 Rows = 1; + repeated string Keys = 2; + //repeated Attr Attrs = 3; +} + message Pair { uint64 ID = 1; string Key = 3; @@ -78,6 +84,7 @@ message QueryResult { ValCount ValCount = 5; repeated uint64 RowIDs = 7; repeated GroupCount GroupCounts = 8; + RowIdentifiers RowIdentifiers = 9; } message ImportRequest { From 0ab3e72520b5a98f8e962b02a2a721afde269010 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 18 Sep 2018 14:09:52 -0500 Subject: [PATCH 7/7] refactor mergeGroupCounts function --- executor.go | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/executor.go b/executor.go index 0fb6f53a7..858c88d9e 100644 --- a/executor.go +++ b/executor.go @@ -834,20 +834,13 @@ type GroupCount struct { } func mergeGroupCounts(gc, other []GroupCount) []GroupCount { - m := make(map[string]struct { - i int - count uint64 - }) + m := make(map[string]int) for i := range gc { - m[uniqueGroupString(gc[i].Group)] = struct { - i int - count uint64 - }{i, gc[i].Count} + m[uniqueGroupString(gc[i].Group)] = i } for i := range other { - o, found := m[uniqueGroupString(other[i].Group)] - if found { - gc[o.i].Count += other[i].Count + if idx, found := m[uniqueGroupString(other[i].Group)]; found { + gc[idx].Count += other[i].Count } else { gc = append(gc, other[i]) }