From 1fa2a65c0b7d751c5118ff86e31802f4260ccde5 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 21 Dec 2020 19:46:26 -0600 Subject: [PATCH 1/7] Test to demonstrate failure running Distinct --- executor_test.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/executor_test.go b/executor_test.go index a20d60495..373d43bdf 100644 --- a/executor_test.go +++ b/executor_test.go @@ -5487,6 +5487,34 @@ func sameStringSlice(x, y []string) bool { return len(diff) == 0 } +func TestExecutor_Execute_DistinctFailure(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + c.CreateField(t, "i", pilosa.IndexOptions{}, "general") + c.CreateField(t, "i", pilosa.IndexOptions{}, "v", pilosa.OptFieldTypeInt(0, 1000)) + c.ImportBits(t, "i", "general", [][2]uint64{ + {10, 0}, + {10, 1}, + {10, ShardWidth + 1}, + {11, 2}, + {11, ShardWidth + 2}, + {12, 2}, + {12, ShardWidth + 2}, + }) + + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(0, v=10)`}); err != nil { + t.Fatal(err) + } else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1, v=100)`}); err != nil { + t.Fatal(err) + } + + t.Run("BasicDistinct", func(t *testing.T) { + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Distinct(field="v")`}); err != nil { + t.Fatalf("unexpected error: \"%v\"", err) + } + }) +} + func TestExecutor_Execute_GroupBy(t *testing.T) { groupByTest := func(t *testing.T, clusterSize int) { c := test.MustRunCluster(t, 1) From 01e6781abd5e1c7229e58aa405bd9f9eae7a5ff2 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Mon, 18 Jan 2021 13:43:53 -0700 Subject: [PATCH 2/7] Update benchmarks to use expvar URL list --- scripts/etc/gloat/gh.1d.yml | 6 ++++-- scripts/etc/gloat/gh.1m.yml | 6 ++++-- scripts/etc/gloat/gh.1w.yml | 6 ++++-- scripts/etc/gloat/query.count.yml | 3 ++- scripts/etc/gloat/query.difference.yml | 3 ++- scripts/etc/gloat/query.groupby.yml | 3 ++- scripts/etc/gloat/query.intersect.yml | 3 ++- scripts/etc/gloat/query.row-bsi.yml | 3 ++- scripts/etc/gloat/query.row-range.yml | 3 ++- scripts/etc/gloat/query.row.yml | 3 ++- scripts/etc/gloat/query.topk.yml | 3 ++- scripts/etc/gloat/query.union.yml | 3 ++- scripts/etc/gloat/query.xor.yml | 3 ++- 13 files changed, 32 insertions(+), 16 deletions(-) diff --git a/scripts/etc/gloat/gh.1d.yml b/scripts/etc/gloat/gh.1d.yml index 93658c44a..f064b2f7a 100644 --- a/scripts/etc/gloat/gh.1d.yml +++ b/scripts/etc/gloat/gh.1d.yml @@ -1,7 +1,9 @@ name: "GitHub Import Load Testing (1 day)" main: "pilosa server --data-dir ${TMPDIR} --txsrc ${TXSRC}" -load: "molecula-consumer-github -i events -d id --record-type event --batch-size=100000 --start-time 2020-01-01T00:00:00Z --end-time 2020-01-01T23:00:00Z --cache-dir .githubarchive" +load: "molecula-consumer-github -i events -d id --record-type event --batch-size=100000 --start-time 2020-01-01T00:00:00Z --end-time 2020-01-01T23:00:00Z --cache-dir ~/.githubarchive" health_url: "http://localhost:10101/status" -debug_url: "http://localhost:10101/debug" +vars_urls: + - http://localhost:10101/debug/vars + - http://localhost:7070/debug/vars diff --git a/scripts/etc/gloat/gh.1m.yml b/scripts/etc/gloat/gh.1m.yml index 9b4dacf4a..5be52cb7a 100644 --- a/scripts/etc/gloat/gh.1m.yml +++ b/scripts/etc/gloat/gh.1m.yml @@ -1,7 +1,9 @@ name: "GitHub Import Load Testing (1 month)" main: "pilosa server --data-dir ${TMPDIR} --txsrc ${TXSRC}" -load: "molecula-consumer-github -i events -d id --record-type event --batch-size=100000 --start-time 2020-01-01T00:00:00Z --end-time 2020-01-31T23:00:00Z --cache-dir .githubarchive" +load: "molecula-consumer-github -i events -d id --record-type event --batch-size=100000 --start-time 2020-01-01T00:00:00Z --end-time 2020-01-31T23:00:00Z --cache-dir ~/.githubarchive" health_url: "http://localhost:10101/status" -debug_url: "http://localhost:10101/debug" +vars_urls: + - http://localhost:10101/debug/vars + - http://localhost:7070/debug/vars diff --git a/scripts/etc/gloat/gh.1w.yml b/scripts/etc/gloat/gh.1w.yml index 23343ae45..084492919 100644 --- a/scripts/etc/gloat/gh.1w.yml +++ b/scripts/etc/gloat/gh.1w.yml @@ -1,7 +1,9 @@ name: "GitHub Import Load Testing (1 week)" main: "pilosa server --data-dir ${TMPDIR} --txsrc ${TXSRC}" -load: "molecula-consumer-github -i events -d id --record-type event --batch-size=100000 --start-time 2020-01-01T00:00:00Z --end-time 2020-01-06T23:00:00Z --cache-dir .githubarchive" +load: "molecula-consumer-github -i events -d id --record-type event --batch-size=100000 --start-time 2020-01-01T00:00:00Z --end-time 2020-01-06T23:00:00Z --cache-dir ~/.githubarchive" health_url: "http://localhost:10101/status" -debug_url: "http://localhost:10101/debug" +vars_urls: + - http://localhost:10101/debug/vars + - http://localhost:7070/debug/vars diff --git a/scripts/etc/gloat/query.count.yml b/scripts/etc/gloat/query.count.yml index 59b2a766f..5690c13a8 100644 --- a/scripts/etc/gloat/query.count.yml +++ b/scripts/etc/gloat/query.count.yml @@ -6,4 +6,5 @@ load: "pilosa-bench -type count -rate 100 -n 3000" health_url: "http://localhost:10101/status" health_regexp: "NORMAL" -debug_url: "http://localhost:10101/debug" +vars_urls: + - http://localhost:10101/debug/vars diff --git a/scripts/etc/gloat/query.difference.yml b/scripts/etc/gloat/query.difference.yml index cac32117c..3a73a8e46 100644 --- a/scripts/etc/gloat/query.difference.yml +++ b/scripts/etc/gloat/query.difference.yml @@ -6,4 +6,5 @@ load: "pilosa-bench -type difference -rate 10 -n 300" health_url: "http://localhost:10101/status" health_regexp: "NORMAL" -debug_url: "http://localhost:10101/debug" +vars_urls: + - http://localhost:10101/debug/vars diff --git a/scripts/etc/gloat/query.groupby.yml b/scripts/etc/gloat/query.groupby.yml index 07dfc5ef9..a5c4c5d00 100644 --- a/scripts/etc/gloat/query.groupby.yml +++ b/scripts/etc/gloat/query.groupby.yml @@ -6,4 +6,5 @@ load: "pilosa-bench -type groupby -rate 100 -n 3000" health_url: "http://localhost:10101/status" health_regexp: "NORMAL" -debug_url: "http://localhost:10101/debug" +vars_urls: + - http://localhost:10101/debug/vars diff --git a/scripts/etc/gloat/query.intersect.yml b/scripts/etc/gloat/query.intersect.yml index 2ae1c3660..00bbfbd2e 100644 --- a/scripts/etc/gloat/query.intersect.yml +++ b/scripts/etc/gloat/query.intersect.yml @@ -6,4 +6,5 @@ load: "pilosa-bench -type intersect -rate 100 -n 3000" health_url: "http://localhost:10101/status" health_regexp: "NORMAL" -debug_url: "http://localhost:10101/debug" +vars_urls: + - http://localhost:10101/debug/vars diff --git a/scripts/etc/gloat/query.row-bsi.yml b/scripts/etc/gloat/query.row-bsi.yml index dfb9dac20..04fe472eb 100644 --- a/scripts/etc/gloat/query.row-bsi.yml +++ b/scripts/etc/gloat/query.row-bsi.yml @@ -6,4 +6,5 @@ load: "pilosa-bench -type row -rate 100 -n 3000" health_url: "http://localhost:10101/status" health_regexp: "NORMAL" -debug_url: "http://localhost:10101/debug" +vars_urls: + - http://localhost:10101/debug/vars diff --git a/scripts/etc/gloat/query.row-range.yml b/scripts/etc/gloat/query.row-range.yml index eb21834cf..f05b5cb4c 100644 --- a/scripts/etc/gloat/query.row-range.yml +++ b/scripts/etc/gloat/query.row-range.yml @@ -6,4 +6,5 @@ load: "pilosa-bench -type row-range -rate 10 -n 300 -from 2020-01-01T00:00:00Z - health_url: "http://localhost:10101/status" health_regexp: "NORMAL" -debug_url: "http://localhost:10101/debug" +vars_urls: + - http://localhost:10101/debug/vars diff --git a/scripts/etc/gloat/query.row.yml b/scripts/etc/gloat/query.row.yml index 8a8ec812c..142f12004 100644 --- a/scripts/etc/gloat/query.row.yml +++ b/scripts/etc/gloat/query.row.yml @@ -6,4 +6,5 @@ load: "pilosa-bench -type row -rate 100 -n 3000" health_url: "http://localhost:10101/status" health_regexp: "NORMAL" -debug_url: "http://localhost:10101/debug" +vars_urls: + - http://localhost:10101/debug/vars diff --git a/scripts/etc/gloat/query.topk.yml b/scripts/etc/gloat/query.topk.yml index 44230ec95..10da63427 100644 --- a/scripts/etc/gloat/query.topk.yml +++ b/scripts/etc/gloat/query.topk.yml @@ -6,4 +6,5 @@ load: "pilosa-bench -type row-range -rate 10 -n 300 -from 2020-01-01T00:00:00Z - health_url: "http://localhost:10101/status" health_regexp: "NORMAL" -debug_url: "http://localhost:10101/debug" +vars_urls: + - http://localhost:10101/debug/vars diff --git a/scripts/etc/gloat/query.union.yml b/scripts/etc/gloat/query.union.yml index 3620fc0fa..a2270d974 100644 --- a/scripts/etc/gloat/query.union.yml +++ b/scripts/etc/gloat/query.union.yml @@ -6,4 +6,5 @@ load: "pilosa-bench -type union -rate 10 -n 300" health_url: "http://localhost:10101/status" health_regexp: "NORMAL" -debug_url: "http://localhost:10101/debug" +vars_urls: + - http://localhost:10101/debug/vars diff --git a/scripts/etc/gloat/query.xor.yml b/scripts/etc/gloat/query.xor.yml index 3c81f5f83..f2eb857b8 100644 --- a/scripts/etc/gloat/query.xor.yml +++ b/scripts/etc/gloat/query.xor.yml @@ -6,4 +6,5 @@ load: "pilosa-bench -type xor -rate 10 -n 300" health_url: "http://localhost:10101/status" health_regexp: "NORMAL" -debug_url: "http://localhost:10101/debug" +vars_urls: + - http://localhost:10101/debug/vars From 4ebf6f6ff7c9f4d4a92f0ae5c2d714a78ade3baf Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 14 Jan 2021 13:26:02 -0600 Subject: [PATCH 3/7] Customize serialization of []GroupCount based on aggregate type/presence --- executor.go | 96 +++++++++++++++++++++++++++++++++++++++++--------- server/grpc.go | 10 ++++-- sql/select.go | 5 ++- 3 files changed, 92 insertions(+), 19 deletions(-) diff --git a/executor.go b/executor.go index 722f63d32..deced589c 100644 --- a/executor.go +++ b/executor.go @@ -21,6 +21,7 @@ import ( "math" "math/bits" "sort" + "strconv" "strings" "sync" "time" @@ -312,6 +313,8 @@ func (e *executor) safeCopy(resp QueryResponse) (out QueryResponse) { out.Results = append(out.Results, x) case []GroupCount: out.Results = append(out.Results, x) + case *GroupCounts: + out.Results = append(out.Results, x) case ExtractedTable: out.Results = append(out.Results, x) case ExtractedIDMatrix: @@ -2702,7 +2705,7 @@ func getSorter(sortSpec string) (*groupCountSorter, error) { return gcs, nil } -func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *execOptions) ([]GroupCount, error) { +func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *execOptions) (*GroupCounts, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGroupBy") defer span.Finish() // validate call @@ -2787,7 +2790,7 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c return nil, errors.Wrap(err, "getting rows for ") } if len(childRows[i]) == 0 { // there are no results because this field has no values. - return []GroupCount{}, nil + return &GroupCounts{}, nil } } } @@ -2907,7 +2910,20 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c } - return results, nil + ret := NewGroupCounts() + if aggregate != nil { + switch aggregate.Name { + case "Sum": + ret.aggregateType = sumAggregate + case "Count": + ret.aggregateType = distinctAggregate + default: + ret.aggregateType = nilAggregate + } + } + ret.Groups = results + + return ret, nil } func applyLimitAndOffsetToGroupByResult(c *pql.Call, results []GroupCount) ([]GroupCount, error) { @@ -2992,17 +3008,32 @@ func (fr FieldRow) String() string { return fmt.Sprintf("%s.%d.%s", fr.Field, fr.RowID, fr.RowKey) } +type aggregateType int + +const ( + nilAggregate aggregateType = 0 + sumAggregate aggregateType = 1 + distinctAggregate aggregateType = 2 +) + // GroupCounts is a list of GroupCount. -type GroupCounts []GroupCount +type GroupCounts struct { + Groups []GroupCount + aggregateType aggregateType +} + +func NewGroupCounts() *GroupCounts { + return &GroupCounts{} +} // ToTable implements the ToTabler interface. -func (g GroupCounts) ToTable() (*pb.TableResponse, error) { - return pb.RowsToTable(&g, len(g)) +func (g *GroupCounts) ToTable() (*pb.TableResponse, error) { + return pb.RowsToTable(g, len(g.Groups)) } // ToRows implements the ToRowser interface. -func (g GroupCounts) ToRows(callback func(*pb.RowResponse) error) error { - for i, gc := range g { +func (g *GroupCounts) ToRows(callback func(*pb.RowResponse) error) error { + for i, gc := range g.Groups { var ci []*pb.ColumnInfo if i == 0 { for _, fieldRow := range gc.Group { @@ -3042,11 +3073,44 @@ func (g GroupCounts) ToRows(callback func(*pb.RowResponse) error) error { return nil } +// MarshalJSON makes GroupCounts satisfy interface json.Marshaler and +// customizes the JSON output of the aggregate field label. +func (g *GroupCounts) MarshalJSON() ([]byte, error) { + if len(g.Groups) == 0 { + return []byte("[]"), nil + } + var aggregateLabel string + switch g.aggregateType { + case sumAggregate: + aggregateLabel = "sum" + case distinctAggregate: + aggregateLabel = "distinct" // TODO: not sure the best name here + } + var out = []byte("[") + for _, group := range g.Groups { + groupJson, err := json.Marshal(group) + if err != nil { + return nil, errors.Wrap(err, "marshaling group") + } + if g.aggregateType != nilAggregate { + // Insert `"aggregatelabel": aggregatevalue` at end of JSON, append comma + groupJson = append(groupJson[:len(groupJson)-1], []byte(",\""+aggregateLabel+"\": "+strconv.Itoa(int(group.Sum))+"},")...) + } else { + // Append comma + groupJson = append(groupJson, byte(',')) + } + out = append(out, groupJson...) + } + // replace final comma with "]" + out[len(out)-1] = byte(']') + return out, nil +} + // GroupCount represents a result item for a group by query. type GroupCount struct { Group []FieldRow `json:"group"` Count uint64 `json:"count"` - Sum int64 `json:"sum"` + Sum int64 `json:"-"` } func (g *GroupCount) Clone() (r *GroupCount) { @@ -6497,10 +6561,10 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index } } - case []GroupCount: + case GroupCounts: fieldIDs := make(map[*Field]map[uint64]struct{}) foreignIDs := make(map[*Field]map[uint64]struct{}) - for _, gl := range result { + for _, gl := range result.Groups { for _, g := range gl.Group { field := idx.Field(g.Field) if field == nil { @@ -6511,7 +6575,7 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index if fi := field.ForeignIndex(); fi != "" { m, ok := foreignIDs[field] if !ok { - m = make(map[uint64]struct{}, len(result)) + m = make(map[uint64]struct{}, len(result.Groups)) foreignIDs[field] = m } @@ -6522,7 +6586,7 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index m, ok := fieldIDs[field] if !ok { - m = make(map[uint64]struct{}, len(result)) + m = make(map[uint64]struct{}, len(result.Groups)) fieldIDs[field] = m } @@ -6549,8 +6613,8 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index foreignTranslations[field.Name()] = trans } - other := make([]GroupCount, 0) - for _, gl := range result { + other := NewGroupCounts() + for _, gl := range result.Groups { group := make([]FieldRow, len(gl.Group)) for i, g := range gl.Group { @@ -6564,7 +6628,7 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index group[i] = g } - other = append(other, GroupCount{ + other.Groups = append(other.Groups, GroupCount{ Group: group, Count: gl.Count, Sum: gl.Sum, diff --git a/server/grpc.go b/server/grpc.go index 55ad7e67d..7f189a730 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -455,8 +455,11 @@ func ToTablerWrapper(result interface{}) (pb.ToTabler, error) { toTabler, ok := result.(pb.ToTabler) if !ok { switch v := result.(type) { + // TODO: replace []pilosa.GroupCount with *pilosa.GroupCounts, handle aggregate case []pilosa.GroupCount: - toTabler = pilosa.GroupCounts(v) + gc := pilosa.NewGroupCounts() + gc.Groups = v + toTabler = gc case uint64: toTabler = ResultUint64(v) case bool: @@ -476,8 +479,11 @@ func ToRowserWrapper(result interface{}) (pb.ToRowser, error) { toRowser, ok := result.(pb.ToRowser) if !ok { switch v := result.(type) { + // TODO: replace []pilosa.GroupCount with *pilosa.GroupCounts, handle aggregate case []pilosa.GroupCount: - toRowser = pilosa.GroupCounts(v) + gc := pilosa.NewGroupCounts() + gc.Groups = v + toRowser = gc case uint64: toRowser = ResultUint64(v) case bool: diff --git a/sql/select.go b/sql/select.go index 9ac7d5011..f053634da 100644 --- a/sql/select.go +++ b/sql/select.go @@ -89,8 +89,11 @@ func (s *SelectHandler) execMappingResult(ctx context.Context, mr *MappingResult switch res := res.(type) { case pproto.ToRowser: result = res + // TODO: replace []pilosa.GroupCount with *pilosa.GroupCounts, handle aggregate case []pilosa.GroupCount: - result = pilosa.GroupCounts(res) + gc := pilosa.NewGroupCounts() + gc.Groups = res + result = gc case uint64: result = pproto.ConstRowser{ { From 932e84b68145c47470274c790f5ae59cae182fbf Mon Sep 17 00:00:00 2001 From: Seebs Date: Fri, 15 Jan 2021 13:10:35 -0600 Subject: [PATCH 4/7] handling aggregate types: add to protobuf, etc We want to distinguish different *kinds* of GroupCounts, so we're making the GroupCounts parent object track its type so we can keep that correct. Adding this to protobuf, etc, then creates some weird behaviors because sometimes we expect []GroupCount, and sometimes we expect *GroupCounts. This implies changes to test cases. Also, the changes to test cases imply that some test cases are probably now wrong; for instance, they're expecting a "sum" column, equal to zero, when no sum was requested. We try to make the encoder handle a []*GroupCount gotten from another node without panicing, and avoid breaking the semantics of the existing messages, renumbering messages or components, etc. Since a previous version, the `.Groups` member has been privatized, and the `.Get()` convenience accessor has been renamed `.Groups()` and is now used consistently in a way that should reduce the risk of nil pointers causing crashes. Also, NewGroupCounts is used in a couple more places. --- encoding/proto/proto.go | 42 +- executor.go | 201 +++++--- executor_internal_test.go | 4 +- executor_test.go | 159 +++---- internal/private.pb.go | 192 ++------ internal/public.pb.go | 933 +++++++++++++++++++++++--------------- internal/public.proto | 18 +- server/grpc.go | 8 +- server/grpc_test.go | 31 +- server/pg.go | 44 +- server/pg_test.go | 5 +- server/server_test.go | 2 +- sql/select.go | 5 +- 13 files changed, 916 insertions(+), 728 deletions(-) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 10a2f05a2..2091cc850 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -536,7 +536,7 @@ func (s Serializer) encodeQueryResponse(m *pilosa.QueryResponse) *internal.Query case pilosa.ExtractedIDMatrix: pb.Results[i].Type = queryResultTypeExtractedIDMatrix pb.Results[i].ExtractedIDMatrix = s.endcodeExtractedIDMatrix(result) - case []pilosa.GroupCount: + case *pilosa.GroupCounts: pb.Results[i].Type = queryResultTypeGroupCounts pb.Results[i].GroupCounts = s.encodeGroupCounts(result) case pilosa.RowIdentifiers: @@ -1423,7 +1423,7 @@ func (s Serializer) decodeQueryResult(pb *internal.QueryResult) interface{} { case queryResultTypeRowIdentifiers: return s.decodeRowIdentifiers(pb.RowIdentifiers) case queryResultTypeGroupCounts: - return s.decodeGroupCounts(pb.GroupCounts) + return s.decodeGroupCounts(pb.GroupCounts, pb.OldGroupCounts) case queryResultTypePair: return s.decodePair(pb.Pairs[0]) case queryResultTypePairField: @@ -1587,16 +1587,22 @@ func (s Serializer) decodeRowIdentifiers(a *internal.RowIdentifiers) *pilosa.Row } } -func (s Serializer) decodeGroupCounts(a []*internal.GroupCount) []pilosa.GroupCount { - other := make([]pilosa.GroupCount, len(a)) - for i := range a { +func (s Serializer) decodeGroupCounts(a *internal.GroupCounts, b []*internal.GroupCount) *pilosa.GroupCounts { + // Workaround: If we get an old-style "[]*GroupCount", we translate it. + if a == nil { + a = &internal.GroupCounts{Aggregate: "", Groups: b} + } + other := make([]pilosa.GroupCount, len(a.Groups)) + for i, gc := range a.Groups { other[i] = pilosa.GroupCount{ - Group: s.decodeFieldRows(a[i].Group), - Count: a[i].Count, - Sum: a[i].Sum, + Group: s.decodeFieldRows(gc.Group), + Count: gc.Count, + // note: not renaming the `internal` structure members now + // to avoid breaking protobuf interactions. + Agg: gc.Agg, } } - return other + return pilosa.NewGroupCounts(a.Aggregate, other...) } func (s Serializer) decodeFieldRows(a []*internal.FieldRow) []pilosa.FieldRow { @@ -1723,13 +1729,17 @@ func (s Serializer) encodeRowIdentifiers(r pilosa.RowIdentifiers) *internal.RowI } } -func (s Serializer) encodeGroupCounts(counts []pilosa.GroupCount) []*internal.GroupCount { - result := make([]*internal.GroupCount, len(counts)) - for i := range counts { - result[i] = &internal.GroupCount{ - Group: s.encodeFieldRows(counts[i].Group), - Count: counts[i].Count, - Sum: counts[i].Sum, +func (s Serializer) encodeGroupCounts(counts *pilosa.GroupCounts) *internal.GroupCounts { + groups := counts.Groups() + result := &internal.GroupCounts{ + Groups: make([]*internal.GroupCount, len(groups)), + Aggregate: counts.AggregateColumn(), + } + for i, gc := range groups { + result.Groups[i] = &internal.GroupCount{ + Group: s.encodeFieldRows(gc.Group), + Count: gc.Count, + Agg: gc.Agg, } } return result diff --git a/executor.go b/executor.go index deced589c..ae1071b33 100644 --- a/executor.go +++ b/executor.go @@ -21,10 +21,10 @@ import ( "math" "math/bits" "sort" - "strconv" "strings" "sync" "time" + "unsafe" "github.com/pilosa/pilosa/v2/pql" pb "github.com/pilosa/pilosa/v2/proto" @@ -2656,10 +2656,10 @@ func (g *groupCountSorter) Less(i, j int) bool { } else if gci.Count > gcj.Count { return fieldOrder == desc } - case -2: // aggregate/Sum - if gci.Sum < gcj.Sum { + case -2: // Aggregate + if gci.Agg < gcj.Agg { return fieldOrder == asc - } else if gci.Sum > gcj.Sum { + } else if gci.Agg > gcj.Agg { return fieldOrder == desc } default: @@ -2705,6 +2705,18 @@ func getSorter(sortSpec string) (*groupCountSorter, error) { return gcs, nil } +// findGroupCounts gets a safe-to-use but possibly empty []GroupCount from +// an interface which might be a *GroupCounts or a []GroupCount. +func findGroupCounts(v interface{}) []GroupCount { + switch gc := v.(type) { + case []GroupCount: + return gc + case *GroupCounts: + return gc.Groups() + } + return nil +} + func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *execOptions) (*GroupCounts, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGroupBy") defer span.Finish() @@ -2801,11 +2813,11 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c } // Merge returned results at coordinating node. reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { - other, _ := prev.([]GroupCount) + other := findGroupCounts(prev) if err := ctx.Err(); err != nil { return err } - return mergeGroupCounts(other, v.([]GroupCount), limit) + return mergeGroupCounts(other, findGroupCounts(v), limit) } // Get full result set. other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) @@ -2872,7 +2884,7 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c if err != nil { return nil, err } - results[n].Sum = int64(aggregateCount[0].(uint64)) + results[n].Agg = int64(aggregateCount[0].(uint64)) } } @@ -2910,20 +2922,16 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c } - ret := NewGroupCounts() + aggType := "" if aggregate != nil { switch aggregate.Name { case "Sum": - ret.aggregateType = sumAggregate + aggType = "sum" case "Count": - ret.aggregateType = distinctAggregate - default: - ret.aggregateType = nilAggregate + aggType = "aggregate" } } - ret.Groups = results - - return ret, nil + return NewGroupCounts(aggType, results...), nil } func applyLimitAndOffsetToGroupByResult(c *pql.Call, results []GroupCount) ([]GroupCount, error) { @@ -3018,22 +3026,60 @@ const ( // GroupCounts is a list of GroupCount. type GroupCounts struct { - Groups []GroupCount + groups []GroupCount aggregateType aggregateType } -func NewGroupCounts() *GroupCounts { - return &GroupCounts{} +// AggregateColumn gives the likely column name to use for aggregates, because +// for historical reasons we used "sum" when it was a sum, but don't want to +// use that when it's something else. This will likely get revisited. +func (g *GroupCounts) AggregateColumn() string { + switch g.aggregateType { + case sumAggregate: + return "sum" + case distinctAggregate: + return "aggregate" + default: + return "" + } +} + +// Groups is a convenience method to let us not worry as much about the +// potentially-nil nature of a *GroupCounts. +func (g *GroupCounts) Groups() []GroupCount { + if g == nil { + return nil + } + return g.groups +} + +// NewGroupCounts creates a GroupCounts with the given type and slice +// of GroupCount objects. There's intentionally no externally-accessible way +// to change the []GroupCount after creation. +func NewGroupCounts(agg string, groups ...GroupCount) *GroupCounts { + var aggType aggregateType + switch agg { + case "sum": + aggType = sumAggregate + case "aggregate": + aggType = distinctAggregate + case "": + aggType = nilAggregate + default: + panic(fmt.Sprintf("invalid aggregate type %q", agg)) + } + return &GroupCounts{aggregateType: aggType, groups: groups} } // ToTable implements the ToTabler interface. func (g *GroupCounts) ToTable() (*pb.TableResponse, error) { - return pb.RowsToTable(g, len(g.Groups)) + return pb.RowsToTable(g, len(g.Groups())) } // ToRows implements the ToRowser interface. func (g *GroupCounts) ToRows(callback func(*pb.RowResponse) error) error { - for i, gc := range g.Groups { + agg := g.AggregateColumn() + for i, gc := range g.Groups() { var ci []*pb.ColumnInfo if i == 0 { for _, fieldRow := range gc.Group { @@ -3046,7 +3092,10 @@ func (g *GroupCounts) ToRows(callback func(*pb.RowResponse) error) error { } } ci = append(ci, &pb.ColumnInfo{Name: "count", Datatype: "uint64"}) - ci = append(ci, &pb.ColumnInfo{Name: "sum", Datatype: "int64"}) + if agg != "" { + ci = append(ci, &pb.ColumnInfo{Name: agg, Datatype: "int64"}) + } + } rowResp := &pb.RowResponse{ Headers: ci, @@ -3063,9 +3112,11 @@ func (g *GroupCounts) ToRows(callback func(*pb.RowResponse) error) error { } } rowResp.Columns = append(rowResp.Columns, - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: gc.Count}}, - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: gc.Sum}}, - ) + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: gc.Count}}) + if agg != "" { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: gc.Agg}}) + } if err := callback(rowResp); err != nil { return errors.Wrap(err, "calling callback") } @@ -3076,48 +3127,48 @@ func (g *GroupCounts) ToRows(callback func(*pb.RowResponse) error) error { // MarshalJSON makes GroupCounts satisfy interface json.Marshaler and // customizes the JSON output of the aggregate field label. func (g *GroupCounts) MarshalJSON() ([]byte, error) { - if len(g.Groups) == 0 { + groups := g.Groups() + var counts interface{} = groups + + if len(groups) == 0 { return []byte("[]"), nil } - var aggregateLabel string switch g.aggregateType { case sumAggregate: - aggregateLabel = "sum" + counts = *(*[]groupCountSum)(unsafe.Pointer(&groups)) case distinctAggregate: - aggregateLabel = "distinct" // TODO: not sure the best name here + counts = *(*[]groupCountAggregate)(unsafe.Pointer(&groups)) } - var out = []byte("[") - for _, group := range g.Groups { - groupJson, err := json.Marshal(group) - if err != nil { - return nil, errors.Wrap(err, "marshaling group") - } - if g.aggregateType != nilAggregate { - // Insert `"aggregatelabel": aggregatevalue` at end of JSON, append comma - groupJson = append(groupJson[:len(groupJson)-1], []byte(",\""+aggregateLabel+"\": "+strconv.Itoa(int(group.Sum))+"},")...) - } else { - // Append comma - groupJson = append(groupJson, byte(',')) - } - out = append(out, groupJson...) - } - // replace final comma with "]" - out[len(out)-1] = byte(']') - return out, nil + return json.Marshal(counts) } // GroupCount represents a result item for a group by query. type GroupCount struct { Group []FieldRow `json:"group"` Count uint64 `json:"count"` - Sum int64 `json:"-"` + Agg int64 `json:"-"` } +type groupCountSum struct { + Group []FieldRow `json:"group"` + Count uint64 `json:"count"` + Agg int64 `json:"sum"` +} + +type groupCountAggregate struct { + Group []FieldRow `json:"group"` + Count uint64 `json:"count"` + Agg int64 `json:"aggregate"` +} + +var _ GroupCount = GroupCount(groupCountSum{}) +var _ GroupCount = GroupCount(groupCountAggregate{}) + func (g *GroupCount) Clone() (r *GroupCount) { r = &GroupCount{ Group: make([]FieldRow, len(g.Group)), Count: g.Count, - Sum: g.Sum, + Agg: g.Agg, } for i := range g.Group { r.Group[i] = *(g.Group[i].Clone()) @@ -3141,7 +3192,7 @@ func mergeGroupCounts(a, b []GroupCount, limit int) []GroupCount { i++ case 0: a[i].Count += b[j].Count - a[i].Sum += b[j].Sum + a[i].Agg += b[j].Agg ret = append(ret, a[i]) i++ j++ @@ -3248,27 +3299,27 @@ func (g GroupCount) satisfiesCondition(subj string, cond *pql.Condition) bool { return false } if cond.Op == pql.EQ { - if g.Sum == val { + if g.Agg == val { return true } } else if cond.Op == pql.NEQ { - if g.Sum != val { + if g.Agg != val { return true } } else if cond.Op == pql.LT { - if g.Sum < val { + if g.Agg < val { return true } } else if cond.Op == pql.LTE { - if g.Sum <= val { + if g.Agg <= val { return true } } else if cond.Op == pql.GT { - if g.Sum > val { + if g.Agg > val { return true } } else if cond.Op == pql.GTE { - if g.Sum >= val { + if g.Agg >= val { return true } } @@ -3278,19 +3329,19 @@ func (g GroupCount) satisfiesCondition(subj string, cond *pql.Condition) bool { return false } if cond.Op == pql.BETWEEN { - if val[0] <= g.Sum && g.Sum <= val[1] { + if val[0] <= g.Agg && g.Agg <= val[1] { return true } } else if cond.Op == pql.BTWN_LT_LTE { - if val[0] < g.Sum && g.Sum <= val[1] { + if val[0] < g.Agg && g.Agg <= val[1] { return true } } else if cond.Op == pql.BTWN_LTE_LT { - if val[0] <= g.Sum && g.Sum < val[1] { + if val[0] <= g.Agg && g.Agg < val[1] { return true } } else if cond.Op == pql.BTWN_LT_LT { - if val[0] < g.Sum && g.Sum < val[1] { + if val[0] < g.Agg && g.Agg < val[1] { return true } } @@ -6561,10 +6612,11 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index } } - case GroupCounts: + case *GroupCounts: fieldIDs := make(map[*Field]map[uint64]struct{}) foreignIDs := make(map[*Field]map[uint64]struct{}) - for _, gl := range result.Groups { + groups := result.Groups() + for _, gl := range groups { for _, g := range gl.Group { field := idx.Field(g.Field) if field == nil { @@ -6575,7 +6627,7 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index if fi := field.ForeignIndex(); fi != "" { m, ok := foreignIDs[field] if !ok { - m = make(map[uint64]struct{}, len(result.Groups)) + m = make(map[uint64]struct{}, len(groups)) foreignIDs[field] = m } @@ -6586,7 +6638,7 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index m, ok := fieldIDs[field] if !ok { - m = make(map[uint64]struct{}, len(result.Groups)) + m = make(map[uint64]struct{}, len(groups)) fieldIDs[field] = m } @@ -6613,8 +6665,11 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index foreignTranslations[field.Name()] = trans } - other := NewGroupCounts() - for _, gl := range result.Groups { + // We are reluctant to smash result, and I'm not sure we need + // to be but I'm not sure we don't need to be. + newGroups := make([]GroupCount, len(groups)) + copy(newGroups, groups) + for gi, gl := range groups { group := make([]FieldRow, len(gl.Group)) for i, g := range gl.Group { @@ -6627,15 +6682,15 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index group[i] = g } - - other.Groups = append(other.Groups, GroupCount{ - Group: group, - Count: gl.Count, - Sum: gl.Sum, - }) + // Replace with translated group. + newGroups[gi].Group = group } + other := &GroupCounts{} + if result != nil { + other.aggregateType = result.aggregateType + } + other.groups = newGroups return other, nil - case RowIDs: fieldName := callArgString(call, "_field") if fieldName == "" { @@ -7502,7 +7557,7 @@ func (gbi *groupByIterator) Next(ctx context.Context) (ret GroupCount, done bool return ret, false, err } ret.Count = uint64(result.Count) - ret.Sum = result.Val + ret.Agg = result.Val } } if ret.Count == 0 { diff --git a/executor_internal_test.go b/executor_internal_test.go index 6de260293..fbc3f55d6 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -164,7 +164,7 @@ func TestExecutor_GroupCountCondition(t *testing.T) { }, }, { - groupCount: GroupCount{Sum: 100}, + groupCount: GroupCount{Agg: 100}, checks: []condCheck{ {cond: "sum == 99", exp: false}, {cond: "sum != 99", exp: true}, @@ -196,7 +196,7 @@ func TestExecutor_GroupCountCondition(t *testing.T) { }, }, { - groupCount: GroupCount{Sum: -100}, + groupCount: GroupCount{Agg: -100}, checks: []condCheck{ {cond: "sum == -99", exp: false}, {cond: "sum != -99", exp: true}, diff --git a/executor_test.go b/executor_test.go index 6db994817..b04beacf8 100644 --- a/executor_test.go +++ b/executor_test.go @@ -3066,7 +3066,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "f", RowID: 7}}, Count: 1}, {Group: []pilosa.FieldRow{{Field: "f", RowID: 10}}, Count: 4}, } - results := res.Results[0].([]pilosa.GroupCount) + results := res.Results[0].(*pilosa.GroupCounts).Groups() test.CheckGroupBy(t, expected, results) } }) @@ -3108,7 +3108,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "fint", Value: &d}}, Count: 1}, } - results := res.Results[0].([]pilosa.GroupCount) + results := res.Results[0].(*pilosa.GroupCounts).Groups() test.CheckGroupBy(t, expected, results) } }) @@ -3139,7 +3139,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "hint", Value: &c}}, Count: 1}, } - results := res.Results[0].([]pilosa.GroupCount) + results := res.Results[0].(*pilosa.GroupCounts).Groups() test.CheckGroupBy(t, expected, results) } }) @@ -5074,20 +5074,20 @@ func TestExecutor_GroupByStrings(t *testing.T) { { query: "GroupBy(Rows(generals), aggregate=Sum(field=v))", expected: []pilosa.GroupCount{ - {Group: []pilosa.FieldRow{{Field: "generals", RowID: 1, RowKey: "r1"}}, Count: 5, Sum: 25}, - {Group: []pilosa.FieldRow{{Field: "generals", RowID: 2, RowKey: "r2"}}, Count: 5, Sum: 30}, + {Group: []pilosa.FieldRow{{Field: "generals", RowID: 1, RowKey: "r1"}}, Count: 5, Agg: 25}, + {Group: []pilosa.FieldRow{{Field: "generals", RowID: 2, RowKey: "r2"}}, Count: 5, Agg: 30}, }, }, { query: "GroupBy(Rows(generals), aggregate=Sum(field=v), having=Condition(sum>25))", expected: []pilosa.GroupCount{ - {Group: []pilosa.FieldRow{{Field: "generals", RowID: 2, RowKey: "r2"}}, Count: 5, Sum: 30}, + {Group: []pilosa.FieldRow{{Field: "generals", RowID: 2, RowKey: "r2"}}, Count: 5, Agg: 30}, }, }, { query: "GroupBy(Rows(generals), aggregate=Sum(field=v), having=Condition(-5 10), aggregate=Count(Distinct(field=v)))`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(general), Rows(sub), filter=Row(v > 10), aggregate=Count(Distinct(field=v)))`).Results[0].(*pilosa.GroupCounts).Groups() test.CheckGroupBy(t, expected, results) }) t.Run("AggregateCountDistinctFilterDistinct", func(t *testing.T) { expected := []pilosa.GroupCount{ - {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Count: 3, Sum: 1}, - {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Count: 1, Sum: 0}, - {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}, {Field: "sub", RowID: 110}}, Count: 1, Sum: 0}, - {Group: []pilosa.FieldRow{{Field: "general", RowID: 12}, {Field: "sub", RowID: 110}}, Count: 1, Sum: 0}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Count: 3, Agg: 1}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Count: 1, Agg: 0}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}, {Field: "sub", RowID: 110}}, Count: 1, Agg: 0}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 12}, {Field: "sub", RowID: 110}}, Count: 1, Agg: 0}, } - results := c.Query(t, "i", `GroupBy(Rows(general), Rows(sub), aggregate=Count(Distinct(Row(v > 10), field=v)))`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(general), Rows(sub), aggregate=Count(Distinct(Row(v > 10), field=v)))`).Results[0].(*pilosa.GroupCounts).Groups() test.CheckGroupBy(t, expected, results) }) @@ -5622,7 +5622,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "general", RowID: 12}}, Count: 2}, } - results := c.Query(t, "i", `GroupBy(Rows(general, previous=10))`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(general, previous=10))`).Results[0].(*pilosa.GroupCounts).Groups() test.CheckGroupBy(t, expected, results) }) @@ -5631,7 +5631,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Count: 2}, } - results := c.Query(t, "i", `GroupBy(Rows(general, previous=10), limit=1)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(general, previous=10), limit=1)`).Results[0].(*pilosa.GroupCounts).Groups() test.CheckGroupBy(t, expected, results) }) @@ -5652,7 +5652,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "a", RowID: 0}, {Field: "b", RowID: 1}}, Count: 1}, } - results := c.Query(t, "i", `GroupBy(Rows(a), Rows(b), limit=1)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(a), Rows(b), limit=1)`).Results[0].(*pilosa.GroupCounts).Groups() test.CheckGroupBy(t, expected, results) }) @@ -5680,7 +5680,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { }) t.Run("test wrapping with previous", func(t *testing.T) { - results := c.Query(t, "i", `GroupBy(Rows(wa), Rows(wb), Rows(wc, previous=1), limit=3)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(wa), Rows(wb), Rows(wc, previous=1), limit=3)`).Results[0].(*pilosa.GroupCounts).Groups() expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "wa", RowID: 0}, {Field: "wb", RowID: 0}, {Field: "wc", RowID: 2}}, Count: 2}, {Group: []pilosa.FieldRow{{Field: "wa", RowID: 0}, {Field: "wb", RowID: 1}, {Field: "wc", RowID: 0}}, Count: 1}, @@ -5690,14 +5690,14 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { }) t.Run("test previous is last result", func(t *testing.T) { - results := c.Query(t, "i", `GroupBy(Rows(wa, previous=3), Rows(wb, previous=3), Rows(wc, previous=3), limit=3)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(wa, previous=3), Rows(wb, previous=3), Rows(wc, previous=3), limit=3)`).Results[0].(*pilosa.GroupCounts).Groups() if len(results) > 0 { t.Fatalf("expected no results because previous specified last result") } }) t.Run("test wrapping multiple", func(t *testing.T) { - results := c.Query(t, "i", `GroupBy(Rows(wa), Rows(wb, previous=2), Rows(wc, previous=2), limit=1)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(wa), Rows(wb, previous=2), Rows(wc, previous=2), limit=1)`).Results[0].(*pilosa.GroupCounts).Groups() expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "wa", RowID: 1}, {Field: "wb", RowID: 0}, {Field: "wc", RowID: 0}}, Count: 1}, } @@ -5721,7 +5721,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {3, ShardWidth}, }) t.Run("distinct rows in different shards", func(t *testing.T) { - results := c.Query(t, "i", `GroupBy(Rows(ma), Rows(mb), limit=5)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(ma), Rows(mb), limit=5)`).Results[0].(*pilosa.GroupCounts).Groups() expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "ma", RowID: 0}, {Field: "mb", RowID: 0}}, Count: 1}, {Group: []pilosa.FieldRow{{Field: "ma", RowID: 0}, {Field: "mb", RowID: 2}}, Count: 1}, @@ -5733,7 +5733,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { }) t.Run("distinct rows in different shards with row limit", func(t *testing.T) { - results := c.Query(t, "i", `GroupBy(Rows(ma), Rows(mb, limit=2), limit=5)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(ma), Rows(mb, limit=2), limit=5)`).Results[0].(*pilosa.GroupCounts).Groups() expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "ma", RowID: 0}, {Field: "mb", RowID: 0}}, Count: 1}, {Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 1}}, Count: 1}, @@ -5744,7 +5744,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { }) t.Run("distinct rows in different shards with column arg", func(t *testing.T) { - results := c.Query(t, "i", fmt.Sprintf(`GroupBy(Rows(ma), Rows(mb, column=%d), limit=5)`, ShardWidth)).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", fmt.Sprintf(`GroupBy(Rows(ma), Rows(mb, column=%d), limit=5)`, ShardWidth)).Results[0].(*pilosa.GroupCounts).Groups() expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 1}}, Count: 1}, {Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 3}}, Count: 1}, @@ -5769,7 +5769,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {1, ShardWidth}, }) t.Run("same rows in different shards", func(t *testing.T) { - results := c.Query(t, "i", `GroupBy(Rows(na), Rows(nb))`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(na), Rows(nb))`).Results[0].(*pilosa.GroupCounts).Groups() expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "na", RowID: 0}, {Field: "nb", RowID: 0}}, Count: 2}, {Group: []pilosa.FieldRow{{Field: "na", RowID: 0}, {Field: "nb", RowID: 1}}, Count: 2}, @@ -5806,12 +5806,12 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { t.Run("test wrapping with previous", func(t *testing.T) { totalResults := make([]pilosa.GroupCount, 0) - results := c.Query(t, "i", `GroupBy(Rows(ppa), Rows(ppb), Rows(ppc), limit=3)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(ppa), Rows(ppb), Rows(ppc), limit=3)`).Results[0].(*pilosa.GroupCounts).Groups() totalResults = append(totalResults, results...) for len(totalResults) < 64 { lastGroup := results[len(results)-1].Group query := fmt.Sprintf("GroupBy(Rows(ppa, previous=%d), Rows(ppb, previous=%d), Rows(ppc, previous=%d), limit=3)", lastGroup[0].RowID, lastGroup[1].RowID, lastGroup[2].RowID) - results = c.Query(t, "i", query).Results[0].([]pilosa.GroupCount) + results = c.Query(t, "i", query).Results[0].(*pilosa.GroupCounts).Groups() totalResults = append(totalResults, results...) } @@ -5853,7 +5853,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "generalk", RowID: 3, RowKey: "twelve"}, {Field: "subk", RowID: 2, RowKey: "one-hundred-ten"}}, Count: 1}, } - results := c.Query(t, "i", `GroupBy(Rows(generalk), Rows(subk))`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(generalk), Rows(subk))`).Results[0].(*pilosa.GroupCounts).Groups() test.CheckGroupBy(t, expected, results) }) @@ -5891,7 +5891,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "child", RowID: 2, RowKey: "three"}}, Count: 2}, } - results := c.Query(t, "fic", `GroupBy(Rows(child))`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "fic", `GroupBy(Rows(child))`).Results[0].(*pilosa.GroupCounts).Groups() test.CheckGroupBy(t, expected, results) }) @@ -6477,10 +6477,11 @@ func TestExecutor_Execute_CountDistinct(t *testing.T) { if err != nil { t.Fatal(err) } - gc, ok := resp.Results[0].([]pilosa.GroupCount) + gcc, ok := resp.Results[0].(*pilosa.GroupCounts) if !ok { t.Fatalf("invalid response type, expected: []pilosa.GroupCount, got: %T", resp.Results[0]) } + gc := gcc.Groups() if len(gc) != 2 { t.Fatalf("invalid group count length, expected: 2, got: %v", len(gc)) } @@ -7011,13 +7012,13 @@ func variousQueries(t *testing.T, clusterSize int) { }, { query: "GroupBy(Rows(field=likes))", - csvVerifier: `molecula,1,0 -pilosa,1,0 -pangolin,1,0 -zebra,1,0 -toucan,1,0 -dog,1,0 -icecream,6,0 + csvVerifier: `molecula,1 +pilosa,1 +pangolin,1 +zebra,1 +toucan,1 +dog,1 +icecream,6 `, }, { @@ -7028,15 +7029,15 @@ zebra,1,1000 }, { query: "GroupBy(Rows(field=likes), having=Condition(count>5))", - csvVerifier: "icecream,6,0\n", + csvVerifier: "icecream,6\n", }, { query: "GroupBy(Rows(field=likes), filter=Row(affinity>-7))", - csvVerifier: `molecula,1,0 -pangolin,1,0 -zebra,1,0 -toucan,1,0 -icecream,4,0 + csvVerifier: `molecula,1 +pangolin,1 +zebra,1 +toucan,1 +icecream,4 `, }, { @@ -7076,13 +7077,13 @@ icecream,5,3 }, { query: "GroupBy(Rows(field=likes), sort=\"count desc\")", - csvVerifier: `icecream,6,0 -molecula,1,0 -pilosa,1,0 -pangolin,1,0 -zebra,1,0 -toucan,1,0 -dog,1,0 + csvVerifier: `icecream,6 +molecula,1 +pilosa,1 +pangolin,1 +zebra,1 +toucan,1 +dog,1 `, }, { diff --git a/internal/private.pb.go b/internal/private.pb.go index 6dd3eb9bc..b3c0fec5b 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -5620,10 +5620,7 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6028,10 +6025,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6114,10 +6108,7 @@ func (m *ImportResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6302,10 +6293,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6508,10 +6496,7 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6638,10 +6623,7 @@ func (m *Cache) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6788,7 +6770,7 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > postIndex { @@ -6805,10 +6787,7 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6942,10 +6921,7 @@ func (m *CreateShardMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7028,10 +7004,7 @@ func (m *DeleteIndexMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7169,10 +7142,7 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7342,10 +7312,7 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7460,10 +7427,7 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7597,10 +7561,7 @@ func (m *DeleteAvailableShardMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7770,10 +7731,7 @@ func (m *Field) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7858,10 +7816,7 @@ func (m *Schema) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8033,10 +7988,7 @@ func (m *Index) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8170,10 +8122,7 @@ func (m *URI) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8380,10 +8329,7 @@ func (m *Node) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8498,10 +8444,7 @@ func (m *NodeStateMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8607,10 +8550,7 @@ func (m *NodeEventMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8767,10 +8707,7 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8906,10 +8843,7 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -9087,10 +9021,7 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -9275,10 +9206,7 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -9431,10 +9359,7 @@ func (m *BSIGroup) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -9581,10 +9506,7 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -9731,10 +9653,7 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10016,10 +9935,7 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10221,10 +10137,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10362,10 +10275,7 @@ func (m *TranslationResizeSource) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10503,10 +10413,7 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10593,10 +10500,7 @@ func (m *SetCoordinatorMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10683,10 +10587,7 @@ func (m *UpdateCoordinatorMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10801,10 +10702,7 @@ func (m *Topology) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10855,10 +10753,7 @@ func (m *RecalculateCaches) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10977,10 +10872,7 @@ func (m *TransactionMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -11177,10 +11069,7 @@ func (m *Transaction) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -11231,10 +11120,7 @@ func (m *TransactionStats) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { diff --git a/internal/public.pb.go b/internal/public.pb.go index 30eddfcb8..e406e06ab 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -1111,7 +1111,7 @@ func (m *FieldRow) GetValue() *Int64 { type GroupCount struct { Group []*FieldRow `protobuf:"bytes,1,rep,name=Group,proto3" json:"Group,omitempty"` Count uint64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` - Sum int64 `protobuf:"varint,3,opt,name=Sum,proto3" json:"Sum,omitempty"` + Agg int64 `protobuf:"varint,3,opt,name=Agg,proto3" json:"Agg,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1164,9 +1164,9 @@ func (m *GroupCount) GetCount() uint64 { return 0 } -func (m *GroupCount) GetSum() int64 { +func (m *GroupCount) GetAgg() int64 { if m != nil { - return m.Sum + return m.Agg } return 0 } @@ -1661,14 +1661,20 @@ func (m *QueryResponse) GetColumnAttrSets() []*ColumnAttrSet { } type QueryResult struct { - Type uint32 `protobuf:"varint,6,opt,name=Type,proto3" json:"Type,omitempty"` - Row *Row `protobuf:"bytes,1,opt,name=Row,proto3" json:"Row,omitempty"` - N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` - Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs,proto3" json:"Pairs,omitempty"` - Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` - ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount,proto3" json:"ValCount,omitempty"` - RowIDs []uint64 `protobuf:"varint,7,rep,packed,name=RowIDs,proto3" json:"RowIDs,omitempty"` - GroupCounts []*GroupCount `protobuf:"bytes,8,rep,name=GroupCounts,proto3" json:"GroupCounts,omitempty"` + Type uint32 `protobuf:"varint,6,opt,name=Type,proto3" json:"Type,omitempty"` + Row *Row `protobuf:"bytes,1,opt,name=Row,proto3" json:"Row,omitempty"` + N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` + Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs,proto3" json:"Pairs,omitempty"` + Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` + ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount,proto3" json:"ValCount,omitempty"` + RowIDs []uint64 `protobuf:"varint,7,rep,packed,name=RowIDs,proto3" json:"RowIDs,omitempty"` + // In the past, GroupCounts was a []GroupCount which did not indicate + // whether it had an aggregate, or which aggregate it had. We've + // updated this, but we keep this here so that messages using the old + // format can get a best-effort treatment rather than causing panics. + // Later this can almost certainly go away, but leave a comment warning + // people that 8 is Spoken For if you do that, please. + OldGroupCounts []*GroupCount `protobuf:"bytes,8,rep,name=OldGroupCounts,proto3" json:"OldGroupCounts,omitempty"` RowIdentifiers *RowIdentifiers `protobuf:"bytes,9,opt,name=RowIdentifiers,proto3" json:"RowIdentifiers,omitempty"` SignedRow *SignedRow `protobuf:"bytes,10,opt,name=SignedRow,proto3" json:"SignedRow,omitempty"` PairsField *PairsField `protobuf:"bytes,11,opt,name=PairsField,proto3" json:"PairsField,omitempty"` @@ -1676,6 +1682,7 @@ type QueryResult struct { ExtractedIDMatrix *ExtractedIDMatrix `protobuf:"bytes,13,opt,name=ExtractedIDMatrix,proto3" json:"ExtractedIDMatrix,omitempty"` ExtractedTable *ExtractedTable `protobuf:"bytes,14,opt,name=ExtractedTable,proto3" json:"ExtractedTable,omitempty"` RowMatrix *RowMatrix `protobuf:"bytes,15,opt,name=RowMatrix,proto3" json:"RowMatrix,omitempty"` + GroupCounts *GroupCounts `protobuf:"bytes,16,opt,name=GroupCounts,proto3" json:"GroupCounts,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1763,9 +1770,9 @@ func (m *QueryResult) GetRowIDs() []uint64 { return nil } -func (m *QueryResult) GetGroupCounts() []*GroupCount { +func (m *QueryResult) GetOldGroupCounts() []*GroupCount { if m != nil { - return m.GroupCounts + return m.OldGroupCounts } return nil } @@ -1819,6 +1826,13 @@ func (m *QueryResult) GetRowMatrix() *RowMatrix { return nil } +func (m *QueryResult) GetGroupCounts() *GroupCounts { + if m != nil { + return m.GroupCounts + } + return nil +} + type ImportRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` @@ -2648,6 +2662,61 @@ func (m *ImportColumnAttrsRequest) GetIndexCreatedAt() int64 { return 0 } +type GroupCounts struct { + Aggregate string `protobuf:"bytes,1,opt,name=Aggregate,proto3" json:"Aggregate,omitempty"` + Groups []*GroupCount `protobuf:"bytes,2,rep,name=Groups,proto3" json:"Groups,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GroupCounts) Reset() { *m = GroupCounts{} } +func (m *GroupCounts) String() string { return proto.CompactTextString(m) } +func (*GroupCounts) ProtoMessage() {} +func (*GroupCounts) Descriptor() ([]byte, []int) { + return fileDescriptor_413a91106d7bcce8, []int{37} +} +func (m *GroupCounts) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *GroupCounts) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_GroupCounts.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *GroupCounts) XXX_Merge(src proto.Message) { + xxx_messageInfo_GroupCounts.Merge(m, src) +} +func (m *GroupCounts) XXX_Size() int { + return m.Size() +} +func (m *GroupCounts) XXX_DiscardUnknown() { + xxx_messageInfo_GroupCounts.DiscardUnknown(m) +} + +var xxx_messageInfo_GroupCounts proto.InternalMessageInfo + +func (m *GroupCounts) GetAggregate() string { + if m != nil { + return m.Aggregate + } + return "" +} + +func (m *GroupCounts) GetGroups() []*GroupCount { + if m != nil { + return m.Groups + } + return nil +} + func init() { proto.RegisterType((*Row)(nil), "internal.Row") proto.RegisterType((*RowMatrix)(nil), "internal.RowMatrix") @@ -2686,119 +2755,123 @@ func init() { proto.RegisterType((*ImportRoaringRequestView)(nil), "internal.ImportRoaringRequestView") proto.RegisterType((*ImportRoaringRequest)(nil), "internal.ImportRoaringRequest") proto.RegisterType((*ImportColumnAttrsRequest)(nil), "internal.ImportColumnAttrsRequest") + proto.RegisterType((*GroupCounts)(nil), "internal.GroupCounts") } func init() { proto.RegisterFile("public.proto", fileDescriptor_413a91106d7bcce8) } var fileDescriptor_413a91106d7bcce8 = []byte{ - // 1709 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x4f, 0x6f, 0x23, 0x4b, - 0x11, 0xcf, 0x78, 0xc6, 0xb1, 0x5d, 0x76, 0xb2, 0x79, 0xbd, 0x7e, 0x8f, 0xd1, 0x92, 0xcd, 0x33, - 0xa3, 0xc0, 0x33, 0x08, 0xe5, 0x29, 0xe1, 0xed, 0x6a, 0x2f, 0xc0, 0x26, 0xeb, 0x2c, 0x19, 0x2d, - 0x09, 0x4b, 0x3b, 0xca, 0xde, 0x90, 0x26, 0x76, 0x93, 0x1d, 0x31, 0xf6, 0x98, 0xf1, 0x78, 0x9d, - 0x5c, 0x90, 0xf8, 0x0c, 0x7b, 0xe1, 0x86, 0xb8, 0x71, 0xe2, 0x43, 0x70, 0x81, 0x23, 0x47, 0x24, - 0x2e, 0x68, 0xe1, 0x83, 0xa0, 0xaa, 0x9e, 0x9e, 0xee, 0x19, 0x4f, 0xb2, 0xd1, 0x8a, 0x5b, 0xd7, - 0x9f, 0xae, 0xee, 0xfa, 0xf5, 0xaf, 0xab, 0x6b, 0x06, 0x3a, 0xb3, 0xc5, 0x65, 0x14, 0x8e, 0xf6, - 0x66, 0x49, 0x9c, 0xc6, 0xac, 0x19, 0x4e, 0x53, 0x91, 0x4c, 0x83, 0xc8, 0xfb, 0xa3, 0x05, 0x36, - 0x8f, 0x97, 0xcc, 0x85, 0xc6, 0x8b, 0x38, 0x5a, 0x4c, 0xa6, 0x73, 0xd7, 0xea, 0xd9, 0x7d, 0x87, - 0x2b, 0x91, 0x31, 0x70, 0x5e, 0x89, 0x9b, 0xb9, 0x6b, 0xf7, 0xec, 0x7e, 0x8b, 0xd3, 0x98, 0xed, - 0x42, 0xfd, 0x30, 0x4d, 0x93, 0xb9, 0x5b, 0xeb, 0xd9, 0xfd, 0xf6, 0xc1, 0xe6, 0x9e, 0x8a, 0xb7, - 0x87, 0x6a, 0x2e, 0x8d, 0x18, 0x93, 0xc7, 0x41, 0x12, 0x4e, 0xaf, 0x5c, 0xa7, 0x67, 0xf5, 0x3b, - 0x5c, 0x89, 0xac, 0x0b, 0x75, 0x7f, 0x3a, 0x16, 0xd7, 0x6e, 0xbd, 0x67, 0xf5, 0x5b, 0x5c, 0x0a, - 0xa8, 0x7d, 0x19, 0x8a, 0x68, 0xec, 0xae, 0x4b, 0x2d, 0x09, 0xde, 0x1e, 0xb4, 0x78, 0xbc, 0x3c, - 0x0d, 0xd2, 0x24, 0xbc, 0x66, 0xdf, 0x01, 0x87, 0xc7, 0x4b, 0xb9, 0xc7, 0xf6, 0xc1, 0x86, 0x5e, - 0x97, 0xc7, 0x4b, 0x4e, 0x26, 0xef, 0x14, 0x5a, 0xc3, 0xf0, 0x6a, 0x2a, 0xc6, 0x98, 0xd6, 0x97, - 0x60, 0xbf, 0x8e, 0xd1, 0xdd, 0x5a, 0x75, 0x47, 0x0b, 0x3a, 0x9c, 0x89, 0x2b, 0xb7, 0x56, 0xe9, - 0x70, 0x26, 0xae, 0xbc, 0x67, 0xb0, 0xc9, 0xe3, 0xa5, 0x3f, 0x16, 0xd3, 0x34, 0xfc, 0x75, 0x28, - 0x12, 0x02, 0x24, 0xdf, 0x83, 0x23, 0x17, 0xcd, 0x41, 0xaa, 0x69, 0x90, 0xbc, 0x47, 0xb0, 0xee, - 0x0f, 0x7e, 0x1e, 0xce, 0x53, 0xb6, 0x05, 0xb6, 0x3f, 0x50, 0x13, 0x70, 0xe8, 0xf9, 0xf0, 0xd9, - 0xf1, 0x75, 0x9a, 0x04, 0xa3, 0x54, 0x8c, 0xfd, 0x81, 0x84, 0x9a, 0x6d, 0x42, 0xcd, 0x1f, 0xd0, - 0x5e, 0x1d, 0x5e, 0xf3, 0x07, 0x6c, 0x17, 0x9c, 0x8b, 0x20, 0x52, 0x20, 0x6f, 0xe9, 0xcd, 0xc9, - 0xb0, 0x9c, 0xac, 0xde, 0x65, 0x21, 0x54, 0x86, 0xd3, 0x17, 0xb0, 0x4e, 0xe8, 0xc9, 0x45, 0x5b, - 0x3c, 0x93, 0xd8, 0x13, 0x7d, 0xcc, 0x32, 0xea, 0xb7, 0x75, 0xd4, 0x95, 0x0d, 0xe5, 0x1c, 0xf0, - 0x1e, 0x43, 0xe3, 0x95, 0xb8, 0xa1, 0x5c, 0x54, 0xa6, 0x96, 0x91, 0xe9, 0xbf, 0x2c, 0x78, 0x98, - 0xcf, 0x3e, 0x0f, 0x2e, 0x23, 0x71, 0x11, 0x44, 0x0b, 0xc1, 0x76, 0x55, 0xde, 0x56, 0xd5, 0xfe, - 0x4f, 0xd6, 0x08, 0x0b, 0xf6, 0x55, 0x8e, 0x1d, 0xba, 0x7d, 0xa6, 0xdd, 0xb2, 0x25, 0x4f, 0xd6, - 0x32, 0xd6, 0x6d, 0x43, 0xf3, 0x68, 0xe8, 0x53, 0x68, 0xd7, 0xee, 0x59, 0x7d, 0xfb, 0x64, 0x8d, - 0xe7, 0x1a, 0xf6, 0x08, 0x1a, 0xa7, 0x8b, 0x54, 0x5c, 0xfb, 0x03, 0x62, 0x9b, 0x73, 0xb2, 0xc6, - 0x95, 0x02, 0x67, 0xd2, 0xf0, 0x95, 0xb8, 0x91, 0x94, 0xc3, 0x99, 0x4a, 0xc3, 0xba, 0xe0, 0x1c, - 0xc5, 0x71, 0x44, 0xb4, 0x6b, 0xe2, 0x6a, 0x28, 0x1d, 0x35, 0xa0, 0x4e, 0x81, 0xbd, 0xdf, 0x41, - 0xb7, 0x98, 0x5c, 0x76, 0x5c, 0x0c, 0x6c, 0x8c, 0x67, 0x65, 0xf1, 0x50, 0x60, 0x5b, 0x74, 0x84, - 0xb5, 0x6c, 0x7d, 0x3c, 0xc4, 0x27, 0xb0, 0x4e, 0x61, 0xe4, 0x05, 0x6a, 0x1f, 0x3c, 0xae, 0x00, - 0x5c, 0x43, 0xc6, 0x33, 0xe7, 0xa3, 0x16, 0x21, 0xfe, 0x8b, 0xc4, 0x1f, 0x78, 0x3f, 0x2e, 0x83, - 0x4b, 0x67, 0x89, 0x07, 0x71, 0x16, 0x4c, 0x84, 0x5c, 0x9f, 0xd3, 0x18, 0x75, 0xe7, 0x37, 0x33, - 0x41, 0x1b, 0x68, 0x71, 0x1a, 0x7b, 0xbf, 0xb7, 0x60, 0xb3, 0x38, 0x1f, 0xf7, 0x64, 0xb0, 0xe3, - 0x8e, 0x3d, 0x91, 0x57, 0x4e, 0x9e, 0x67, 0x65, 0xf2, 0xec, 0xdc, 0x36, 0xaf, 0xcc, 0x9f, 0x9f, - 0x80, 0xf3, 0x3a, 0x08, 0x93, 0x15, 0x86, 0x6f, 0x49, 0x08, 0x6d, 0xda, 0xae, 0x2d, 0xcf, 0xa2, - 0xfe, 0x22, 0x5e, 0x4c, 0x53, 0x89, 0x21, 0x97, 0x82, 0x77, 0x0c, 0x2d, 0x9c, 0x2f, 0x13, 0xf7, - 0x64, 0xb0, 0x8c, 0x56, 0x46, 0xed, 0x41, 0x2d, 0x97, 0x0b, 0xe5, 0xa5, 0xa4, 0x66, 0x96, 0x92, - 0x13, 0x00, 0xb4, 0xce, 0x65, 0x9c, 0x5d, 0xa8, 0x93, 0x94, 0x81, 0x50, 0x0e, 0x24, 0x8d, 0xb7, - 0x44, 0x7a, 0x8c, 0x05, 0x2c, 0x7d, 0xfa, 0x0d, 0x9a, 0x25, 0x21, 0x71, 0x37, 0x36, 0xcf, 0x28, - 0xb3, 0x80, 0xa6, 0x84, 0x2e, 0x5e, 0xea, 0x00, 0x96, 0x11, 0x00, 0xb5, 0x58, 0x56, 0x06, 0x2a, - 0x4f, 0x12, 0xf0, 0xda, 0xf2, 0x78, 0xa9, 0x21, 0xc9, 0x24, 0xf6, 0x5d, 0xb5, 0x8a, 0x43, 0x39, - 0x3f, 0x30, 0xae, 0x12, 0xee, 0x42, 0x2d, 0xfb, 0x2b, 0x80, 0x9f, 0x25, 0xf1, 0x62, 0x46, 0xa0, - 0xb1, 0x3e, 0xd4, 0x49, 0xca, 0xf2, 0x63, 0x7a, 0x92, 0xda, 0x1b, 0x97, 0x0e, 0xd5, 0xa0, 0xe3, - 0xe1, 0x0c, 0x17, 0x13, 0x79, 0xd3, 0x38, 0x0e, 0x91, 0x4a, 0xcd, 0x8b, 0x20, 0xca, 0xcd, 0x17, - 0x41, 0x94, 0xe5, 0x8d, 0xc3, 0x62, 0x18, 0x5b, 0x85, 0x79, 0x04, 0xcd, 0x97, 0x51, 0x1c, 0xa4, - 0xe8, 0x8c, 0xb1, 0x2c, 0x9e, 0xcb, 0x6c, 0x1f, 0x60, 0x20, 0x46, 0xe1, 0x24, 0x88, 0xd0, 0xea, - 0x94, 0x0b, 0x40, 0x66, 0xe3, 0x86, 0x93, 0xf7, 0x04, 0x1a, 0x99, 0x54, 0x8d, 0x3d, 0x6a, 0x87, - 0xa3, 0x20, 0x12, 0x6a, 0x17, 0x24, 0x78, 0x6f, 0x60, 0x43, 0x92, 0x11, 0x9f, 0xa6, 0xa1, 0x48, - 0xef, 0x41, 0xc5, 0x7b, 0x3d, 0x72, 0xde, 0x9f, 0x2d, 0x70, 0x70, 0xa4, 0x02, 0x58, 0x3a, 0x80, - 0x79, 0x1b, 0x1d, 0x79, 0x1b, 0x59, 0x0f, 0xda, 0xc3, 0x14, 0xdf, 0x40, 0x5d, 0xc6, 0x5a, 0xdc, - 0x54, 0x21, 0x5e, 0xfe, 0x34, 0xd5, 0xc7, 0x6d, 0xf3, 0x5c, 0x66, 0xdb, 0xd0, 0xc2, 0xda, 0x24, - 0x8d, 0x58, 0xc8, 0x9a, 0x5c, 0x2b, 0xd8, 0x0e, 0x80, 0x42, 0x76, 0x21, 0xa8, 0x9a, 0x59, 0xdc, - 0xd0, 0x78, 0x5f, 0x43, 0x03, 0x77, 0x7a, 0x1a, 0xcc, 0x74, 0x6e, 0xd6, 0x5d, 0xb9, 0xfd, 0xa9, - 0x06, 0x9d, 0x5f, 0x2e, 0x44, 0x72, 0xc3, 0xc5, 0x6f, 0x17, 0x62, 0x9e, 0x22, 0xb6, 0x24, 0x2b, - 0x2e, 0x93, 0x80, 0xac, 0x1d, 0xbe, 0x0d, 0x92, 0xb1, 0x44, 0xca, 0xe1, 0x99, 0x84, 0xb9, 0x6a, - 0xcc, 0xe7, 0x94, 0x6b, 0x93, 0x9b, 0x2a, 0xe2, 0xbb, 0x98, 0xc4, 0xa9, 0x4a, 0x26, 0x93, 0x58, - 0x1f, 0x1e, 0x1c, 0x5f, 0x8f, 0xa2, 0xc5, 0x58, 0xf0, 0x78, 0x29, 0x67, 0x53, 0x71, 0xe6, 0x65, - 0x35, 0xfb, 0x1e, 0x16, 0x37, 0x52, 0xa9, 0xd2, 0xd4, 0x20, 0xc7, 0x92, 0x96, 0xed, 0x43, 0xe7, - 0x78, 0x72, 0x29, 0xc6, 0x63, 0x31, 0x1e, 0x04, 0x69, 0xe0, 0x36, 0xab, 0x1a, 0x88, 0x82, 0x0b, - 0xdb, 0x85, 0x8d, 0xd7, 0x89, 0x38, 0x4f, 0x82, 0xe9, 0x3c, 0x0a, 0x52, 0x31, 0x76, 0x5b, 0x14, - 0xb9, 0xa8, 0xf4, 0xde, 0x5b, 0xb0, 0x91, 0x61, 0x34, 0x9f, 0xc5, 0xd3, 0xb9, 0x40, 0x22, 0x1c, - 0x27, 0x89, 0x22, 0xc2, 0x71, 0x92, 0xb0, 0xaf, 0xa1, 0xc1, 0xc5, 0x7c, 0x11, 0xa5, 0x8a, 0x4b, - 0x9f, 0xeb, 0x75, 0xd5, 0xdc, 0x45, 0x94, 0x72, 0xe5, 0xc5, 0x7e, 0x0a, 0x9b, 0x05, 0xb6, 0xaa, - 0xc7, 0xe3, 0x5b, 0x7a, 0x5e, 0xc1, 0xce, 0x4b, 0xee, 0xde, 0x5f, 0xea, 0xd0, 0x36, 0x22, 0xe7, - 0x54, 0x44, 0x14, 0x37, 0x32, 0x2a, 0x7e, 0x49, 0x9d, 0xdf, 0x2d, 0xbd, 0x11, 0x56, 0xae, 0x0e, - 0x58, 0x67, 0x19, 0x79, 0xad, 0x33, 0x5d, 0x2e, 0xed, 0xbb, 0xca, 0x25, 0xf6, 0x91, 0x6f, 0x83, - 0xe9, 0x95, 0x18, 0x13, 0x79, 0x9b, 0x5c, 0x89, 0x6c, 0x4f, 0xd7, 0x0e, 0x3a, 0xed, 0x42, 0x45, - 0x52, 0x16, 0xae, 0xeb, 0x8b, 0xac, 0x85, 0xd8, 0x3f, 0x34, 0x24, 0xab, 0xa4, 0xc4, 0x9e, 0x42, - 0x5b, 0x17, 0xb9, 0x79, 0x76, 0x90, 0x5d, 0x1d, 0x4a, 0x1b, 0xb9, 0xe9, 0xc8, 0x9e, 0x97, 0x1b, - 0x39, 0x3a, 0xcf, 0xf6, 0x81, 0x5b, 0xc8, 0xdc, 0xb0, 0xf3, 0x72, 0xe3, 0xb7, 0x6f, 0x74, 0x96, - 0x2e, 0xd0, 0xe4, 0x87, 0x7a, 0x72, 0x6e, 0xe2, 0x46, 0xff, 0xf9, 0x8d, 0xf9, 0xe2, 0xb8, 0x6d, - 0x9a, 0xd3, 0x2d, 0x22, 0x27, 0x6d, 0xdc, 0x7c, 0x99, 0xf6, 0x8d, 0xe7, 0xce, 0xed, 0x94, 0x17, - 0xca, 0x4d, 0xdc, 0x78, 0x14, 0xfd, 0x8a, 0x2e, 0xd0, 0xdd, 0xa0, 0xa9, 0xd5, 0x2d, 0x9e, 0x74, - 0xe1, 0x15, 0xbd, 0xe3, 0xf3, 0x72, 0xbf, 0xe0, 0x6e, 0x96, 0x81, 0x2a, 0xda, 0x79, 0xb9, 0xbf, - 0xd8, 0x37, 0x5a, 0x76, 0xf7, 0x41, 0x79, 0xff, 0xb9, 0x89, 0x6b, 0x2f, 0xef, 0x6f, 0x35, 0xd8, - 0xf0, 0x27, 0xb3, 0x38, 0x49, 0x8d, 0x5a, 0x23, 0xbf, 0x11, 0xac, 0xca, 0x6f, 0x84, 0x5a, 0xe9, - 0x35, 0xa5, 0x9a, 0x43, 0x35, 0xc6, 0xe1, 0x52, 0x30, 0x18, 0xe4, 0x14, 0x18, 0xb4, 0x0d, 0x2d, - 0x79, 0x5d, 0xd0, 0x54, 0x27, 0x93, 0x56, 0xc8, 0xaf, 0x96, 0x25, 0x75, 0xa4, 0x0d, 0xea, 0x71, - 0x95, 0x88, 0xf5, 0x55, 0xba, 0x91, 0xb1, 0x49, 0x46, 0x43, 0x83, 0xf6, 0xf3, 0x70, 0x22, 0xe6, - 0x69, 0x30, 0x99, 0x61, 0xc1, 0xb2, 0xfb, 0x36, 0x37, 0x34, 0x58, 0xab, 0x28, 0x89, 0x17, 0x89, - 0xc0, 0xd2, 0x71, 0x98, 0x12, 0x03, 0x6d, 0x5e, 0xd2, 0xa2, 0x1f, 0xa5, 0xa5, 0xfd, 0x40, 0xfa, - 0x15, 0xb5, 0xf4, 0xde, 0x46, 0x22, 0x48, 0x88, 0x57, 0x4d, 0x2e, 0x05, 0xef, 0x9f, 0x35, 0x60, - 0x12, 0x49, 0xd9, 0x51, 0xfe, 0xdf, 0xe0, 0xbc, 0x1b, 0xb6, 0x22, 0x38, 0x8d, 0x15, 0x70, 0xbe, - 0xc8, 0xfb, 0x60, 0x09, 0x4c, 0x26, 0xe1, 0x23, 0xa1, 0x9f, 0x28, 0x89, 0xaa, 0xc5, 0x4d, 0x15, - 0xf3, 0xa0, 0x63, 0xbc, 0x8f, 0x78, 0x6d, 0x31, 0x76, 0x41, 0x57, 0x01, 0x2d, 0xdc, 0x13, 0xda, - 0xf6, 0xdd, 0xd0, 0x76, 0x4c, 0x68, 0xdf, 0x5b, 0xd0, 0x39, 0x4c, 0xe3, 0x49, 0x38, 0xe2, 0x62, - 0x14, 0x27, 0xe3, 0xdb, 0x41, 0x95, 0xf0, 0xd5, 0x4c, 0xf8, 0xf6, 0xc0, 0xf6, 0xdf, 0x25, 0x59, - 0xf5, 0xdc, 0x36, 0x3a, 0xb8, 0x95, 0xb3, 0xe2, 0xe8, 0xc8, 0xbe, 0x82, 0x9a, 0x9f, 0x10, 0x73, - 0x0b, 0x75, 0xbf, 0x70, 0x49, 0x78, 0xcd, 0x4f, 0xbc, 0x1f, 0x42, 0x57, 0x6e, 0x4a, 0x99, 0xb2, - 0x77, 0xa8, 0x0b, 0xf5, 0xe3, 0x24, 0x89, 0xd5, 0x4b, 0x24, 0x05, 0xef, 0x1a, 0xba, 0xf9, 0xeb, - 0x85, 0x07, 0xf3, 0x29, 0xfc, 0xa8, 0xfa, 0x25, 0xd0, 0x83, 0xf6, 0x59, 0x9c, 0xbe, 0x49, 0xc2, - 0x94, 0x4a, 0x86, 0x2c, 0xfe, 0xa6, 0xca, 0xfb, 0x3e, 0x7c, 0x5e, 0x5a, 0x59, 0x3f, 0x98, 0x48, - 0x29, 0x5b, 0x7f, 0x1e, 0x0f, 0xe1, 0x61, 0xee, 0xea, 0x0f, 0x3e, 0x69, 0x8f, 0xab, 0x41, 0x7f, - 0x60, 0x64, 0x4e, 0x41, 0xb3, 0xe5, 0x2b, 0xb2, 0xf1, 0x8e, 0xc0, 0xcd, 0xd0, 0x94, 0x7f, 0x2c, - 0xb2, 0x1d, 0x5c, 0x84, 0x62, 0x79, 0xdb, 0x87, 0x17, 0xb5, 0x15, 0x35, 0xfa, 0xcf, 0x41, 0x63, - 0xef, 0xbf, 0x16, 0x74, 0xab, 0x82, 0x68, 0x72, 0x59, 0x06, 0xb9, 0xd8, 0x33, 0xa8, 0xbf, 0x0b, - 0xc5, 0x52, 0xb5, 0x08, 0xde, 0xca, 0x91, 0xaf, 0xec, 0x84, 0xcb, 0x09, 0x78, 0xb5, 0x0e, 0x47, - 0x69, 0x18, 0x4f, 0xd5, 0x57, 0x83, 0x94, 0x70, 0x9d, 0xa3, 0x28, 0x1e, 0xfd, 0x46, 0x7e, 0x0f, - 0x73, 0x29, 0x54, 0x5c, 0x95, 0xfa, 0x3d, 0xaf, 0xca, 0x7a, 0xd5, 0x55, 0xf1, 0xfe, 0x6a, 0x29, - 0xac, 0x8c, 0xce, 0xee, 0xa3, 0x27, 0xa6, 0x2f, 0x88, 0xad, 0x2e, 0x88, 0x2b, 0xdb, 0x53, 0xdd, - 0x85, 0x2b, 0x11, 0x5b, 0x62, 0x1c, 0xd2, 0xcf, 0x10, 0x87, 0x4e, 0x29, 0x97, 0x3f, 0x52, 0x95, - 0x56, 0x93, 0x5d, 0xaf, 0x4a, 0xf6, 0x68, 0xeb, 0xef, 0x1f, 0x76, 0xac, 0x7f, 0x7c, 0xd8, 0xb1, - 0xfe, 0xfd, 0x61, 0xc7, 0xfa, 0xc3, 0x7f, 0x76, 0xd6, 0x2e, 0xd7, 0xe9, 0x4f, 0xd9, 0x8f, 0xfe, - 0x17, 0x00, 0x00, 0xff, 0xff, 0x97, 0x50, 0x77, 0xb2, 0x39, 0x13, 0x00, 0x00, + // 1757 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x4f, 0x6f, 0xe4, 0x48, + 0x15, 0x8f, 0xdb, 0xee, 0x7f, 0xaf, 0x3b, 0x99, 0x6c, 0x4d, 0xcf, 0x62, 0x0d, 0x99, 0x6c, 0x63, + 0x05, 0xb6, 0x41, 0xab, 0xac, 0x12, 0x76, 0x60, 0x0e, 0xfc, 0xd9, 0x64, 0x3a, 0x4b, 0xac, 0x21, + 0xd9, 0xa1, 0x12, 0x65, 0xc5, 0x05, 0xc9, 0xe9, 0x2e, 0x7a, 0x2d, 0xdc, 0xed, 0xc6, 0xed, 0xde, + 0x4e, 0x2e, 0x48, 0x7c, 0x86, 0xbd, 0x70, 0x43, 0xdc, 0xf8, 0x1c, 0x5c, 0xe0, 0xc8, 0x11, 0x89, + 0x0b, 0x1a, 0xb8, 0xf2, 0x1d, 0xd0, 0x7b, 0xe5, 0x72, 0x95, 0xdd, 0x4e, 0x26, 0x1a, 0x71, 0xab, + 0xf7, 0xa7, 0x5e, 0xd5, 0xfb, 0xbd, 0x57, 0xef, 0x3d, 0x1b, 0xba, 0xf3, 0xe5, 0x75, 0x14, 0x8e, + 0xf6, 0xe7, 0x49, 0x9c, 0xc6, 0xac, 0x15, 0xce, 0x52, 0x91, 0xcc, 0x82, 0xc8, 0xfb, 0xa3, 0x05, + 0x36, 0x8f, 0x57, 0xcc, 0x85, 0xe6, 0xcb, 0x38, 0x5a, 0x4e, 0x67, 0x0b, 0xd7, 0xea, 0xdb, 0x03, + 0x87, 0x2b, 0x92, 0x31, 0x70, 0x5e, 0x89, 0xdb, 0x85, 0x6b, 0xf7, 0xed, 0x41, 0x9b, 0xd3, 0x9a, + 0xed, 0x41, 0xfd, 0x28, 0x4d, 0x93, 0x85, 0x5b, 0xeb, 0xdb, 0x83, 0xce, 0xe1, 0xd6, 0xbe, 0xb2, + 0xb7, 0x8f, 0x6c, 0x2e, 0x85, 0x68, 0x93, 0xc7, 0x41, 0x12, 0xce, 0x26, 0xae, 0xd3, 0xb7, 0x06, + 0x5d, 0xae, 0x48, 0xd6, 0x83, 0xba, 0x3f, 0x1b, 0x8b, 0x1b, 0xb7, 0xde, 0xb7, 0x06, 0x6d, 0x2e, + 0x09, 0xe4, 0x7e, 0x16, 0x8a, 0x68, 0xec, 0x36, 0x24, 0x97, 0x08, 0x6f, 0x1f, 0xda, 0x3c, 0x5e, + 0x9d, 0x05, 0x69, 0x12, 0xde, 0xb0, 0x6f, 0x81, 0xc3, 0xe3, 0x95, 0xbc, 0x63, 0xe7, 0x70, 0x53, + 0x9f, 0xcb, 0xe3, 0x15, 0x27, 0x91, 0x77, 0x06, 0xed, 0x8b, 0x70, 0x32, 0x13, 0x63, 0x74, 0xeb, + 0x03, 0xb0, 0x5f, 0xc7, 0xa8, 0x6e, 0xad, 0xab, 0xa3, 0x04, 0x15, 0xce, 0xc5, 0xc4, 0xad, 0x55, + 0x2a, 0x9c, 0x8b, 0x89, 0xf7, 0x02, 0xb6, 0x78, 0xbc, 0xf2, 0xc7, 0x62, 0x96, 0x86, 0xbf, 0x0e, + 0x45, 0x42, 0x80, 0xe4, 0x77, 0x70, 0xe4, 0xa1, 0x39, 0x48, 0x35, 0x0d, 0x92, 0xf7, 0x14, 0x1a, + 0xfe, 0xf0, 0xe7, 0xe1, 0x22, 0x65, 0xdb, 0x60, 0xfb, 0x43, 0xb5, 0x01, 0x97, 0x9e, 0x0f, 0xef, + 0x9d, 0xdc, 0xa4, 0x49, 0x30, 0x4a, 0xc5, 0xd8, 0x1f, 0x4a, 0xa8, 0xd9, 0x16, 0xd4, 0xfc, 0x21, + 0xdd, 0xd5, 0xe1, 0x35, 0x7f, 0xc8, 0xf6, 0xc0, 0xb9, 0x0a, 0x22, 0x05, 0xf2, 0xb6, 0xbe, 0x9c, + 0x34, 0xcb, 0x49, 0xea, 0x5d, 0x17, 0x4c, 0x65, 0x38, 0xbd, 0x0f, 0x0d, 0x42, 0x4f, 0x1e, 0xda, + 0xe6, 0x19, 0xc5, 0x9e, 0xeb, 0x30, 0x4b, 0xab, 0xdf, 0xd4, 0x56, 0xd7, 0x2e, 0x94, 0xe7, 0x80, + 0xf7, 0x0c, 0x9a, 0xaf, 0xc4, 0x2d, 0xf9, 0xa2, 0x3c, 0xb5, 0x0c, 0x4f, 0xff, 0x69, 0xc1, 0xe3, + 0x7c, 0xf7, 0x65, 0x70, 0x1d, 0x89, 0xab, 0x20, 0x5a, 0x0a, 0xb6, 0xa7, 0xfc, 0xb6, 0xaa, 0xee, + 0x7f, 0xba, 0x41, 0x58, 0xb0, 0x0f, 0x73, 0xec, 0x50, 0xed, 0x3d, 0xad, 0x96, 0x1d, 0x79, 0xba, + 0x91, 0x65, 0xdd, 0x0e, 0xb4, 0x8e, 0x2f, 0x7c, 0x32, 0xed, 0xda, 0x7d, 0x6b, 0x60, 0x9f, 0x6e, + 0xf0, 0x9c, 0xc3, 0x9e, 0x42, 0xf3, 0x6c, 0x99, 0x8a, 0x1b, 0x7f, 0x48, 0xd9, 0xe6, 0x9c, 0x6e, + 0x70, 0xc5, 0xc0, 0x9d, 0xb4, 0x7c, 0x25, 0x6e, 0x65, 0xca, 0xe1, 0x4e, 0xc5, 0x61, 0x3d, 0x70, + 0x8e, 0xe3, 0x38, 0xa2, 0xb4, 0x6b, 0xe1, 0x69, 0x48, 0x1d, 0x37, 0xa1, 0x4e, 0x86, 0xbd, 0xdf, + 0x41, 0xaf, 0xe8, 0x5c, 0x16, 0x2e, 0x06, 0x36, 0xda, 0xb3, 0x32, 0x7b, 0x48, 0xb0, 0x6d, 0x0a, + 0x61, 0x2d, 0x3b, 0x1f, 0x83, 0xf8, 0x1c, 0x1a, 0x64, 0x46, 0x3e, 0xa0, 0xce, 0xe1, 0xb3, 0x0a, + 0xc0, 0x35, 0x64, 0x3c, 0x53, 0x3e, 0x6e, 0x13, 0xe2, 0x9f, 0x27, 0xfe, 0xd0, 0xfb, 0x71, 0x19, + 0x5c, 0x8a, 0x25, 0x06, 0xe2, 0x3c, 0x98, 0x0a, 0x79, 0x3e, 0xa7, 0x35, 0xf2, 0x2e, 0x6f, 0xe7, + 0x82, 0x2e, 0xd0, 0xe6, 0xb4, 0xf6, 0x7e, 0x6f, 0xc1, 0x56, 0x71, 0x3f, 0xde, 0xc9, 0xc8, 0x8e, + 0x7b, 0xee, 0x44, 0x5a, 0x79, 0xf2, 0xbc, 0x28, 0x27, 0xcf, 0xee, 0x5d, 0xfb, 0xca, 0xf9, 0xf3, + 0x13, 0x70, 0x5e, 0x07, 0x61, 0xb2, 0x96, 0xe1, 0xdb, 0x12, 0x42, 0x9b, 0xae, 0x6b, 0xcb, 0x58, + 0xd4, 0x5f, 0xc6, 0xcb, 0x59, 0x2a, 0x31, 0xe4, 0x92, 0xf0, 0x4e, 0xa0, 0x8d, 0xfb, 0xa5, 0xe3, + 0x9e, 0x34, 0x96, 0xa5, 0x95, 0x51, 0x7b, 0x90, 0xcb, 0xe5, 0x41, 0x79, 0x29, 0xa9, 0x99, 0xa5, + 0xe4, 0x14, 0x00, 0xa5, 0x0b, 0x69, 0x67, 0x0f, 0xea, 0x44, 0x65, 0x20, 0x94, 0x0d, 0x49, 0xe1, + 0x1d, 0x96, 0x9e, 0x61, 0x01, 0x4b, 0x7f, 0xf0, 0x09, 0x8a, 0x65, 0x42, 0xe2, 0x6d, 0x6c, 0x9e, + 0xa5, 0xcc, 0x12, 0x5a, 0x12, 0xba, 0x78, 0xa5, 0x0d, 0x58, 0x86, 0x01, 0xe4, 0x62, 0x59, 0x19, + 0x2a, 0x3f, 0x89, 0xc0, 0x67, 0xcb, 0xe3, 0x95, 0x86, 0x24, 0xa3, 0xd8, 0xb7, 0xd5, 0x29, 0x0e, + 0xf9, 0xfc, 0xc8, 0x78, 0x4a, 0x78, 0x0b, 0x75, 0xec, 0xaf, 0x00, 0x7e, 0x96, 0xc4, 0xcb, 0x39, + 0x81, 0xc6, 0x06, 0x50, 0x27, 0x2a, 0xf3, 0x8f, 0xe9, 0x4d, 0xea, 0x6e, 0x5c, 0x2a, 0x54, 0x83, + 0x8e, 0xc1, 0x39, 0x9a, 0x4c, 0xe4, 0x4b, 0xe3, 0xb8, 0xc4, 0x54, 0x6a, 0x5d, 0x05, 0x51, 0x2e, + 0xbe, 0x0a, 0xa2, 0xcc, 0x6f, 0x5c, 0x16, 0xcd, 0xd8, 0xca, 0xcc, 0x53, 0x68, 0x7d, 0x16, 0xc5, + 0x41, 0x8a, 0xca, 0x68, 0xcb, 0xe2, 0x39, 0xcd, 0x0e, 0x00, 0x86, 0x62, 0x14, 0x4e, 0x83, 0x08, + 0xa5, 0x4e, 0xb9, 0x00, 0x64, 0x32, 0x6e, 0x28, 0x79, 0xcf, 0xa1, 0x99, 0x51, 0xd5, 0xd8, 0x23, + 0xf7, 0x62, 0x14, 0x44, 0x42, 0xdd, 0x82, 0x08, 0xef, 0x0b, 0xd8, 0x94, 0xc9, 0x88, 0xad, 0xe9, + 0x42, 0xa4, 0x0f, 0x48, 0xc5, 0x07, 0x35, 0x39, 0xef, 0xcf, 0x16, 0x38, 0xb8, 0x52, 0x06, 0x2c, + 0x6d, 0xc0, 0x7c, 0x8d, 0x8e, 0x7c, 0x8d, 0xac, 0x0f, 0x9d, 0x8b, 0x14, 0x7b, 0xa0, 0x2e, 0x63, + 0x6d, 0x6e, 0xb2, 0x10, 0x2f, 0x7f, 0x96, 0xea, 0x70, 0xdb, 0x3c, 0xa7, 0xd9, 0x0e, 0xb4, 0xb1, + 0x36, 0x49, 0x21, 0x16, 0xb2, 0x16, 0xd7, 0x0c, 0xb6, 0x0b, 0xa0, 0x90, 0x5d, 0x0a, 0xaa, 0x66, + 0x16, 0x37, 0x38, 0xde, 0xc7, 0xd0, 0xc4, 0x9b, 0x9e, 0x05, 0x73, 0xed, 0x9b, 0x75, 0x9f, 0x6f, + 0x7f, 0xaa, 0x41, 0xf7, 0x17, 0x4b, 0x91, 0xdc, 0x72, 0xf1, 0xdb, 0xa5, 0x58, 0xa4, 0x88, 0x2d, + 0xd1, 0x2a, 0x97, 0x89, 0xc0, 0xac, 0xbd, 0xf8, 0x32, 0x48, 0xc6, 0x12, 0x29, 0x87, 0x67, 0x14, + 0xfa, 0xaa, 0x31, 0x5f, 0x90, 0xaf, 0x2d, 0x6e, 0xb2, 0x28, 0xdf, 0xc5, 0x34, 0x4e, 0x95, 0x33, + 0x19, 0xc5, 0x06, 0xf0, 0xe8, 0xe4, 0x66, 0x14, 0x2d, 0xc7, 0x82, 0xc7, 0x2b, 0xb9, 0x9b, 0x8a, + 0x33, 0x2f, 0xb3, 0xd9, 0x77, 0xb0, 0xb8, 0x11, 0x4b, 0x95, 0xa6, 0x26, 0x29, 0x96, 0xb8, 0xec, + 0x00, 0xba, 0x27, 0xd3, 0x6b, 0x31, 0x1e, 0x8b, 0xf1, 0x30, 0x48, 0x03, 0xb7, 0x55, 0x35, 0x40, + 0x14, 0x54, 0xd8, 0x1e, 0x6c, 0xbe, 0x4e, 0xc4, 0x65, 0x12, 0xcc, 0x16, 0x51, 0x90, 0x8a, 0xb1, + 0xdb, 0x26, 0xcb, 0x45, 0xa6, 0xf7, 0xb5, 0x05, 0x9b, 0x19, 0x46, 0x8b, 0x79, 0x3c, 0x5b, 0x08, + 0x4c, 0x84, 0x93, 0x24, 0x51, 0x89, 0x70, 0x92, 0x24, 0xec, 0x63, 0x68, 0x72, 0xb1, 0x58, 0x46, + 0xa9, 0xca, 0xa5, 0x27, 0xfa, 0x5c, 0xb5, 0x77, 0x19, 0xa5, 0x5c, 0x69, 0xb1, 0x9f, 0xc2, 0x56, + 0x21, 0x5b, 0x55, 0xf3, 0xf8, 0x86, 0xde, 0x57, 0x90, 0xf3, 0x92, 0xba, 0xf7, 0xdf, 0x3a, 0x74, + 0x0c, 0xcb, 0x79, 0x2a, 0x22, 0x8a, 0x9b, 0x59, 0x2a, 0x7e, 0x40, 0x93, 0xdf, 0x1d, 0xb3, 0x11, + 0x56, 0xae, 0x2e, 0x58, 0xe7, 0x59, 0xf2, 0x5a, 0xe7, 0xba, 0x5c, 0xda, 0xf7, 0x95, 0x4b, 0x9c, + 0x23, 0xbf, 0x0c, 0x66, 0x13, 0x31, 0xa6, 0xe4, 0x6d, 0x71, 0x45, 0xb2, 0x7d, 0x5d, 0x3b, 0x28, + 0xda, 0x85, 0x8a, 0xa4, 0x24, 0x5c, 0xd7, 0x17, 0x59, 0x0b, 0x71, 0x7e, 0x68, 0xca, 0xac, 0x92, + 0x14, 0xfb, 0x11, 0x6c, 0x7d, 0x1e, 0x8d, 0x75, 0x9d, 0x5b, 0x64, 0xb1, 0xec, 0x69, 0x6b, 0x5a, + 0xc8, 0x4b, 0xba, 0xec, 0xd3, 0xf2, 0x38, 0x47, 0x51, 0xed, 0x1c, 0xba, 0x05, 0xff, 0x0d, 0x39, + 0x2f, 0x8f, 0x7f, 0x07, 0xc6, 0x7c, 0xe9, 0x02, 0x6d, 0x7e, 0xac, 0x37, 0xe7, 0x22, 0x6e, 0x4c, + 0xa1, 0x9f, 0x98, 0x7d, 0xc7, 0xed, 0xd0, 0x9e, 0x5e, 0x11, 0x3f, 0x29, 0xe3, 0x66, 0x7f, 0x3a, + 0x30, 0x9a, 0x9e, 0xdb, 0x2d, 0x1f, 0x94, 0x8b, 0xb8, 0xd1, 0x1a, 0xfd, 0x8a, 0x59, 0xd0, 0xdd, + 0xa4, 0xad, 0xd5, 0x83, 0x9e, 0x54, 0xe1, 0x15, 0x13, 0xe4, 0xa7, 0xe5, 0xa9, 0xc1, 0xdd, 0x2a, + 0x03, 0x55, 0x94, 0xf3, 0xf2, 0x94, 0x71, 0x60, 0x0c, 0xee, 0xee, 0xa3, 0xf2, 0xfd, 0x73, 0x11, + 0x37, 0xc6, 0xfb, 0x1f, 0x42, 0xc7, 0x0c, 0xec, 0x36, 0x6d, 0x7a, 0x52, 0x15, 0xd8, 0x05, 0x37, + 0x35, 0xbd, 0xbf, 0xd6, 0x60, 0xd3, 0x9f, 0xce, 0xe3, 0x24, 0x35, 0x4a, 0x95, 0xfc, 0xc4, 0xb0, + 0x2a, 0x3f, 0x31, 0x6a, 0xa5, 0x66, 0x4c, 0x25, 0x8b, 0x4a, 0x94, 0xc3, 0x25, 0x61, 0x24, 0xa0, + 0x53, 0x48, 0xc0, 0x1d, 0x68, 0xcb, 0xd7, 0x86, 0xa2, 0x3a, 0x89, 0x34, 0x43, 0x7e, 0xf4, 0xac, + 0x68, 0xa0, 0x6d, 0xd2, 0x88, 0xac, 0x48, 0x2c, 0xcf, 0x52, 0x8d, 0x84, 0x2d, 0x12, 0x1a, 0x1c, + 0x94, 0x5f, 0x86, 0x53, 0xb1, 0x48, 0x83, 0xe9, 0x1c, 0xeb, 0x9d, 0x3d, 0xb0, 0xb9, 0xc1, 0xc1, + 0x52, 0x47, 0x4e, 0xbc, 0x4c, 0x04, 0x56, 0x9e, 0xa3, 0x94, 0x52, 0xd7, 0xe6, 0x25, 0x2e, 0xea, + 0x91, 0x5b, 0x5a, 0x0f, 0xa4, 0x5e, 0x91, 0x4b, 0xed, 0x3a, 0x12, 0x41, 0x42, 0x09, 0xd9, 0xe2, + 0x92, 0xf0, 0xfe, 0x51, 0x03, 0x26, 0x91, 0x94, 0x03, 0xe9, 0xff, 0x0d, 0xce, 0xfb, 0x61, 0x2b, + 0x82, 0xd3, 0x5c, 0x03, 0xe7, 0xfd, 0x7c, 0x8c, 0x96, 0xc0, 0x64, 0x14, 0xf6, 0x18, 0xdd, 0xe1, + 0x24, 0xaa, 0x16, 0x37, 0x59, 0xcc, 0x83, 0xae, 0xd1, 0x5e, 0xf1, 0xbd, 0xa3, 0xed, 0x02, 0xaf, + 0x02, 0x5a, 0x78, 0x20, 0xb4, 0x9d, 0xfb, 0xa1, 0xed, 0x9a, 0xd0, 0x7e, 0x6d, 0x41, 0xf7, 0x28, + 0x8d, 0xa7, 0xe1, 0x88, 0x8b, 0x51, 0x9c, 0x8c, 0xef, 0x06, 0x55, 0xc2, 0x57, 0x33, 0xe1, 0xdb, + 0x07, 0xdb, 0xff, 0x2a, 0xc9, 0x8a, 0xef, 0x8e, 0x31, 0x00, 0xae, 0xc5, 0x8a, 0xa3, 0x22, 0xfb, + 0x10, 0x6a, 0x7e, 0x42, 0x99, 0x5b, 0x68, 0x1b, 0x85, 0x47, 0xc2, 0x6b, 0x7e, 0xe2, 0x7d, 0x04, + 0x3d, 0x79, 0x29, 0x25, 0xca, 0xda, 0x58, 0x0f, 0xea, 0x27, 0x49, 0x12, 0xab, 0x46, 0x26, 0x09, + 0xef, 0x06, 0x7a, 0x79, 0xf3, 0xc3, 0xc0, 0xbc, 0x4b, 0x7e, 0x54, 0xfd, 0x51, 0xe8, 0x43, 0xe7, + 0x3c, 0x4e, 0xbf, 0x48, 0xc2, 0x94, 0x6a, 0x8d, 0xec, 0x1d, 0x26, 0xcb, 0xfb, 0x2e, 0x3c, 0x29, + 0x9d, 0xac, 0xfb, 0x2d, 0xa6, 0x94, 0xad, 0xbf, 0xae, 0x2f, 0xe0, 0x71, 0xae, 0xea, 0x0f, 0xdf, + 0xe9, 0x8e, 0xeb, 0x46, 0xbf, 0x67, 0x78, 0x4e, 0x46, 0xb3, 0xe3, 0x2b, 0xbc, 0xf1, 0x8e, 0xc1, + 0xcd, 0xd0, 0x94, 0x3f, 0x3c, 0xb2, 0x1b, 0x5c, 0x85, 0x62, 0x75, 0xd7, 0x77, 0x1b, 0x4d, 0x25, + 0x35, 0xfa, 0x4d, 0x42, 0x6b, 0xef, 0x3f, 0x16, 0xf4, 0xaa, 0x8c, 0xe8, 0xe4, 0xb2, 0x8c, 0xe4, + 0x62, 0x2f, 0xa0, 0xfe, 0x55, 0x28, 0x56, 0x6a, 0xc2, 0xf0, 0xd6, 0x42, 0xbe, 0x76, 0x13, 0x2e, + 0x37, 0xe0, 0xd3, 0x3a, 0x1a, 0xa5, 0x61, 0x3c, 0x53, 0x1f, 0x1d, 0x92, 0xc2, 0x73, 0x8e, 0xa3, + 0x78, 0xf4, 0x1b, 0xf9, 0x39, 0xcd, 0x25, 0x51, 0xf1, 0x54, 0xea, 0x0f, 0x7c, 0x2a, 0x8d, 0xaa, + 0xa7, 0xe2, 0xfd, 0xc5, 0x52, 0x58, 0x19, 0x83, 0xe1, 0x5b, 0x23, 0xa6, 0x1f, 0x88, 0xad, 0x1e, + 0x88, 0x2b, 0xa7, 0x5b, 0x3d, 0xc4, 0x2b, 0x12, 0x27, 0x6a, 0x5c, 0xd2, 0xbf, 0x14, 0x87, 0xa2, + 0x94, 0xd3, 0x6f, 0xa9, 0x4a, 0xeb, 0xce, 0x36, 0xaa, 0x9c, 0xf5, 0x7e, 0x59, 0xe8, 0x5b, 0x68, + 0xf4, 0x68, 0x32, 0x49, 0xc4, 0x24, 0x48, 0x55, 0x9c, 0x35, 0x83, 0x7d, 0x04, 0x0d, 0x52, 0x56, + 0xa1, 0xaa, 0x1e, 0x5c, 0x32, 0x9d, 0xe3, 0xed, 0xbf, 0xbd, 0xd9, 0xb5, 0xfe, 0xfe, 0x66, 0xd7, + 0xfa, 0xd7, 0x9b, 0x5d, 0xeb, 0x0f, 0xff, 0xde, 0xdd, 0xb8, 0x6e, 0xd0, 0x3f, 0xbc, 0xef, 0xff, + 0x2f, 0x00, 0x00, 0xff, 0xff, 0x0a, 0x79, 0xf3, 0x2b, 0xd3, 0x13, 0x00, 0x00, } func (m *Row) Marshal() (dAtA []byte, err error) { @@ -3770,8 +3843,8 @@ func (m *GroupCount) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } - if m.Sum != 0 { - i = encodeVarintPublic(dAtA, i, uint64(m.Sum)) + if m.Agg != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.Agg)) i-- dAtA[i] = 0x18 } @@ -4252,6 +4325,20 @@ func (m *QueryResult) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.GroupCounts != nil { + { + size, err := m.GroupCounts.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPublic(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x82 + } if m.RowMatrix != nil { { size, err := m.RowMatrix.MarshalToSizedBuffer(dAtA[:i]) @@ -4336,10 +4423,10 @@ func (m *QueryResult) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0x4a } - if len(m.GroupCounts) > 0 { - for iNdEx := len(m.GroupCounts) - 1; iNdEx >= 0; iNdEx-- { + if len(m.OldGroupCounts) > 0 { + for iNdEx := len(m.OldGroupCounts) - 1; iNdEx >= 0; iNdEx-- { { - size, err := m.GroupCounts[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + size, err := m.OldGroupCounts[iNdEx].MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } @@ -4351,20 +4438,20 @@ func (m *QueryResult) MarshalToSizedBuffer(dAtA []byte) (int, error) { } } if len(m.RowIDs) > 0 { - dAtA24 := make([]byte, len(m.RowIDs)*10) - var j23 int + dAtA25 := make([]byte, len(m.RowIDs)*10) + var j24 int for _, num := range m.RowIDs { for num >= 1<<7 { - dAtA24[j23] = uint8(uint64(num)&0x7f | 0x80) + dAtA25[j24] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j23++ + j24++ } - dAtA24[j23] = uint8(num) - j23++ + dAtA25[j24] = uint8(num) + j24++ } - i -= j23 - copy(dAtA[i:], dAtA24[:j23]) - i = encodeVarintPublic(dAtA, i, uint64(j23)) + i -= j24 + copy(dAtA[i:], dAtA25[:j24]) + i = encodeVarintPublic(dAtA, i, uint64(j24)) i-- dAtA[i] = 0x3a } @@ -4492,57 +4579,57 @@ func (m *ImportRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { } } if len(m.Timestamps) > 0 { - dAtA28 := make([]byte, len(m.Timestamps)*10) - var j27 int + dAtA29 := make([]byte, len(m.Timestamps)*10) + var j28 int for _, num1 := range m.Timestamps { num := uint64(num1) for num >= 1<<7 { - dAtA28[j27] = uint8(uint64(num)&0x7f | 0x80) + dAtA29[j28] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j27++ + j28++ } - dAtA28[j27] = uint8(num) - j27++ + dAtA29[j28] = uint8(num) + j28++ } - i -= j27 - copy(dAtA[i:], dAtA28[:j27]) - i = encodeVarintPublic(dAtA, i, uint64(j27)) + i -= j28 + copy(dAtA[i:], dAtA29[:j28]) + i = encodeVarintPublic(dAtA, i, uint64(j28)) i-- dAtA[i] = 0x32 } if len(m.ColumnIDs) > 0 { - dAtA30 := make([]byte, len(m.ColumnIDs)*10) - var j29 int + dAtA31 := make([]byte, len(m.ColumnIDs)*10) + var j30 int for _, num := range m.ColumnIDs { for num >= 1<<7 { - dAtA30[j29] = uint8(uint64(num)&0x7f | 0x80) + dAtA31[j30] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j29++ + j30++ } - dAtA30[j29] = uint8(num) - j29++ + dAtA31[j30] = uint8(num) + j30++ } - i -= j29 - copy(dAtA[i:], dAtA30[:j29]) - i = encodeVarintPublic(dAtA, i, uint64(j29)) + i -= j30 + copy(dAtA[i:], dAtA31[:j30]) + i = encodeVarintPublic(dAtA, i, uint64(j30)) i-- dAtA[i] = 0x2a } if len(m.RowIDs) > 0 { - dAtA32 := make([]byte, len(m.RowIDs)*10) - var j31 int + dAtA33 := make([]byte, len(m.RowIDs)*10) + var j32 int for _, num := range m.RowIDs { for num >= 1<<7 { - dAtA32[j31] = uint8(uint64(num)&0x7f | 0x80) + dAtA33[j32] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j31++ + j32++ } - dAtA32[j31] = uint8(num) - j31++ + dAtA33[j32] = uint8(num) + j32++ } - i -= j31 - copy(dAtA[i:], dAtA32[:j31]) - i = encodeVarintPublic(dAtA, i, uint64(j31)) + i -= j32 + copy(dAtA[i:], dAtA33[:j32]) + i = encodeVarintPublic(dAtA, i, uint64(j32)) i-- dAtA[i] = 0x22 } @@ -4623,9 +4710,9 @@ func (m *ImportValueRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { } if len(m.FloatValues) > 0 { for iNdEx := len(m.FloatValues) - 1; iNdEx >= 0; iNdEx-- { - f33 := math.Float64bits(float64(m.FloatValues[iNdEx])) + f34 := math.Float64bits(float64(m.FloatValues[iNdEx])) i -= 8 - encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(f33)) + encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(f34)) } i = encodeVarintPublic(dAtA, i, uint64(len(m.FloatValues)*8)) i-- @@ -4641,39 +4728,39 @@ func (m *ImportValueRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { } } if len(m.Values) > 0 { - dAtA35 := make([]byte, len(m.Values)*10) - var j34 int + dAtA36 := make([]byte, len(m.Values)*10) + var j35 int for _, num1 := range m.Values { num := uint64(num1) for num >= 1<<7 { - dAtA35[j34] = uint8(uint64(num)&0x7f | 0x80) + dAtA36[j35] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j34++ + j35++ } - dAtA35[j34] = uint8(num) - j34++ + dAtA36[j35] = uint8(num) + j35++ } - i -= j34 - copy(dAtA[i:], dAtA35[:j34]) - i = encodeVarintPublic(dAtA, i, uint64(j34)) + i -= j35 + copy(dAtA[i:], dAtA36[:j35]) + i = encodeVarintPublic(dAtA, i, uint64(j35)) i-- dAtA[i] = 0x32 } if len(m.ColumnIDs) > 0 { - dAtA37 := make([]byte, len(m.ColumnIDs)*10) - var j36 int + dAtA38 := make([]byte, len(m.ColumnIDs)*10) + var j37 int for _, num := range m.ColumnIDs { for num >= 1<<7 { - dAtA37[j36] = uint8(uint64(num)&0x7f | 0x80) + dAtA38[j37] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j36++ + j37++ } - dAtA37[j36] = uint8(num) - j36++ + dAtA38[j37] = uint8(num) + j37++ } - i -= j36 - copy(dAtA[i:], dAtA37[:j36]) - i = encodeVarintPublic(dAtA, i, uint64(j36)) + i -= j37 + copy(dAtA[i:], dAtA38[:j37]) + i = encodeVarintPublic(dAtA, i, uint64(j37)) i-- dAtA[i] = 0x2a } @@ -4885,20 +4972,20 @@ func (m *TranslateKeysResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { copy(dAtA[i:], m.XXX_unrecognized) } if len(m.IDs) > 0 { - dAtA39 := make([]byte, len(m.IDs)*10) - var j38 int + dAtA40 := make([]byte, len(m.IDs)*10) + var j39 int for _, num := range m.IDs { for num >= 1<<7 { - dAtA39[j38] = uint8(uint64(num)&0x7f | 0x80) + dAtA40[j39] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j38++ + j39++ } - dAtA39[j38] = uint8(num) - j38++ + dAtA40[j39] = uint8(num) + j39++ } - i -= j38 - copy(dAtA[i:], dAtA39[:j38]) - i = encodeVarintPublic(dAtA, i, uint64(j38)) + i -= j39 + copy(dAtA[i:], dAtA40[:j39]) + i = encodeVarintPublic(dAtA, i, uint64(j39)) i-- dAtA[i] = 0x1a } @@ -4930,20 +5017,20 @@ func (m *TranslateIDsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { copy(dAtA[i:], m.XXX_unrecognized) } if len(m.IDs) > 0 { - dAtA41 := make([]byte, len(m.IDs)*10) - var j40 int + dAtA42 := make([]byte, len(m.IDs)*10) + var j41 int for _, num := range m.IDs { for num >= 1<<7 { - dAtA41[j40] = uint8(uint64(num)&0x7f | 0x80) + dAtA42[j41] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j40++ + j41++ } - dAtA41[j40] = uint8(num) - j40++ + dAtA42[j41] = uint8(num) + j41++ } - i -= j40 - copy(dAtA[i:], dAtA41[:j40]) - i = encodeVarintPublic(dAtA, i, uint64(j40)) + i -= j41 + copy(dAtA[i:], dAtA42[:j41]) + i = encodeVarintPublic(dAtA, i, uint64(j41)) i-- dAtA[i] = 0x1a } @@ -5144,20 +5231,20 @@ func (m *ImportColumnAttrsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error dAtA[i] = 0x30 } if len(m.ColumnIDs) > 0 { - dAtA43 := make([]byte, len(m.ColumnIDs)*10) - var j42 int + dAtA44 := make([]byte, len(m.ColumnIDs)*10) + var j43 int for _, num := range m.ColumnIDs { for num >= 1<<7 { - dAtA43[j42] = uint8(uint64(num)&0x7f | 0x80) + dAtA44[j43] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j42++ + j43++ } - dAtA43[j42] = uint8(num) - j42++ + dAtA44[j43] = uint8(num) + j43++ } - i -= j42 - copy(dAtA[i:], dAtA43[:j42]) - i = encodeVarintPublic(dAtA, i, uint64(j42)) + i -= j43 + copy(dAtA[i:], dAtA44[:j43]) + i = encodeVarintPublic(dAtA, i, uint64(j43)) i-- dAtA[i] = 0x2a } @@ -5192,6 +5279,54 @@ func (m *ImportColumnAttrsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error return len(dAtA) - i, nil } +func (m *GroupCounts) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *GroupCounts) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *GroupCounts) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Groups) > 0 { + for iNdEx := len(m.Groups) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Groups[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPublic(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + } + if len(m.Aggregate) > 0 { + i -= len(m.Aggregate) + copy(dAtA[i:], m.Aggregate) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Aggregate))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func encodeVarintPublic(dAtA []byte, offset int, v uint64) int { offset -= sovPublic(v) base := offset @@ -5672,8 +5807,8 @@ func (m *GroupCount) Size() (n int) { if m.Count != 0 { n += 1 + sovPublic(uint64(m.Count)) } - if m.Sum != 0 { - n += 1 + sovPublic(uint64(m.Sum)) + if m.Agg != 0 { + n += 1 + sovPublic(uint64(m.Agg)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) @@ -5907,8 +6042,8 @@ func (m *QueryResult) Size() (n int) { } n += 1 + sovPublic(uint64(l)) + l } - if len(m.GroupCounts) > 0 { - for _, e := range m.GroupCounts { + if len(m.OldGroupCounts) > 0 { + for _, e := range m.OldGroupCounts { l = e.Size() n += 1 + l + sovPublic(uint64(l)) } @@ -5941,6 +6076,10 @@ func (m *QueryResult) Size() (n int) { l = m.RowMatrix.Size() n += 1 + l + sovPublic(uint64(l)) } + if m.GroupCounts != nil { + l = m.GroupCounts.Size() + n += 2 + l + sovPublic(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -6306,6 +6445,28 @@ func (m *ImportColumnAttrsRequest) Size() (n int) { return n } +func (m *GroupCounts) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Aggregate) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } + if len(m.Groups) > 0 { + for _, e := range m.Groups { + l = e.Size() + n += 1 + l + sovPublic(uint64(l)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func sovPublic(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } @@ -6587,10 +6748,7 @@ func (m *Row) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -6675,10 +6833,7 @@ func (m *RowMatrix) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -6801,10 +6956,7 @@ func (m *SignedRow) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -6963,10 +7115,7 @@ func (m *RowIdentifiers) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7093,10 +7242,7 @@ func (m *IDList) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7200,10 +7346,7 @@ func (m *ExtractedIDColumn) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7320,10 +7463,7 @@ func (m *ExtractedIDMatrix) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7406,10 +7546,7 @@ func (m *KeyList) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7623,10 +7760,7 @@ func (m *ExtractedTableValue) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7763,10 +7897,7 @@ func (m *ExtractedTableColumn) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7881,10 +8012,7 @@ func (m *ExtractedTableField) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8003,10 +8131,7 @@ func (m *ExtractedTable) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8127,10 +8252,7 @@ func (m *Pair) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8249,10 +8371,7 @@ func (m *PairField) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8369,10 +8488,7 @@ func (m *PairsField) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8442,10 +8558,7 @@ func (m *Int64) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8615,10 +8728,7 @@ func (m *FieldRow) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8718,9 +8828,9 @@ func (m *GroupCount) Unmarshal(dAtA []byte) error { } case 3: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Sum", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Agg", wireType) } - m.Sum = 0 + m.Agg = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPublic @@ -8730,7 +8840,7 @@ func (m *GroupCount) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Sum |= int64(b&0x7F) << shift + m.Agg |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -8741,10 +8851,7 @@ func (m *GroupCount) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8880,10 +8987,7 @@ func (m *ValCount) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8972,10 +9076,7 @@ func (m *Decimal) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -9111,10 +9212,7 @@ func (m *ColumnAttrSet) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -9298,10 +9396,7 @@ func (m *Attr) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -9386,10 +9481,7 @@ func (m *AttrMap) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -9682,10 +9774,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -9836,10 +9925,7 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -10126,7 +10212,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } case 8: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field GroupCounts", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field OldGroupCounts", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -10153,8 +10239,8 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.GroupCounts = append(m.GroupCounts, &GroupCount{}) - if err := m.GroupCounts[len(m.GroupCounts)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.OldGroupCounts = append(m.OldGroupCounts, &GroupCount{}) + if err := m.OldGroupCounts[len(m.OldGroupCounts)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -10410,16 +10496,49 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 16: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field GroupCounts", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPublic + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.GroupCounts == nil { + m.GroupCounts = &GroupCounts{} + } + if err := m.GroupCounts.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -10903,10 +11022,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -11368,10 +11484,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -11541,10 +11654,7 @@ func (m *AtomicRecord) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -11627,10 +11737,7 @@ func (m *AtomicImportResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -11797,10 +11904,7 @@ func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -11927,10 +12031,7 @@ func (m *TranslateKeysResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -12121,10 +12222,7 @@ func (m *TranslateIDsRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -12207,10 +12305,7 @@ func (m *TranslateIDsResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -12327,10 +12422,7 @@ func (m *ImportRoaringRequestView) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -12524,10 +12616,7 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -12788,10 +12877,124 @@ func (m *ImportColumnAttrsRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } - if (iNdEx + skippy) < 0 { + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *GroupCounts) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: GroupCounts: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: GroupCounts: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Aggregate", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Aggregate = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Groups", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPublic + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Groups = append(m.Groups, &GroupCount{}) + if err := m.Groups[len(m.Groups)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPublic(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { diff --git a/internal/public.proto b/internal/public.proto index 3a4a251b5..51de362bd 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -102,7 +102,7 @@ message FieldRow { message GroupCount{ repeated FieldRow Group = 1; uint64 Count = 2; - int64 Sum = 3; + int64 Agg = 3; } message ValCount { @@ -159,9 +159,15 @@ message QueryResult { uint64 N = 2; repeated Pair Pairs = 3; bool Changed = 4; - ValCount ValCount = 5; + ValCount ValCount = 5; repeated uint64 RowIDs = 7; - repeated GroupCount GroupCounts = 8; + // In the past, GroupCounts was a []GroupCount which did not indicate + // whether it had an aggregate, or which aggregate it had. We've + // updated this, but we keep this here so that messages using the old + // format can get a best-effort treatment rather than causing panics. + // Later this can almost certainly go away, but leave a comment warning + // people that 8 is Spoken For if you do that, please. + repeated GroupCount OldGroupCounts = 8; RowIdentifiers RowIdentifiers = 9; SignedRow SignedRow = 10; PairsField PairsField = 11; @@ -169,6 +175,7 @@ message QueryResult { ExtractedIDMatrix ExtractedIDMatrix = 13; ExtractedTable ExtractedTable = 14; RowMatrix RowMatrix = 15; + GroupCounts GroupCounts = 16; } message ImportRequest { @@ -254,3 +261,8 @@ message ImportColumnAttrsRequest { repeated uint64 ColumnIDs = 5; int64 IndexCreatedAt = 6; } + +message GroupCounts{ + string Aggregate = 1; + repeated GroupCount Groups = 2; +} diff --git a/server/grpc.go b/server/grpc.go index 7f189a730..c2e68bb1f 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -455,10 +455,8 @@ func ToTablerWrapper(result interface{}) (pb.ToTabler, error) { toTabler, ok := result.(pb.ToTabler) if !ok { switch v := result.(type) { - // TODO: replace []pilosa.GroupCount with *pilosa.GroupCounts, handle aggregate case []pilosa.GroupCount: - gc := pilosa.NewGroupCounts() - gc.Groups = v + gc := pilosa.NewGroupCounts("", v...) toTabler = gc case uint64: toTabler = ResultUint64(v) @@ -479,10 +477,8 @@ func ToRowserWrapper(result interface{}) (pb.ToRowser, error) { toRowser, ok := result.(pb.ToRowser) if !ok { switch v := result.(type) { - // TODO: replace []pilosa.GroupCount with *pilosa.GroupCounts, handle aggregate case []pilosa.GroupCount: - gc := pilosa.NewGroupCounts() - gc.Groups = v + gc := pilosa.NewGroupCounts("", v...) toRowser = gc case uint64: toRowser = ResultUint64(v) diff --git a/server/grpc_test.go b/server/grpc_test.go index 49bf03103..3cc808bb2 100644 --- a/server/grpc_test.go +++ b/server/grpc_test.go @@ -138,7 +138,7 @@ func TestGRPC(t *testing.T) { }, // []GroupCount (uint64) { - []pilosa.GroupCount{ + pilosa.NewGroupCounts("", []pilosa.GroupCount{ pilosa.GroupCount{ Group: []pilosa.FieldRow{ {Field: "a", RowID: 10}, @@ -160,22 +160,21 @@ func TestGRPC(t *testing.T) { }, Count: 789, }, - }, + }...), []expHeader{ {"a", "uint64"}, {"b", "uint64"}, {"count", "uint64"}, - {"sum", "int64"}, }, [][]expColumn{ - {uint64(10), uint64(11), uint64(123), int64(0)}, - {uint64(10), uint64(12), uint64(456), int64(0)}, - {int64(va), int64(vb), uint64(789), int64(0)}, + {uint64(10), uint64(11), uint64(123)}, + {uint64(10), uint64(12), uint64(456)}, + {int64(va), int64(vb), uint64(789)}, }, }, - // []GroupCount (string) + // []GroupCount (string) + sum { - []pilosa.GroupCount{ + pilosa.NewGroupCounts("sum", []pilosa.GroupCount{ pilosa.GroupCount{ Group: []pilosa.FieldRow{ {Field: "a", RowKey: "ten"}, @@ -190,7 +189,7 @@ func TestGRPC(t *testing.T) { }, Count: 456, }, - }, + }...), []expHeader{ {"a", "string"}, {"b", "string"}, @@ -292,7 +291,12 @@ func TestGRPC(t *testing.T) { } // Ensure headers match. - for i, header := range table.GetHeaders() { + headers := table.GetHeaders() + if len(headers) < len(test.expHeaders) { + t.Fatalf("test %d expected %d headers, got %d, first missing header %q", + ti, len(test.expHeaders), len(headers), test.expHeaders[len(headers)].name) + } + for i, header := range headers { if header.Name != test.expHeaders[i].name { t.Fatalf("test %d expected header name: %s, but got: %s", ti, test.expHeaders[i].name, header.Name) } @@ -303,7 +307,12 @@ func TestGRPC(t *testing.T) { // Ensure column data matches. for i, row := range table.GetRows() { - for j, column := range row.GetColumns() { + columns := row.GetColumns() + if len(columns) != len(test.expColumns[i]) { + t.Fatalf("test %d expected %d columns, got %d in row %d", + ti, len(test.expColumns[i]), len(columns), i) + } + for j, column := range columns { switch v := test.expColumns[i][j].(type) { case string: val := column.GetStringVal() diff --git a/server/pg.go b/server/pg.go index 43fb22454..8938cc44b 100644 --- a/server/pg.go +++ b/server/pg.go @@ -252,27 +252,38 @@ func pgWriteExtractedTable(w pg.QueryResultWriter, tbl pilosa.ExtractedTable) er return nil } -func pgWriteGroupCount(w pg.QueryResultWriter, counts []pilosa.GroupCount) error { - if len(counts) == 0 { +func pgWriteGroupCount(w pg.QueryResultWriter, counts *pilosa.GroupCounts) error { + groups := counts.Groups() + if len(groups) == 0 { // Not enough information is available to construct the header. // This is a significant flaw in the data type. return nil } + expectedLen := len(groups[0].Group) + 1 - headers := make([]pg.ColumnInfo, len(counts[0].Group)+2) - for i, g := range counts[0].Group { + agg := counts.AggregateColumn() + if agg != "" { + expectedLen++ + } + + headers := make([]pg.ColumnInfo, expectedLen) + for i, g := range groups[0].Group { headers[i] = pg.ColumnInfo{ Name: g.Field, Type: pg.TypeCharoid, } } - headers[len(headers)-2] = pg.ColumnInfo{ + next := len(groups[0].Group) + headers[next] = pg.ColumnInfo{ Name: "count", Type: pg.TypeCharoid, } - headers[len(headers)-1] = pg.ColumnInfo{ - Name: "sum", - Type: pg.TypeCharoid, + if agg != "" { + next++ + headers[next] = pg.ColumnInfo{ + Name: agg, + Type: pg.TypeCharoid, + } } err := w.WriteHeader(headers...) if err != nil { @@ -280,8 +291,10 @@ func pgWriteGroupCount(w pg.QueryResultWriter, counts []pilosa.GroupCount) error } vals := make([]string, len(headers)) - for _, gc := range counts { - for j, g := range gc.Group { + for _, gc := range groups { + var j int + var g pilosa.FieldRow + for j, g = range gc.Group { var v string switch { case g.Value != nil: @@ -293,8 +306,12 @@ func pgWriteGroupCount(w pg.QueryResultWriter, counts []pilosa.GroupCount) error } vals[j] = v } - vals[len(vals)-2] = strconv.FormatUint(gc.Count, 10) - vals[len(vals)-1] = strconv.FormatInt(gc.Sum, 10) + j++ + vals[j] = strconv.FormatUint(gc.Count, 10) + if agg != "" { + j++ + vals[j] = strconv.FormatInt(gc.Agg, 10) + } err := w.WriteRowText(vals...) if err != nil { @@ -370,6 +387,9 @@ func pgWriteResult(w pg.QueryResultWriter, result interface{}) error { case pilosa.ExtractedTable: return pgWriteExtractedTable(w, result) case []pilosa.GroupCount: + gc := pilosa.NewGroupCounts("", result...) + return pgWriteGroupCount(w, gc) + case *pilosa.GroupCounts: return pgWriteGroupCount(w, result) case pb.ToRowser: // we should avoid protobuf where we can... return pgWriteRowser(w, result) diff --git a/server/pg_test.go b/server/pg_test.go index 86d743ade..59768dfe9 100644 --- a/server/pg_test.go +++ b/server/pg_test.go @@ -169,11 +169,10 @@ func TestPostgresHandler(t *testing.T) { Columns: []pg.ColumnInfo{ {Name: "set", Type: pg.TypeCharoid}, {Name: "count", Type: pg.TypeCharoid}, - {Name: "sum", Type: pg.TypeCharoid}, }, Data: [][]string{ - {"4", "3", "0"}, - {"5", "3", "0"}, + {"4", "3"}, + {"5", "3"}, }, }, }, diff --git a/server/server_test.go b/server/server_test.go index 46e79daac..78fe7d24e 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -298,7 +298,7 @@ func TestMain_GroupBy(t *testing.T) { if res, err := m.QueryProtobuf("i", `GroupBy(Rows(generalk), Rows(subk))`); err != nil { t.Fatal(err) } else { - test.CheckGroupBy(t, expected, res.Results[0].([]pilosa.GroupCount)) + test.CheckGroupBy(t, expected, res.Results[0].(*pilosa.GroupCounts).Groups()) } } diff --git a/sql/select.go b/sql/select.go index f053634da..b54eee0b4 100644 --- a/sql/select.go +++ b/sql/select.go @@ -89,11 +89,8 @@ func (s *SelectHandler) execMappingResult(ctx context.Context, mr *MappingResult switch res := res.(type) { case pproto.ToRowser: result = res - // TODO: replace []pilosa.GroupCount with *pilosa.GroupCounts, handle aggregate case []pilosa.GroupCount: - gc := pilosa.NewGroupCounts() - gc.Groups = res - result = gc + result = pilosa.NewGroupCounts("", res...) case uint64: result = pproto.ConstRowser{ { From a0d6c253d1a6f875cacd62e91a9903c0fc77b159 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 13 Jan 2021 11:02:54 -0600 Subject: [PATCH 5/7] Pass SQL query string from mapper to tracker --- api.go | 2 +- handler.go | 3 +++ http/handler.go | 2 +- sql/mapper.go | 2 ++ sql/select.go | 8 +++++--- tracker.go | 26 +++++++++++++++----------- 6 files changed, 27 insertions(+), 16 deletions(-) diff --git a/api.go b/api.go index 9140153a1..2bd8eb2e4 100644 --- a/api.go +++ b/api.go @@ -161,7 +161,7 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er } if !req.Remote { - defer api.tracker.Finish(api.tracker.Start(req.Query, api.server.nodeID, req.Index, start)) + defer api.tracker.Finish(api.tracker.Start(req.Query, req.SQLQuery, api.server.nodeID, req.Index, start)) } // TODO can we get rid of exec options and pass the QueryRequest directly to executor? execOpts := &execOptions{ diff --git a/handler.go b/handler.go index d8e370d56..b50e6e946 100644 --- a/handler.go +++ b/handler.go @@ -29,6 +29,9 @@ type QueryRequest struct { // The query string to parse and execute. Query string + // The SQL source query, if applicable. + SQLQuery string + // The shards to include in the query execution. // If empty, all shards are included. Shards []uint64 diff --git a/http/handler.go b/http/handler.go index bc2fbb1ab..a6dfd5828 100644 --- a/http/handler.go +++ b/http/handler.go @@ -1182,7 +1182,7 @@ func (h *Handler) handleGetActiveQueries(w http.ResponseWriter, r *http.Request) } } for i, q := range queries { - _, err := fmt.Fprintf(w, "%*s%q\n", -(maxlen + 2), durations[i], q.Query) + _, err := fmt.Fprintf(w, "%*s%q\n", -(maxlen + 2), durations[i], q.PQL) if err != nil { h.logger.Printf("sending GetActiveQueries response: %s", err) return diff --git a/sql/mapper.go b/sql/mapper.go index a43dba06d..eb6c23f0c 100644 --- a/sql/mapper.go +++ b/sql/mapper.go @@ -40,6 +40,7 @@ type MappedSQL struct { Statement sqlparser.Statement Mask QueryMask Tables []string + SQL string } // Mapper is responsible for mapping a SQL query to structure representation @@ -109,5 +110,6 @@ func (m *Mapper) MapSQL(sql string) (*MappedSQL, error) { Statement: stmt, Mask: qm, Tables: tableNames, + SQL: sql, }, nil } diff --git a/sql/select.go b/sql/select.go index b54eee0b4..7e1418b14 100644 --- a/sql/select.go +++ b/sql/select.go @@ -43,6 +43,7 @@ func NewSelectHandler(api *pilosa.API) *SelectHandler { // Handle executes mapped SQL func (s *SelectHandler) Handle(ctx context.Context, mapped *MappedSQL) (pproto.ToRowser, error) { stmt, ok := mapped.Statement.(*sqlparser.Select) + if !ok { return nil, fmt.Errorf("statement is not type select: %T", mapped.Statement) } @@ -50,7 +51,7 @@ func (s *SelectHandler) Handle(ctx context.Context, mapped *MappedSQL) (pproto.T if err != nil { return nil, errors.Wrap(err, "mapping select") } - return s.execMappingResult(ctx, mr) + return s.execMappingResult(ctx, mr, mapped.SQL) } func (s *SelectHandler) mapSelect(ctx context.Context, selectStmt *sqlparser.Select, qm QueryMask) (*MappingResult, error) { @@ -74,12 +75,13 @@ func (s *SelectHandler) mapSelect(ctx context.Context, selectStmt *sqlparser.Sel return mr, nil } -func (s *SelectHandler) execMappingResult(ctx context.Context, mr *MappingResult) (pproto.ToRowser, error) { +func (s *SelectHandler) execMappingResult(ctx context.Context, mr *MappingResult, sql string) (pproto.ToRowser, error) { if mr.Query == "" { return nil, errors.New("no pql query created") } + fmt.Printf("execMappingResult: %+v\n", sql) - resp, err := s.api.Query(ctx, &pilosa.QueryRequest{Index: mr.IndexName, Query: mr.Query}) + resp, err := s.api.Query(ctx, &pilosa.QueryRequest{Index: mr.IndexName, Query: mr.Query, SQLQuery: sql}) if err != nil { return nil, errors.Wrap(err, "doing pql query") } diff --git a/tracker.go b/tracker.go index 359850005..bec0cf4db 100644 --- a/tracker.go +++ b/tracker.go @@ -21,14 +21,16 @@ import ( ) type ActiveQueryStatus struct { - Query string `json:"query"` + PQL string `json:"PQL"` + SQL string `json:"SQL,omitempty"` Node string `json:"node"` Index string `json:"index"` Age time.Duration `json:"age"` } type PastQueryStatus struct { - Query string `json:"query"` + PQL string `json:"PQL"` + SQL string `json:"SQL,omitempty"` Node string `json:"nodeID"` Index string `json:"index"` Start time.Time `json:"start"` @@ -36,14 +38,16 @@ type PastQueryStatus struct { } type activeQuery struct { - query string + PQL string + SQL string node string index string started time.Time } type pastQuery struct { - query string + PQL string + SQL string node string index string started time.Time @@ -123,7 +127,7 @@ func newQueryTracker(historyLength int) *queryTracker { select { case update := <-updates: if update.end { - pq := pastQuery{update.q.query, update.q.node, update.q.index, update.q.started, update.endTime.Sub(update.q.started)} + pq := pastQuery{update.q.PQL, update.q.SQL, update.q.node, update.q.index, update.q.started, update.endTime.Sub(update.q.started)} tracker.history.add(pq) delete(activeQueries, update.q) } else { @@ -146,8 +150,8 @@ func newQueryTracker(historyLength int) *queryTracker { return tracker } -func (t *queryTracker) Start(query, nodeID, index string, start time.Time) *activeQuery { - q := &activeQuery{query, nodeID, index, start} +func (t *queryTracker) Start(pql, sql, nodeID, index string, start time.Time) *activeQuery { + q := &activeQuery{pql, sql, nodeID, index, start} t.updates <- queryStatusUpdate{q, false, time.Time{}} return q } @@ -166,9 +170,9 @@ func (t *queryTracker) ActiveQueries() []ActiveQueryStatus { return true case queries[i].started.After(queries[j].started): return false - case queries[i].query < queries[j].query: + case queries[i].PQL < queries[j].PQL: return true - case queries[i].query > queries[j].query: + case queries[i].PQL > queries[j].PQL: return false default: return false @@ -177,7 +181,7 @@ func (t *queryTracker) ActiveQueries() []ActiveQueryStatus { now := time.Now() out := make([]ActiveQueryStatus, len(queries)) for i, v := range queries { - out[i] = ActiveQueryStatus{v.query, v.node, v.index, now.Sub(v.started)} + out[i] = ActiveQueryStatus{v.PQL, v.SQL, v.node, v.index, now.Sub(v.started)} } return out } @@ -186,7 +190,7 @@ func (t *queryTracker) PastQueries() []PastQueryStatus { queries := t.history.slice() out := make([]PastQueryStatus, len(queries)) for i, v := range queries { - out[i] = PastQueryStatus{v.query, v.node, v.index, v.started, v.runtime} + out[i] = PastQueryStatus{v.PQL, v.SQL, v.node, v.index, v.started, v.runtime} } return out From bed03ba490fe9f93f59bf590c840db0319367fdd Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 15 Jan 2021 03:07:55 -0600 Subject: [PATCH 6/7] Update tests --- server/handler_test.go | 5 +++-- tracker_test.go | 10 +++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/server/handler_test.go b/server/handler_test.go index 67b9af8e5..7c3c024b4 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -1505,6 +1505,7 @@ func TestQueryHistory(t *testing.T) { test.Do(t, "POST", cmd.URL()+"/index/i0/query", "Set(0, f0=0)") test.Do(t, "POST", cmd.URL()+"/index/i0/query", "Set(3000000, f0=0)") test.Do(t, "POST", cmd.URL()+"/index/i0/query", "TopN(f0)") + // TODO SQL h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/query-history", nil)) if w.Code != gohttp.StatusOK { @@ -1543,8 +1544,8 @@ func TestQueryHistory(t *testing.T) { if ret[0].Node != cluster.GetNode(0).Server.NodeID() { t.Fatalf("response value for 'Node' was '%s', expected '%s'", ret[0].Node, cluster.GetNode(0).Server.NodeID()) } - if ret[0].Query != "TopN(f0)" { - t.Fatalf("response value for 'Query' was '%s', expected 'TopN(f0)'", ret[0].Query) + if ret[0].PQL != "TopN(f0)" { + t.Fatalf("response value for 'PQL' was '%s', expected 'TopN(f0)'", ret[0].PQL) } } diff --git a/tracker_test.go b/tracker_test.go index a5d8df418..569a7b609 100644 --- a/tracker_test.go +++ b/tracker_test.go @@ -47,11 +47,11 @@ func TestRingBuffer(t *testing.T) { } for n, q := range buffer.slice() { - if q.query != tests[k].queries[n] { - t.Fatalf("test[%d], buffer[%d] expected querystring '%s', found '%s'", k, n, tests[k].queries[n], q.query) + if q.PQL != tests[k].queries[n] { + t.Fatalf("test[%d], buffer[%d] expected querystring '%s', found '%s'", k, n, tests[k].queries[n], q.PQL) } } - buffer.add(pastQuery{query: fmt.Sprintf("%d", k)}) + buffer.add(pastQuery{PQL: fmt.Sprintf("%d", k)}) } } @@ -63,13 +63,13 @@ func TestQueryTracker(t *testing.T) { t.Fatalf("expected no active queries; found %v", queries) } - qs := tracker.Start("test query", "node0", "i", time.Now()) + qs := tracker.Start("test query", "test SQL", "node0", "i", time.Now()) var queries []ActiveQueryStatus for len(queries) < 1 { queries = tracker.ActiveQueries() } - if len(queries) > 1 || queries[0].Query != "test query" { + if len(queries) > 1 || queries[0].PQL != "test query" { t.Fatalf("unexpected queries: %v", queries) } From eaf3ef153e9d4b08056293cc4151a72bacd0313f Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 20 Jan 2021 15:58:55 -0600 Subject: [PATCH 7/7] Test SQL behavior --- server/handler_test.go | 24 ++++++++++++++++++++---- sql/select.go | 1 - 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/server/handler_test.go b/server/handler_test.go index 7c3c024b4..705cfdb5c 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -37,6 +37,7 @@ import ( "github.com/pilosa/pilosa/v2/encoding/proto" "github.com/pilosa/pilosa/v2/http" "github.com/pilosa/pilosa/v2/pql" + pb "github.com/pilosa/pilosa/v2/proto" "github.com/pilosa/pilosa/v2/server" "github.com/pilosa/pilosa/v2/test" ) @@ -1502,17 +1503,26 @@ func TestQueryHistory(t *testing.T) { test.Do(t, "POST", cmd.URL()+"/index/i0", "") test.Do(t, "POST", cmd.URL()+"/index/i0/field/f0", "") + + gh := server.NewGRPCHandler(cmd.API) + _, err = gh.QuerySQLUnary(context.Background(), &pb.QuerySQLRequest{ + Sql: `select * from i0`, + }) + + if err != nil { + t.Fatalf("QuerySQLUnary failed: %v", err) + } + test.Do(t, "POST", cmd.URL()+"/index/i0/query", "Set(0, f0=0)") test.Do(t, "POST", cmd.URL()+"/index/i0/query", "Set(3000000, f0=0)") test.Do(t, "POST", cmd.URL()+"/index/i0/query", "TopN(f0)") - // TODO SQL h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/query-history", nil)) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } - ret := make([]pilosa.PastQueryStatus, 3) + ret := make([]pilosa.PastQueryStatus, 4) b, err := ioutil.ReadAll(w.Body) if err != nil { t.Fatalf("reading: %v", err) @@ -1523,10 +1533,10 @@ func TestQueryHistory(t *testing.T) { } // verify result length - if len(ret) != 3 { + if len(ret) != 4 { // each set query executes on both nodes once // topn query gets added to history on node0 once, node1 twice - t.Fatalf("expected list of length 3, got %d", len(ret)) + t.Fatalf("expected list of length 4, got %d\n%+v", len(ret), ret) } // verify sort order @@ -1544,6 +1554,12 @@ func TestQueryHistory(t *testing.T) { if ret[0].Node != cluster.GetNode(0).Server.NodeID() { t.Fatalf("response value for 'Node' was '%s', expected '%s'", ret[0].Node, cluster.GetNode(0).Server.NodeID()) } + if ret[3].PQL != "Extract(All(),Rows(f0))" { + t.Fatalf("response value for 'PQL' was '%s', expected 'Extract(All(),Rows(f0))'", ret[0].PQL) + } + if ret[3].SQL != "select * from i0" { + t.Fatalf("response value for 'SQL' was '%s', expected 'select * from i0'", ret[0].SQL) + } if ret[0].PQL != "TopN(f0)" { t.Fatalf("response value for 'PQL' was '%s', expected 'TopN(f0)'", ret[0].PQL) } diff --git a/sql/select.go b/sql/select.go index 7e1418b14..b403f80c9 100644 --- a/sql/select.go +++ b/sql/select.go @@ -79,7 +79,6 @@ func (s *SelectHandler) execMappingResult(ctx context.Context, mr *MappingResult if mr.Query == "" { return nil, errors.New("no pql query created") } - fmt.Printf("execMappingResult: %+v\n", sql) resp, err := s.api.Query(ctx, &pilosa.QueryRequest{Index: mr.IndexName, Query: mr.Query, SQLQuery: sql}) if err != nil {