diff --git a/Makefile b/Makefile index abd07304d..9114fda54 100644 --- a/Makefile +++ b/Makefile @@ -152,6 +152,9 @@ prerelease-upload: install: go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa +install-bench: + go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-bench + lattice: git clone git@github.com:molecula/lattice.git diff --git a/api.go b/api.go index 177f57c94..9140153a1 100644 --- a/api.go +++ b/api.go @@ -163,13 +163,15 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er if !req.Remote { defer api.tracker.Finish(api.tracker.Start(req.Query, api.server.nodeID, req.Index, start)) } + // TODO can we get rid of exec options and pass the QueryRequest directly to executor? execOpts := &execOptions{ Remote: req.Remote, Profile: req.Profile, ExcludeRowAttrs: req.ExcludeRowAttrs, // NOTE: Kept for Pilosa 1.x compat. ExcludeColumns: req.ExcludeColumns, // NOTE: Kept for Pilosa 1.x compat. ColumnAttrs: req.ColumnAttrs, // NOTE: Kept for Pilosa 1.x compat. - EmbeddedData: req.EmbeddedData, // precomputed values that needed to be passed with the request + PreTranslated: req.PreTranslated, + EmbeddedData: req.EmbeddedData, // precomputed values that needed to be passed with the request } resp, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts) if err != nil { diff --git a/cluster.go b/cluster.go index dbf2ad67b..3a9976e72 100644 --- a/cluster.go +++ b/cluster.go @@ -1047,7 +1047,7 @@ func (c *cluster) ShardNodes(index string, shard uint64) []*Node { return c.shardNodes(index, shard) } -// shardNodes returns a list of nodes that own a fragment. unprotected +// shardNodes returns a list of nodes that own a shard. unprotected func (c *cluster) shardNodes(index string, shard uint64) []*Node { return c.partitionNodes(c.shardToShardPartition(index, shard)) } diff --git a/cmd/pilosa-bench/main.go b/cmd/pilosa-bench/main.go new file mode 100644 index 000000000..b33bf0fcf --- /dev/null +++ b/cmd/pilosa-bench/main.go @@ -0,0 +1,313 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "flag" + "fmt" + "io/ioutil" + "log" + "math/rand" + "net/http" + "os" + "sort" + "strings" + "time" + + "github.com/pilosa/pilosa/v2" + phttp "github.com/pilosa/pilosa/v2/http" + "golang.org/x/sync/errgroup" +) + +func main() { + if err := run(context.Background(), os.Args[1:]); err == flag.ErrHelp { + os.Exit(1) + } else if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(ctx context.Context, args []string) (err error) { + fs := flag.NewFlagSet("pilosa-bench", flag.ContinueOnError) + hostport := fs.String("hostport", "localhost:10101", "") + typ := fs.String("type", "row", "query type (row)") + n := fs.Int("n", 1000, "number of queries") + rate := fs.Int("rate", 1, "number of queries per second") + verbose := fs.Bool("v", false, "verbose logging") + from := fs.String("from", "", "from time for row-range queries (ISO 8601)") + to := fs.String("to", "", "to time for row-range queries (ISO 8601)") + if err := fs.Parse(args); err != nil { + return err + } + + // Parse from/to time. + var opt queryOptions + if *from != "" { + if opt.from, err = time.Parse(time.RFC3339, *from); err != nil { + return fmt.Errorf("cannot parse -from time") + } + } + if *to != "" { + if opt.to, err = time.Parse(time.RFC3339, *to); err != nil { + return fmt.Errorf("cannot parse -to time") + } + } + + if (*typ == "row-range" || *typ == "topk") && (opt.from.IsZero() || opt.to.IsZero()) { + return fmt.Errorf("-from and -to flags must be specified for topk & row-range queries") + } + + // Clear time prefix on log. + log.SetFlags(0) + if !*verbose { + log.SetOutput(ioutil.Discard) + } + + // Setup PRNG to have consistent values for the same set of data. + rand.Seed(0) + + // Setup connection to pilosa. + client, err := phttp.NewInternalClient(*hostport, http.DefaultClient) + if err != nil { + return err + } + + // Load all id/keys for each field. + log.Printf("loading field identifiers") + fieldIDMap, err := loadFields(ctx, client) + if err != nil { + return fmt.Errorf("cannot load field identifiers: %w", err) + } else if len(fieldIDMap) == 0 { + return fmt.Errorf("no field identifiers available, please verify data exists") + } + + // Generate list of sorted keys. + fieldKeys := make([]fieldKey, 0, len(fieldIDMap)) + for k, f := range fieldIDMap { + switch *typ { + case "row-bsi": + if f.info.Options.Type != "int" { + continue + } + case "row-range", "topk": + if f.info.Options.Type != "time" { + continue + } + default: + if f.info.Options.Type == "int" || f.info.Options.Type == "time" { + continue + } + } + + fieldKeys = append(fieldKeys, k) + } + sort.Slice(fieldKeys, func(i, j int) bool { + return compareFieldKeys(fieldKeys[i], fieldKeys[j]) == -1 + }) + + // Ensure we have appropriate fields for our query type. + if len(fieldKeys) == 0 { + return fmt.Errorf("no available fields are appropriate for %q queries", *typ) + } + + log.Printf("issuing %d queries at %d query/sec", *n, *rate) + + // Repeatedly issue queries based on available row data. + ticker := time.NewTicker(time.Second / time.Duration(*rate)) + var g errgroup.Group + for i := 0; i < *n; i++ { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + + key := fieldKeys[rand.Intn(len(fieldKeys))] + q, err := generateQuery(*typ, key.index, key.field, fieldIDMap[key].info, fieldIDMap[key].identifiers, opt) + if err != nil { + return fmt.Errorf("cannot generate query: %w", err) + } + + log.Printf("[query] %s", q) + + g.Go(func() error { + _, err = client.Query(ctx, key.index, &pilosa.QueryRequest{Index: key.index, Query: q}) + if err != nil { + return err + } + return nil + }) + } + + return g.Wait() +} + +func generateQuery(typ, index, field string, info *pilosa.FieldInfo, identifiers *pilosa.RowIdentifiers, opt queryOptions) (string, error) { + switch typ { + case "row": + return generateRowQuery(index, field, identifiers), nil + case "row-bsi": + return generateRowBSIQuery(index, field), nil + case "row-range": + return generateRowRangeQuery(index, field, identifiers, opt.from, opt.to), nil + case "count": + return generateCountQuery(index, field, identifiers), nil + case "intersect": + return generateIntersectQuery(index, field, identifiers), nil + case "union": + return generateUnionQuery(index, field, identifiers), nil + case "difference": + return generateDifferenceQuery(index, field, identifiers), nil + case "xor": + return generateXorQuery(index, field, identifiers), nil + case "groupby": + return generateGroupByQuery(index, field), nil + case "topk": + return generateTopKQuery(index, field, opt.from, opt.to), nil + default: + return "", fmt.Errorf("invalid query type: %q", typ) + } +} + +func generateRowQuery(index, field string, identifiers *pilosa.RowIdentifiers) string { + if len(identifiers.Rows) > 0 { + return fmt.Sprintf("Row(%s=%d)", field, chooseRowID(identifiers)) + } + return fmt.Sprintf("Row(%s=%q)", field, chooseRowKey(identifiers)) +} + +func generateRowBSIQuery(index, field string) string { + return fmt.Sprintf("Row(%s > 0)", field) +} + +func generateRowRangeQuery(index, field string, identifiers *pilosa.RowIdentifiers, from, to time.Time) string { + if len(identifiers.Rows) > 0 { + return fmt.Sprintf("Row(%s=%d, from='%s', to='%s')", field, chooseRowID(identifiers), from.Format("2006-01-02T15:04"), to.Format("2006-01-02T15:04")) + } + return fmt.Sprintf("Row(%s=%q, from='%s', to='%s')", field, chooseRowKey(identifiers), from.Format("2006-01-02T15:04"), to.Format("2006-01-02T15:04")) +} + +func generateRowQueries(index, field string, identifiers *pilosa.RowIdentifiers) string { + a := make([]string, rand.Intn(9)+1) + for i := range a { + a[i] = generateRowQuery(index, field, identifiers) + } + return strings.Join(a, ", ") +} + +func generateCountQuery(index, field string, identifiers *pilosa.RowIdentifiers) string { + return fmt.Sprintf("Count(%s)", generateRowQuery(index, field, identifiers)) +} + +func generateIntersectQuery(index, field string, identifiers *pilosa.RowIdentifiers) string { + return fmt.Sprintf("Intersect(%s)", generateRowQueries(index, field, identifiers)) +} + +func generateUnionQuery(index, field string, identifiers *pilosa.RowIdentifiers) string { + return fmt.Sprintf("Union(%s)", generateRowQueries(index, field, identifiers)) +} + +func generateDifferenceQuery(index, field string, identifiers *pilosa.RowIdentifiers) string { + return fmt.Sprintf("Difference(%s)", generateRowQueries(index, field, identifiers)) +} + +func generateXorQuery(index, field string, identifiers *pilosa.RowIdentifiers) string { + return fmt.Sprintf("Xor(%s)", generateRowQueries(index, field, identifiers)) +} + +func generateGroupByQuery(index, field string) string { + return fmt.Sprintf("GroupBy(Rows(%s))", field) +} + +func generateTopKQuery(index, field string, from, to time.Time) string { + return fmt.Sprintf("TopK(%s, from='%s', to='%s')", field, from.Format("2006-01-02T15:04"), to.Format("2006-01-02T15:04")) +} + +// loadFields returns a mapping of index/field names to field info & identifiers. +func loadFields(ctx context.Context, client *phttp.InternalClient) (map[fieldKey]*fieldInfo, error) { + indexes, err := client.Schema(ctx) + if err != nil { + return nil, err + } + + m := make(map[fieldKey]*fieldInfo) + for _, ii := range indexes { + for _, f := range ii.Fields { + log.Printf("field: index=%s name=%s type=%s", ii.Name, f.Name, f.Options.Type) + + switch f.Options.Type { + case "set", "mutex", "time": + identifiers, err := fetchFieldIDs(ctx, client, ii.Name, f.Name) + if err != nil { + return nil, fmt.Errorf("fetch fields: %w", err) + } else if len(identifiers.Rows) > 0 || len(identifiers.Keys) > 0 { + m[fieldKey{ii.Name, f.Name}] = &fieldInfo{f, identifiers} + } + + case "int": + m[fieldKey{ii.Name, f.Name}] = &fieldInfo{info: f} + } + } + } + return m, nil +} + +// fetchFieldIDs returns a list of field IDs or keys. +func fetchFieldIDs(ctx context.Context, client *phttp.InternalClient, indexName, fieldName string) (*pilosa.RowIdentifiers, error) { + resp, err := client.Query(ctx, indexName, &pilosa.QueryRequest{Index: indexName, Query: `Rows(` + fieldName + `)`}) + if err != nil { + return nil, err + } + + switch result := resp.Results[0].(type) { + case *pilosa.RowIdentifiers: + return result, nil + case pilosa.RowIdentifiers: + return &result, nil + default: + return nil, fmt.Errorf("unexpected result type: %T", result) + } +} + +func chooseRowID(identifiers *pilosa.RowIdentifiers) uint64 { + return identifiers.Rows[rand.Intn(len(identifiers.Rows))] +} + +func chooseRowKey(identifiers *pilosa.RowIdentifiers) string { + return identifiers.Keys[rand.Intn(len(identifiers.Keys))] +} + +type fieldKey struct { + index string + field string +} + +type fieldInfo struct { + info *pilosa.FieldInfo + identifiers *pilosa.RowIdentifiers +} + +func compareFieldKeys(x, y fieldKey) int { + if cmp := strings.Compare(x.index, y.index); cmp != 0 { + return cmp + } + return strings.Compare(x.field, y.field) +} + +type queryOptions struct { + from, to time.Time +} diff --git a/cmd/root_test.go b/cmd/root_test.go index a4e931a78..caf53d74b 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -148,6 +148,10 @@ func (ct *commandTest) setupCommand(t *testing.T) *cobra.Command { err = os.Setenv(name, val) failErr(t, err, fmt.Sprintf("setting environment variable '%s' to '%s'", name, val)) } + // address common case where system might have postgres bind + // setting that an existing running Pilosa is using (causing test + // failures due to port conflict) + os.Setenv("PILOSA_POSTGRES_BIND", "") // make command and set args rc := cmd.NewRootCommand(strings.NewReader(""), ioutil.Discard, ioutil.Discard) diff --git a/cmd/server_test.go b/cmd/server_test.go index ab7541c84..dc8e72b99 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -15,6 +15,7 @@ package cmd_test import ( + "fmt" "io/ioutil" "strings" "testing" @@ -169,38 +170,49 @@ func TestServerConfig(t *testing.T) { // run server tests for i, test := range tests { - com := test.setupCommand(t) - executed := make(chan struct{}) - var execErr error - go func() { - execErr = com.Execute() - close(executed) - }() - select { - case <-cmd.Server.Started: - case <-executed: - } - if execErr != nil { - t.Fatalf("executing server command: %v", execErr) - } - err := cmd.Server.Close() - failErr(t, err, "closing pilosa server command") - <-executed - failErr(t, execErr, "executing command") + t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { + com := test.setupCommand(t) + executed := make(chan struct{}) + var execErr error + go func() { + execErr = com.Execute() + close(executed) + }() + select { + case <-cmd.Server.Started: + case <-executed: + } + if execErr != nil { + t.Fatalf("executing server command: %v", execErr) + } + err := cmd.Server.Close() + failErr(t, err, "closing pilosa server command") + <-executed + failErr(t, execErr, "executing command") - if err := test.validation(); err != nil { - t.Fatalf("Failed test %d due to: %v", i, err) - } - test.reset() + if err := test.validation(); err != nil { + t.Fatalf("Failed test %d due to: %v", i, err) + } + test.reset() + }) } } func TestServerConfig_DeprecateLongQueryTime(t *testing.T) { + actualDataDir, err := ioutil.TempDir("", "") + failErr(t, err, "making data dir") + tests := []commandTest{ // TEST 0 { - args: []string{"server", "--long-query-time", "1m10s"}, - env: map[string]string{}, - cfgFileContent: "", + args: []string{"server", "--long-query-time", "1m10s"}, + env: map[string]string{}, + cfgFileContent: ` + bind = "localhost:0" + bind-grpc = "localhost:0" + data-dir = "` + actualDataDir + `" + [gossip] + port = "14321" +`, validation: func() error { v := validator{} v.Check(cmd.Server.Config.LongQueryTime, toml.Duration(time.Second*70)) @@ -210,9 +222,14 @@ func TestServerConfig_DeprecateLongQueryTime(t *testing.T) { }, // TEST 1 { - args: []string{"server", "--cluster.long-query-time", "1m20s"}, - env: map[string]string{}, - cfgFileContent: "", + args: []string{"server", "--cluster.long-query-time", "1m20s"}, + env: map[string]string{}, + cfgFileContent: ` + bind = "localhost:0" + bind-grpc = "localhost:0" + [gossip] + port = "14321" +`, validation: func() error { v := validator{} v.Check(cmd.Server.Config.Cluster.LongQueryTime, toml.Duration(time.Second*80)) @@ -222,9 +239,14 @@ func TestServerConfig_DeprecateLongQueryTime(t *testing.T) { }, // TEST 2: Use old value if both are provided because it is the simplest implementation { - args: []string{"server", "--long-query-time", "50s", "--cluster.long-query-time", "1m30s"}, - env: map[string]string{}, - cfgFileContent: "", + args: []string{"server", "--long-query-time", "50s", "--cluster.long-query-time", "1m30s"}, + env: map[string]string{}, + cfgFileContent: ` + bind = "localhost:0" + bind-grpc = "localhost:0" + [gossip] + port = "14321" +`, validation: func() error { v := validator{} v.Check(cmd.Server.Config.LongQueryTime, toml.Duration(time.Second*50)) @@ -237,28 +259,30 @@ func TestServerConfig_DeprecateLongQueryTime(t *testing.T) { // run server tests for i, test := range tests { - com := test.setupCommand(t) - executed := make(chan struct{}) - var execErr error - go func() { - execErr = com.Execute() - close(executed) - }() - select { - case <-cmd.Server.Started: - case <-executed: - } - if execErr != nil { - t.Fatalf("executing server command: %v", execErr) - } - err := cmd.Server.Close() - failErr(t, err, "closing pilosa server command") - <-executed - failErr(t, execErr, "executing command") + t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { + com := test.setupCommand(t) + executed := make(chan struct{}) + var execErr error + go func() { + execErr = com.Execute() + close(executed) + }() + select { + case <-cmd.Server.Started: + case <-executed: + } + if execErr != nil { + t.Fatalf("executing server command: %v", execErr) + } + err := cmd.Server.Close() + failErr(t, err, "closing pilosa server command") + <-executed + failErr(t, execErr, "executing command") - if err := test.validation(); err != nil { - t.Fatalf("Failed test %d due to: %v", i, err) - } - test.reset() + if err := test.validation(); err != nil { + t.Fatalf("Failed test %d due to: %v", i, err) + } + test.reset() + }) } } diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index ec3e3c950..2091cc850 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -490,6 +490,7 @@ func (s Serializer) encodeQueryRequest(m *pilosa.QueryRequest) *internal.QueryRe Remote: m.Remote, ExcludeRowAttrs: m.ExcludeRowAttrs, ExcludeColumns: m.ExcludeColumns, + PreTranslated: m.PreTranslated, EmbeddedData: make([]*internal.Row, len(m.EmbeddedData)), } for i := range m.EmbeddedData { @@ -535,7 +536,7 @@ func (s Serializer) encodeQueryResponse(m *pilosa.QueryResponse) *internal.Query case pilosa.ExtractedIDMatrix: pb.Results[i].Type = queryResultTypeExtractedIDMatrix pb.Results[i].ExtractedIDMatrix = s.endcodeExtractedIDMatrix(result) - case []pilosa.GroupCount: + case *pilosa.GroupCounts: pb.Results[i].Type = queryResultTypeGroupCounts pb.Results[i].GroupCounts = s.encodeGroupCounts(result) case pilosa.RowIdentifiers: @@ -1210,6 +1211,7 @@ func (s Serializer) decodeQueryRequest(pb *internal.QueryRequest, m *pilosa.Quer m.ExcludeRowAttrs = pb.ExcludeRowAttrs m.ExcludeColumns = pb.ExcludeColumns m.EmbeddedData = make([]*pilosa.Row, len(pb.EmbeddedData)) + m.PreTranslated = pb.PreTranslated for i := range pb.EmbeddedData { m.EmbeddedData[i] = s.decodeRow(pb.EmbeddedData[i]) } @@ -1421,7 +1423,7 @@ func (s Serializer) decodeQueryResult(pb *internal.QueryResult) interface{} { case queryResultTypeRowIdentifiers: return s.decodeRowIdentifiers(pb.RowIdentifiers) case queryResultTypeGroupCounts: - return s.decodeGroupCounts(pb.GroupCounts) + return s.decodeGroupCounts(pb.GroupCounts, pb.OldGroupCounts) case queryResultTypePair: return s.decodePair(pb.Pairs[0]) case queryResultTypePairField: @@ -1585,16 +1587,22 @@ func (s Serializer) decodeRowIdentifiers(a *internal.RowIdentifiers) *pilosa.Row } } -func (s Serializer) decodeGroupCounts(a []*internal.GroupCount) []pilosa.GroupCount { - other := make([]pilosa.GroupCount, len(a)) - for i := range a { +func (s Serializer) decodeGroupCounts(a *internal.GroupCounts, b []*internal.GroupCount) *pilosa.GroupCounts { + // Workaround: If we get an old-style "[]*GroupCount", we translate it. + if a == nil { + a = &internal.GroupCounts{Aggregate: "", Groups: b} + } + other := make([]pilosa.GroupCount, len(a.Groups)) + for i, gc := range a.Groups { other[i] = pilosa.GroupCount{ - Group: s.decodeFieldRows(a[i].Group), - Count: a[i].Count, - Sum: a[i].Sum, + Group: s.decodeFieldRows(gc.Group), + Count: gc.Count, + // note: not renaming the `internal` structure members now + // to avoid breaking protobuf interactions. + Agg: gc.Agg, } } - return other + return pilosa.NewGroupCounts(a.Aggregate, other...) } func (s Serializer) decodeFieldRows(a []*internal.FieldRow) []pilosa.FieldRow { @@ -1696,7 +1704,7 @@ func (s Serializer) encodeSignedRow(r pilosa.SignedRow) *internal.SignedRow { func (s Serializer) encodeRow(r *pilosa.Row) *internal.Row { if r == nil { - return nil + return &internal.Row{} // Generated proto code doesn't like a nil Row. } ir := &internal.Row{ @@ -1721,13 +1729,17 @@ func (s Serializer) encodeRowIdentifiers(r pilosa.RowIdentifiers) *internal.RowI } } -func (s Serializer) encodeGroupCounts(counts []pilosa.GroupCount) []*internal.GroupCount { - result := make([]*internal.GroupCount, len(counts)) - for i := range counts { - result[i] = &internal.GroupCount{ - Group: s.encodeFieldRows(counts[i].Group), - Count: counts[i].Count, - Sum: counts[i].Sum, +func (s Serializer) encodeGroupCounts(counts *pilosa.GroupCounts) *internal.GroupCounts { + groups := counts.Groups() + result := &internal.GroupCounts{ + Groups: make([]*internal.GroupCount, len(groups)), + Aggregate: counts.AggregateColumn(), + } + for i, gc := range groups { + result.Groups[i] = &internal.GroupCount{ + Group: s.encodeFieldRows(gc.Group), + Count: gc.Count, + Agg: gc.Agg, } } return result diff --git a/executor.go b/executor.go index 1910d42e8..ae1071b33 100644 --- a/executor.go +++ b/executor.go @@ -24,6 +24,7 @@ import ( "strings" "sync" "time" + "unsafe" "github.com/pilosa/pilosa/v2/pql" pb "github.com/pilosa/pilosa/v2/proto" @@ -312,6 +313,8 @@ func (e *executor) safeCopy(resp QueryResponse) (out QueryResponse) { out.Results = append(out.Results, x) case []GroupCount: out.Results = append(out.Results, x) + case *GroupCounts: + out.Results = append(out.Results, x) case ExtractedTable: out.Results = append(out.Results, x) case ExtractedIDMatrix: @@ -441,9 +444,11 @@ func (e *executor) handlePreCalls(ctx context.Context, qcx *Qcx, index string, c // shards if the query has to go to them opt.EmbeddedData = append(opt.EmbeddedData, row) // and stash a copy locally, so local calls can use it - c.Precomputed = make(map[uint64]interface{}, len(row.segments)) - for _, segment := range row.segments { - c.Precomputed[segment.shard] = &Row{segments: []rowSegment{segment}} + if row != nil { + c.Precomputed = make(map[uint64]interface{}, len(row.segments)) + for _, segment := range row.segments { + c.Precomputed[segment.shard] = &Row{segments: []rowSegment{segment}} + } } return nil } @@ -468,7 +473,11 @@ func (e *executor) handlePreCallChildren(ctx context.Context, qcx *Qcx, index st return err } } - for _, val := range c.Args { + for key, val := range c.Args { + // Do not precompute GroupBy aggregates + if key == "aggregate" { + continue + } // Handle Call() operations which exist inside named arguments, too. if call, ok := val.(*pql.Call); ok { if err := ctx.Err(); err != nil { @@ -528,7 +537,7 @@ func (e *executor) execute(ctx context.Context, qcx *Qcx, index string, q *pql.Q } // Apply call translation. - if !opt.Remote { + if !opt.Remote && !opt.PreTranslated { translated, err := e.translateCall(call, index, colTranslations, rowTranslations) if err != nil { return nil, errors.Wrap(err, "translating call") @@ -672,77 +681,101 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p switch c.Name { case "Sum": statFn() - return e.executeSum(ctx, qcx, index, c, shards, opt) + res, err := e.executeSum(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeSum %v", shards) case "Min": statFn() - return e.executeMin(ctx, qcx, index, c, shards, opt) + res, err := e.executeMin(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeMin %v", shards) case "Max": statFn() - return e.executeMax(ctx, qcx, index, c, shards, opt) + res, err := e.executeMax(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeMax %v", shards) case "MinRow": statFn() - return e.executeMinRow(ctx, qcx, index, c, shards, opt) + res, err := e.executeMinRow(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeMinRow %v", shards) case "MaxRow": statFn() - return e.executeMaxRow(ctx, qcx, index, c, shards, opt) + res, err := e.executeMaxRow(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeMaxRow %v", shards) case "Clear": statFn() - return e.executeClearBit(ctx, qcx, index, c, opt) + res, err := e.executeClearBit(ctx, qcx, index, c, opt) + return res, errors.Wrapf(err, "executeClearBit %v", shards) case "ClearRow": statFn() - return e.executeClearRow(ctx, qcx, index, c, shards, opt) + res, err := e.executeClearRow(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeClearRow %v", shards) case "Distinct": statFn() - return e.executeDistinct(ctx, qcx, index, c, shards, opt) + res, err := e.executeDistinct(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeDistinct %v", shards) case "Store": statFn() - return e.executeSetRow(ctx, qcx, index, c, shards, opt) + res, err := e.executeSetRow(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeSetRow %v", shards) case "Count": statFn() - return e.executeCount(ctx, qcx, index, c, shards, opt) + res, err := e.executeCount(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeCount %v", shards) case "Set": statFn() - return e.executeSet(ctx, qcx, index, c, opt) + res, err := e.executeSet(ctx, qcx, index, c, opt) + return res, errors.Wrapf(err, "executeSet %v", shards) case "SetRowAttrs": statFn() - return nil, e.executeSetRowAttrs(ctx, qcx, index, c, opt) + return nil, errors.Wrap(e.executeSetRowAttrs(ctx, qcx, index, c, opt), "executeSetRowAttrs") case "SetColumnAttrs": statFn() - return nil, e.executeSetColumnAttrs(ctx, qcx, index, c, opt) + return nil, errors.Wrap(e.executeSetColumnAttrs(ctx, qcx, index, c, opt), "executeSetColumnAttrs") case "TopK": statFn() - return e.executeTopK(ctx, qcx, index, c, shards, opt) + res, err := e.executeTopK(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeTopK %v", shards) case "TopN": statFn() - return e.executeTopN(ctx, qcx, index, c, shards, opt) + res, err := e.executeTopN(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeTopN %v", shards) case "Rows": statFn() - return e.executeRows(ctx, qcx, index, c, shards, opt) + res, err := e.executeRows(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeRows %v", shards) case "Extract": statFn() - return e.executeExtract(ctx, qcx, index, c, shards, opt) + res, err := e.executeExtract(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeExtract %v", shards) case "GroupBy": statFn() - return e.executeGroupBy(ctx, qcx, index, c, shards, opt) + res, err := e.executeGroupBy(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeGroupBy %v", shards) case "Options": statFn() - return e.executeOptionsCall(ctx, qcx, index, c, shards, opt) + res, err := e.executeOptionsCall(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeOptionsCall %v", shards) case "IncludesColumn": - return e.executeIncludesColumnCall(ctx, qcx, index, c, shards, opt) + res, err := e.executeIncludesColumnCall(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeIncludesColumnCall %v", shards) case "FieldValue": statFn() - return e.executeFieldValueCall(ctx, qcx, index, c, shards, opt) + res, err := e.executeFieldValueCall(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeFieldValueCall %v", shards) case "Precomputed": - return e.executePrecomputedCall(ctx, qcx, index, c, shards, opt) + res, err := e.executePrecomputedCall(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executePrecomputedCall %v", shards) case "UnionRows": - return e.executeUnionRows(ctx, qcx, index, c, shards, opt) + res, err := e.executeUnionRows(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeUnionRows %v", shards) case "ConstRow": - return e.executeConstRow(ctx, index, c) + res, err := e.executeConstRow(ctx, index, c) + return res, errors.Wrapf(err, "executeConstRow %v", shards) case "Limit": - return e.executeLimitCall(ctx, qcx, index, c, shards, opt) + res, err := e.executeLimitCall(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeLimitCall %v", shards) default: // e.g. "Row", "Union", "Intersect" or anything that returns a bitmap. statFn() - return e.executeBitmapCall(ctx, qcx, index, c, shards, opt) + res, err := e.executeBitmapCall(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeBitmapCall %v", shards) } } @@ -1107,6 +1140,11 @@ func (e *executor) executeDistinct(ctx context.Context, qcx *Qcx, index string, case SignedRow: return other.union(v.(SignedRow)) case *Row: + if other == nil { + return v + } else if v.(*Row) == nil { + return other + } return other.Union(v.(*Row)) case nil: return v @@ -1117,13 +1155,12 @@ func (e *executor) executeDistinct(ctx context.Context, qcx *Qcx, index string, result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { - return nil, err + return nil, errors.Wrap(err, "mapReduce") } if other, ok := result.(SignedRow); ok { other.field = field } - return result, nil } @@ -1466,10 +1503,15 @@ func executeDistinctShardSet(ctx context.Context, qcx *Qcx, idx *Index, fieldNam defer finisher(&err0) fragData, _, err := tx.ContainerIterator(index, fieldName, "standard", shard, 0) - if err != nil { + switch errors.Cause(err) { + case ViewNotFound, FragmentNotFound: + return nil, nil + case nil: + default: return nil, errors.Wrap(err, "getting fragment data") } defer fragData.Close() + // We can't grab the containers "for each row" from the set-type field, // because we don't know how many rows there are, and some of them // might be empty, so really, we're going to iterate through the @@ -2576,7 +2618,106 @@ func (r RowIDs) merge(other RowIDs, limit int) RowIDs { return result } -func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *execOptions) ([]GroupCount, error) { +// order denotes sort order—can be asc or desc (see constants below). +type order bool + +const ( + asc order = true + desc order = false +) + +// groupCountSorter sorts the output of a GroupBy request (a +// []GroupCount) according to sorting instructions encoded in "fields" +// and "order". +// +// Each field in "fields" is an integer which can be -1 to denote +// sorting on the Count and -2 to denote sorting on the +// sum/aggregate. Currently nothing else is supported, but the idea +// was that if there were positive integers they would be indexes into +// GroupCount.FieldRow and allowing sorting on the values of different +// fields in the group. Each item in "order" corresponds to the same +// index in "fields" and denotes the order of the sort. +type groupCountSorter struct { + fields []int + order []order + data []GroupCount +} + +func (g *groupCountSorter) Len() int { return len(g.data) } +func (g *groupCountSorter) Swap(i, j int) { g.data[i], g.data[j] = g.data[j], g.data[i] } +func (g *groupCountSorter) Less(i, j int) bool { + gci, gcj := g.data[i], g.data[j] + for idx, fieldIndex := range g.fields { + fieldOrder := g.order[idx] + switch fieldIndex { + case -1: // Count + if gci.Count < gcj.Count { + return fieldOrder == asc + } else if gci.Count > gcj.Count { + return fieldOrder == desc + } + case -2: // Aggregate + if gci.Agg < gcj.Agg { + return fieldOrder == asc + } else if gci.Agg > gcj.Agg { + return fieldOrder == desc + } + default: + panic("impossible") + } + } + return false +} + +// getSorter hackily parses the sortSpec and figures out how to sort +// the GroupBy results. +func getSorter(sortSpec string) (*groupCountSorter, error) { + gcs := &groupCountSorter{ + fields: []int{}, + order: []order{}, + } + sortOn := strings.Split(sortSpec, ",") + for _, sortField := range sortOn { + sortField = strings.TrimSpace(sortField) + fieldDir := strings.Fields(sortField) + if len(fieldDir) == 0 { + return nil, errors.Errorf("invalid sorting directive: '%s'", sortField) + } else if fieldDir[0] == "count" { + gcs.fields = append(gcs.fields, -1) + } else if fieldDir[0] == "aggregate" || fieldDir[0] == "sum" { + gcs.fields = append(gcs.fields, -2) + } else { + return nil, errors.Errorf("sorting is only supported on count, aggregate, or sum, not '%s'", fieldDir[0]) + } + + if len(fieldDir) == 1 { + gcs.order = append(gcs.order, desc) + } else if len(fieldDir) > 2 { + return nil, errors.Errorf("parsing sort directive: '%s': too many elements", sortField) + } else if fieldDir[1] == "asc" { + gcs.order = append(gcs.order, asc) + } else if fieldDir[1] == "desc" { + gcs.order = append(gcs.order, desc) + } else { + return nil, errors.Errorf("unknown sort direction '%s'", fieldDir[1]) + } + } + return gcs, nil +} + +// findGroupCounts gets a safe-to-use but possibly empty []GroupCount from +// an interface which might be a *GroupCounts or a []GroupCount. +func findGroupCounts(v interface{}) []GroupCount { + switch gc := v.(type) { + case []GroupCount: + return gc + case *GroupCounts: + return gc.Groups() + } + return nil +} + +func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shards []uint64, opt *execOptions) (*GroupCounts, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGroupBy") defer span.Finish() // validate call @@ -2594,6 +2735,25 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c return nil, err } + var sorter *groupCountSorter + if sortSpec, found, err := c.StringArg("sort"); err != nil { + return nil, errors.Wrap(err, "getting sort arg") + } else if found { + sorter, err = getSorter(sortSpec) + if err != nil { + return nil, errors.Wrap(err, "parsing sort spec") + } + // don't want to prematurely limit the results if we're sorting + limit = int(^uint(0) >> 1) + } + having, hasHaving, err := c.CallArg("having") + if err != nil { + return nil, errors.Wrap(err, "getting 'having' argument") + } else if hasHaving { + // don't want to prematurely limit the results if we're filtering some out + limit = int(^uint(0) >> 1) + } + idx := e.Holder.Index(index) if idx == nil { return nil, newNotFoundError(ErrIndexNotFound, index) @@ -2642,7 +2802,7 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c return nil, errors.Wrap(err, "getting rows for ") } if len(childRows[i]) == 0 { // there are no results because this field has no values. - return []GroupCount{}, nil + return &GroupCounts{}, nil } } } @@ -2653,23 +2813,83 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c } // Merge returned results at coordinating node. reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { - other, _ := prev.([]GroupCount) + other := findGroupCounts(prev) if err := ctx.Err(); err != nil { return err } - return mergeGroupCounts(other, v.([]GroupCount), limit) + return mergeGroupCounts(other, findGroupCounts(v), limit) } // Get full result set. other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "mapReduce shards: %v", shards) } results, _ := other.([]GroupCount) + // If there's no sorting, we want to apply limits before + // calculating the Distinct aggregate which is expensive on a + // per-result basis. + if sorter == nil && !hasHaving { + results, err = applyLimitAndOffsetToGroupByResult(c, results) + if err != nil { + return nil, errors.Wrap(err, "applying limit/offset") + } + } + + // TODO as an optimization, we could apply some "having" + // conditions here long as they aren't on the Count(Distinct) + // aggregate + + // Calculate Count(Distinct) aggregate if requested. + aggregate, _, err := c.CallArg("aggregate") + if err == nil && aggregate != nil && aggregate.Name == "Count" && len(aggregate.Children) > 0 && aggregate.Children[0].Name == "Distinct" && !opt.Remote { + for n, gc := range results { + intersectRows := make([]*pql.Call, 0, len(gc.Group)) + for _, fr := range gc.Group { + var value interface{} = fr.RowID + // use fr.Value instead of fr.RowID if set (from int fields) + if fr.Value != nil { + value = &pql.Condition{Op: pql.EQ, Value: *fr.Value} + } + intersectRows = append(intersectRows, &pql.Call{Name: "Row", Args: map[string]interface{}{fr.Field: value}}) + } + // apply any filter, if present + if filter != nil { + intersectRows = append(intersectRows, filter) + } + // also intersect with any children of Distinct + if len(aggregate.Children[0].Children) > 0 { + intersectRows = append(intersectRows, aggregate.Children[0].Children[0]) + } + + countDistinctIntersect := &pql.Call{ + Name: "Count", + Children: []*pql.Call{ + { + Name: "Distinct", + Children: []*pql.Call{ + { + Name: "Intersect", + Children: intersectRows, + }, + }, + Args: aggregate.Children[0].Args, + Type: pql.PrecallGlobal, + }, + }, + } + + opt.PreTranslated = true + aggregateCount, err := e.execute(ctx, qcx, index, &pql.Query{Calls: []*pql.Call{countDistinctIntersect}}, []uint64{}, opt) + if err != nil { + return nil, err + } + results[n].Agg = int64(aggregateCount[0].(uint64)) + } + } + // Apply having. - if having, hasHaving, err := c.CallArg("having"); err != nil { - return nil, err - } else if hasHaving { + if hasHaving && !opt.Remote { // parse the condition as PQL if having.Name != "Condition" { return nil, errors.New("the only supported having call is Condition()") @@ -2687,6 +2907,34 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c } } + if sorter != nil && !opt.Remote { + sorter.data = results + sort.Stable(sorter) + results, err = applyLimitAndOffsetToGroupByResult(c, results) + if err != nil { + return nil, errors.Wrap(err, "applying limit/offset") + } + } else if hasHaving && !opt.Remote { + results, err = applyLimitAndOffsetToGroupByResult(c, results) + if err != nil { + return nil, errors.Wrap(err, "applying limit/offset") + } + + } + + aggType := "" + if aggregate != nil { + switch aggregate.Name { + case "Sum": + aggType = "sum" + case "Count": + aggType = "aggregate" + } + } + return NewGroupCounts(aggType, results...), nil +} + +func applyLimitAndOffsetToGroupByResult(c *pql.Call, results []GroupCount) ([]GroupCount, error) { // Apply offset. if offset, hasOffset, err := c.UintArg("offset"); err != nil { return nil, err @@ -2768,17 +3016,70 @@ func (fr FieldRow) String() string { return fmt.Sprintf("%s.%d.%s", fr.Field, fr.RowID, fr.RowKey) } +type aggregateType int + +const ( + nilAggregate aggregateType = 0 + sumAggregate aggregateType = 1 + distinctAggregate aggregateType = 2 +) + // GroupCounts is a list of GroupCount. -type GroupCounts []GroupCount +type GroupCounts struct { + groups []GroupCount + aggregateType aggregateType +} + +// AggregateColumn gives the likely column name to use for aggregates, because +// for historical reasons we used "sum" when it was a sum, but don't want to +// use that when it's something else. This will likely get revisited. +func (g *GroupCounts) AggregateColumn() string { + switch g.aggregateType { + case sumAggregate: + return "sum" + case distinctAggregate: + return "aggregate" + default: + return "" + } +} + +// Groups is a convenience method to let us not worry as much about the +// potentially-nil nature of a *GroupCounts. +func (g *GroupCounts) Groups() []GroupCount { + if g == nil { + return nil + } + return g.groups +} + +// NewGroupCounts creates a GroupCounts with the given type and slice +// of GroupCount objects. There's intentionally no externally-accessible way +// to change the []GroupCount after creation. +func NewGroupCounts(agg string, groups ...GroupCount) *GroupCounts { + var aggType aggregateType + switch agg { + case "sum": + aggType = sumAggregate + case "aggregate": + aggType = distinctAggregate + case "": + aggType = nilAggregate + default: + panic(fmt.Sprintf("invalid aggregate type %q", agg)) + } + return &GroupCounts{aggregateType: aggType, groups: groups} +} // ToTable implements the ToTabler interface. -func (g GroupCounts) ToTable() (*pb.TableResponse, error) { - return pb.RowsToTable(&g, len(g)) +func (g *GroupCounts) ToTable() (*pb.TableResponse, error) { + return pb.RowsToTable(g, len(g.Groups())) } // ToRows implements the ToRowser interface. -func (g GroupCounts) ToRows(callback func(*pb.RowResponse) error) error { - for i, gc := range g { +func (g *GroupCounts) ToRows(callback func(*pb.RowResponse) error) error { + agg := g.AggregateColumn() + for i, gc := range g.Groups() { var ci []*pb.ColumnInfo if i == 0 { for _, fieldRow := range gc.Group { @@ -2791,7 +3092,10 @@ func (g GroupCounts) ToRows(callback func(*pb.RowResponse) error) error { } } ci = append(ci, &pb.ColumnInfo{Name: "count", Datatype: "uint64"}) - ci = append(ci, &pb.ColumnInfo{Name: "sum", Datatype: "int64"}) + if agg != "" { + ci = append(ci, &pb.ColumnInfo{Name: agg, Datatype: "int64"}) + } + } rowResp := &pb.RowResponse{ Headers: ci, @@ -2808,9 +3112,11 @@ func (g GroupCounts) ToRows(callback func(*pb.RowResponse) error) error { } } rowResp.Columns = append(rowResp.Columns, - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: gc.Count}}, - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: gc.Sum}}, - ) + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: gc.Count}}) + if agg != "" { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: gc.Agg}}) + } if err := callback(rowResp); err != nil { return errors.Wrap(err, "calling callback") } @@ -2818,18 +3124,51 @@ func (g GroupCounts) ToRows(callback func(*pb.RowResponse) error) error { return nil } +// MarshalJSON makes GroupCounts satisfy interface json.Marshaler and +// customizes the JSON output of the aggregate field label. +func (g *GroupCounts) MarshalJSON() ([]byte, error) { + groups := g.Groups() + var counts interface{} = groups + + if len(groups) == 0 { + return []byte("[]"), nil + } + switch g.aggregateType { + case sumAggregate: + counts = *(*[]groupCountSum)(unsafe.Pointer(&groups)) + case distinctAggregate: + counts = *(*[]groupCountAggregate)(unsafe.Pointer(&groups)) + } + return json.Marshal(counts) +} + // GroupCount represents a result item for a group by query. type GroupCount struct { Group []FieldRow `json:"group"` Count uint64 `json:"count"` - Sum int64 `json:"sum"` + Agg int64 `json:"-"` } +type groupCountSum struct { + Group []FieldRow `json:"group"` + Count uint64 `json:"count"` + Agg int64 `json:"sum"` +} + +type groupCountAggregate struct { + Group []FieldRow `json:"group"` + Count uint64 `json:"count"` + Agg int64 `json:"aggregate"` +} + +var _ GroupCount = GroupCount(groupCountSum{}) +var _ GroupCount = GroupCount(groupCountAggregate{}) + func (g *GroupCount) Clone() (r *GroupCount) { r = &GroupCount{ Group: make([]FieldRow, len(g.Group)), Count: g.Count, - Sum: g.Sum, + Agg: g.Agg, } for i := range g.Group { r.Group[i] = *(g.Group[i].Clone()) @@ -2853,7 +3192,7 @@ func mergeGroupCounts(a, b []GroupCount, limit int) []GroupCount { i++ case 0: a[i].Count += b[j].Count - a[i].Sum += b[j].Sum + a[i].Agg += b[j].Agg ret = append(ret, a[i]) i++ j++ @@ -2960,27 +3299,27 @@ func (g GroupCount) satisfiesCondition(subj string, cond *pql.Condition) bool { return false } if cond.Op == pql.EQ { - if g.Sum == val { + if g.Agg == val { return true } } else if cond.Op == pql.NEQ { - if g.Sum != val { + if g.Agg != val { return true } } else if cond.Op == pql.LT { - if g.Sum < val { + if g.Agg < val { return true } } else if cond.Op == pql.LTE { - if g.Sum <= val { + if g.Agg <= val { return true } } else if cond.Op == pql.GT { - if g.Sum > val { + if g.Agg > val { return true } } else if cond.Op == pql.GTE { - if g.Sum >= val { + if g.Agg >= val { return true } } @@ -2990,19 +3329,19 @@ func (g GroupCount) satisfiesCondition(subj string, cond *pql.Condition) bool { return false } if cond.Op == pql.BETWEEN { - if val[0] <= g.Sum && g.Sum <= val[1] { + if val[0] <= g.Agg && g.Agg <= val[1] { return true } } else if cond.Op == pql.BTWN_LT_LTE { - if val[0] < g.Sum && g.Sum <= val[1] { + if val[0] < g.Agg && g.Agg <= val[1] { return true } } else if cond.Op == pql.BTWN_LTE_LT { - if val[0] <= g.Sum && g.Sum < val[1] { + if val[0] <= g.Agg && g.Agg < val[1] { return true } } else if cond.Op == pql.BTWN_LT_LT { - if val[0] < g.Sum && g.Sum < val[1] { + if val[0] < g.Agg && g.Agg < val[1] { return true } } @@ -5102,7 +5441,7 @@ loop: continue loop } } - return nil, errShardUnavailable + return nil, errors.Wrapf(errShardUnavailable, "%s:%d:%v:%v", index, shard, shards, nodes) } return m, nil } @@ -5225,7 +5564,7 @@ func (e *executor) mapper(ctx context.Context, cancel context.CancelFunc, ch cha // Group shards together by nodes. m, err := e.shardsByNode(nodes, index, shards) if err != nil { - return errors.Wrap(err, "shards by node") + return errors.Wrapf(err, "shards by node %v", shards) } // Execute each node in a separate goroutine. @@ -6273,10 +6612,11 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index } } - case []GroupCount: + case *GroupCounts: fieldIDs := make(map[*Field]map[uint64]struct{}) foreignIDs := make(map[*Field]map[uint64]struct{}) - for _, gl := range result { + groups := result.Groups() + for _, gl := range groups { for _, g := range gl.Group { field := idx.Field(g.Field) if field == nil { @@ -6287,7 +6627,7 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index if fi := field.ForeignIndex(); fi != "" { m, ok := foreignIDs[field] if !ok { - m = make(map[uint64]struct{}, len(result)) + m = make(map[uint64]struct{}, len(groups)) foreignIDs[field] = m } @@ -6298,7 +6638,7 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index m, ok := fieldIDs[field] if !ok { - m = make(map[uint64]struct{}, len(result)) + m = make(map[uint64]struct{}, len(groups)) fieldIDs[field] = m } @@ -6325,8 +6665,11 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index foreignTranslations[field.Name()] = trans } - other := make([]GroupCount, 0) - for _, gl := range result { + // We are reluctant to smash result, and I'm not sure we need + // to be but I'm not sure we don't need to be. + newGroups := make([]GroupCount, len(groups)) + copy(newGroups, groups) + for gi, gl := range groups { group := make([]FieldRow, len(gl.Group)) for i, g := range gl.Group { @@ -6339,15 +6682,15 @@ func (e *executor) translateResult(ctx context.Context, index string, idx *Index group[i] = g } - - other = append(other, GroupCount{ - Group: group, - Count: gl.Count, - Sum: gl.Sum, - }) + // Replace with translated group. + newGroups[gi].Group = group } + other := &GroupCounts{} + if result != nil { + other.aggregateType = result.aggregateType + } + other.groups = newGroups return other, nil - case RowIDs: fieldName := callArgString(call, "_field") if fieldName == "" { @@ -6628,6 +6971,7 @@ type execOptions struct { ExcludeRowAttrs bool ExcludeColumns bool ColumnAttrs bool + PreTranslated bool EmbeddedData []*Row } @@ -7193,7 +7537,7 @@ func (gbi *groupByIterator) Next(ctx context.Context) (ret GroupCount, done bool if gbi.done { return ret, true, nil } - if gbi.aggregate == nil { + if gbi.aggregate == nil || gbi.aggregate.Name == "Count" { if len(gbi.rows) == 1 { ret.Count = gbi.rows[len(gbi.rows)-1].row.Count() } else { @@ -7213,7 +7557,7 @@ func (gbi *groupByIterator) Next(ctx context.Context) (ret GroupCount, done bool return ret, false, err } ret.Count = uint64(result.Count) - ret.Sum = result.Val + ret.Agg = result.Val } } if ret.Count == 0 { diff --git a/executor_internal_test.go b/executor_internal_test.go index 06e271fde..fbc3f55d6 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -18,6 +18,7 @@ import ( "context" "encoding/json" "fmt" + "reflect" "strconv" "testing" @@ -163,7 +164,7 @@ func TestExecutor_GroupCountCondition(t *testing.T) { }, }, { - groupCount: GroupCount{Sum: 100}, + groupCount: GroupCount{Agg: 100}, checks: []condCheck{ {cond: "sum == 99", exp: false}, {cond: "sum != 99", exp: true}, @@ -195,7 +196,7 @@ func TestExecutor_GroupCountCondition(t *testing.T) { }, }, { - groupCount: GroupCount{Sum: -100}, + groupCount: GroupCount{Agg: -100}, checks: []condCheck{ {cond: "sum == -99", exp: false}, {cond: "sum != -99", exp: true}, @@ -384,3 +385,93 @@ func TestToInt64(t *testing.T) { } } } + +func TestGetSorter(t *testing.T) { + tests := []struct { + sortSpec string + expGCS *groupCountSorter + expErr string + }{ + { + sortSpec: "count asc", + expGCS: &groupCountSorter{fields: []int{-1}, order: []order{asc}}, + }, + { + sortSpec: "count asc", + expGCS: &groupCountSorter{fields: []int{-1}, order: []order{asc}}, + }, + { + sortSpec: " count asc", + expGCS: &groupCountSorter{fields: []int{-1}, order: []order{asc}}, + }, + { + sortSpec: " count asc ", + expGCS: &groupCountSorter{fields: []int{-1}, order: []order{asc}}, + }, + { + sortSpec: "count", + expGCS: &groupCountSorter{fields: []int{-1}, order: []order{desc}}, + }, + { + sortSpec: "sum asc", + expGCS: &groupCountSorter{fields: []int{-2}, order: []order{asc}}, + }, + { + sortSpec: "aggregate asc", + expGCS: &groupCountSorter{fields: []int{-2}, order: []order{asc}}, + }, + { + sortSpec: "boondoggle asc", + expErr: "sorting is only supported on count, aggregate, or sum, not 'boondoggle'", + }, + { + sortSpec: "sum asc, count desc", + expGCS: &groupCountSorter{fields: []int{-2, -1}, order: []order{asc, desc}}, + }, + { + sortSpec: "count asc, sum desc", + expGCS: &groupCountSorter{fields: []int{-1, -2}, order: []order{asc, desc}}, + }, + { + sortSpec: " count asc , sum desc ", + expGCS: &groupCountSorter{fields: []int{-1, -2}, order: []order{asc, desc}}, + }, + { + sortSpec: " count asc , sum desc blah", + expErr: "parsing sort directive: 'sum desc blah': too many elements", + }, + { + sortSpec: "count asc, sum fesc", + expErr: "unknown sort direction 'fesc'", + }, + { + sortSpec: " , sum fesc", + expErr: "invalid sorting directive: ''", + }, + { + // weird and useless, but I guess fine? + sortSpec: "count asc,count asc ", + expGCS: &groupCountSorter{fields: []int{-1, -1}, order: []order{asc, asc}}, + }, + } + + for i, tst := range tests { + t.Run(fmt.Sprintf("%s_%d", tst.sortSpec, i), func(t *testing.T) { + gcs, err := getSorter(tst.sortSpec) + if err != nil { + if tst.expErr == "" { + t.Errorf("unexpected error: %v", err) + return + } + if tst.expErr != err.Error() { + t.Errorf("mismatched errors got: '%v', exp: '%s'", err, tst.expErr) + } + return + } + if !reflect.DeepEqual(gcs, tst.expGCS) { + t.Errorf("exp:\n%+v\ngot:\n%v\n", tst.expGCS, gcs) + } + + }) + } +} diff --git a/executor_test.go b/executor_test.go index 373d43bdf..552aed964 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1343,7 +1343,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Fatal(err) } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0, 100)); err != nil { t.Fatal(err) - } else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err == nil || err.Error() != `executing: finding top results: cannot compute TopN() on integer field: "f"` { + } else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err == nil || !strings.Contains(err.Error(), `finding top results: cannot compute TopN() on integer field: "f"`) { t.Fatalf("unexpected error: %v", err) } }) @@ -1362,7 +1362,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { Set(0, f=1) `}); err != nil { t.Fatal(err) - } else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err == nil || err.Error() != `executing: finding top results: cannot compute TopN(), field has no cache: "f"` { + } else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err == nil || !strings.Contains(err.Error(), `finding top results: cannot compute TopN(), field has no cache: "f"`) { t.Fatalf("unexpected error: %v", err) } }) @@ -3066,7 +3066,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "f", RowID: 7}}, Count: 1}, {Group: []pilosa.FieldRow{{Field: "f", RowID: 10}}, Count: 4}, } - results := res.Results[0].([]pilosa.GroupCount) + results := res.Results[0].(*pilosa.GroupCounts).Groups() test.CheckGroupBy(t, expected, results) } }) @@ -3108,7 +3108,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "fint", Value: &d}}, Count: 1}, } - results := res.Results[0].([]pilosa.GroupCount) + results := res.Results[0].(*pilosa.GroupCounts).Groups() test.CheckGroupBy(t, expected, results) } }) @@ -3139,7 +3139,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "hint", Value: &c}}, Count: 1}, } - results := res.Results[0].([]pilosa.GroupCount) + results := res.Results[0].(*pilosa.GroupCounts).Groups() test.CheckGroupBy(t, expected, results) } }) @@ -5074,20 +5074,20 @@ func TestExecutor_GroupByStrings(t *testing.T) { { query: "GroupBy(Rows(generals), aggregate=Sum(field=v))", expected: []pilosa.GroupCount{ - {Group: []pilosa.FieldRow{{Field: "generals", RowID: 1, RowKey: "r1"}}, Count: 5, Sum: 25}, - {Group: []pilosa.FieldRow{{Field: "generals", RowID: 2, RowKey: "r2"}}, Count: 5, Sum: 30}, + {Group: []pilosa.FieldRow{{Field: "generals", RowID: 1, RowKey: "r1"}}, Count: 5, Agg: 25}, + {Group: []pilosa.FieldRow{{Field: "generals", RowID: 2, RowKey: "r2"}}, Count: 5, Agg: 30}, }, }, { query: "GroupBy(Rows(generals), aggregate=Sum(field=v), having=Condition(sum>25))", expected: []pilosa.GroupCount{ - {Group: []pilosa.FieldRow{{Field: "generals", RowID: 2, RowKey: "r2"}}, Count: 5, Sum: 30}, + {Group: []pilosa.FieldRow{{Field: "generals", RowID: 2, RowKey: "r2"}}, Count: 5, Agg: 30}, }, }, { query: "GroupBy(Rows(generals), aggregate=Sum(field=v), having=Condition(-5 10), aggregate=Count(Distinct(field=v)))`).Results[0].(*pilosa.GroupCounts).Groups() + test.CheckGroupBy(t, expected, results) + }) + + t.Run("AggregateCountDistinctFilterDistinct", func(t *testing.T) { + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Count: 3, Agg: 1}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Count: 1, Agg: 0}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}, {Field: "sub", RowID: 110}}, Count: 1, Agg: 0}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 12}, {Field: "sub", RowID: 110}}, Count: 1, Agg: 0}, + } + + results := c.Query(t, "i", `GroupBy(Rows(general), Rows(sub), aggregate=Count(Distinct(Row(v > 10), field=v)))`).Results[0].(*pilosa.GroupCounts).Groups() test.CheckGroupBy(t, expected, results) }) @@ -5615,7 +5650,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "general", RowID: 12}}, Count: 2}, } - results := c.Query(t, "i", `GroupBy(Rows(general, previous=10))`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(general, previous=10))`).Results[0].(*pilosa.GroupCounts).Groups() test.CheckGroupBy(t, expected, results) }) @@ -5624,7 +5659,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Count: 2}, } - results := c.Query(t, "i", `GroupBy(Rows(general, previous=10), limit=1)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(general, previous=10), limit=1)`).Results[0].(*pilosa.GroupCounts).Groups() test.CheckGroupBy(t, expected, results) }) @@ -5645,7 +5680,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "a", RowID: 0}, {Field: "b", RowID: 1}}, Count: 1}, } - results := c.Query(t, "i", `GroupBy(Rows(a), Rows(b), limit=1)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(a), Rows(b), limit=1)`).Results[0].(*pilosa.GroupCounts).Groups() test.CheckGroupBy(t, expected, results) }) @@ -5673,7 +5708,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { }) t.Run("test wrapping with previous", func(t *testing.T) { - results := c.Query(t, "i", `GroupBy(Rows(wa), Rows(wb), Rows(wc, previous=1), limit=3)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(wa), Rows(wb), Rows(wc, previous=1), limit=3)`).Results[0].(*pilosa.GroupCounts).Groups() expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "wa", RowID: 0}, {Field: "wb", RowID: 0}, {Field: "wc", RowID: 2}}, Count: 2}, {Group: []pilosa.FieldRow{{Field: "wa", RowID: 0}, {Field: "wb", RowID: 1}, {Field: "wc", RowID: 0}}, Count: 1}, @@ -5683,14 +5718,14 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { }) t.Run("test previous is last result", func(t *testing.T) { - results := c.Query(t, "i", `GroupBy(Rows(wa, previous=3), Rows(wb, previous=3), Rows(wc, previous=3), limit=3)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(wa, previous=3), Rows(wb, previous=3), Rows(wc, previous=3), limit=3)`).Results[0].(*pilosa.GroupCounts).Groups() if len(results) > 0 { t.Fatalf("expected no results because previous specified last result") } }) t.Run("test wrapping multiple", func(t *testing.T) { - results := c.Query(t, "i", `GroupBy(Rows(wa), Rows(wb, previous=2), Rows(wc, previous=2), limit=1)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(wa), Rows(wb, previous=2), Rows(wc, previous=2), limit=1)`).Results[0].(*pilosa.GroupCounts).Groups() expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "wa", RowID: 1}, {Field: "wb", RowID: 0}, {Field: "wc", RowID: 0}}, Count: 1}, } @@ -5714,7 +5749,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {3, ShardWidth}, }) t.Run("distinct rows in different shards", func(t *testing.T) { - results := c.Query(t, "i", `GroupBy(Rows(ma), Rows(mb), limit=5)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(ma), Rows(mb), limit=5)`).Results[0].(*pilosa.GroupCounts).Groups() expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "ma", RowID: 0}, {Field: "mb", RowID: 0}}, Count: 1}, {Group: []pilosa.FieldRow{{Field: "ma", RowID: 0}, {Field: "mb", RowID: 2}}, Count: 1}, @@ -5726,7 +5761,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { }) t.Run("distinct rows in different shards with row limit", func(t *testing.T) { - results := c.Query(t, "i", `GroupBy(Rows(ma), Rows(mb, limit=2), limit=5)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(ma), Rows(mb, limit=2), limit=5)`).Results[0].(*pilosa.GroupCounts).Groups() expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "ma", RowID: 0}, {Field: "mb", RowID: 0}}, Count: 1}, {Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 1}}, Count: 1}, @@ -5737,7 +5772,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { }) t.Run("distinct rows in different shards with column arg", func(t *testing.T) { - results := c.Query(t, "i", fmt.Sprintf(`GroupBy(Rows(ma), Rows(mb, column=%d), limit=5)`, ShardWidth)).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", fmt.Sprintf(`GroupBy(Rows(ma), Rows(mb, column=%d), limit=5)`, ShardWidth)).Results[0].(*pilosa.GroupCounts).Groups() expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 1}}, Count: 1}, {Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 3}}, Count: 1}, @@ -5762,7 +5797,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {1, ShardWidth}, }) t.Run("same rows in different shards", func(t *testing.T) { - results := c.Query(t, "i", `GroupBy(Rows(na), Rows(nb))`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(na), Rows(nb))`).Results[0].(*pilosa.GroupCounts).Groups() expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "na", RowID: 0}, {Field: "nb", RowID: 0}}, Count: 2}, {Group: []pilosa.FieldRow{{Field: "na", RowID: 0}, {Field: "nb", RowID: 1}}, Count: 2}, @@ -5799,12 +5834,12 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { t.Run("test wrapping with previous", func(t *testing.T) { totalResults := make([]pilosa.GroupCount, 0) - results := c.Query(t, "i", `GroupBy(Rows(ppa), Rows(ppb), Rows(ppc), limit=3)`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(ppa), Rows(ppb), Rows(ppc), limit=3)`).Results[0].(*pilosa.GroupCounts).Groups() totalResults = append(totalResults, results...) for len(totalResults) < 64 { lastGroup := results[len(results)-1].Group query := fmt.Sprintf("GroupBy(Rows(ppa, previous=%d), Rows(ppb, previous=%d), Rows(ppc, previous=%d), limit=3)", lastGroup[0].RowID, lastGroup[1].RowID, lastGroup[2].RowID) - results = c.Query(t, "i", query).Results[0].([]pilosa.GroupCount) + results = c.Query(t, "i", query).Results[0].(*pilosa.GroupCounts).Groups() totalResults = append(totalResults, results...) } @@ -5846,7 +5881,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "generalk", RowID: 3, RowKey: "twelve"}, {Field: "subk", RowID: 2, RowKey: "one-hundred-ten"}}, Count: 1}, } - results := c.Query(t, "i", `GroupBy(Rows(generalk), Rows(subk))`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "i", `GroupBy(Rows(generalk), Rows(subk))`).Results[0].(*pilosa.GroupCounts).Groups() test.CheckGroupBy(t, expected, results) }) @@ -5884,7 +5919,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "child", RowID: 2, RowKey: "three"}}, Count: 2}, } - results := c.Query(t, "fic", `GroupBy(Rows(child))`).Results[0].([]pilosa.GroupCount) + results := c.Query(t, "fic", `GroupBy(Rows(child))`).Results[0].(*pilosa.GroupCounts).Groups() test.CheckGroupBy(t, expected, results) }) @@ -6470,10 +6505,11 @@ func TestExecutor_Execute_CountDistinct(t *testing.T) { if err != nil { t.Fatal(err) } - gc, ok := resp.Results[0].([]pilosa.GroupCount) + gcc, ok := resp.Results[0].(*pilosa.GroupCounts) if !ok { t.Fatalf("invalid response type, expected: []pilosa.GroupCount, got: %T", resp.Results[0]) } + gc := gcc.Groups() if len(gc) != 2 { t.Fatalf("invalid group count length, expected: 2, got: %v", len(gc)) } @@ -6780,7 +6816,15 @@ func TestMissingKeyRegression(t *testing.T) { // (single and multi-node clusters, different endpoints for the // queries (HTTP, GRPC, Postgres), etc.). func TestVariousQueries(t *testing.T) { - c := test.MustRunCluster(t, 3) + for _, clusterSize := range []int{1, 3, 4, 7} { + t.Run(fmt.Sprintf("%d-node", clusterSize), func(t *testing.T) { + variousQueries(t, clusterSize) + }) + } +} + +func variousQueries(t *testing.T, clusterSize int) { + c := test.MustRunCluster(t, clusterSize) defer c.Close() // Create and populate "likenums" similar to "likes", but without keys on the field. @@ -6796,7 +6840,9 @@ func TestVariousQueries(t *testing.T) { {ID: 7, Key: "userB"}, {ID: 7, Key: "userC"}, {ID: 7, Key: "userD"}, - {ID: 7, Key: "userE"}, + // we intentionally leave user E out because then there is no + // data for userE's shard for this field, which triggered a + // "fragment not found" problem {ID: 7, Key: "userF"}, }) @@ -6827,6 +6873,28 @@ func TestVariousQueries(t *testing.T) { {Val: 0, Key: "userE"}, }) + // Create and populate "affinity" int field with negative, positive, zero and null values. + + c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "net_worth", pilosa.OptFieldTypeInt(-100000000, 100000000)) + c.ImportIntKey(t, "users", "net_worth", []test.IntKey{ + {Val: 1, Key: "userA"}, + {Val: 10, Key: "userB"}, + {Val: 100, Key: "userC"}, + {Val: 1000, Key: "userD"}, + {Val: 10000, Key: "userE"}, + {Val: 100000, Key: "userF"}, + }) + + c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "zip_code", pilosa.OptFieldTypeInt(0, 100000)) + c.ImportIntKey(t, "users", "zip_code", []test.IntKey{ + {Val: 78739, Key: "userA"}, + {Val: 78739, Key: "userB"}, + {Val: 19707, Key: "userC"}, + {Val: 19707, Key: "userD"}, + {Val: 86753, Key: "userE"}, + {Val: 78739, Key: "userG"}, + }) + tests := []struct { query string qrVerifier func(t *testing.T, resp pilosa.QueryResponse) @@ -6835,11 +6903,11 @@ func TestVariousQueries(t *testing.T) { { query: "Count(All())", qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) { - if resp.Results[0].(uint64) != 6 { - t.Errorf("expected 6, got %+v", resp.Results[0]) + if resp.Results[0].(uint64) != 7 { + t.Errorf("expected 7, got %+v", resp.Results[0]) } }, - csvVerifier: "6\n", + csvVerifier: "7\n", }, { query: "Count(Distinct(field=likenums))", @@ -6970,6 +7038,116 @@ func TestVariousQueries(t *testing.T) { }, csvVerifier: "molecula\npilosa\npangolin\nzebra\ntoucan\ndog\nicecream\n", }, + { + query: "GroupBy(Rows(field=likes))", + csvVerifier: `molecula,1 +pilosa,1 +pangolin,1 +zebra,1 +toucan,1 +dog,1 +icecream,6 +`, + }, + { + query: "GroupBy(Rows(field=likes), aggregate=Sum(field=net_worth), limit=2, having=Condition(sum>10))", + csvVerifier: `pangolin,1,100 +zebra,1,1000 +`, + }, + { + query: "GroupBy(Rows(field=likes), having=Condition(count>5))", + csvVerifier: "icecream,6\n", + }, + { + query: "GroupBy(Rows(field=likes), filter=Row(affinity>-7))", + csvVerifier: `molecula,1 +pangolin,1 +zebra,1 +toucan,1 +icecream,4 +`, + }, + { + query: "GroupBy(Rows(field=likes), aggregate=Count(Distinct(field=zip_code)))", + csvVerifier: `molecula,1,1 +pilosa,1,1 +pangolin,1,1 +zebra,1,1 +toucan,1,1 +dog,1,0 +icecream,6,3 +`, + }, + { + query: "GroupBy(Rows(field=likes), aggregate=Count(Distinct(field=zip_code)), having=Condition(sum>2))", + csvVerifier: "icecream,6,3\n", + }, + { + query: "GroupBy(Rows(field=likes), filter=Row(affinity>-11), aggregate=Count(Distinct(field=zip_code)))", + csvVerifier: `molecula,1,1 +pilosa,1,1 +pangolin,1,1 +zebra,1,1 +toucan,1,1 +icecream,5,3 +`, + }, + { + query: "GroupBy(Rows(field=likes), filter=Row(affinity>-11), aggregate=Count(Distinct(Row(affinity>-7), field=zip_code)))", + csvVerifier: `molecula,1,1 +pilosa,1,0 +pangolin,1,1 +zebra,1,1 +toucan,1,1 +icecream,5,3 +`, + }, + { + query: "GroupBy(Rows(field=likes), sort=\"count desc\")", + csvVerifier: `icecream,6 +molecula,1 +pilosa,1 +pangolin,1 +zebra,1 +toucan,1 +dog,1 +`, + }, + { + query: "GroupBy(Rows(field=likes), aggregate=Sum(field=net_worth), sort=\"aggregate desc, count asc\")", + csvVerifier: `icecream,6,111111 +dog,1,100000 +toucan,1,10000 +zebra,1,1000 +pangolin,1,100 +pilosa,1,10 +molecula,1,1 +`, + }, + { + query: "GroupBy(Rows(field=likes), aggregate=Sum(field=net_worth), sort=\"aggregate desc, count asc\", limit=3)", + csvVerifier: `icecream,6,111111 +dog,1,100000 +toucan,1,10000 +`, + }, + { + query: "GroupBy(Rows(field=likes), aggregate=Sum(field=net_worth),sort=\"aggregate desc, count asc\",limit=3,offset=2)", + csvVerifier: `toucan,1,10000 +zebra,1,1000 +pangolin,1,100 +`, + }, + { + query: "GroupBy(Rows(field=affinity), aggregate=Count(Distinct(field=zip_code)))", + csvVerifier: `-10,1,1 +-5,1,1 +0,1,1 +5,1,1 +10,1,1 +`, + }, } for i, tst := range tests { @@ -6986,7 +7164,7 @@ func TestVariousQueries(t *testing.T) { // verify everything after header got := csvString[strings.Index(csvString, "\n")+1:] if got != tst.csvVerifier { - t.Errorf("expected '%s', got '%s'", tst.csvVerifier, got) + t.Errorf("expected:\n%s\ngot:\n%s", tst.csvVerifier, got) } // TODO: add HTTP and Postgres and ability to convert @@ -6995,6 +7173,88 @@ func TestVariousQueries(t *testing.T) { } } +// TestVariousSingleShardQueries tests queries on a dataset which +// consists of an unkeyed index, and data only in the first +// shard. Turns out that there are some interesting failure modes +// which only crop up with one shard. An example is that the mapReduce +// logic does its first reduce call with a nil interface{} value, and +// an actual result value. If this is the only reduce that gets done +// (because there's only one shard), the result might never go through +// normal merge/reduction logic and might e.g. be nil instead of an +// empty struct resulting in an NPE later on. +func TestVariousSingleShardQueries(t *testing.T) { + for _, clusterSize := range []int{1, 4} { + t.Run(fmt.Sprintf("%d-node", clusterSize), func(t *testing.T) { + variousSingleShardQueries(t, clusterSize) + }) + } +} + +func variousSingleShardQueries(t *testing.T, clusterSize int) { + c := test.MustRunCluster(t, clusterSize) + defer c.Close() + + // Create and populate "likenums" similar to "likes", but without keys on the field. + c.CreateField(t, "events", pilosa.IndexOptions{Keys: false, TrackExistence: true}, "lostcount", pilosa.OptFieldTypeInt(0, 1000000000)) + c.ImportIntID(t, "events", "lostcount", []test.IntID{ + {Val: 0, ID: 1}, + {Val: 1, ID: 2}, + {Val: 0, ID: 3}, + {Val: 2, ID: 4}, + {Val: 2, ID: 5}, + {Val: 0, ID: 6}, + {Val: 3, ID: 7}, + {Val: 3, ID: 8}, + {Val: 3, ID: 9}, + {Val: 0, ID: 10}, + }) + + c.CreateField(t, "events", pilosa.IndexOptions{Keys: false, TrackExistence: true}, "jittermax", pilosa.OptFieldTypeInt(0, 1000000000)) + c.ImportIntID(t, "events", "jittermax", []test.IntID{ + {Val: 17, ID: 1}, + {Val: 3, ID: 2}, + {Val: 42, ID: 3}, + {Val: 9, ID: 4}, + {Val: 17, ID: 5}, + {Val: 3, ID: 6}, + {Val: 42, ID: 7}, + {Val: 9, ID: 8}, + {Val: 17, ID: 9}, + {Val: 3, ID: 10}, + }) + + tests := []struct { + query string + csvVerifier string + }{ + { + query: "GroupBy(Rows(lostcount), aggregate=Count(Distinct(field=jittermax)))", + csvVerifier: `0,4,3 +1,1,1 +2,2,2 +3,3,3 +`, + }, + } + + for i, tst := range tests { + t.Run(fmt.Sprintf("%d-%s", i, tst.query), func(t *testing.T) { + tr := c.QueryGRPC(t, "events", tst.query) + csvString, err := tableResponseToCSVString(tr) + if err != nil { + t.Fatal(err) + } + // verify everything after header + got := csvString[strings.Index(csvString, "\n")+1:] + if got != tst.csvVerifier { + t.Errorf("expected:\n%s\ngot:\n%s", tst.csvVerifier, got) + } + + }) + } + +} + // tableResponseToCSV converts a generic TableResponse to a CSV format // and writes it to the writer. func tableResponseToCSV(m *proto.TableResponse, w io.Writer) error { diff --git a/handler.go b/handler.go index fd937e01d..d8e370d56 100644 --- a/handler.go +++ b/handler.go @@ -46,6 +46,10 @@ type QueryRequest struct { // If false, this request is on the originating node. Remote bool + // Query has already been translated. This is only used if Remote + // is false, Remote=true implies this. + PreTranslated bool + // Should we profile this query? Profile bool diff --git a/http/client.go b/http/client.go index 0f3cab4ec..4f37d7a36 100644 --- a/http/client.go +++ b/http/client.go @@ -286,7 +286,6 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index s } else if queryRequest.Query == "" { return nil, pilosa.ErrQueryRequired } - buf, err := c.serializer.Marshal(queryRequest) if err != nil { return nil, errors.Wrap(err, "marshaling queryRequest") @@ -308,7 +307,7 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index s // Execute request against the host. resp, err := c.executeRequest(req.WithContext(ctx)) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "'%s', shards %v", queryRequest.Query, queryRequest.Shards) } defer resp.Body.Close() @@ -1714,12 +1713,15 @@ func (c *InternalClient) executeRequest(req *http.Request, opts ...executeReques var msg string // try to decode a JSON response var sr successResponse + qr := &pilosa.QueryResponse{} if err = json.Unmarshal(buf, &sr); err == nil { msg = sr.Error.Error() + } else if err := c.serializer.Unmarshal(buf, qr); err == nil { + msg = qr.Err.Error() } else { msg = string(buf) } - return resp, errors.Errorf("server error %s: '%s'", resp.Status, msg) + return resp, errors.Errorf("against %s %s: '%s'", req.URL.String(), resp.Status, msg) } return resp, nil } diff --git a/internal/private.pb.go b/internal/private.pb.go index 6dd3eb9bc..b3c0fec5b 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -5620,10 +5620,7 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6028,10 +6025,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6114,10 +6108,7 @@ func (m *ImportResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6302,10 +6293,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6508,10 +6496,7 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6638,10 +6623,7 @@ func (m *Cache) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6788,7 +6770,7 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > postIndex { @@ -6805,10 +6787,7 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6942,10 +6921,7 @@ func (m *CreateShardMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7028,10 +7004,7 @@ func (m *DeleteIndexMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7169,10 +7142,7 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7342,10 +7312,7 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7460,10 +7427,7 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7597,10 +7561,7 @@ func (m *DeleteAvailableShardMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7770,10 +7731,7 @@ func (m *Field) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7858,10 +7816,7 @@ func (m *Schema) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8033,10 +7988,7 @@ func (m *Index) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8170,10 +8122,7 @@ func (m *URI) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8380,10 +8329,7 @@ func (m *Node) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8498,10 +8444,7 @@ func (m *NodeStateMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8607,10 +8550,7 @@ func (m *NodeEventMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8767,10 +8707,7 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8906,10 +8843,7 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -9087,10 +9021,7 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -9275,10 +9206,7 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -9431,10 +9359,7 @@ func (m *BSIGroup) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -9581,10 +9506,7 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -9731,10 +9653,7 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10016,10 +9935,7 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10221,10 +10137,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10362,10 +10275,7 @@ func (m *TranslationResizeSource) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10503,10 +10413,7 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10593,10 +10500,7 @@ func (m *SetCoordinatorMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10683,10 +10587,7 @@ func (m *UpdateCoordinatorMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10801,10 +10702,7 @@ func (m *Topology) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10855,10 +10753,7 @@ func (m *RecalculateCaches) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10977,10 +10872,7 @@ func (m *TransactionMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -11177,10 +11069,7 @@ func (m *Transaction) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -11231,10 +11120,7 @@ func (m *TransactionStats) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { diff --git a/internal/public.pb.go b/internal/public.pb.go index b334a48e3..e406e06ab 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -1111,7 +1111,7 @@ func (m *FieldRow) GetValue() *Int64 { type GroupCount struct { Group []*FieldRow `protobuf:"bytes,1,rep,name=Group,proto3" json:"Group,omitempty"` Count uint64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` - Sum int64 `protobuf:"varint,3,opt,name=Sum,proto3" json:"Sum,omitempty"` + Agg int64 `protobuf:"varint,3,opt,name=Agg,proto3" json:"Agg,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1164,9 +1164,9 @@ func (m *GroupCount) GetCount() uint64 { return 0 } -func (m *GroupCount) GetSum() int64 { +func (m *GroupCount) GetAgg() int64 { if m != nil { - return m.Sum + return m.Agg } return 0 } @@ -1502,6 +1502,7 @@ type QueryRequest struct { ExcludeRowAttrs bool `protobuf:"varint,6,opt,name=ExcludeRowAttrs,proto3" json:"ExcludeRowAttrs,omitempty"` ExcludeColumns bool `protobuf:"varint,7,opt,name=ExcludeColumns,proto3" json:"ExcludeColumns,omitempty"` EmbeddedData []*Row `protobuf:"bytes,8,rep,name=EmbeddedData,proto3" json:"EmbeddedData,omitempty"` + PreTranslated bool `protobuf:"varint,9,opt,name=PreTranslated,proto3" json:"PreTranslated,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1589,6 +1590,13 @@ func (m *QueryRequest) GetEmbeddedData() []*Row { return nil } +func (m *QueryRequest) GetPreTranslated() bool { + if m != nil { + return m.PreTranslated + } + return false +} + type QueryResponse struct { Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` Results []*QueryResult `protobuf:"bytes,2,rep,name=Results,proto3" json:"Results,omitempty"` @@ -1653,14 +1661,20 @@ func (m *QueryResponse) GetColumnAttrSets() []*ColumnAttrSet { } type QueryResult struct { - Type uint32 `protobuf:"varint,6,opt,name=Type,proto3" json:"Type,omitempty"` - Row *Row `protobuf:"bytes,1,opt,name=Row,proto3" json:"Row,omitempty"` - N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` - Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs,proto3" json:"Pairs,omitempty"` - Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` - ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount,proto3" json:"ValCount,omitempty"` - RowIDs []uint64 `protobuf:"varint,7,rep,packed,name=RowIDs,proto3" json:"RowIDs,omitempty"` - GroupCounts []*GroupCount `protobuf:"bytes,8,rep,name=GroupCounts,proto3" json:"GroupCounts,omitempty"` + Type uint32 `protobuf:"varint,6,opt,name=Type,proto3" json:"Type,omitempty"` + Row *Row `protobuf:"bytes,1,opt,name=Row,proto3" json:"Row,omitempty"` + N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` + Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs,proto3" json:"Pairs,omitempty"` + Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` + ValCount *ValCount `protobuf:"bytes,5,opt,name=ValCount,proto3" json:"ValCount,omitempty"` + RowIDs []uint64 `protobuf:"varint,7,rep,packed,name=RowIDs,proto3" json:"RowIDs,omitempty"` + // In the past, GroupCounts was a []GroupCount which did not indicate + // whether it had an aggregate, or which aggregate it had. We've + // updated this, but we keep this here so that messages using the old + // format can get a best-effort treatment rather than causing panics. + // Later this can almost certainly go away, but leave a comment warning + // people that 8 is Spoken For if you do that, please. + OldGroupCounts []*GroupCount `protobuf:"bytes,8,rep,name=OldGroupCounts,proto3" json:"OldGroupCounts,omitempty"` RowIdentifiers *RowIdentifiers `protobuf:"bytes,9,opt,name=RowIdentifiers,proto3" json:"RowIdentifiers,omitempty"` SignedRow *SignedRow `protobuf:"bytes,10,opt,name=SignedRow,proto3" json:"SignedRow,omitempty"` PairsField *PairsField `protobuf:"bytes,11,opt,name=PairsField,proto3" json:"PairsField,omitempty"` @@ -1668,6 +1682,7 @@ type QueryResult struct { ExtractedIDMatrix *ExtractedIDMatrix `protobuf:"bytes,13,opt,name=ExtractedIDMatrix,proto3" json:"ExtractedIDMatrix,omitempty"` ExtractedTable *ExtractedTable `protobuf:"bytes,14,opt,name=ExtractedTable,proto3" json:"ExtractedTable,omitempty"` RowMatrix *RowMatrix `protobuf:"bytes,15,opt,name=RowMatrix,proto3" json:"RowMatrix,omitempty"` + GroupCounts *GroupCounts `protobuf:"bytes,16,opt,name=GroupCounts,proto3" json:"GroupCounts,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1755,9 +1770,9 @@ func (m *QueryResult) GetRowIDs() []uint64 { return nil } -func (m *QueryResult) GetGroupCounts() []*GroupCount { +func (m *QueryResult) GetOldGroupCounts() []*GroupCount { if m != nil { - return m.GroupCounts + return m.OldGroupCounts } return nil } @@ -1811,6 +1826,13 @@ func (m *QueryResult) GetRowMatrix() *RowMatrix { return nil } +func (m *QueryResult) GetGroupCounts() *GroupCounts { + if m != nil { + return m.GroupCounts + } + return nil +} + type ImportRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` @@ -2640,6 +2662,61 @@ func (m *ImportColumnAttrsRequest) GetIndexCreatedAt() int64 { return 0 } +type GroupCounts struct { + Aggregate string `protobuf:"bytes,1,opt,name=Aggregate,proto3" json:"Aggregate,omitempty"` + Groups []*GroupCount `protobuf:"bytes,2,rep,name=Groups,proto3" json:"Groups,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GroupCounts) Reset() { *m = GroupCounts{} } +func (m *GroupCounts) String() string { return proto.CompactTextString(m) } +func (*GroupCounts) ProtoMessage() {} +func (*GroupCounts) Descriptor() ([]byte, []int) { + return fileDescriptor_413a91106d7bcce8, []int{37} +} +func (m *GroupCounts) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *GroupCounts) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_GroupCounts.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *GroupCounts) XXX_Merge(src proto.Message) { + xxx_messageInfo_GroupCounts.Merge(m, src) +} +func (m *GroupCounts) XXX_Size() int { + return m.Size() +} +func (m *GroupCounts) XXX_DiscardUnknown() { + xxx_messageInfo_GroupCounts.DiscardUnknown(m) +} + +var xxx_messageInfo_GroupCounts proto.InternalMessageInfo + +func (m *GroupCounts) GetAggregate() string { + if m != nil { + return m.Aggregate + } + return "" +} + +func (m *GroupCounts) GetGroups() []*GroupCount { + if m != nil { + return m.Groups + } + return nil +} + func init() { proto.RegisterType((*Row)(nil), "internal.Row") proto.RegisterType((*RowMatrix)(nil), "internal.RowMatrix") @@ -2678,118 +2755,123 @@ func init() { proto.RegisterType((*ImportRoaringRequestView)(nil), "internal.ImportRoaringRequestView") proto.RegisterType((*ImportRoaringRequest)(nil), "internal.ImportRoaringRequest") proto.RegisterType((*ImportColumnAttrsRequest)(nil), "internal.ImportColumnAttrsRequest") + proto.RegisterType((*GroupCounts)(nil), "internal.GroupCounts") } func init() { proto.RegisterFile("public.proto", fileDescriptor_413a91106d7bcce8) } var fileDescriptor_413a91106d7bcce8 = []byte{ - // 1694 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x4f, 0x73, 0x1b, 0x4b, - 0x11, 0xf7, 0x6a, 0x57, 0x96, 0xd4, 0x92, 0x1d, 0xbf, 0x89, 0xde, 0x63, 0x2b, 0x38, 0x7e, 0x62, - 0xcb, 0xf0, 0x04, 0x45, 0x39, 0xe5, 0x90, 0xa4, 0x72, 0x01, 0x62, 0x47, 0x0e, 0xde, 0x0a, 0x36, - 0x61, 0xe4, 0x72, 0x6e, 0x54, 0xad, 0xa5, 0xc1, 0xd9, 0x62, 0xa5, 0x15, 0xab, 0x55, 0x64, 0x5f, - 0xa8, 0xe2, 0x33, 0xe4, 0xc2, 0x8d, 0x2b, 0x27, 0x3e, 0x04, 0x17, 0x38, 0x72, 0xa4, 0x8a, 0x0b, - 0x15, 0xf8, 0x16, 0x5c, 0xa8, 0xee, 0xd9, 0xd9, 0x99, 0x5d, 0xad, 0x1d, 0x57, 0x8a, 0xdb, 0xf4, - 0x9f, 0xe9, 0xe9, 0xfe, 0x75, 0x4f, 0x4f, 0xef, 0x42, 0x67, 0xb6, 0xb8, 0x88, 0xc2, 0xd1, 0xde, - 0x2c, 0x89, 0xd3, 0x98, 0x35, 0xc3, 0x69, 0x2a, 0x92, 0x69, 0x10, 0x79, 0x7f, 0xb4, 0xc0, 0xe6, - 0xf1, 0x92, 0xb9, 0xd0, 0x78, 0x19, 0x47, 0x8b, 0xc9, 0x74, 0xee, 0x5a, 0x3d, 0xbb, 0xef, 0x70, - 0x45, 0x32, 0x06, 0xce, 0x6b, 0x71, 0x3d, 0x77, 0xed, 0x9e, 0xdd, 0x6f, 0x71, 0x5a, 0xb3, 0x5d, - 0xa8, 0x1f, 0xa4, 0x69, 0x32, 0x77, 0x6b, 0x3d, 0xbb, 0xdf, 0x7e, 0xbc, 0xb9, 0xa7, 0xec, 0xed, - 0x21, 0x9b, 0x4b, 0x21, 0xda, 0xe4, 0x71, 0x90, 0x84, 0xd3, 0x4b, 0xd7, 0xe9, 0x59, 0xfd, 0x0e, - 0x57, 0x24, 0xeb, 0x42, 0xdd, 0x9f, 0x8e, 0xc5, 0x95, 0x5b, 0xef, 0x59, 0xfd, 0x16, 0x97, 0x04, - 0x72, 0x5f, 0x85, 0x22, 0x1a, 0xbb, 0xeb, 0x92, 0x4b, 0x84, 0xb7, 0x07, 0x2d, 0x1e, 0x2f, 0x4f, - 0x82, 0x34, 0x09, 0xaf, 0xd8, 0x77, 0xc0, 0xe1, 0xf1, 0x52, 0xfa, 0xd8, 0x7e, 0xbc, 0xa1, 0xcf, - 0xe5, 0xf1, 0x92, 0x93, 0xc8, 0x3b, 0x81, 0xd6, 0x30, 0xbc, 0x9c, 0x8a, 0x31, 0x86, 0xf5, 0x35, - 0xd8, 0x6f, 0x62, 0x54, 0xb7, 0x56, 0xd5, 0x51, 0x82, 0x0a, 0xa7, 0xe2, 0xd2, 0xad, 0x55, 0x2a, - 0x9c, 0x8a, 0x4b, 0xef, 0x39, 0x6c, 0xf2, 0x78, 0xe9, 0x8f, 0xc5, 0x34, 0x0d, 0x7f, 0x1d, 0x8a, - 0x84, 0x00, 0xc9, 0x7d, 0x70, 0xe4, 0xa1, 0x39, 0x48, 0x35, 0x0d, 0x92, 0xf7, 0x00, 0xd6, 0xfd, - 0xc1, 0xcf, 0xc3, 0x79, 0xca, 0xb6, 0xc0, 0xf6, 0x07, 0x6a, 0x03, 0x2e, 0x3d, 0x1f, 0xbe, 0x38, - 0xba, 0x4a, 0x93, 0x60, 0x94, 0x8a, 0xb1, 0x3f, 0x90, 0x50, 0xb3, 0x4d, 0xa8, 0xf9, 0x03, 0xf2, - 0xd5, 0xe1, 0x35, 0x7f, 0xc0, 0x76, 0xc1, 0x39, 0x0f, 0x22, 0x05, 0xf2, 0x96, 0x76, 0x4e, 0x9a, - 0xe5, 0x24, 0xf5, 0x2e, 0x0a, 0xa6, 0x32, 0x9c, 0xbe, 0x82, 0x75, 0x42, 0x4f, 0x1e, 0xda, 0xe2, - 0x19, 0xc5, 0x9e, 0xea, 0x34, 0x4b, 0xab, 0xdf, 0xd6, 0x56, 0x57, 0x1c, 0xca, 0x6b, 0xc0, 0x7b, - 0x08, 0x8d, 0xd7, 0xe2, 0x9a, 0x62, 0x51, 0x91, 0x5a, 0x46, 0xa4, 0xff, 0xb4, 0xe0, 0x7e, 0xbe, - 0xfb, 0x2c, 0xb8, 0x88, 0xc4, 0x79, 0x10, 0x2d, 0x04, 0xdb, 0x55, 0x71, 0x5b, 0x55, 0xfe, 0x1f, - 0xaf, 0x11, 0x16, 0xec, 0x9b, 0x1c, 0x3b, 0x54, 0xfb, 0x42, 0xab, 0x65, 0x47, 0x1e, 0xaf, 0x65, - 0x55, 0xb7, 0x0d, 0xcd, 0xc3, 0xa1, 0x4f, 0xa6, 0x5d, 0xbb, 0x67, 0xf5, 0xed, 0xe3, 0x35, 0x9e, - 0x73, 0xd8, 0x03, 0x68, 0x9c, 0x2c, 0x52, 0x71, 0xe5, 0x0f, 0xa8, 0xda, 0x9c, 0xe3, 0x35, 0xae, - 0x18, 0xb8, 0x93, 0x96, 0xaf, 0xc5, 0xb5, 0x2c, 0x39, 0xdc, 0xa9, 0x38, 0xac, 0x0b, 0xce, 0x61, - 0x1c, 0x47, 0x54, 0x76, 0x4d, 0x3c, 0x0d, 0xa9, 0xc3, 0x06, 0xd4, 0xc9, 0xb0, 0xf7, 0x3b, 0xe8, - 0x16, 0x83, 0xcb, 0xd2, 0xc5, 0xc0, 0x46, 0x7b, 0x56, 0x66, 0x0f, 0x09, 0xb6, 0x45, 0x29, 0xac, - 0x65, 0xe7, 0x63, 0x12, 0x9f, 0xc2, 0x3a, 0x99, 0x91, 0x17, 0xa8, 0xfd, 0xf8, 0x61, 0x05, 0xe0, - 0x1a, 0x32, 0x9e, 0x29, 0x1f, 0xb6, 0x08, 0xf1, 0x5f, 0x24, 0xfe, 0xc0, 0xfb, 0x71, 0x19, 0x5c, - 0xca, 0x25, 0x26, 0xe2, 0x34, 0x98, 0x08, 0x79, 0x3e, 0xa7, 0x35, 0xf2, 0xce, 0xae, 0x67, 0x82, - 0x1c, 0x68, 0x71, 0x5a, 0x7b, 0xbf, 0xb7, 0x60, 0xb3, 0xb8, 0x1f, 0x7d, 0x32, 0xaa, 0xe3, 0x16, - 0x9f, 0x48, 0x2b, 0x2f, 0x9e, 0xe7, 0xe5, 0xe2, 0xd9, 0xb9, 0x69, 0x5f, 0xb9, 0x7e, 0x7e, 0x02, - 0xce, 0x9b, 0x20, 0x4c, 0x56, 0x2a, 0x7c, 0x4b, 0x42, 0x68, 0x93, 0xbb, 0xb6, 0xcc, 0x45, 0xfd, - 0x65, 0xbc, 0x98, 0xa6, 0x12, 0x43, 0x2e, 0x09, 0xef, 0x08, 0x5a, 0xb8, 0x5f, 0x06, 0xee, 0x49, - 0x63, 0x59, 0x59, 0x19, 0xbd, 0x07, 0xb9, 0x5c, 0x1e, 0x94, 0xb7, 0x92, 0x9a, 0xd9, 0x4a, 0x8e, - 0x01, 0x50, 0x3a, 0x97, 0x76, 0x76, 0xa1, 0x4e, 0x54, 0x06, 0x42, 0xd9, 0x90, 0x14, 0xde, 0x60, - 0xe9, 0x21, 0x36, 0xb0, 0xf4, 0xd9, 0x13, 0x14, 0xcb, 0x82, 0x44, 0x6f, 0x6c, 0x9e, 0x95, 0xcc, - 0x02, 0x9a, 0x12, 0xba, 0x78, 0xa9, 0x0d, 0x58, 0x86, 0x01, 0xe4, 0x62, 0x5b, 0x19, 0xa8, 0x38, - 0x89, 0xc0, 0x6b, 0xcb, 0xe3, 0xa5, 0x86, 0x24, 0xa3, 0xd8, 0x77, 0xd5, 0x29, 0x0e, 0xc5, 0x7c, - 0xcf, 0xb8, 0x4a, 0xe8, 0x85, 0x3a, 0xf6, 0x57, 0x00, 0x3f, 0x4b, 0xe2, 0xc5, 0x8c, 0x40, 0x63, - 0x7d, 0xa8, 0x13, 0x95, 0xc5, 0xc7, 0xf4, 0x26, 0xe5, 0x1b, 0x97, 0x0a, 0xd5, 0xa0, 0x63, 0x72, - 0x86, 0x8b, 0x89, 0xbc, 0x69, 0x1c, 0x97, 0x58, 0x4a, 0xcd, 0xf3, 0x20, 0xca, 0xc5, 0xe7, 0x41, - 0x94, 0xc5, 0x8d, 0xcb, 0xa2, 0x19, 0x5b, 0x99, 0x79, 0x00, 0xcd, 0x57, 0x51, 0x1c, 0xa4, 0xa8, - 0x8c, 0xb6, 0x2c, 0x9e, 0xd3, 0x6c, 0x1f, 0x60, 0x20, 0x46, 0xe1, 0x24, 0x88, 0x50, 0xea, 0x94, - 0x1b, 0x40, 0x26, 0xe3, 0x86, 0x92, 0xf7, 0x14, 0x1a, 0x19, 0x55, 0x8d, 0x3d, 0x72, 0x87, 0xa3, - 0x20, 0x12, 0xca, 0x0b, 0x22, 0xbc, 0xb7, 0xb0, 0x21, 0x8b, 0x11, 0x9f, 0xa6, 0xa1, 0x48, 0xef, - 0x50, 0x8a, 0x77, 0x7a, 0xe4, 0xbc, 0x3f, 0x59, 0xe0, 0xe0, 0x4a, 0x19, 0xb0, 0xb4, 0x01, 0xf3, - 0x36, 0x3a, 0xf2, 0x36, 0xb2, 0x1e, 0xb4, 0x87, 0x29, 0xbe, 0x81, 0xba, 0x8d, 0xb5, 0xb8, 0xc9, - 0x42, 0xbc, 0xfc, 0x69, 0xaa, 0xd3, 0x6d, 0xf3, 0x9c, 0x66, 0xdb, 0xd0, 0xc2, 0xde, 0x24, 0x85, - 0xd8, 0xc8, 0x9a, 0x5c, 0x33, 0xd8, 0x0e, 0x80, 0x42, 0x76, 0x21, 0xa8, 0x9b, 0x59, 0xdc, 0xe0, - 0x78, 0x8f, 0xa0, 0x81, 0x9e, 0x9e, 0x04, 0x33, 0x1d, 0x9b, 0x75, 0x5b, 0x6c, 0xff, 0xb5, 0xa0, - 0xf3, 0xcb, 0x85, 0x48, 0xae, 0xb9, 0xf8, 0xed, 0x42, 0xcc, 0x53, 0xc4, 0x96, 0x68, 0x55, 0xcb, - 0x44, 0x60, 0xd5, 0x0e, 0xdf, 0x05, 0xc9, 0x58, 0x22, 0xe5, 0xf0, 0x8c, 0xc2, 0x58, 0x35, 0xe6, - 0x73, 0x8a, 0xb5, 0xc9, 0x4d, 0x16, 0xd5, 0xbb, 0x98, 0xc4, 0xa9, 0x0a, 0x26, 0xa3, 0x58, 0x1f, - 0xee, 0x1d, 0x5d, 0x8d, 0xa2, 0xc5, 0x58, 0xf0, 0x78, 0x29, 0x77, 0x53, 0x73, 0xe6, 0x65, 0x36, - 0xfb, 0x1e, 0x36, 0x37, 0x62, 0xa9, 0xd6, 0xd4, 0x20, 0xc5, 0x12, 0x97, 0xed, 0x43, 0xe7, 0x68, - 0x72, 0x21, 0xc6, 0x63, 0x31, 0x1e, 0x04, 0x69, 0xe0, 0x36, 0xab, 0x06, 0x88, 0x82, 0x8a, 0xf7, - 0xc1, 0x82, 0x8d, 0x2c, 0xfa, 0xf9, 0x2c, 0x9e, 0xce, 0x05, 0xa6, 0xf8, 0x28, 0x49, 0x54, 0x8a, - 0x8f, 0x92, 0x84, 0x3d, 0x82, 0x06, 0x17, 0xf3, 0x45, 0x94, 0xaa, 0x2a, 0xf9, 0x52, 0x5b, 0x54, - 0x7b, 0x17, 0x51, 0xca, 0x95, 0x16, 0xfb, 0x29, 0x6c, 0x16, 0xea, 0x50, 0x3d, 0x0b, 0xdf, 0xd2, - 0xfb, 0x0a, 0x72, 0x5e, 0x52, 0xf7, 0xfe, 0x5c, 0x87, 0xb6, 0x61, 0x39, 0x2f, 0x32, 0xc4, 0x67, - 0x23, 0x2b, 0xb2, 0xaf, 0x69, 0xa6, 0xbb, 0x61, 0xea, 0xc1, 0x9e, 0xd4, 0x01, 0xeb, 0x34, 0x2b, - 0x4b, 0xeb, 0x54, 0x37, 0x42, 0xfb, 0xb6, 0x46, 0x88, 0x13, 0xe2, 0xbb, 0x60, 0x7a, 0x29, 0xc6, - 0x54, 0x96, 0x4d, 0xae, 0x48, 0xb6, 0xa7, 0xbb, 0x02, 0xe5, 0xb1, 0xd0, 0x6b, 0x94, 0x84, 0xeb, - 0xce, 0x21, 0xbb, 0x1c, 0x4e, 0x06, 0x0d, 0x59, 0x2f, 0x92, 0x62, 0xcf, 0xa0, 0xad, 0xdb, 0xd7, - 0x3c, 0x4b, 0x51, 0x57, 0x9b, 0xd2, 0x42, 0x6e, 0x2a, 0xb2, 0x17, 0xe5, 0x11, 0xcd, 0x6d, 0x91, - 0x17, 0x6e, 0x21, 0x72, 0x43, 0xce, 0xcb, 0x23, 0xdd, 0xbe, 0x31, 0x33, 0xba, 0x40, 0x9b, 0xef, - 0xeb, 0xcd, 0xb9, 0x88, 0x1b, 0x93, 0xe5, 0x13, 0xf3, 0x2d, 0x71, 0xdb, 0xb4, 0xa7, 0x5b, 0x44, - 0x4e, 0xca, 0xb8, 0xf9, 0xe6, 0xec, 0x1b, 0x0f, 0x99, 0xdb, 0x29, 0x1f, 0x94, 0x8b, 0xb8, 0xf1, - 0xdc, 0xf9, 0x15, 0xf3, 0x9d, 0xbb, 0x41, 0x5b, 0xab, 0x87, 0x37, 0xa9, 0xc2, 0x2b, 0xa6, 0xc2, - 0x17, 0xe5, 0x49, 0xc0, 0xdd, 0x2c, 0x03, 0x55, 0x94, 0xf3, 0xf2, 0xe4, 0xb0, 0x6f, 0x0c, 0xe3, - 0xee, 0xbd, 0xb2, 0xff, 0xb9, 0x88, 0x6b, 0x2d, 0xef, 0xaf, 0x35, 0xd8, 0xf0, 0x27, 0xb3, 0x38, - 0x49, 0x8d, 0x2e, 0x22, 0xa7, 0x7f, 0xab, 0x72, 0xfa, 0xaf, 0x95, 0xde, 0x49, 0xea, 0x26, 0xd4, - 0x3d, 0x1c, 0x2e, 0x09, 0xa3, 0x82, 0x9c, 0x42, 0x05, 0x6d, 0x43, 0x4b, 0x5e, 0x17, 0x14, 0xd5, - 0x49, 0xa4, 0x19, 0xf2, 0x7b, 0x64, 0x49, 0xb3, 0x66, 0x83, 0xa6, 0x57, 0x45, 0x62, 0xe7, 0x94, - 0x6a, 0x24, 0x6c, 0x92, 0xd0, 0xe0, 0xa0, 0xfc, 0x2c, 0x9c, 0x88, 0x79, 0x1a, 0x4c, 0x66, 0xd8, - 0x8a, 0xec, 0xbe, 0xcd, 0x0d, 0x0e, 0x76, 0x21, 0x0a, 0xe2, 0x65, 0x22, 0x82, 0x54, 0x8c, 0x0f, - 0x52, 0xaa, 0x40, 0x9b, 0x97, 0xb8, 0xa8, 0x47, 0x61, 0x69, 0x3d, 0x90, 0x7a, 0x45, 0x2e, 0xbd, - 0xa4, 0x91, 0x08, 0x12, 0xaa, 0xab, 0x26, 0x97, 0x84, 0xf7, 0x8f, 0x1a, 0x30, 0x89, 0xa4, 0x9c, - 0x15, 0xff, 0x6f, 0x70, 0xde, 0x0e, 0x5b, 0x11, 0x9c, 0xc6, 0x0a, 0x38, 0x5f, 0xe5, 0x13, 0xae, - 0x04, 0x26, 0xa3, 0xb0, 0xfd, 0xeb, 0xc7, 0x47, 0xa2, 0x6a, 0x71, 0x93, 0xc5, 0x3c, 0xe8, 0x18, - 0x2f, 0x1f, 0x5e, 0x5b, 0xb4, 0x5d, 0xe0, 0x55, 0x40, 0x0b, 0x77, 0x84, 0xb6, 0x7d, 0x3b, 0xb4, - 0x1d, 0x13, 0xda, 0x0f, 0x16, 0x74, 0x0e, 0xd2, 0x78, 0x12, 0x8e, 0xb8, 0x18, 0xc5, 0xc9, 0xf8, - 0x66, 0x50, 0x25, 0x7c, 0x35, 0x13, 0xbe, 0x3d, 0xb0, 0xfd, 0xf7, 0x49, 0xd6, 0x3d, 0xb7, 0x8d, - 0xd9, 0x6c, 0x25, 0x57, 0x1c, 0x15, 0xd9, 0x37, 0x50, 0xf3, 0x13, 0xaa, 0xdc, 0x42, 0xdf, 0x2f, - 0x5c, 0x12, 0x5e, 0xf3, 0x13, 0xef, 0x87, 0xd0, 0x95, 0x4e, 0x29, 0x51, 0xf6, 0x0e, 0x75, 0xa1, - 0x7e, 0x94, 0x24, 0xb1, 0x7a, 0x89, 0x24, 0xe1, 0x5d, 0x41, 0xf7, 0x2c, 0x09, 0xa6, 0xf3, 0x28, - 0x48, 0x05, 0x26, 0xe6, 0x73, 0xea, 0xa3, 0xea, 0x63, 0xbf, 0x07, 0xed, 0xd3, 0x38, 0x7d, 0x9b, - 0x84, 0x29, 0xb5, 0x0c, 0xd9, 0xfc, 0x4d, 0x96, 0xf7, 0x7d, 0xf8, 0xb2, 0x74, 0xb2, 0x7e, 0x30, - 0xb1, 0xa4, 0x6c, 0xfd, 0xe1, 0x3b, 0x84, 0xfb, 0xb9, 0xaa, 0x3f, 0xf8, 0x2c, 0x1f, 0x57, 0x8d, - 0xfe, 0xc0, 0x88, 0x9c, 0x8c, 0x66, 0xc7, 0x57, 0x44, 0xe3, 0x1d, 0x82, 0x9b, 0xa1, 0x29, 0xff, - 0x45, 0x64, 0x1e, 0x9c, 0x87, 0x62, 0x79, 0xd3, 0x27, 0x15, 0x0d, 0x0c, 0x35, 0xfa, 0x83, 0x41, - 0x6b, 0xef, 0x3f, 0x16, 0x74, 0xab, 0x8c, 0xe8, 0xe2, 0xb2, 0x8c, 0xe2, 0x62, 0xcf, 0xa1, 0xfe, - 0x3e, 0x14, 0x4b, 0x35, 0x22, 0x78, 0x2b, 0x29, 0x5f, 0xf1, 0x84, 0xcb, 0x0d, 0x78, 0xb5, 0x0e, - 0x46, 0x69, 0x18, 0x4f, 0xd5, 0xf7, 0x80, 0xa4, 0xf0, 0x9c, 0xc3, 0x28, 0x1e, 0xfd, 0x46, 0x7e, - 0xe9, 0x72, 0x49, 0x54, 0x5c, 0x95, 0xfa, 0x1d, 0xaf, 0xca, 0x7a, 0xd5, 0x55, 0xf1, 0xfe, 0x62, - 0x29, 0xac, 0x8c, 0x99, 0xed, 0x93, 0x19, 0xd3, 0x17, 0xc4, 0x56, 0x17, 0xc4, 0x95, 0x83, 0xa7, - 0x9e, 0xaf, 0x15, 0x89, 0xc3, 0x2e, 0x2e, 0xe9, 0x37, 0x87, 0x43, 0x59, 0xca, 0xe9, 0x4f, 0x74, - 0xa5, 0xd5, 0x60, 0xd7, 0xab, 0x82, 0x3d, 0xdc, 0xfa, 0xdb, 0xc7, 0x1d, 0xeb, 0xef, 0x1f, 0x77, - 0xac, 0x7f, 0x7d, 0xdc, 0xb1, 0xfe, 0xf0, 0xef, 0x9d, 0xb5, 0x8b, 0x75, 0xfa, 0x07, 0xf6, 0xa3, - 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0x2d, 0xb9, 0x97, 0xfb, 0x13, 0x13, 0x00, 0x00, + // 1757 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x4f, 0x6f, 0xe4, 0x48, + 0x15, 0x8f, 0xdb, 0xee, 0x7f, 0xaf, 0x3b, 0x99, 0x6c, 0x4d, 0xcf, 0x62, 0x0d, 0x99, 0x6c, 0x63, + 0x05, 0xb6, 0x41, 0xab, 0xac, 0x12, 0x76, 0x60, 0x0e, 0xfc, 0xd9, 0x64, 0x3a, 0x4b, 0xac, 0x21, + 0xd9, 0xa1, 0x12, 0x65, 0xc5, 0x05, 0xc9, 0xe9, 0x2e, 0x7a, 0x2d, 0xdc, 0xed, 0xc6, 0xed, 0xde, + 0x4e, 0x2e, 0x48, 0x7c, 0x86, 0xbd, 0x70, 0x43, 0xdc, 0xf8, 0x1c, 0x5c, 0xe0, 0xc8, 0x11, 0x89, + 0x0b, 0x1a, 0xb8, 0xf2, 0x1d, 0xd0, 0x7b, 0xe5, 0x72, 0x95, 0xdd, 0x4e, 0x26, 0x1a, 0x71, 0xab, + 0xf7, 0xa7, 0x5e, 0xd5, 0xfb, 0xbd, 0x57, 0xef, 0x3d, 0x1b, 0xba, 0xf3, 0xe5, 0x75, 0x14, 0x8e, + 0xf6, 0xe7, 0x49, 0x9c, 0xc6, 0xac, 0x15, 0xce, 0x52, 0x91, 0xcc, 0x82, 0xc8, 0xfb, 0xa3, 0x05, + 0x36, 0x8f, 0x57, 0xcc, 0x85, 0xe6, 0xcb, 0x38, 0x5a, 0x4e, 0x67, 0x0b, 0xd7, 0xea, 0xdb, 0x03, + 0x87, 0x2b, 0x92, 0x31, 0x70, 0x5e, 0x89, 0xdb, 0x85, 0x6b, 0xf7, 0xed, 0x41, 0x9b, 0xd3, 0x9a, + 0xed, 0x41, 0xfd, 0x28, 0x4d, 0x93, 0x85, 0x5b, 0xeb, 0xdb, 0x83, 0xce, 0xe1, 0xd6, 0xbe, 0xb2, + 0xb7, 0x8f, 0x6c, 0x2e, 0x85, 0x68, 0x93, 0xc7, 0x41, 0x12, 0xce, 0x26, 0xae, 0xd3, 0xb7, 0x06, + 0x5d, 0xae, 0x48, 0xd6, 0x83, 0xba, 0x3f, 0x1b, 0x8b, 0x1b, 0xb7, 0xde, 0xb7, 0x06, 0x6d, 0x2e, + 0x09, 0xe4, 0x7e, 0x16, 0x8a, 0x68, 0xec, 0x36, 0x24, 0x97, 0x08, 0x6f, 0x1f, 0xda, 0x3c, 0x5e, + 0x9d, 0x05, 0x69, 0x12, 0xde, 0xb0, 0x6f, 0x81, 0xc3, 0xe3, 0x95, 0xbc, 0x63, 0xe7, 0x70, 0x53, + 0x9f, 0xcb, 0xe3, 0x15, 0x27, 0x91, 0x77, 0x06, 0xed, 0x8b, 0x70, 0x32, 0x13, 0x63, 0x74, 0xeb, + 0x03, 0xb0, 0x5f, 0xc7, 0xa8, 0x6e, 0xad, 0xab, 0xa3, 0x04, 0x15, 0xce, 0xc5, 0xc4, 0xad, 0x55, + 0x2a, 0x9c, 0x8b, 0x89, 0xf7, 0x02, 0xb6, 0x78, 0xbc, 0xf2, 0xc7, 0x62, 0x96, 0x86, 0xbf, 0x0e, + 0x45, 0x42, 0x80, 0xe4, 0x77, 0x70, 0xe4, 0xa1, 0x39, 0x48, 0x35, 0x0d, 0x92, 0xf7, 0x14, 0x1a, + 0xfe, 0xf0, 0xe7, 0xe1, 0x22, 0x65, 0xdb, 0x60, 0xfb, 0x43, 0xb5, 0x01, 0x97, 0x9e, 0x0f, 0xef, + 0x9d, 0xdc, 0xa4, 0x49, 0x30, 0x4a, 0xc5, 0xd8, 0x1f, 0x4a, 0xa8, 0xd9, 0x16, 0xd4, 0xfc, 0x21, + 0xdd, 0xd5, 0xe1, 0x35, 0x7f, 0xc8, 0xf6, 0xc0, 0xb9, 0x0a, 0x22, 0x05, 0xf2, 0xb6, 0xbe, 0x9c, + 0x34, 0xcb, 0x49, 0xea, 0x5d, 0x17, 0x4c, 0x65, 0x38, 0xbd, 0x0f, 0x0d, 0x42, 0x4f, 0x1e, 0xda, + 0xe6, 0x19, 0xc5, 0x9e, 0xeb, 0x30, 0x4b, 0xab, 0xdf, 0xd4, 0x56, 0xd7, 0x2e, 0x94, 0xe7, 0x80, + 0xf7, 0x0c, 0x9a, 0xaf, 0xc4, 0x2d, 0xf9, 0xa2, 0x3c, 0xb5, 0x0c, 0x4f, 0xff, 0x69, 0xc1, 0xe3, + 0x7c, 0xf7, 0x65, 0x70, 0x1d, 0x89, 0xab, 0x20, 0x5a, 0x0a, 0xb6, 0xa7, 0xfc, 0xb6, 0xaa, 0xee, + 0x7f, 0xba, 0x41, 0x58, 0xb0, 0x0f, 0x73, 0xec, 0x50, 0xed, 0x3d, 0xad, 0x96, 0x1d, 0x79, 0xba, + 0x91, 0x65, 0xdd, 0x0e, 0xb4, 0x8e, 0x2f, 0x7c, 0x32, 0xed, 0xda, 0x7d, 0x6b, 0x60, 0x9f, 0x6e, + 0xf0, 0x9c, 0xc3, 0x9e, 0x42, 0xf3, 0x6c, 0x99, 0x8a, 0x1b, 0x7f, 0x48, 0xd9, 0xe6, 0x9c, 0x6e, + 0x70, 0xc5, 0xc0, 0x9d, 0xb4, 0x7c, 0x25, 0x6e, 0x65, 0xca, 0xe1, 0x4e, 0xc5, 0x61, 0x3d, 0x70, + 0x8e, 0xe3, 0x38, 0xa2, 0xb4, 0x6b, 0xe1, 0x69, 0x48, 0x1d, 0x37, 0xa1, 0x4e, 0x86, 0xbd, 0xdf, + 0x41, 0xaf, 0xe8, 0x5c, 0x16, 0x2e, 0x06, 0x36, 0xda, 0xb3, 0x32, 0x7b, 0x48, 0xb0, 0x6d, 0x0a, + 0x61, 0x2d, 0x3b, 0x1f, 0x83, 0xf8, 0x1c, 0x1a, 0x64, 0x46, 0x3e, 0xa0, 0xce, 0xe1, 0xb3, 0x0a, + 0xc0, 0x35, 0x64, 0x3c, 0x53, 0x3e, 0x6e, 0x13, 0xe2, 0x9f, 0x27, 0xfe, 0xd0, 0xfb, 0x71, 0x19, + 0x5c, 0x8a, 0x25, 0x06, 0xe2, 0x3c, 0x98, 0x0a, 0x79, 0x3e, 0xa7, 0x35, 0xf2, 0x2e, 0x6f, 0xe7, + 0x82, 0x2e, 0xd0, 0xe6, 0xb4, 0xf6, 0x7e, 0x6f, 0xc1, 0x56, 0x71, 0x3f, 0xde, 0xc9, 0xc8, 0x8e, + 0x7b, 0xee, 0x44, 0x5a, 0x79, 0xf2, 0xbc, 0x28, 0x27, 0xcf, 0xee, 0x5d, 0xfb, 0xca, 0xf9, 0xf3, + 0x13, 0x70, 0x5e, 0x07, 0x61, 0xb2, 0x96, 0xe1, 0xdb, 0x12, 0x42, 0x9b, 0xae, 0x6b, 0xcb, 0x58, + 0xd4, 0x5f, 0xc6, 0xcb, 0x59, 0x2a, 0x31, 0xe4, 0x92, 0xf0, 0x4e, 0xa0, 0x8d, 0xfb, 0xa5, 0xe3, + 0x9e, 0x34, 0x96, 0xa5, 0x95, 0x51, 0x7b, 0x90, 0xcb, 0xe5, 0x41, 0x79, 0x29, 0xa9, 0x99, 0xa5, + 0xe4, 0x14, 0x00, 0xa5, 0x0b, 0x69, 0x67, 0x0f, 0xea, 0x44, 0x65, 0x20, 0x94, 0x0d, 0x49, 0xe1, + 0x1d, 0x96, 0x9e, 0x61, 0x01, 0x4b, 0x7f, 0xf0, 0x09, 0x8a, 0x65, 0x42, 0xe2, 0x6d, 0x6c, 0x9e, + 0xa5, 0xcc, 0x12, 0x5a, 0x12, 0xba, 0x78, 0xa5, 0x0d, 0x58, 0x86, 0x01, 0xe4, 0x62, 0x59, 0x19, + 0x2a, 0x3f, 0x89, 0xc0, 0x67, 0xcb, 0xe3, 0x95, 0x86, 0x24, 0xa3, 0xd8, 0xb7, 0xd5, 0x29, 0x0e, + 0xf9, 0xfc, 0xc8, 0x78, 0x4a, 0x78, 0x0b, 0x75, 0xec, 0xaf, 0x00, 0x7e, 0x96, 0xc4, 0xcb, 0x39, + 0x81, 0xc6, 0x06, 0x50, 0x27, 0x2a, 0xf3, 0x8f, 0xe9, 0x4d, 0xea, 0x6e, 0x5c, 0x2a, 0x54, 0x83, + 0x8e, 0xc1, 0x39, 0x9a, 0x4c, 0xe4, 0x4b, 0xe3, 0xb8, 0xc4, 0x54, 0x6a, 0x5d, 0x05, 0x51, 0x2e, + 0xbe, 0x0a, 0xa2, 0xcc, 0x6f, 0x5c, 0x16, 0xcd, 0xd8, 0xca, 0xcc, 0x53, 0x68, 0x7d, 0x16, 0xc5, + 0x41, 0x8a, 0xca, 0x68, 0xcb, 0xe2, 0x39, 0xcd, 0x0e, 0x00, 0x86, 0x62, 0x14, 0x4e, 0x83, 0x08, + 0xa5, 0x4e, 0xb9, 0x00, 0x64, 0x32, 0x6e, 0x28, 0x79, 0xcf, 0xa1, 0x99, 0x51, 0xd5, 0xd8, 0x23, + 0xf7, 0x62, 0x14, 0x44, 0x42, 0xdd, 0x82, 0x08, 0xef, 0x0b, 0xd8, 0x94, 0xc9, 0x88, 0xad, 0xe9, + 0x42, 0xa4, 0x0f, 0x48, 0xc5, 0x07, 0x35, 0x39, 0xef, 0xcf, 0x16, 0x38, 0xb8, 0x52, 0x06, 0x2c, + 0x6d, 0xc0, 0x7c, 0x8d, 0x8e, 0x7c, 0x8d, 0xac, 0x0f, 0x9d, 0x8b, 0x14, 0x7b, 0xa0, 0x2e, 0x63, + 0x6d, 0x6e, 0xb2, 0x10, 0x2f, 0x7f, 0x96, 0xea, 0x70, 0xdb, 0x3c, 0xa7, 0xd9, 0x0e, 0xb4, 0xb1, + 0x36, 0x49, 0x21, 0x16, 0xb2, 0x16, 0xd7, 0x0c, 0xb6, 0x0b, 0xa0, 0x90, 0x5d, 0x0a, 0xaa, 0x66, + 0x16, 0x37, 0x38, 0xde, 0xc7, 0xd0, 0xc4, 0x9b, 0x9e, 0x05, 0x73, 0xed, 0x9b, 0x75, 0x9f, 0x6f, + 0x7f, 0xaa, 0x41, 0xf7, 0x17, 0x4b, 0x91, 0xdc, 0x72, 0xf1, 0xdb, 0xa5, 0x58, 0xa4, 0x88, 0x2d, + 0xd1, 0x2a, 0x97, 0x89, 0xc0, 0xac, 0xbd, 0xf8, 0x32, 0x48, 0xc6, 0x12, 0x29, 0x87, 0x67, 0x14, + 0xfa, 0xaa, 0x31, 0x5f, 0x90, 0xaf, 0x2d, 0x6e, 0xb2, 0x28, 0xdf, 0xc5, 0x34, 0x4e, 0x95, 0x33, + 0x19, 0xc5, 0x06, 0xf0, 0xe8, 0xe4, 0x66, 0x14, 0x2d, 0xc7, 0x82, 0xc7, 0x2b, 0xb9, 0x9b, 0x8a, + 0x33, 0x2f, 0xb3, 0xd9, 0x77, 0xb0, 0xb8, 0x11, 0x4b, 0x95, 0xa6, 0x26, 0x29, 0x96, 0xb8, 0xec, + 0x00, 0xba, 0x27, 0xd3, 0x6b, 0x31, 0x1e, 0x8b, 0xf1, 0x30, 0x48, 0x03, 0xb7, 0x55, 0x35, 0x40, + 0x14, 0x54, 0xd8, 0x1e, 0x6c, 0xbe, 0x4e, 0xc4, 0x65, 0x12, 0xcc, 0x16, 0x51, 0x90, 0x8a, 0xb1, + 0xdb, 0x26, 0xcb, 0x45, 0xa6, 0xf7, 0xb5, 0x05, 0x9b, 0x19, 0x46, 0x8b, 0x79, 0x3c, 0x5b, 0x08, + 0x4c, 0x84, 0x93, 0x24, 0x51, 0x89, 0x70, 0x92, 0x24, 0xec, 0x63, 0x68, 0x72, 0xb1, 0x58, 0x46, + 0xa9, 0xca, 0xa5, 0x27, 0xfa, 0x5c, 0xb5, 0x77, 0x19, 0xa5, 0x5c, 0x69, 0xb1, 0x9f, 0xc2, 0x56, + 0x21, 0x5b, 0x55, 0xf3, 0xf8, 0x86, 0xde, 0x57, 0x90, 0xf3, 0x92, 0xba, 0xf7, 0xdf, 0x3a, 0x74, + 0x0c, 0xcb, 0x79, 0x2a, 0x22, 0x8a, 0x9b, 0x59, 0x2a, 0x7e, 0x40, 0x93, 0xdf, 0x1d, 0xb3, 0x11, + 0x56, 0xae, 0x2e, 0x58, 0xe7, 0x59, 0xf2, 0x5a, 0xe7, 0xba, 0x5c, 0xda, 0xf7, 0x95, 0x4b, 0x9c, + 0x23, 0xbf, 0x0c, 0x66, 0x13, 0x31, 0xa6, 0xe4, 0x6d, 0x71, 0x45, 0xb2, 0x7d, 0x5d, 0x3b, 0x28, + 0xda, 0x85, 0x8a, 0xa4, 0x24, 0x5c, 0xd7, 0x17, 0x59, 0x0b, 0x71, 0x7e, 0x68, 0xca, 0xac, 0x92, + 0x14, 0xfb, 0x11, 0x6c, 0x7d, 0x1e, 0x8d, 0x75, 0x9d, 0x5b, 0x64, 0xb1, 0xec, 0x69, 0x6b, 0x5a, + 0xc8, 0x4b, 0xba, 0xec, 0xd3, 0xf2, 0x38, 0x47, 0x51, 0xed, 0x1c, 0xba, 0x05, 0xff, 0x0d, 0x39, + 0x2f, 0x8f, 0x7f, 0x07, 0xc6, 0x7c, 0xe9, 0x02, 0x6d, 0x7e, 0xac, 0x37, 0xe7, 0x22, 0x6e, 0x4c, + 0xa1, 0x9f, 0x98, 0x7d, 0xc7, 0xed, 0xd0, 0x9e, 0x5e, 0x11, 0x3f, 0x29, 0xe3, 0x66, 0x7f, 0x3a, + 0x30, 0x9a, 0x9e, 0xdb, 0x2d, 0x1f, 0x94, 0x8b, 0xb8, 0xd1, 0x1a, 0xfd, 0x8a, 0x59, 0xd0, 0xdd, + 0xa4, 0xad, 0xd5, 0x83, 0x9e, 0x54, 0xe1, 0x15, 0x13, 0xe4, 0xa7, 0xe5, 0xa9, 0xc1, 0xdd, 0x2a, + 0x03, 0x55, 0x94, 0xf3, 0xf2, 0x94, 0x71, 0x60, 0x0c, 0xee, 0xee, 0xa3, 0xf2, 0xfd, 0x73, 0x11, + 0x37, 0xc6, 0xfb, 0x1f, 0x42, 0xc7, 0x0c, 0xec, 0x36, 0x6d, 0x7a, 0x52, 0x15, 0xd8, 0x05, 0x37, + 0x35, 0xbd, 0xbf, 0xd6, 0x60, 0xd3, 0x9f, 0xce, 0xe3, 0x24, 0x35, 0x4a, 0x95, 0xfc, 0xc4, 0xb0, + 0x2a, 0x3f, 0x31, 0x6a, 0xa5, 0x66, 0x4c, 0x25, 0x8b, 0x4a, 0x94, 0xc3, 0x25, 0x61, 0x24, 0xa0, + 0x53, 0x48, 0xc0, 0x1d, 0x68, 0xcb, 0xd7, 0x86, 0xa2, 0x3a, 0x89, 0x34, 0x43, 0x7e, 0xf4, 0xac, + 0x68, 0xa0, 0x6d, 0xd2, 0x88, 0xac, 0x48, 0x2c, 0xcf, 0x52, 0x8d, 0x84, 0x2d, 0x12, 0x1a, 0x1c, + 0x94, 0x5f, 0x86, 0x53, 0xb1, 0x48, 0x83, 0xe9, 0x1c, 0xeb, 0x9d, 0x3d, 0xb0, 0xb9, 0xc1, 0xc1, + 0x52, 0x47, 0x4e, 0xbc, 0x4c, 0x04, 0x56, 0x9e, 0xa3, 0x94, 0x52, 0xd7, 0xe6, 0x25, 0x2e, 0xea, + 0x91, 0x5b, 0x5a, 0x0f, 0xa4, 0x5e, 0x91, 0x4b, 0xed, 0x3a, 0x12, 0x41, 0x42, 0x09, 0xd9, 0xe2, + 0x92, 0xf0, 0xfe, 0x51, 0x03, 0x26, 0x91, 0x94, 0x03, 0xe9, 0xff, 0x0d, 0xce, 0xfb, 0x61, 0x2b, + 0x82, 0xd3, 0x5c, 0x03, 0xe7, 0xfd, 0x7c, 0x8c, 0x96, 0xc0, 0x64, 0x14, 0xf6, 0x18, 0xdd, 0xe1, + 0x24, 0xaa, 0x16, 0x37, 0x59, 0xcc, 0x83, 0xae, 0xd1, 0x5e, 0xf1, 0xbd, 0xa3, 0xed, 0x02, 0xaf, + 0x02, 0x5a, 0x78, 0x20, 0xb4, 0x9d, 0xfb, 0xa1, 0xed, 0x9a, 0xd0, 0x7e, 0x6d, 0x41, 0xf7, 0x28, + 0x8d, 0xa7, 0xe1, 0x88, 0x8b, 0x51, 0x9c, 0x8c, 0xef, 0x06, 0x55, 0xc2, 0x57, 0x33, 0xe1, 0xdb, + 0x07, 0xdb, 0xff, 0x2a, 0xc9, 0x8a, 0xef, 0x8e, 0x31, 0x00, 0xae, 0xc5, 0x8a, 0xa3, 0x22, 0xfb, + 0x10, 0x6a, 0x7e, 0x42, 0x99, 0x5b, 0x68, 0x1b, 0x85, 0x47, 0xc2, 0x6b, 0x7e, 0xe2, 0x7d, 0x04, + 0x3d, 0x79, 0x29, 0x25, 0xca, 0xda, 0x58, 0x0f, 0xea, 0x27, 0x49, 0x12, 0xab, 0x46, 0x26, 0x09, + 0xef, 0x06, 0x7a, 0x79, 0xf3, 0xc3, 0xc0, 0xbc, 0x4b, 0x7e, 0x54, 0xfd, 0x51, 0xe8, 0x43, 0xe7, + 0x3c, 0x4e, 0xbf, 0x48, 0xc2, 0x94, 0x6a, 0x8d, 0xec, 0x1d, 0x26, 0xcb, 0xfb, 0x2e, 0x3c, 0x29, + 0x9d, 0xac, 0xfb, 0x2d, 0xa6, 0x94, 0xad, 0xbf, 0xae, 0x2f, 0xe0, 0x71, 0xae, 0xea, 0x0f, 0xdf, + 0xe9, 0x8e, 0xeb, 0x46, 0xbf, 0x67, 0x78, 0x4e, 0x46, 0xb3, 0xe3, 0x2b, 0xbc, 0xf1, 0x8e, 0xc1, + 0xcd, 0xd0, 0x94, 0x3f, 0x3c, 0xb2, 0x1b, 0x5c, 0x85, 0x62, 0x75, 0xd7, 0x77, 0x1b, 0x4d, 0x25, + 0x35, 0xfa, 0x4d, 0x42, 0x6b, 0xef, 0x3f, 0x16, 0xf4, 0xaa, 0x8c, 0xe8, 0xe4, 0xb2, 0x8c, 0xe4, + 0x62, 0x2f, 0xa0, 0xfe, 0x55, 0x28, 0x56, 0x6a, 0xc2, 0xf0, 0xd6, 0x42, 0xbe, 0x76, 0x13, 0x2e, + 0x37, 0xe0, 0xd3, 0x3a, 0x1a, 0xa5, 0x61, 0x3c, 0x53, 0x1f, 0x1d, 0x92, 0xc2, 0x73, 0x8e, 0xa3, + 0x78, 0xf4, 0x1b, 0xf9, 0x39, 0xcd, 0x25, 0x51, 0xf1, 0x54, 0xea, 0x0f, 0x7c, 0x2a, 0x8d, 0xaa, + 0xa7, 0xe2, 0xfd, 0xc5, 0x52, 0x58, 0x19, 0x83, 0xe1, 0x5b, 0x23, 0xa6, 0x1f, 0x88, 0xad, 0x1e, + 0x88, 0x2b, 0xa7, 0x5b, 0x3d, 0xc4, 0x2b, 0x12, 0x27, 0x6a, 0x5c, 0xd2, 0xbf, 0x14, 0x87, 0xa2, + 0x94, 0xd3, 0x6f, 0xa9, 0x4a, 0xeb, 0xce, 0x36, 0xaa, 0x9c, 0xf5, 0x7e, 0x59, 0xe8, 0x5b, 0x68, + 0xf4, 0x68, 0x32, 0x49, 0xc4, 0x24, 0x48, 0x55, 0x9c, 0x35, 0x83, 0x7d, 0x04, 0x0d, 0x52, 0x56, + 0xa1, 0xaa, 0x1e, 0x5c, 0x32, 0x9d, 0xe3, 0xed, 0xbf, 0xbd, 0xd9, 0xb5, 0xfe, 0xfe, 0x66, 0xd7, + 0xfa, 0xd7, 0x9b, 0x5d, 0xeb, 0x0f, 0xff, 0xde, 0xdd, 0xb8, 0x6e, 0xd0, 0x3f, 0xbc, 0xef, 0xff, + 0x2f, 0x00, 0x00, 0xff, 0xff, 0x0a, 0x79, 0xf3, 0x2b, 0xd3, 0x13, 0x00, 0x00, } func (m *Row) Marshal() (dAtA []byte, err error) { @@ -3761,8 +3843,8 @@ func (m *GroupCount) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } - if m.Sum != 0 { - i = encodeVarintPublic(dAtA, i, uint64(m.Sum)) + if m.Agg != 0 { + i = encodeVarintPublic(dAtA, i, uint64(m.Agg)) i-- dAtA[i] = 0x18 } @@ -4065,6 +4147,16 @@ func (m *QueryRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.PreTranslated { + i-- + if m.PreTranslated { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x48 + } if len(m.EmbeddedData) > 0 { for iNdEx := len(m.EmbeddedData) - 1; iNdEx >= 0; iNdEx-- { { @@ -4233,6 +4325,20 @@ func (m *QueryResult) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.GroupCounts != nil { + { + size, err := m.GroupCounts.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPublic(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x82 + } if m.RowMatrix != nil { { size, err := m.RowMatrix.MarshalToSizedBuffer(dAtA[:i]) @@ -4317,10 +4423,10 @@ func (m *QueryResult) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0x4a } - if len(m.GroupCounts) > 0 { - for iNdEx := len(m.GroupCounts) - 1; iNdEx >= 0; iNdEx-- { + if len(m.OldGroupCounts) > 0 { + for iNdEx := len(m.OldGroupCounts) - 1; iNdEx >= 0; iNdEx-- { { - size, err := m.GroupCounts[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + size, err := m.OldGroupCounts[iNdEx].MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } @@ -4332,20 +4438,20 @@ func (m *QueryResult) MarshalToSizedBuffer(dAtA []byte) (int, error) { } } if len(m.RowIDs) > 0 { - dAtA24 := make([]byte, len(m.RowIDs)*10) - var j23 int + dAtA25 := make([]byte, len(m.RowIDs)*10) + var j24 int for _, num := range m.RowIDs { for num >= 1<<7 { - dAtA24[j23] = uint8(uint64(num)&0x7f | 0x80) + dAtA25[j24] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j23++ + j24++ } - dAtA24[j23] = uint8(num) - j23++ + dAtA25[j24] = uint8(num) + j24++ } - i -= j23 - copy(dAtA[i:], dAtA24[:j23]) - i = encodeVarintPublic(dAtA, i, uint64(j23)) + i -= j24 + copy(dAtA[i:], dAtA25[:j24]) + i = encodeVarintPublic(dAtA, i, uint64(j24)) i-- dAtA[i] = 0x3a } @@ -4473,57 +4579,57 @@ func (m *ImportRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { } } if len(m.Timestamps) > 0 { - dAtA28 := make([]byte, len(m.Timestamps)*10) - var j27 int + dAtA29 := make([]byte, len(m.Timestamps)*10) + var j28 int for _, num1 := range m.Timestamps { num := uint64(num1) for num >= 1<<7 { - dAtA28[j27] = uint8(uint64(num)&0x7f | 0x80) + dAtA29[j28] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j27++ + j28++ } - dAtA28[j27] = uint8(num) - j27++ + dAtA29[j28] = uint8(num) + j28++ } - i -= j27 - copy(dAtA[i:], dAtA28[:j27]) - i = encodeVarintPublic(dAtA, i, uint64(j27)) + i -= j28 + copy(dAtA[i:], dAtA29[:j28]) + i = encodeVarintPublic(dAtA, i, uint64(j28)) i-- dAtA[i] = 0x32 } if len(m.ColumnIDs) > 0 { - dAtA30 := make([]byte, len(m.ColumnIDs)*10) - var j29 int + dAtA31 := make([]byte, len(m.ColumnIDs)*10) + var j30 int for _, num := range m.ColumnIDs { for num >= 1<<7 { - dAtA30[j29] = uint8(uint64(num)&0x7f | 0x80) + dAtA31[j30] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j29++ + j30++ } - dAtA30[j29] = uint8(num) - j29++ + dAtA31[j30] = uint8(num) + j30++ } - i -= j29 - copy(dAtA[i:], dAtA30[:j29]) - i = encodeVarintPublic(dAtA, i, uint64(j29)) + i -= j30 + copy(dAtA[i:], dAtA31[:j30]) + i = encodeVarintPublic(dAtA, i, uint64(j30)) i-- dAtA[i] = 0x2a } if len(m.RowIDs) > 0 { - dAtA32 := make([]byte, len(m.RowIDs)*10) - var j31 int + dAtA33 := make([]byte, len(m.RowIDs)*10) + var j32 int for _, num := range m.RowIDs { for num >= 1<<7 { - dAtA32[j31] = uint8(uint64(num)&0x7f | 0x80) + dAtA33[j32] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j31++ + j32++ } - dAtA32[j31] = uint8(num) - j31++ + dAtA33[j32] = uint8(num) + j32++ } - i -= j31 - copy(dAtA[i:], dAtA32[:j31]) - i = encodeVarintPublic(dAtA, i, uint64(j31)) + i -= j32 + copy(dAtA[i:], dAtA33[:j32]) + i = encodeVarintPublic(dAtA, i, uint64(j32)) i-- dAtA[i] = 0x22 } @@ -4604,9 +4710,9 @@ func (m *ImportValueRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { } if len(m.FloatValues) > 0 { for iNdEx := len(m.FloatValues) - 1; iNdEx >= 0; iNdEx-- { - f33 := math.Float64bits(float64(m.FloatValues[iNdEx])) + f34 := math.Float64bits(float64(m.FloatValues[iNdEx])) i -= 8 - encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(f33)) + encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(f34)) } i = encodeVarintPublic(dAtA, i, uint64(len(m.FloatValues)*8)) i-- @@ -4622,39 +4728,39 @@ func (m *ImportValueRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { } } if len(m.Values) > 0 { - dAtA35 := make([]byte, len(m.Values)*10) - var j34 int + dAtA36 := make([]byte, len(m.Values)*10) + var j35 int for _, num1 := range m.Values { num := uint64(num1) for num >= 1<<7 { - dAtA35[j34] = uint8(uint64(num)&0x7f | 0x80) + dAtA36[j35] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j34++ + j35++ } - dAtA35[j34] = uint8(num) - j34++ + dAtA36[j35] = uint8(num) + j35++ } - i -= j34 - copy(dAtA[i:], dAtA35[:j34]) - i = encodeVarintPublic(dAtA, i, uint64(j34)) + i -= j35 + copy(dAtA[i:], dAtA36[:j35]) + i = encodeVarintPublic(dAtA, i, uint64(j35)) i-- dAtA[i] = 0x32 } if len(m.ColumnIDs) > 0 { - dAtA37 := make([]byte, len(m.ColumnIDs)*10) - var j36 int + dAtA38 := make([]byte, len(m.ColumnIDs)*10) + var j37 int for _, num := range m.ColumnIDs { for num >= 1<<7 { - dAtA37[j36] = uint8(uint64(num)&0x7f | 0x80) + dAtA38[j37] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j36++ + j37++ } - dAtA37[j36] = uint8(num) - j36++ + dAtA38[j37] = uint8(num) + j37++ } - i -= j36 - copy(dAtA[i:], dAtA37[:j36]) - i = encodeVarintPublic(dAtA, i, uint64(j36)) + i -= j37 + copy(dAtA[i:], dAtA38[:j37]) + i = encodeVarintPublic(dAtA, i, uint64(j37)) i-- dAtA[i] = 0x2a } @@ -4866,20 +4972,20 @@ func (m *TranslateKeysResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { copy(dAtA[i:], m.XXX_unrecognized) } if len(m.IDs) > 0 { - dAtA39 := make([]byte, len(m.IDs)*10) - var j38 int + dAtA40 := make([]byte, len(m.IDs)*10) + var j39 int for _, num := range m.IDs { for num >= 1<<7 { - dAtA39[j38] = uint8(uint64(num)&0x7f | 0x80) + dAtA40[j39] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j38++ + j39++ } - dAtA39[j38] = uint8(num) - j38++ + dAtA40[j39] = uint8(num) + j39++ } - i -= j38 - copy(dAtA[i:], dAtA39[:j38]) - i = encodeVarintPublic(dAtA, i, uint64(j38)) + i -= j39 + copy(dAtA[i:], dAtA40[:j39]) + i = encodeVarintPublic(dAtA, i, uint64(j39)) i-- dAtA[i] = 0x1a } @@ -4911,20 +5017,20 @@ func (m *TranslateIDsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { copy(dAtA[i:], m.XXX_unrecognized) } if len(m.IDs) > 0 { - dAtA41 := make([]byte, len(m.IDs)*10) - var j40 int + dAtA42 := make([]byte, len(m.IDs)*10) + var j41 int for _, num := range m.IDs { for num >= 1<<7 { - dAtA41[j40] = uint8(uint64(num)&0x7f | 0x80) + dAtA42[j41] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j40++ + j41++ } - dAtA41[j40] = uint8(num) - j40++ + dAtA42[j41] = uint8(num) + j41++ } - i -= j40 - copy(dAtA[i:], dAtA41[:j40]) - i = encodeVarintPublic(dAtA, i, uint64(j40)) + i -= j41 + copy(dAtA[i:], dAtA42[:j41]) + i = encodeVarintPublic(dAtA, i, uint64(j41)) i-- dAtA[i] = 0x1a } @@ -5125,20 +5231,20 @@ func (m *ImportColumnAttrsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error dAtA[i] = 0x30 } if len(m.ColumnIDs) > 0 { - dAtA43 := make([]byte, len(m.ColumnIDs)*10) - var j42 int + dAtA44 := make([]byte, len(m.ColumnIDs)*10) + var j43 int for _, num := range m.ColumnIDs { for num >= 1<<7 { - dAtA43[j42] = uint8(uint64(num)&0x7f | 0x80) + dAtA44[j43] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j42++ + j43++ } - dAtA43[j42] = uint8(num) - j42++ + dAtA44[j43] = uint8(num) + j43++ } - i -= j42 - copy(dAtA[i:], dAtA43[:j42]) - i = encodeVarintPublic(dAtA, i, uint64(j42)) + i -= j43 + copy(dAtA[i:], dAtA44[:j43]) + i = encodeVarintPublic(dAtA, i, uint64(j43)) i-- dAtA[i] = 0x2a } @@ -5173,6 +5279,54 @@ func (m *ImportColumnAttrsRequest) MarshalToSizedBuffer(dAtA []byte) (int, error return len(dAtA) - i, nil } +func (m *GroupCounts) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *GroupCounts) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *GroupCounts) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Groups) > 0 { + for iNdEx := len(m.Groups) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Groups[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPublic(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + } + if len(m.Aggregate) > 0 { + i -= len(m.Aggregate) + copy(dAtA[i:], m.Aggregate) + i = encodeVarintPublic(dAtA, i, uint64(len(m.Aggregate))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func encodeVarintPublic(dAtA []byte, offset int, v uint64) int { offset -= sovPublic(v) base := offset @@ -5653,8 +5807,8 @@ func (m *GroupCount) Size() (n int) { if m.Count != 0 { n += 1 + sovPublic(uint64(m.Count)) } - if m.Sum != 0 { - n += 1 + sovPublic(uint64(m.Sum)) + if m.Agg != 0 { + n += 1 + sovPublic(uint64(m.Agg)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) @@ -5815,6 +5969,9 @@ func (m *QueryRequest) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } + if m.PreTranslated { + n += 2 + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -5885,8 +6042,8 @@ func (m *QueryResult) Size() (n int) { } n += 1 + sovPublic(uint64(l)) + l } - if len(m.GroupCounts) > 0 { - for _, e := range m.GroupCounts { + if len(m.OldGroupCounts) > 0 { + for _, e := range m.OldGroupCounts { l = e.Size() n += 1 + l + sovPublic(uint64(l)) } @@ -5919,6 +6076,10 @@ func (m *QueryResult) Size() (n int) { l = m.RowMatrix.Size() n += 1 + l + sovPublic(uint64(l)) } + if m.GroupCounts != nil { + l = m.GroupCounts.Size() + n += 2 + l + sovPublic(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -6284,6 +6445,28 @@ func (m *ImportColumnAttrsRequest) Size() (n int) { return n } +func (m *GroupCounts) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Aggregate) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } + if len(m.Groups) > 0 { + for _, e := range m.Groups { + l = e.Size() + n += 1 + l + sovPublic(uint64(l)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func sovPublic(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } @@ -6565,10 +6748,7 @@ func (m *Row) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -6653,10 +6833,7 @@ func (m *RowMatrix) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -6779,10 +6956,7 @@ func (m *SignedRow) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -6941,10 +7115,7 @@ func (m *RowIdentifiers) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7071,10 +7242,7 @@ func (m *IDList) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7178,10 +7346,7 @@ func (m *ExtractedIDColumn) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7298,10 +7463,7 @@ func (m *ExtractedIDMatrix) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7384,10 +7546,7 @@ func (m *KeyList) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7601,10 +7760,7 @@ func (m *ExtractedTableValue) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7741,10 +7897,7 @@ func (m *ExtractedTableColumn) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7859,10 +8012,7 @@ func (m *ExtractedTableField) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7981,10 +8131,7 @@ func (m *ExtractedTable) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8105,10 +8252,7 @@ func (m *Pair) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8227,10 +8371,7 @@ func (m *PairField) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8347,10 +8488,7 @@ func (m *PairsField) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8420,10 +8558,7 @@ func (m *Int64) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8593,10 +8728,7 @@ func (m *FieldRow) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8696,9 +8828,9 @@ func (m *GroupCount) Unmarshal(dAtA []byte) error { } case 3: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Sum", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Agg", wireType) } - m.Sum = 0 + m.Agg = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPublic @@ -8708,7 +8840,7 @@ func (m *GroupCount) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Sum |= int64(b&0x7F) << shift + m.Agg |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -8719,10 +8851,7 @@ func (m *GroupCount) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8858,10 +8987,7 @@ func (m *ValCount) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8950,10 +9076,7 @@ func (m *Decimal) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -9089,10 +9212,7 @@ func (m *ColumnAttrSet) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -9276,10 +9396,7 @@ func (m *Attr) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -9364,10 +9481,7 @@ func (m *AttrMap) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -9634,16 +9748,33 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 9: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field PreTranslated", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.PreTranslated = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -9794,10 +9925,7 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -10084,7 +10212,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { } case 8: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field GroupCounts", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field OldGroupCounts", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -10111,8 +10239,8 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.GroupCounts = append(m.GroupCounts, &GroupCount{}) - if err := m.GroupCounts[len(m.GroupCounts)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.OldGroupCounts = append(m.OldGroupCounts, &GroupCount{}) + if err := m.OldGroupCounts[len(m.OldGroupCounts)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -10368,16 +10496,49 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 16: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field GroupCounts", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPublic + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.GroupCounts == nil { + m.GroupCounts = &GroupCounts{} + } + if err := m.GroupCounts.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -10861,10 +11022,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -11326,10 +11484,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -11499,10 +11654,7 @@ func (m *AtomicRecord) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -11585,10 +11737,7 @@ func (m *AtomicImportResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -11755,10 +11904,7 @@ func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -11885,10 +12031,7 @@ func (m *TranslateKeysResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -12079,10 +12222,7 @@ func (m *TranslateIDsRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -12165,10 +12305,7 @@ func (m *TranslateIDsResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -12285,10 +12422,7 @@ func (m *ImportRoaringRequestView) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -12482,10 +12616,7 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -12746,10 +12877,124 @@ func (m *ImportColumnAttrsRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } - if (iNdEx + skippy) < 0 { + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *GroupCounts) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: GroupCounts: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: GroupCounts: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Aggregate", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPublic + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Aggregate = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Groups", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPublic + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Groups = append(m.Groups, &GroupCount{}) + if err := m.Groups[len(m.Groups)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPublic(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { diff --git a/internal/public.proto b/internal/public.proto index 9e3b4241e..51de362bd 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -102,7 +102,7 @@ message FieldRow { message GroupCount{ repeated FieldRow Group = 1; uint64 Count = 2; - int64 Sum = 3; + int64 Agg = 3; } message ValCount { @@ -144,6 +144,7 @@ message QueryRequest { bool ExcludeRowAttrs = 6; bool ExcludeColumns = 7; repeated Row EmbeddedData = 8; + bool PreTranslated = 9; } message QueryResponse { @@ -158,9 +159,15 @@ message QueryResult { uint64 N = 2; repeated Pair Pairs = 3; bool Changed = 4; - ValCount ValCount = 5; + ValCount ValCount = 5; repeated uint64 RowIDs = 7; - repeated GroupCount GroupCounts = 8; + // In the past, GroupCounts was a []GroupCount which did not indicate + // whether it had an aggregate, or which aggregate it had. We've + // updated this, but we keep this here so that messages using the old + // format can get a best-effort treatment rather than causing panics. + // Later this can almost certainly go away, but leave a comment warning + // people that 8 is Spoken For if you do that, please. + repeated GroupCount OldGroupCounts = 8; RowIdentifiers RowIdentifiers = 9; SignedRow SignedRow = 10; PairsField PairsField = 11; @@ -168,6 +175,7 @@ message QueryResult { ExtractedIDMatrix ExtractedIDMatrix = 13; ExtractedTable ExtractedTable = 14; RowMatrix RowMatrix = 15; + GroupCounts GroupCounts = 16; } message ImportRequest { @@ -253,3 +261,8 @@ message ImportColumnAttrsRequest { repeated uint64 ColumnIDs = 5; int64 IndexCreatedAt = 6; } + +message GroupCounts{ + string Aggregate = 1; + repeated GroupCount Groups = 2; +} diff --git a/pql/ast.go b/pql/ast.go index 50b38e5c5..40e2acaba 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -441,9 +441,11 @@ var callInfoByFunc = map[string]callInfo{ prototypes: map[string]interface{}{ "filter": nil, "limit": int64(0), + "offset": int64(0), "previous": nil, "aggregate": nil, "having": nil, + "sort": "", }, }, "Options": { diff --git a/proto/pilosa.pb.go b/proto/pilosa.pb.go index 9ac5095f2..4cd578f2c 100644 --- a/proto/pilosa.pb.go +++ b/proto/pilosa.pb.go @@ -838,6 +838,341 @@ func (*IdsOrKeys) XXX_OneofWrappers() []interface{} { } } +type Index struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Index) Reset() { *m = Index{} } +func (m *Index) String() string { return proto.CompactTextString(m) } +func (*Index) ProtoMessage() {} +func (*Index) Descriptor() ([]byte, []int) { + return fileDescriptor_ef0691a44d1e275c, []int{13} +} + +func (m *Index) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Index.Unmarshal(m, b) +} +func (m *Index) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Index.Marshal(b, m, deterministic) +} +func (m *Index) XXX_Merge(src proto.Message) { + xxx_messageInfo_Index.Merge(m, src) +} +func (m *Index) XXX_Size() int { + return xxx_messageInfo_Index.Size(m) +} +func (m *Index) XXX_DiscardUnknown() { + xxx_messageInfo_Index.DiscardUnknown(m) +} + +var xxx_messageInfo_Index proto.InternalMessageInfo + +func (m *Index) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +type CreateIndexRequest struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Keys bool `protobuf:"varint,2,opt,name=keys,proto3" json:"keys,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *CreateIndexRequest) Reset() { *m = CreateIndexRequest{} } +func (m *CreateIndexRequest) String() string { return proto.CompactTextString(m) } +func (*CreateIndexRequest) ProtoMessage() {} +func (*CreateIndexRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_ef0691a44d1e275c, []int{14} +} + +func (m *CreateIndexRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_CreateIndexRequest.Unmarshal(m, b) +} +func (m *CreateIndexRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_CreateIndexRequest.Marshal(b, m, deterministic) +} +func (m *CreateIndexRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_CreateIndexRequest.Merge(m, src) +} +func (m *CreateIndexRequest) XXX_Size() int { + return xxx_messageInfo_CreateIndexRequest.Size(m) +} +func (m *CreateIndexRequest) XXX_DiscardUnknown() { + xxx_messageInfo_CreateIndexRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_CreateIndexRequest proto.InternalMessageInfo + +func (m *CreateIndexRequest) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *CreateIndexRequest) GetKeys() bool { + if m != nil { + return m.Keys + } + return false +} + +type CreateIndexResponse struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *CreateIndexResponse) Reset() { *m = CreateIndexResponse{} } +func (m *CreateIndexResponse) String() string { return proto.CompactTextString(m) } +func (*CreateIndexResponse) ProtoMessage() {} +func (*CreateIndexResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_ef0691a44d1e275c, []int{15} +} + +func (m *CreateIndexResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_CreateIndexResponse.Unmarshal(m, b) +} +func (m *CreateIndexResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_CreateIndexResponse.Marshal(b, m, deterministic) +} +func (m *CreateIndexResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CreateIndexResponse.Merge(m, src) +} +func (m *CreateIndexResponse) XXX_Size() int { + return xxx_messageInfo_CreateIndexResponse.Size(m) +} +func (m *CreateIndexResponse) XXX_DiscardUnknown() { + xxx_messageInfo_CreateIndexResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_CreateIndexResponse proto.InternalMessageInfo + +type GetIndexRequest struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetIndexRequest) Reset() { *m = GetIndexRequest{} } +func (m *GetIndexRequest) String() string { return proto.CompactTextString(m) } +func (*GetIndexRequest) ProtoMessage() {} +func (*GetIndexRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_ef0691a44d1e275c, []int{16} +} + +func (m *GetIndexRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetIndexRequest.Unmarshal(m, b) +} +func (m *GetIndexRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetIndexRequest.Marshal(b, m, deterministic) +} +func (m *GetIndexRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetIndexRequest.Merge(m, src) +} +func (m *GetIndexRequest) XXX_Size() int { + return xxx_messageInfo_GetIndexRequest.Size(m) +} +func (m *GetIndexRequest) XXX_DiscardUnknown() { + xxx_messageInfo_GetIndexRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_GetIndexRequest proto.InternalMessageInfo + +func (m *GetIndexRequest) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +type GetIndexResponse struct { + Index *Index `protobuf:"bytes,1,opt,name=index,proto3" json:"index,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetIndexResponse) Reset() { *m = GetIndexResponse{} } +func (m *GetIndexResponse) String() string { return proto.CompactTextString(m) } +func (*GetIndexResponse) ProtoMessage() {} +func (*GetIndexResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_ef0691a44d1e275c, []int{17} +} + +func (m *GetIndexResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetIndexResponse.Unmarshal(m, b) +} +func (m *GetIndexResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetIndexResponse.Marshal(b, m, deterministic) +} +func (m *GetIndexResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetIndexResponse.Merge(m, src) +} +func (m *GetIndexResponse) XXX_Size() int { + return xxx_messageInfo_GetIndexResponse.Size(m) +} +func (m *GetIndexResponse) XXX_DiscardUnknown() { + xxx_messageInfo_GetIndexResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_GetIndexResponse proto.InternalMessageInfo + +func (m *GetIndexResponse) GetIndex() *Index { + if m != nil { + return m.Index + } + return nil +} + +type GetIndexesRequest struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetIndexesRequest) Reset() { *m = GetIndexesRequest{} } +func (m *GetIndexesRequest) String() string { return proto.CompactTextString(m) } +func (*GetIndexesRequest) ProtoMessage() {} +func (*GetIndexesRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_ef0691a44d1e275c, []int{18} +} + +func (m *GetIndexesRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetIndexesRequest.Unmarshal(m, b) +} +func (m *GetIndexesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetIndexesRequest.Marshal(b, m, deterministic) +} +func (m *GetIndexesRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetIndexesRequest.Merge(m, src) +} +func (m *GetIndexesRequest) XXX_Size() int { + return xxx_messageInfo_GetIndexesRequest.Size(m) +} +func (m *GetIndexesRequest) XXX_DiscardUnknown() { + xxx_messageInfo_GetIndexesRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_GetIndexesRequest proto.InternalMessageInfo + +type GetIndexesResponse struct { + Indexes []*Index `protobuf:"bytes,1,rep,name=indexes,proto3" json:"indexes,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetIndexesResponse) Reset() { *m = GetIndexesResponse{} } +func (m *GetIndexesResponse) String() string { return proto.CompactTextString(m) } +func (*GetIndexesResponse) ProtoMessage() {} +func (*GetIndexesResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_ef0691a44d1e275c, []int{19} +} + +func (m *GetIndexesResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetIndexesResponse.Unmarshal(m, b) +} +func (m *GetIndexesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetIndexesResponse.Marshal(b, m, deterministic) +} +func (m *GetIndexesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetIndexesResponse.Merge(m, src) +} +func (m *GetIndexesResponse) XXX_Size() int { + return xxx_messageInfo_GetIndexesResponse.Size(m) +} +func (m *GetIndexesResponse) XXX_DiscardUnknown() { + xxx_messageInfo_GetIndexesResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_GetIndexesResponse proto.InternalMessageInfo + +func (m *GetIndexesResponse) GetIndexes() []*Index { + if m != nil { + return m.Indexes + } + return nil +} + +type DeleteIndexRequest struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DeleteIndexRequest) Reset() { *m = DeleteIndexRequest{} } +func (m *DeleteIndexRequest) String() string { return proto.CompactTextString(m) } +func (*DeleteIndexRequest) ProtoMessage() {} +func (*DeleteIndexRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_ef0691a44d1e275c, []int{20} +} + +func (m *DeleteIndexRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DeleteIndexRequest.Unmarshal(m, b) +} +func (m *DeleteIndexRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DeleteIndexRequest.Marshal(b, m, deterministic) +} +func (m *DeleteIndexRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteIndexRequest.Merge(m, src) +} +func (m *DeleteIndexRequest) XXX_Size() int { + return xxx_messageInfo_DeleteIndexRequest.Size(m) +} +func (m *DeleteIndexRequest) XXX_DiscardUnknown() { + xxx_messageInfo_DeleteIndexRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_DeleteIndexRequest proto.InternalMessageInfo + +func (m *DeleteIndexRequest) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +type DeleteIndexResponse struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DeleteIndexResponse) Reset() { *m = DeleteIndexResponse{} } +func (m *DeleteIndexResponse) String() string { return proto.CompactTextString(m) } +func (*DeleteIndexResponse) ProtoMessage() {} +func (*DeleteIndexResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_ef0691a44d1e275c, []int{21} +} + +func (m *DeleteIndexResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DeleteIndexResponse.Unmarshal(m, b) +} +func (m *DeleteIndexResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DeleteIndexResponse.Marshal(b, m, deterministic) +} +func (m *DeleteIndexResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteIndexResponse.Merge(m, src) +} +func (m *DeleteIndexResponse) XXX_Size() int { + return xxx_messageInfo_DeleteIndexResponse.Size(m) +} +func (m *DeleteIndexResponse) XXX_DiscardUnknown() { + xxx_messageInfo_DeleteIndexResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_DeleteIndexResponse proto.InternalMessageInfo + func init() { proto.RegisterType((*QueryPQLRequest)(nil), "pilosa.QueryPQLRequest") proto.RegisterType((*QuerySQLRequest)(nil), "pilosa.QuerySQLRequest") @@ -852,60 +1187,79 @@ func init() { proto.RegisterType((*Uint64Array)(nil), "pilosa.Uint64Array") proto.RegisterType((*StringArray)(nil), "pilosa.StringArray") proto.RegisterType((*IdsOrKeys)(nil), "pilosa.IdsOrKeys") + proto.RegisterType((*Index)(nil), "pilosa.Index") + proto.RegisterType((*CreateIndexRequest)(nil), "pilosa.CreateIndexRequest") + proto.RegisterType((*CreateIndexResponse)(nil), "pilosa.CreateIndexResponse") + proto.RegisterType((*GetIndexRequest)(nil), "pilosa.GetIndexRequest") + proto.RegisterType((*GetIndexResponse)(nil), "pilosa.GetIndexResponse") + proto.RegisterType((*GetIndexesRequest)(nil), "pilosa.GetIndexesRequest") + proto.RegisterType((*GetIndexesResponse)(nil), "pilosa.GetIndexesResponse") + proto.RegisterType((*DeleteIndexRequest)(nil), "pilosa.DeleteIndexRequest") + proto.RegisterType((*DeleteIndexResponse)(nil), "pilosa.DeleteIndexResponse") } func init() { proto.RegisterFile("pilosa.proto", fileDescriptor_ef0691a44d1e275c) } var fileDescriptor_ef0691a44d1e275c = []byte{ - // 761 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x55, 0xcd, 0x72, 0xd3, 0x48, - 0x10, 0xb6, 0x22, 0xc5, 0xb6, 0xda, 0xf9, 0xdb, 0xc9, 0x6e, 0x56, 0x95, 0xda, 0xda, 0x55, 0x94, - 0xc3, 0x7a, 0x6b, 0xb7, 0x92, 0xac, 0x77, 0x03, 0x05, 0x84, 0x43, 0x12, 0xa0, 0x9c, 0x02, 0x0a, - 0x67, 0x42, 0x72, 0xe0, 0x36, 0xb6, 0xc6, 0x8e, 0x8a, 0xb1, 0xc6, 0xd6, 0x48, 0x09, 0x7e, 0x01, - 0xde, 0x87, 0x33, 0x17, 0x4e, 0x3c, 0x17, 0x35, 0x33, 0x1a, 0xd9, 0x0a, 0x98, 0x0a, 0x54, 0x71, - 0xf2, 0x74, 0x7f, 0x5f, 0xb7, 0xfa, 0x9b, 0xee, 0x69, 0xc3, 0xd2, 0x28, 0x62, 0x5c, 0x90, 0x9d, - 0x51, 0xc2, 0x53, 0x8e, 0xaa, 0xda, 0x0a, 0xee, 0xc1, 0xea, 0x69, 0x46, 0x93, 0x49, 0xe7, 0xf4, - 0x19, 0xa6, 0xe3, 0x8c, 0x8a, 0x14, 0xfd, 0x0c, 0x8b, 0x51, 0x1c, 0xd2, 0x37, 0x9e, 0xe5, 0x5b, - 0x4d, 0x17, 0x6b, 0x03, 0xad, 0x81, 0x3d, 0x1a, 0x33, 0x6f, 0x41, 0xf9, 0xe4, 0x31, 0xd8, 0xce, - 0x43, 0xcf, 0xa6, 0xa1, 0x6b, 0x60, 0x8b, 0x31, 0xcb, 0x03, 0xe5, 0x31, 0x78, 0x00, 0x8d, 0xb3, - 0x94, 0xa4, 0x99, 0x78, 0x9c, 0x24, 0x3c, 0x41, 0x08, 0x9c, 0x63, 0x1e, 0x52, 0xc5, 0x58, 0xc6, - 0xea, 0x8c, 0x3c, 0xa8, 0x3d, 0xa7, 0x42, 0x90, 0x01, 0xcd, 0xb3, 0x1b, 0x33, 0xf8, 0x60, 0x41, - 0x03, 0xf3, 0x6b, 0x4c, 0xc5, 0x88, 0xc7, 0x82, 0xa2, 0x7f, 0xa0, 0x76, 0x49, 0x49, 0x48, 0x13, - 0xe1, 0x59, 0xbe, 0xdd, 0x6c, 0xb4, 0xd0, 0x4e, 0x2e, 0xea, 0x98, 0xb3, 0x6c, 0x18, 0x9f, 0xc4, - 0x7d, 0x8e, 0x0d, 0x05, 0xed, 0x41, 0xad, 0xa7, 0xdc, 0xc2, 0x5b, 0x50, 0xec, 0x8d, 0x32, 0xdb, - 0xa4, 0xc5, 0x86, 0x86, 0xf6, 0x4b, 0xc5, 0x7a, 0xb6, 0x6f, 0x35, 0x1b, 0xad, 0x75, 0x13, 0x35, - 0x03, 0xe1, 0x92, 0xa8, 0x4d, 0xa8, 0x87, 0x59, 0x42, 0xd2, 0x88, 0xc7, 0x9e, 0xe3, 0x5b, 0x4d, - 0x1b, 0x17, 0x76, 0x70, 0x17, 0x6c, 0xcc, 0xaf, 0x67, 0x6b, 0xb1, 0x6e, 0x55, 0x4b, 0xf0, 0xce, - 0x82, 0xe5, 0x97, 0xa4, 0xcb, 0xe8, 0x77, 0xaa, 0xff, 0x03, 0x9c, 0x84, 0x5f, 0x1b, 0xe9, 0x0d, - 0x43, 0x95, 0xd7, 0xa9, 0x80, 0x1f, 0x21, 0xf6, 0x00, 0x60, 0x5a, 0x8a, 0xec, 0x75, 0x4c, 0x86, - 0x34, 0x9f, 0x06, 0x75, 0x56, 0xd1, 0x24, 0x25, 0xe9, 0x64, 0x64, 0x9a, 0x5d, 0xd8, 0xc1, 0x5b, - 0x1b, 0x56, 0xca, 0xb7, 0x81, 0x7e, 0x07, 0x57, 0xa4, 0x49, 0x14, 0x0f, 0x2e, 0x48, 0x3e, 0x55, - 0xed, 0x0a, 0x9e, 0xba, 0x24, 0x9e, 0x45, 0x71, 0x7a, 0xe7, 0x7f, 0x89, 0xcb, 0x7c, 0x8e, 0xc4, - 0x0b, 0x17, 0xfa, 0x0d, 0xea, 0x05, 0x2c, 0x05, 0xda, 0xed, 0x0a, 0x2e, 0x3c, 0x68, 0x13, 0x6a, - 0x5d, 0xce, 0x99, 0x04, 0xa5, 0x92, 0x7a, 0xbb, 0x82, 0x8d, 0x43, 0x61, 0x8c, 0x77, 0x25, 0xb6, - 0xe8, 0x5b, 0xcd, 0x25, 0x85, 0x69, 0x07, 0x7a, 0x08, 0x2b, 0xfa, 0x13, 0x87, 0x49, 0x42, 0x26, - 0x92, 0x52, 0x2d, 0x5f, 0xde, 0xf9, 0x14, 0x6d, 0x57, 0xf0, 0x0d, 0xb2, 0x0c, 0xd7, 0x0a, 0x8a, - 0xf0, 0xda, 0xcd, 0xbb, 0x2f, 0x50, 0x19, 0x5e, 0x26, 0x23, 0x1f, 0xa0, 0xcf, 0x38, 0xc9, 0x55, - 0xd5, 0x7d, 0xab, 0x69, 0xb5, 0x2b, 0x78, 0xc6, 0x87, 0xfe, 0x05, 0x08, 0x69, 0x2f, 0x1a, 0x12, - 0x25, 0xcd, 0x55, 0xc9, 0x57, 0x4d, 0xf2, 0x47, 0x1a, 0x91, 0x21, 0x53, 0xd2, 0x51, 0x03, 0x5c, - 0x3d, 0x78, 0x17, 0x84, 0x05, 0xfb, 0x50, 0xcb, 0x59, 0x72, 0x17, 0x5c, 0x11, 0x96, 0xe9, 0x26, - 0xda, 0x58, 0x1b, 0xd2, 0x2b, 0x7a, 0x84, 0xe9, 0x16, 0xda, 0x58, 0x1b, 0xc1, 0x7b, 0x0b, 0x56, - 0x4e, 0x62, 0x31, 0xa2, 0xbd, 0xf4, 0xeb, 0xab, 0xe4, 0xef, 0xd9, 0x87, 0x29, 0x8b, 0xfb, 0xc9, - 0x14, 0x77, 0x12, 0x8a, 0x17, 0xc9, 0x53, 0x3a, 0x11, 0xd3, 0x37, 0x19, 0xc0, 0x52, 0x3f, 0x62, - 0x29, 0x4d, 0x9e, 0x44, 0x94, 0x85, 0xc2, 0xb3, 0x7d, 0xbb, 0xe9, 0xe2, 0x92, 0x4f, 0x7e, 0x86, - 0x45, 0xc3, 0x28, 0x55, 0x6d, 0x74, 0xb0, 0x36, 0xd0, 0x06, 0x54, 0x79, 0xbf, 0x2f, 0x68, 0xaa, - 0x3a, 0xe8, 0xe0, 0xdc, 0x92, 0xec, 0xb1, 0xdc, 0x5b, 0xaa, 0x6b, 0x2e, 0xd6, 0x46, 0xb0, 0x05, - 0x8d, 0x99, 0xb6, 0xc9, 0xe1, 0xbd, 0x22, 0x4c, 0xbf, 0x34, 0x07, 0xab, 0xb3, 0xa4, 0xcc, 0xb4, - 0xa6, 0x44, 0x71, 0x73, 0xca, 0x00, 0xdc, 0x42, 0x03, 0xfa, 0x13, 0xec, 0x28, 0x14, 0x4a, 0xfb, - 0xdc, 0xe1, 0x90, 0x0c, 0xf4, 0x17, 0x38, 0xaf, 0xe9, 0xc4, 0xdc, 0xc6, 0x9c, 0x39, 0x50, 0x94, - 0xa3, 0x2a, 0x38, 0xf2, 0xb1, 0xb4, 0x3e, 0x2e, 0x40, 0xb5, 0xa3, 0x68, 0xe8, 0x00, 0xea, 0x66, - 0x0f, 0xa3, 0x5f, 0x4d, 0xec, 0x8d, 0xcd, 0xbc, 0xb9, 0x3e, 0xbb, 0x00, 0xf2, 0xe7, 0x15, 0x54, - 0xf6, 0x2c, 0x74, 0x08, 0xcb, 0x86, 0x7b, 0x1e, 0x93, 0x64, 0x32, 0x3f, 0xc5, 0x2f, 0x06, 0x28, - 0xad, 0xa5, 0xa0, 0x52, 0x14, 0xd0, 0xf9, 0xac, 0x80, 0xce, 0x37, 0x14, 0xd0, 0xf9, 0x72, 0x01, - 0x9d, 0x5b, 0x14, 0x70, 0x1f, 0x6a, 0xf9, 0xe0, 0xa1, 0x62, 0xaf, 0x96, 0x27, 0x71, 0xee, 0xe7, - 0x8f, 0xb6, 0x5f, 0x6d, 0x0d, 0xa2, 0xf4, 0x32, 0xeb, 0xee, 0xf4, 0xf8, 0x70, 0x57, 0x93, 0xcc, - 0xcf, 0x55, 0x6b, 0x57, 0xfd, 0x5b, 0x76, 0xab, 0xea, 0xe7, 0xbf, 0x4f, 0x01, 0x00, 0x00, 0xff, - 0xff, 0x18, 0x84, 0x1c, 0xb4, 0x44, 0x07, 0x00, 0x00, + // 923 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x56, 0xdb, 0x72, 0x1b, 0x45, + 0x10, 0xd5, 0x66, 0xd7, 0xba, 0xf4, 0xfa, 0x96, 0x31, 0x31, 0x8b, 0x42, 0x81, 0x32, 0x2e, 0x2a, + 0xa2, 0xa0, 0x9c, 0x20, 0x08, 0x29, 0xc0, 0x29, 0x2a, 0x76, 0x02, 0x72, 0x01, 0x85, 0x32, 0x21, + 0x79, 0xe0, 0x6d, 0x24, 0x8d, 0x9c, 0x2d, 0x56, 0x3b, 0xf2, 0xce, 0xc8, 0x46, 0x3f, 0xc0, 0xff, + 0xf0, 0xcc, 0x0b, 0xdf, 0xc2, 0x97, 0x50, 0x73, 0xdb, 0x8b, 0x2e, 0x24, 0x50, 0xc5, 0x93, 0xa6, + 0xfb, 0x9c, 0xee, 0xe9, 0xd3, 0x3d, 0x33, 0x5a, 0xd8, 0x9e, 0xc5, 0x09, 0x17, 0xf4, 0x78, 0x96, + 0x71, 0xc9, 0x51, 0xdd, 0x58, 0xf8, 0x0b, 0xd8, 0x7b, 0x36, 0x67, 0xd9, 0x62, 0xf0, 0xec, 0x7b, + 0xc2, 0x2e, 0xe7, 0x4c, 0x48, 0xf4, 0x16, 0x6c, 0xc5, 0xe9, 0x98, 0xfd, 0x1a, 0x79, 0x1d, 0xaf, + 0xdb, 0x22, 0xc6, 0x40, 0xfb, 0xe0, 0xcf, 0x2e, 0x93, 0xe8, 0x86, 0xf6, 0xa9, 0x25, 0x3e, 0xb2, + 0xa1, 0xcf, 0x8b, 0xd0, 0x7d, 0xf0, 0xc5, 0x65, 0x62, 0x03, 0xd5, 0x12, 0x7f, 0x05, 0xe1, 0x73, + 0x49, 0xe5, 0x5c, 0x3c, 0xcd, 0x32, 0x9e, 0x21, 0x04, 0xc1, 0x19, 0x1f, 0x33, 0xcd, 0xd8, 0x21, + 0x7a, 0x8d, 0x22, 0x68, 0xfc, 0xc0, 0x84, 0xa0, 0x17, 0xcc, 0x66, 0x77, 0x26, 0xfe, 0xd3, 0x83, + 0x90, 0xf0, 0x6b, 0xc2, 0xc4, 0x8c, 0xa7, 0x82, 0xa1, 0x8f, 0xa1, 0xf1, 0x8a, 0xd1, 0x31, 0xcb, + 0x44, 0xe4, 0x75, 0xfc, 0x6e, 0xd8, 0x43, 0xc7, 0x56, 0xd4, 0x19, 0x4f, 0xe6, 0xd3, 0xf4, 0x3c, + 0x9d, 0x70, 0xe2, 0x28, 0xe8, 0x3e, 0x34, 0x46, 0xda, 0x2d, 0xa2, 0x1b, 0x9a, 0x7d, 0x58, 0x65, + 0xbb, 0xb4, 0xc4, 0xd1, 0xd0, 0x83, 0x4a, 0xb1, 0x91, 0xdf, 0xf1, 0xba, 0x61, 0xef, 0xc0, 0x45, + 0x95, 0x20, 0x52, 0x11, 0xd5, 0x86, 0xe6, 0x78, 0x9e, 0x51, 0x19, 0xf3, 0x34, 0x0a, 0x3a, 0x5e, + 0xd7, 0x27, 0xb9, 0x8d, 0x1f, 0x82, 0x4f, 0xf8, 0x75, 0xb9, 0x16, 0xef, 0x8d, 0x6a, 0xc1, 0xbf, + 0x7b, 0xb0, 0xf3, 0x13, 0x1d, 0x26, 0xec, 0x3f, 0xaa, 0x7f, 0x1f, 0x82, 0x8c, 0x5f, 0x3b, 0xe9, + 0xa1, 0xa3, 0xaa, 0x76, 0x6a, 0xe0, 0xff, 0x10, 0x7b, 0x02, 0x50, 0x94, 0xa2, 0x66, 0x9d, 0xd2, + 0x29, 0xb3, 0xa7, 0x41, 0xaf, 0x75, 0x34, 0x95, 0x54, 0x2e, 0x66, 0x6e, 0xd8, 0xb9, 0x8d, 0x7f, + 0xf3, 0x61, 0xb7, 0xda, 0x0d, 0xf4, 0x1e, 0xb4, 0x84, 0xcc, 0xe2, 0xf4, 0xe2, 0x25, 0xb5, 0xa7, + 0xaa, 0x5f, 0x23, 0x85, 0x4b, 0xe1, 0xf3, 0x38, 0x95, 0x9f, 0x7f, 0xa6, 0x70, 0x95, 0x2f, 0x50, + 0x78, 0xee, 0x42, 0xef, 0x42, 0x33, 0x87, 0x95, 0x40, 0xbf, 0x5f, 0x23, 0xb9, 0x07, 0xb5, 0xa1, + 0x31, 0xe4, 0x3c, 0x51, 0xa0, 0x52, 0xd2, 0xec, 0xd7, 0x88, 0x73, 0x68, 0x2c, 0xe1, 0x43, 0x85, + 0x6d, 0x75, 0xbc, 0xee, 0xb6, 0xc6, 0x8c, 0x03, 0x3d, 0x82, 0x5d, 0xb3, 0xc5, 0xe3, 0x2c, 0xa3, + 0x0b, 0x45, 0xa9, 0x57, 0x9b, 0xf7, 0xa2, 0x40, 0xfb, 0x35, 0xb2, 0x44, 0x56, 0xe1, 0x46, 0x41, + 0x1e, 0xde, 0x58, 0xee, 0x7d, 0x8e, 0xaa, 0xf0, 0x2a, 0x19, 0x75, 0x00, 0x26, 0x09, 0xa7, 0x56, + 0x55, 0xb3, 0xe3, 0x75, 0xbd, 0x7e, 0x8d, 0x94, 0x7c, 0xe8, 0x13, 0x80, 0x31, 0x1b, 0xc5, 0x53, + 0xaa, 0xa5, 0xb5, 0x74, 0xf2, 0x3d, 0x97, 0xfc, 0x89, 0x41, 0x54, 0x48, 0x41, 0x3a, 0x0d, 0xa1, + 0x65, 0x0e, 0xde, 0x4b, 0x9a, 0xe0, 0x07, 0xd0, 0xb0, 0x2c, 0xf5, 0x16, 0x5c, 0xd1, 0x64, 0x6e, + 0x86, 0xe8, 0x13, 0x63, 0x28, 0xaf, 0x18, 0xd1, 0xc4, 0x8c, 0xd0, 0x27, 0xc6, 0xc0, 0x7f, 0x78, + 0xb0, 0x7b, 0x9e, 0x8a, 0x19, 0x1b, 0xc9, 0x7f, 0x7e, 0x4a, 0x3e, 0x2a, 0x5f, 0x4c, 0x55, 0xdc, + 0x4d, 0x57, 0xdc, 0xf9, 0x58, 0xfc, 0x98, 0x7d, 0xc7, 0x16, 0xa2, 0xb8, 0x93, 0x18, 0xb6, 0x27, + 0x71, 0x22, 0x59, 0xf6, 0x4d, 0xcc, 0x92, 0xb1, 0x88, 0xfc, 0x8e, 0xdf, 0x6d, 0x91, 0x8a, 0x4f, + 0x6d, 0x93, 0xc4, 0xd3, 0x58, 0xea, 0x31, 0x06, 0xc4, 0x18, 0xe8, 0x10, 0xea, 0x7c, 0x32, 0x11, + 0x4c, 0xea, 0x09, 0x06, 0xc4, 0x5a, 0x8a, 0x7d, 0xa9, 0xde, 0x2d, 0x3d, 0xb5, 0x16, 0x31, 0x06, + 0xbe, 0x03, 0x61, 0x69, 0x6c, 0xea, 0xf0, 0x5e, 0xd1, 0xc4, 0xdc, 0xb4, 0x80, 0xe8, 0xb5, 0xa2, + 0x94, 0x46, 0x53, 0xa1, 0xb4, 0x2c, 0xe5, 0x02, 0x5a, 0xb9, 0x06, 0x74, 0x17, 0xfc, 0x78, 0x2c, + 0xb4, 0xf6, 0x8d, 0x87, 0x43, 0x31, 0xd0, 0x87, 0x10, 0xfc, 0xc2, 0x16, 0xae, 0x1b, 0x1b, 0xce, + 0x81, 0xa6, 0x9c, 0xd6, 0x21, 0xd0, 0x97, 0xe5, 0x36, 0x6c, 0x9d, 0xeb, 0x66, 0xae, 0xb9, 0x65, + 0xf8, 0x04, 0xd0, 0x59, 0xc6, 0xa8, 0x64, 0x9a, 0xe2, 0x86, 0xb1, 0xee, 0x3e, 0xa2, 0xd2, 0xce, + 0x4d, 0xb3, 0x05, 0xbe, 0x05, 0x07, 0x95, 0x68, 0x73, 0x17, 0xf1, 0x07, 0xb0, 0xf7, 0x2d, 0x93, + 0xaf, 0xcb, 0x88, 0x1f, 0xc2, 0x7e, 0x41, 0xb3, 0xd7, 0xf8, 0xa8, 0x7c, 0x0c, 0xc2, 0xde, 0x4e, + 0x3e, 0x6e, 0xcd, 0x32, 0x18, 0x3e, 0x80, 0x9b, 0x2e, 0x90, 0x09, 0xbb, 0x03, 0x7e, 0x04, 0xa8, + 0xec, 0xb4, 0xf9, 0xee, 0x42, 0x23, 0x36, 0x2e, 0xfb, 0x12, 0x2e, 0x65, 0x74, 0x28, 0xee, 0x02, + 0x7a, 0xc2, 0x12, 0xf6, 0xfa, 0x46, 0x28, 0xd1, 0x15, 0xa6, 0xd9, 0xa9, 0xf7, 0x57, 0x00, 0xf5, + 0x81, 0x4e, 0x8d, 0xfa, 0x10, 0x96, 0xda, 0x82, 0xda, 0xf9, 0xe3, 0xbb, 0xd2, 0xe9, 0xf6, 0xed, + 0xb5, 0x98, 0xed, 0x63, 0x0d, 0x3d, 0x05, 0x28, 0x44, 0xa1, 0x77, 0x1c, 0x79, 0x45, 0x7d, 0xbb, + 0xbd, 0x0e, 0xca, 0xd3, 0x7c, 0x0d, 0x4d, 0xe7, 0x47, 0x6f, 0x2f, 0x33, 0x5d, 0x8a, 0x68, 0x15, + 0xc8, 0x13, 0xf4, 0x21, 0x2c, 0x69, 0x2e, 0x14, 0xad, 0xb6, 0xac, 0x50, 0xb4, 0xa6, 0x49, 0xb8, + 0x86, 0x4e, 0xa0, 0xe9, 0x3e, 0x05, 0x8a, 0x52, 0x96, 0x3e, 0x0e, 0xda, 0x07, 0xe5, 0xff, 0xa0, + 0x3c, 0xf6, 0xbe, 0x87, 0x1e, 0xc3, 0x8e, 0xe3, 0xbe, 0x48, 0x69, 0xb6, 0xd8, 0x9c, 0xe2, 0x96, + 0x03, 0x2a, 0xff, 0x8c, 0xa5, 0x02, 0x06, 0x2b, 0x05, 0x0c, 0xfe, 0x45, 0x01, 0x83, 0xf5, 0x05, + 0x0c, 0xde, 0xa0, 0x80, 0x2f, 0xa1, 0x61, 0xdf, 0x3e, 0x74, 0x58, 0x1c, 0xc6, 0xf2, 0x63, 0xb8, + 0x71, 0xfb, 0xd3, 0xa3, 0x9f, 0xef, 0x5c, 0xc4, 0xf2, 0xd5, 0x7c, 0x78, 0x3c, 0xe2, 0xd3, 0x7b, + 0x86, 0xe4, 0x7e, 0xae, 0x7a, 0xf7, 0xf4, 0x07, 0xdb, 0xb0, 0xae, 0x7f, 0x3e, 0xfd, 0x3b, 0x00, + 0x00, 0xff, 0xff, 0x75, 0x3f, 0x36, 0x94, 0xc7, 0x09, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -920,6 +1274,10 @@ const _ = grpc.SupportPackageIsVersion4 // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. type PilosaClient interface { + CreateIndex(ctx context.Context, in *CreateIndexRequest, opts ...grpc.CallOption) (*CreateIndexResponse, error) + GetIndexes(ctx context.Context, in *GetIndexesRequest, opts ...grpc.CallOption) (*GetIndexesResponse, error) + GetIndex(ctx context.Context, in *GetIndexRequest, opts ...grpc.CallOption) (*GetIndexResponse, error) + DeleteIndex(ctx context.Context, in *DeleteIndexRequest, opts ...grpc.CallOption) (*DeleteIndexResponse, error) QuerySQL(ctx context.Context, in *QuerySQLRequest, opts ...grpc.CallOption) (Pilosa_QuerySQLClient, error) QuerySQLUnary(ctx context.Context, in *QuerySQLRequest, opts ...grpc.CallOption) (*TableResponse, error) QueryPQL(ctx context.Context, in *QueryPQLRequest, opts ...grpc.CallOption) (Pilosa_QueryPQLClient, error) @@ -935,6 +1293,42 @@ func NewPilosaClient(cc *grpc.ClientConn) PilosaClient { return &pilosaClient{cc} } +func (c *pilosaClient) CreateIndex(ctx context.Context, in *CreateIndexRequest, opts ...grpc.CallOption) (*CreateIndexResponse, error) { + out := new(CreateIndexResponse) + err := c.cc.Invoke(ctx, "/pilosa.Pilosa/CreateIndex", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *pilosaClient) GetIndexes(ctx context.Context, in *GetIndexesRequest, opts ...grpc.CallOption) (*GetIndexesResponse, error) { + out := new(GetIndexesResponse) + err := c.cc.Invoke(ctx, "/pilosa.Pilosa/GetIndexes", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *pilosaClient) GetIndex(ctx context.Context, in *GetIndexRequest, opts ...grpc.CallOption) (*GetIndexResponse, error) { + out := new(GetIndexResponse) + err := c.cc.Invoke(ctx, "/pilosa.Pilosa/GetIndex", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *pilosaClient) DeleteIndex(ctx context.Context, in *DeleteIndexRequest, opts ...grpc.CallOption) (*DeleteIndexResponse, error) { + out := new(DeleteIndexResponse) + err := c.cc.Invoke(ctx, "/pilosa.Pilosa/DeleteIndex", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *pilosaClient) QuerySQL(ctx context.Context, in *QuerySQLRequest, opts ...grpc.CallOption) (Pilosa_QuerySQLClient, error) { stream, err := c.cc.NewStream(ctx, &_Pilosa_serviceDesc.Streams[0], "/pilosa.Pilosa/QuerySQL", opts...) if err != nil { @@ -1051,6 +1445,10 @@ func (x *pilosaInspectClient) Recv() (*RowResponse, error) { // PilosaServer is the server API for Pilosa service. type PilosaServer interface { + CreateIndex(context.Context, *CreateIndexRequest) (*CreateIndexResponse, error) + GetIndexes(context.Context, *GetIndexesRequest) (*GetIndexesResponse, error) + GetIndex(context.Context, *GetIndexRequest) (*GetIndexResponse, error) + DeleteIndex(context.Context, *DeleteIndexRequest) (*DeleteIndexResponse, error) QuerySQL(*QuerySQLRequest, Pilosa_QuerySQLServer) error QuerySQLUnary(context.Context, *QuerySQLRequest) (*TableResponse, error) QueryPQL(*QueryPQLRequest, Pilosa_QueryPQLServer) error @@ -1062,6 +1460,18 @@ type PilosaServer interface { type UnimplementedPilosaServer struct { } +func (*UnimplementedPilosaServer) CreateIndex(ctx context.Context, req *CreateIndexRequest) (*CreateIndexResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method CreateIndex not implemented") +} +func (*UnimplementedPilosaServer) GetIndexes(ctx context.Context, req *GetIndexesRequest) (*GetIndexesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetIndexes not implemented") +} +func (*UnimplementedPilosaServer) GetIndex(ctx context.Context, req *GetIndexRequest) (*GetIndexResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetIndex not implemented") +} +func (*UnimplementedPilosaServer) DeleteIndex(ctx context.Context, req *DeleteIndexRequest) (*DeleteIndexResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteIndex not implemented") +} func (*UnimplementedPilosaServer) QuerySQL(req *QuerySQLRequest, srv Pilosa_QuerySQLServer) error { return status.Errorf(codes.Unimplemented, "method QuerySQL not implemented") } @@ -1082,6 +1492,78 @@ func RegisterPilosaServer(s *grpc.Server, srv PilosaServer) { s.RegisterService(&_Pilosa_serviceDesc, srv) } +func _Pilosa_CreateIndex_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateIndexRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(PilosaServer).CreateIndex(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/pilosa.Pilosa/CreateIndex", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(PilosaServer).CreateIndex(ctx, req.(*CreateIndexRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Pilosa_GetIndexes_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetIndexesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(PilosaServer).GetIndexes(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/pilosa.Pilosa/GetIndexes", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(PilosaServer).GetIndexes(ctx, req.(*GetIndexesRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Pilosa_GetIndex_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetIndexRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(PilosaServer).GetIndex(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/pilosa.Pilosa/GetIndex", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(PilosaServer).GetIndex(ctx, req.(*GetIndexRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Pilosa_DeleteIndex_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteIndexRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(PilosaServer).DeleteIndex(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/pilosa.Pilosa/DeleteIndex", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(PilosaServer).DeleteIndex(ctx, req.(*DeleteIndexRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _Pilosa_QuerySQL_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(QuerySQLRequest) if err := stream.RecvMsg(m); err != nil { @@ -1185,6 +1667,22 @@ var _Pilosa_serviceDesc = grpc.ServiceDesc{ ServiceName: "pilosa.Pilosa", HandlerType: (*PilosaServer)(nil), Methods: []grpc.MethodDesc{ + { + MethodName: "CreateIndex", + Handler: _Pilosa_CreateIndex_Handler, + }, + { + MethodName: "GetIndexes", + Handler: _Pilosa_GetIndexes_Handler, + }, + { + MethodName: "GetIndex", + Handler: _Pilosa_GetIndex_Handler, + }, + { + MethodName: "DeleteIndex", + Handler: _Pilosa_DeleteIndex_Handler, + }, { MethodName: "QuerySQLUnary", Handler: _Pilosa_QuerySQLUnary_Handler, diff --git a/proto/pilosa.proto b/proto/pilosa.proto index 898614478..80b82de86 100644 --- a/proto/pilosa.proto +++ b/proto/pilosa.proto @@ -85,7 +85,45 @@ message IdsOrKeys { } } +message Index { + string name = 1; +} + +message CreateIndexRequest { + string name = 1; + bool keys = 2; +} + +message CreateIndexResponse { +} + +message GetIndexRequest { + string name = 1; +} + +message GetIndexResponse { + Index index = 1; +} + +message GetIndexesRequest { +} + +message GetIndexesResponse { + repeated Index indexes = 1; +} + +message DeleteIndexRequest { + string name = 1; +} + +message DeleteIndexResponse { +} + service Pilosa { + rpc CreateIndex(CreateIndexRequest) returns (CreateIndexResponse) {}; + rpc GetIndexes(GetIndexesRequest) returns (GetIndexesResponse) {}; + rpc GetIndex(GetIndexRequest) returns (GetIndexResponse) {}; + rpc DeleteIndex(DeleteIndexRequest) returns (DeleteIndexResponse) {}; rpc QuerySQL(QuerySQLRequest) returns (stream RowResponse) {}; rpc QuerySQLUnary(QuerySQLRequest) returns (TableResponse) {}; rpc QueryPQL(QueryPQLRequest) returns (stream RowResponse) {}; diff --git a/rbf/cursor.go b/rbf/cursor.go index 3920ffaec..4c1a05dbc 100644 --- a/rbf/cursor.go +++ b/rbf/cursor.go @@ -1386,21 +1386,8 @@ func (c *Cursor) difference(key uint64, data *roaring.Container) (bool, error) { } func (c *Cursor) Close() { - if c == nil { - panic("cannot Close nil Cursor") - } - tx := c.tx - c.tx = nil // allow tx to be garbage collected. - - if tx.db.cfg.CursorCacheSize == 0 { - globalCursorSyncPool.Put(c) - return - } - - select { - case tx.db.cursorArenaCh <- c: - case <-tx.db.cursorCleaner.ReqStop.Chan: - } + assert(c != nil) + cursorSyncPool.Put(c) } func keysFromParents(parents []branchCell) (ckeys []int) { diff --git a/rbf/db.go b/rbf/db.go index 6946fdc65..5976d5cc6 100644 --- a/rbf/db.go +++ b/rbf/db.go @@ -24,7 +24,6 @@ import ( "syscall" "github.com/benbjohnson/immutable" - "github.com/glycerine/idem" rbfcfg "github.com/pilosa/pilosa/v2/rbf/cfg" "github.com/pilosa/pilosa/v2/syswrap" ) @@ -33,9 +32,9 @@ var ( ErrClosed = errors.New("rbf: database closed") ) -// global in the sense that it is shared among all instances -// of rbf.DBs in this process. This is deliberate. -var globalCursorSyncPool = &sync.Pool{ +// shared cursor pool across all DB instances. +// Cursors are returned on Cursor.Close(). +var cursorSyncPool = &sync.Pool{ New: func() interface{} { return &Cursor{} }, @@ -63,9 +62,6 @@ type DB struct { // Path represents the path to the database file. Path string - - cursorArenaCh chan *Cursor - cursorCleaner *idem.Halter } // NewDB returns a new instance of DB. @@ -79,12 +75,6 @@ func NewDB(path string, cfg *rbfcfg.Config) *DB { txs: make(map[*Tx]struct{}), pageMap: NewPageMap(), Path: path, - - cursorArenaCh: make(chan *Cursor, cfg.CursorCacheSize), - cursorCleaner: idem.NewHalter(), - } - for i := int64(0); i < cfg.CursorCacheSize; i++ { - db.cursorArenaCh <- &Cursor{} } db.haltCond = sync.NewCond(&db.mu) @@ -272,8 +262,6 @@ func (db *DB) Close() (err error) { db.mu.Lock() defer db.mu.Unlock() - defer db.cursorCleaner.RequestStop() - db.opened = false // Close mmap handle. @@ -580,24 +568,10 @@ func (db *DB) readMetaPage() ([]byte, error) { return db.readDBPage(0) } -func (db *DB) getCursor(tx *Tx) (c *Cursor) { - if db.cfg.CursorCacheSize == 0 { - c = globalCursorSyncPool.Get().(*Cursor) - c.tx = tx - return - } - - n := len(db.cursorArenaCh) - if n < 10 { - vv("warning, db.cursorArenaCh is low! %v left", n) - } - select { - case c = <-db.cursorArenaCh: - c.tx = tx - return - case <-db.cursorCleaner.ReqStop.Chan: - return nil - } +func (db *DB) getCursor(tx *Tx) *Cursor { + c := cursorSyncPool.Get().(*Cursor) + c.tx = tx + return c } // Shared pool for in-memory database pages. diff --git a/rrtx.go b/rrtx.go index 6113067c4..8c75a2924 100644 --- a/rrtx.go +++ b/rrtx.go @@ -250,7 +250,7 @@ func (tx *RoaringTx) Contains(index, field, view string, shard uint64, v uint64) func (tx *RoaringTx) ContainerIterator(index, field, view string, shard uint64, key uint64) (citer roaring.ContainerIterator, found bool, err error) { b, err := tx.bitmap(index, field, view, shard) if err != nil { - return nil, false, err + return nil, false, errors.Wrap(err, "getting bitmap") } //vv("b bitmap back from bitmap(index='%v', field='%v', view='%v', shard='%v')='%#v'", index, field, view, shard, b.Slice()) citer, found = b.Containers.Iterator(key) @@ -395,7 +395,7 @@ const FragmentNotFound = Error("fragment not found") func (tx *RoaringTx) bitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) { frag, err := tx.getFragment(index, field, view, shard) if err != nil { - return nil, err + return nil, errors.Wrap(err, "getFragment") } return frag.storage, nil } diff --git a/scripts/bench_read.sh b/scripts/bench_read.sh new file mode 100755 index 000000000..bfbf6baac --- /dev/null +++ b/scripts/bench_read.sh @@ -0,0 +1,42 @@ +#!/bin/bash -x +set -e + +# This script runs benchmarks and posts them to Slack's #nightly channel. +# The caller of the script should `git pull` on the pilosa repo before executing. +# +# Environment variables: +# - PILOSA_SRC: Path to pilosa src directory. +# - SLACK_OAUTH_TOKEN: Token used to post to Slack. + +# Require environment variables. +: "${PILOSA_SRC:?Must set PILOSA_SRC environment variable}" +: "${SLACK_OAUTH_TOKEN:?Must set SLACK_OAUTH_TOKEN environment variable}" + +# Build pilosa into GOBIN. +make -C $PILOSA_SRC install install-bench + +# Retrieve current SHA. +SHA=$(git -C $PILOSA_SRC rev-parse HEAD) + +# Format current date. +DATE=$(date '+%Y%m%d') + +for TYPE in row row-bsi row-range count intersect union difference xor groupby topk +do + WORKFLOW_PATH="${BASH_SOURCE%/*}/etc/gloat/query.${TYPE}.yml" + WORKFLOW_NAME="$(gloat workflow name $WORKFLOW_PATH)" + TITLE="$WORKFLOW_NAME, $DATE ($SHA)" + + # Execute RBF/Roaring benchmark. + RBF_PATH=gloat/data/query/${TYPE}/rbf/${DATE}.tar.gz + TXSRC=rbf gloat run -v -o "$RBF_PATH" $WORKFLOW_PATH + + ROARING_PATH=gloat/data/query/${TYPE}/roaring/${DATE}.tar.gz + TXSRC=roaring gloat run -v -o "$ROARING_PATH" $WORKFLOW_PATH + + # Generate graph from results. + gloat graph -layout 2,5 -size 5120,820 -title "$TITLE" -name utime,stime,heap_alloc,heap_inuse,heap_objects,num_gc,rchar,wchar,syscr,syscw -series rbf,roaring -o /tmp/output.png $RBF_PATH $ROARING_PATH + + # Post graph to Slack with SHA. + curl -F file=@/tmp/output.png -F channels=C01HBFKRLGH -F "initial_comment=$TITLE" -H "Authorization: Bearer $SLACK_OAUTH_TOKEN" https://slack.com/api/files.upload +done \ No newline at end of file diff --git a/scripts/bench.sh b/scripts/bench_write.sh similarity index 85% rename from scripts/bench.sh rename to scripts/bench_write.sh index 875bbccba..02e8067ca 100755 --- a/scripts/bench.sh +++ b/scripts/bench_write.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -e # This script runs benchmarks and posts them to Slack's #nightly channel. # The caller of the script should `git pull` on the pilosa repo before executing. @@ -32,7 +33,7 @@ ROARING_PATH=gloat/data/1m/roaring/${DATE}.tar.gz TXSRC=roaring gloat run -v -o $ROARING_PATH $WORKFLOW_PATH # Generate graph from results. -gloat graph -layout 5,2 -size 2048,2048 -title "$TITLE" -name utime,stime,heap_alloc,heap_inuse,heap_objects,num_gc,rchar,wchar,syscr,syscw -series rbf,roaring -o /tmp/output.png $RBF_PATH $ROARING_PATH +gloat graph -layout 2,5 -size 5120,820 -title "$TITLE" -name utime,stime,heap_alloc,heap_inuse,heap_objects,num_gc,rchar,wchar,syscr,syscw -series rbf,roaring -o /tmp/output.png $RBF_PATH $ROARING_PATH # Post graph to Slack with SHA. curl -F file=@/tmp/output.png -F channels=C01HBFKRLGH -F "initial_comment=$TITLE" -H "Authorization: Bearer $SLACK_OAUTH_TOKEN" https://slack.com/api/files.upload diff --git a/scripts/etc/gloat/gh.1d.yml b/scripts/etc/gloat/gh.1d.yml index 93658c44a..f064b2f7a 100644 --- a/scripts/etc/gloat/gh.1d.yml +++ b/scripts/etc/gloat/gh.1d.yml @@ -1,7 +1,9 @@ name: "GitHub Import Load Testing (1 day)" main: "pilosa server --data-dir ${TMPDIR} --txsrc ${TXSRC}" -load: "molecula-consumer-github -i events -d id --record-type event --batch-size=100000 --start-time 2020-01-01T00:00:00Z --end-time 2020-01-01T23:00:00Z --cache-dir .githubarchive" +load: "molecula-consumer-github -i events -d id --record-type event --batch-size=100000 --start-time 2020-01-01T00:00:00Z --end-time 2020-01-01T23:00:00Z --cache-dir ~/.githubarchive" health_url: "http://localhost:10101/status" -debug_url: "http://localhost:10101/debug" +vars_urls: + - http://localhost:10101/debug/vars + - http://localhost:7070/debug/vars diff --git a/scripts/etc/gloat/gh.1m.yml b/scripts/etc/gloat/gh.1m.yml index 9b4dacf4a..5be52cb7a 100644 --- a/scripts/etc/gloat/gh.1m.yml +++ b/scripts/etc/gloat/gh.1m.yml @@ -1,7 +1,9 @@ name: "GitHub Import Load Testing (1 month)" main: "pilosa server --data-dir ${TMPDIR} --txsrc ${TXSRC}" -load: "molecula-consumer-github -i events -d id --record-type event --batch-size=100000 --start-time 2020-01-01T00:00:00Z --end-time 2020-01-31T23:00:00Z --cache-dir .githubarchive" +load: "molecula-consumer-github -i events -d id --record-type event --batch-size=100000 --start-time 2020-01-01T00:00:00Z --end-time 2020-01-31T23:00:00Z --cache-dir ~/.githubarchive" health_url: "http://localhost:10101/status" -debug_url: "http://localhost:10101/debug" +vars_urls: + - http://localhost:10101/debug/vars + - http://localhost:7070/debug/vars diff --git a/scripts/etc/gloat/gh.1w.yml b/scripts/etc/gloat/gh.1w.yml index 23343ae45..084492919 100644 --- a/scripts/etc/gloat/gh.1w.yml +++ b/scripts/etc/gloat/gh.1w.yml @@ -1,7 +1,9 @@ name: "GitHub Import Load Testing (1 week)" main: "pilosa server --data-dir ${TMPDIR} --txsrc ${TXSRC}" -load: "molecula-consumer-github -i events -d id --record-type event --batch-size=100000 --start-time 2020-01-01T00:00:00Z --end-time 2020-01-06T23:00:00Z --cache-dir .githubarchive" +load: "molecula-consumer-github -i events -d id --record-type event --batch-size=100000 --start-time 2020-01-01T00:00:00Z --end-time 2020-01-06T23:00:00Z --cache-dir ~/.githubarchive" health_url: "http://localhost:10101/status" -debug_url: "http://localhost:10101/debug" +vars_urls: + - http://localhost:10101/debug/vars + - http://localhost:7070/debug/vars diff --git a/scripts/etc/gloat/query.count.yml b/scripts/etc/gloat/query.count.yml new file mode 100644 index 000000000..5690c13a8 --- /dev/null +++ b/scripts/etc/gloat/query.count.yml @@ -0,0 +1,10 @@ +name: "Count() Load Testing" + +main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}" +load: "pilosa-bench -type count -rate 100 -n 3000" + +health_url: "http://localhost:10101/status" +health_regexp: "NORMAL" + +vars_urls: + - http://localhost:10101/debug/vars diff --git a/scripts/etc/gloat/query.difference.yml b/scripts/etc/gloat/query.difference.yml new file mode 100644 index 000000000..3a73a8e46 --- /dev/null +++ b/scripts/etc/gloat/query.difference.yml @@ -0,0 +1,10 @@ +name: "Difference() Load Testing" + +main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}" +load: "pilosa-bench -type difference -rate 10 -n 300" + +health_url: "http://localhost:10101/status" +health_regexp: "NORMAL" + +vars_urls: + - http://localhost:10101/debug/vars diff --git a/scripts/etc/gloat/query.groupby.yml b/scripts/etc/gloat/query.groupby.yml new file mode 100644 index 000000000..a5c4c5d00 --- /dev/null +++ b/scripts/etc/gloat/query.groupby.yml @@ -0,0 +1,10 @@ +name: "GroupBy() Load Testing" + +main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}" +load: "pilosa-bench -type groupby -rate 100 -n 3000" + +health_url: "http://localhost:10101/status" +health_regexp: "NORMAL" + +vars_urls: + - http://localhost:10101/debug/vars diff --git a/scripts/etc/gloat/query.intersect.yml b/scripts/etc/gloat/query.intersect.yml new file mode 100644 index 000000000..00bbfbd2e --- /dev/null +++ b/scripts/etc/gloat/query.intersect.yml @@ -0,0 +1,10 @@ +name: "Intersect() Load Testing" + +main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}" +load: "pilosa-bench -type intersect -rate 100 -n 3000" + +health_url: "http://localhost:10101/status" +health_regexp: "NORMAL" + +vars_urls: + - http://localhost:10101/debug/vars diff --git a/scripts/etc/gloat/query.row-bsi.yml b/scripts/etc/gloat/query.row-bsi.yml new file mode 100644 index 000000000..04fe472eb --- /dev/null +++ b/scripts/etc/gloat/query.row-bsi.yml @@ -0,0 +1,10 @@ +name: "Row(BSI) Load Testing" + +main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}" +load: "pilosa-bench -type row -rate 100 -n 3000" + +health_url: "http://localhost:10101/status" +health_regexp: "NORMAL" + +vars_urls: + - http://localhost:10101/debug/vars diff --git a/scripts/etc/gloat/query.row-range.yml b/scripts/etc/gloat/query.row-range.yml new file mode 100644 index 000000000..f05b5cb4c --- /dev/null +++ b/scripts/etc/gloat/query.row-range.yml @@ -0,0 +1,10 @@ +name: "Time-based Row() Load Testing" + +main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}" +load: "pilosa-bench -type row-range -rate 10 -n 300 -from 2020-01-01T00:00:00Z -to 2020-01-31T23:00:00Z" + +health_url: "http://localhost:10101/status" +health_regexp: "NORMAL" + +vars_urls: + - http://localhost:10101/debug/vars diff --git a/scripts/etc/gloat/query.row.yml b/scripts/etc/gloat/query.row.yml new file mode 100644 index 000000000..142f12004 --- /dev/null +++ b/scripts/etc/gloat/query.row.yml @@ -0,0 +1,10 @@ +name: "Row() Load Testing" + +main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}" +load: "pilosa-bench -type row -rate 100 -n 3000" + +health_url: "http://localhost:10101/status" +health_regexp: "NORMAL" + +vars_urls: + - http://localhost:10101/debug/vars diff --git a/scripts/etc/gloat/query.topk.yml b/scripts/etc/gloat/query.topk.yml new file mode 100644 index 000000000..10da63427 --- /dev/null +++ b/scripts/etc/gloat/query.topk.yml @@ -0,0 +1,10 @@ +name: "Time-based TopK() Load Testing" + +main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}" +load: "pilosa-bench -type row-range -rate 10 -n 300 -from 2020-01-01T00:00:00Z -to 2020-01-31T23:00:00Z" + +health_url: "http://localhost:10101/status" +health_regexp: "NORMAL" + +vars_urls: + - http://localhost:10101/debug/vars diff --git a/scripts/etc/gloat/query.union.yml b/scripts/etc/gloat/query.union.yml new file mode 100644 index 000000000..a2270d974 --- /dev/null +++ b/scripts/etc/gloat/query.union.yml @@ -0,0 +1,10 @@ +name: "Union() Load Testing" + +main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}" +load: "pilosa-bench -type union -rate 10 -n 300" + +health_url: "http://localhost:10101/status" +health_regexp: "NORMAL" + +vars_urls: + - http://localhost:10101/debug/vars diff --git a/scripts/etc/gloat/query.xor.yml b/scripts/etc/gloat/query.xor.yml new file mode 100644 index 000000000..f2eb857b8 --- /dev/null +++ b/scripts/etc/gloat/query.xor.yml @@ -0,0 +1,10 @@ +name: "Xor() Load Testing" + +main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}" +load: "pilosa-bench -type xor -rate 10 -n 300" + +health_url: "http://localhost:10101/status" +health_regexp: "NORMAL" + +vars_urls: + - http://localhost:10101/debug/vars diff --git a/scripts/populate_query_db.sh b/scripts/populate_query_db.sh new file mode 100755 index 000000000..b225026e3 --- /dev/null +++ b/scripts/populate_query_db.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -e + +# This script generates data query load testing to be run against. +# +# Environment variables: +# - TXSRC: Transaction store type ("roaring", "rbf") +# - CACHEDIR: Path to local GitHub Archive data, if available. + +# Require environment variables. +: "${TXSRC:?Must set TXSRC environment variable}" +: "${GHCACHEDIR:''}" + +echo "Starting pilosa" +pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC} & pid_pilosa=$! +sleep 5 + +echo "" +echo "Importing GitHub Archive" +molecula-consumer-github -i events -d id --record-type event --batch-size=100000 \ + --start-time 2020-01-01T00:00:00Z --end-time 2020-01-31T23:00:00Z \ + --cache-dir "$GHCACHEDIR" + +echo "" +echo "Import complete, shutting down pilosa" + +sleep 5 +kill $pid_pilosa diff --git a/server/grpc.go b/server/grpc.go index d8544ed97..c2e68bb1f 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -68,7 +68,7 @@ func errToStatusError(err error) error { } // Check error string. - switch errors.Cause(err) { + switch cause := errors.Cause(err); cause { case pilosa.ErrIndexNotFound, pilosa.ErrFieldNotFound, pilosa.ErrForeignIndexNotFound, @@ -123,6 +123,10 @@ func errToStatusError(err error) error { pilosa.ErrTooManyWrites, pilosa.ErrNodeIDNotExists: return status.Error(codes.Internal, err.Error()) + default: + if _, ok := cause.(pilosa.ConflictError); ok { + return status.Error(codes.AlreadyExists, err.Error()) + } } return status.Error(codes.Unknown, err.Error()) @@ -267,6 +271,47 @@ func (h *GRPCHandler) QueryPQLUnary(ctx context.Context, req *pb.QueryPQLRequest return table, errToStatusError(nil) } +// CreateIndex creates a new Index +func (h *GRPCHandler) CreateIndex(ctx context.Context, req *pb.CreateIndexRequest) (*pb.CreateIndexResponse, error) { + // Always enable TrackExistence for gRPC-created indexes + opts := pilosa.IndexOptions{Keys: req.Keys, TrackExistence: true} + _, err := h.api.CreateIndex(ctx, req.Name, opts) + if err != nil { + return nil, errToStatusError(err) + } + return &pb.CreateIndexResponse{}, nil +} + +// GetIndex returns a single Index given a name +func (h *GRPCHandler) GetIndex(ctx context.Context, req *pb.GetIndexRequest) (*pb.GetIndexResponse, error) { + schema := h.api.Schema(ctx) + for _, index := range schema { + if req.Name == index.Name { + return &pb.GetIndexResponse{Index: &pb.Index{Name: index.Name}}, nil + } + } + return nil, status.Error(codes.NotFound, fmt.Sprintf("Index with name %s not found", req.Name)) +} + +// GetIndexes returns a list of all Indexes +func (h *GRPCHandler) GetIndexes(ctx context.Context, req *pb.GetIndexesRequest) (*pb.GetIndexesResponse, error) { + schema := h.api.Schema(ctx) + indexes := make([]*pb.Index, len(schema)) + for i, index := range schema { + indexes[i] = &pb.Index{Name: index.Name} + } + return &pb.GetIndexesResponse{Indexes: indexes}, nil +} + +// DeleteIndex deletes an Index +func (h *GRPCHandler) DeleteIndex(ctx context.Context, req *pb.DeleteIndexRequest) (*pb.DeleteIndexResponse, error) { + err := h.api.DeleteIndex(ctx, req.Name) + if err != nil { + return nil, errToStatusError(err) + } + return &pb.DeleteIndexResponse{}, nil +} + // VDSMGRPCHandler contains methods which handle the various gRPC requests, ported from VDSM. type VDSMGRPCHandler struct { grpcHandler *GRPCHandler @@ -411,7 +456,8 @@ func ToTablerWrapper(result interface{}) (pb.ToTabler, error) { if !ok { switch v := result.(type) { case []pilosa.GroupCount: - toTabler = pilosa.GroupCounts(v) + gc := pilosa.NewGroupCounts("", v...) + toTabler = gc case uint64: toTabler = ResultUint64(v) case bool: @@ -432,7 +478,8 @@ func ToRowserWrapper(result interface{}) (pb.ToRowser, error) { if !ok { switch v := result.(type) { case []pilosa.GroupCount: - toRowser = pilosa.GroupCounts(v) + gc := pilosa.NewGroupCounts("", v...) + toRowser = gc case uint64: toRowser = ResultUint64(v) case bool: diff --git a/server/grpc_test.go b/server/grpc_test.go index 79d40b0f1..3cc808bb2 100644 --- a/server/grpc_test.go +++ b/server/grpc_test.go @@ -138,7 +138,7 @@ func TestGRPC(t *testing.T) { }, // []GroupCount (uint64) { - []pilosa.GroupCount{ + pilosa.NewGroupCounts("", []pilosa.GroupCount{ pilosa.GroupCount{ Group: []pilosa.FieldRow{ {Field: "a", RowID: 10}, @@ -160,22 +160,21 @@ func TestGRPC(t *testing.T) { }, Count: 789, }, - }, + }...), []expHeader{ {"a", "uint64"}, {"b", "uint64"}, {"count", "uint64"}, - {"sum", "int64"}, }, [][]expColumn{ - {uint64(10), uint64(11), uint64(123), int64(0)}, - {uint64(10), uint64(12), uint64(456), int64(0)}, - {int64(va), int64(vb), uint64(789), int64(0)}, + {uint64(10), uint64(11), uint64(123)}, + {uint64(10), uint64(12), uint64(456)}, + {int64(va), int64(vb), uint64(789)}, }, }, - // []GroupCount (string) + // []GroupCount (string) + sum { - []pilosa.GroupCount{ + pilosa.NewGroupCounts("sum", []pilosa.GroupCount{ pilosa.GroupCount{ Group: []pilosa.FieldRow{ {Field: "a", RowKey: "ten"}, @@ -190,7 +189,7 @@ func TestGRPC(t *testing.T) { }, Count: 456, }, - }, + }...), []expHeader{ {"a", "string"}, {"b", "string"}, @@ -292,7 +291,12 @@ func TestGRPC(t *testing.T) { } // Ensure headers match. - for i, header := range table.GetHeaders() { + headers := table.GetHeaders() + if len(headers) < len(test.expHeaders) { + t.Fatalf("test %d expected %d headers, got %d, first missing header %q", + ti, len(test.expHeaders), len(headers), test.expHeaders[len(headers)].name) + } + for i, header := range headers { if header.Name != test.expHeaders[i].name { t.Fatalf("test %d expected header name: %s, but got: %s", ti, test.expHeaders[i].name, header.Name) } @@ -303,7 +307,12 @@ func TestGRPC(t *testing.T) { // Ensure column data matches. for i, row := range table.GetRows() { - for j, column := range row.GetColumns() { + columns := row.GetColumns() + if len(columns) != len(test.expColumns[i]) { + t.Fatalf("test %d expected %d columns, got %d in row %d", + ti, len(test.expColumns[i]), len(columns), i) + } + for j, column := range columns { switch v := test.expColumns[i][j].(type) { case string: val := column.GetStringVal() @@ -986,6 +995,171 @@ func TestQuerySQLUnaryWithError(t *testing.T) { } } +func TestCRUDIndexes(t *testing.T) { + m := test.RunCommand(t) + defer m.Close() + + ctx := context.Background() + gh := server.NewGRPCHandler(m.API) + + t.Run("CreateIndex", func(t *testing.T) { + // Try CreateIndex for testindex1 + _, err := gh.CreateIndex(ctx, &pb.CreateIndexRequest{Name: "testindex1", Keys: true}) + if err != nil { + t.Fatal(err) + } + + schema := m.API.Schema(ctx) + if len(schema) != 1 { + t.Fatal("Schema should include one index") + } + if schema[0].Name != "testindex1" { + t.Fatal("Index name not set correctly") + } + if schema[0].Options.Keys != true { + t.Fatal("Index Keys not set correctly") + } + if schema[0].Options.TrackExistence != true { + t.Fatal("Index TrackExistence should be true when created by gRPC") + } + + // Try CreateIndex for testindex2 + _, err = gh.CreateIndex(ctx, &pb.CreateIndexRequest{Name: "testindex2"}) + if err != nil { + t.Fatal(err) + } + + schema = m.API.Schema(ctx) + if len(schema) != 2 { + t.Fatal("Schema should include two indexes") + } + + _ = m.API.DeleteIndex(ctx, "testindex1") + + schema = m.API.Schema(ctx) + if len(schema) != 1 { + t.Fatal("Schema should include one index") + } + if schema[0].Name != "testindex2" { + t.Fatal("Index name not set correctly") + } + if schema[0].Options.Keys != false { + t.Fatal("Index Keys not set correctly") + } + + // Check errors for CreateIndex: create index with same name + _, err = gh.CreateIndex(ctx, &pb.CreateIndexRequest{Name: "testindex2"}) + errStatus, _ := status.FromError(err) + if errStatus.Code() != codes.AlreadyExists { + t.Fatalf("Error code should be codes.AlreadyExists, but is %v", errStatus.Code()) + } + + // Check errors for CreateIndex: create index with no name + _, err = gh.CreateIndex(ctx, &pb.CreateIndexRequest{Name: ""}) + errStatus, _ = status.FromError(err) + if errStatus.Code() != codes.Unknown { + t.Fatalf("Error code should be codes.Unknown, but is %v", errStatus.Code()) + } + + // Check errors for CreateIndex: create index with invalid name + _, err = gh.CreateIndex(ctx, &pb.CreateIndexRequest{Name: "💩"}) + errStatus, _ = status.FromError(err) + if errStatus.Code() != codes.FailedPrecondition { + t.Fatalf("Error code should be codes.FailedPrecondition, but is %v", errStatus.Code()) + } + + _ = m.API.DeleteIndex(ctx, "testindex2") + }) + + t.Run("GetIndex", func(t *testing.T) { + _, err := m.API.CreateIndex(ctx, "testindex1", pilosa.IndexOptions{}) + if err != nil { + t.Fatal(err) + } + + // Check GetIndex for testindex1 + resp, err := gh.GetIndex(ctx, &pb.GetIndexRequest{Name: "testindex1"}) + if err != nil { + t.Fatal(err) + } + if resp.Index.Name != "testindex1" { + t.Fatalf("Index name does not match: %s", resp.Index.Name) + } + + // Check errors for GetIndex: get index that doesn't exist + _, err = gh.GetIndex(ctx, &pb.GetIndexRequest{Name: "wrongname"}) + errStatus, _ := status.FromError(err) + if errStatus.Code() != codes.NotFound { + t.Fatalf("Error code should be codes.NotFound, but is %v", errStatus.Code()) + } + + // Check errors for GetIndex: get index with invalid name + _, err = gh.GetIndex(ctx, &pb.GetIndexRequest{Name: "💩"}) + errStatus, _ = status.FromError(err) + if errStatus.Code() != codes.NotFound { + t.Fatalf("Error code should be codes.NotFound, but is %v", errStatus.Code()) + } + _ = m.API.DeleteIndex(ctx, "testindex1") + }) + + t.Run("GetIndexes", func(t *testing.T) { + _, err := m.API.CreateIndex(ctx, "testindex1", pilosa.IndexOptions{}) + if err != nil { + t.Fatal(err) + } + + // Check GetIndexes + resp2, err := gh.GetIndexes(ctx, &pb.GetIndexesRequest{}) + if err != nil { + t.Fatal(err) + } + if len(resp2.Indexes) != 1 && resp2.Indexes[0].Name != "testindex1" { + t.Fatalf("GetIndexes did not produce the correct result set: %v", resp2.Indexes) + } + + _, err = m.API.CreateIndex(ctx, "testindex2", pilosa.IndexOptions{}) + if err != nil { + t.Fatal(err) + } + + // Check GetIndexes again + resp, err := gh.GetIndexes(ctx, &pb.GetIndexesRequest{}) + if err != nil { + t.Fatal(err) + } + if len(resp.Indexes) != 2 { + t.Fatalf("GetIndexes did not produce the correct result set: %v", resp.Indexes) + } + _ = m.API.DeleteIndex(ctx, "testindex1") + _ = m.API.DeleteIndex(ctx, "testindex2") + }) + + t.Run("DeleteIndexes", func(t *testing.T) { + _, err := m.API.CreateIndex(ctx, "testindex1", pilosa.IndexOptions{}) + if err != nil { + t.Fatal(err) + } + + // Try to delete index + _, err = gh.DeleteIndex(ctx, &pb.DeleteIndexRequest{Name: "testindex1"}) + if err != nil { + t.Fatal(err) + } + + schema := m.API.Schema(ctx) + if len(schema) != 0 { + t.Fatal("Schema should include no index") + } + + // Try to delete non-existing index + _, err = gh.DeleteIndex(ctx, &pb.DeleteIndexRequest{Name: "doesnotexist"}) + errStatus, _ := status.FromError(err) + if errStatus.Code() != codes.NotFound { + t.Fatalf("Error code should be codes.NotFound, but is %v", errStatus.Code()) + } + }) +} + func setUpTestQuerySQLUnary(ctx context.Context, t *testing.T) (gh *server.GRPCHandler, tearDownFunc func()) { t.Helper() diff --git a/server/pg.go b/server/pg.go index 43fb22454..8938cc44b 100644 --- a/server/pg.go +++ b/server/pg.go @@ -252,27 +252,38 @@ func pgWriteExtractedTable(w pg.QueryResultWriter, tbl pilosa.ExtractedTable) er return nil } -func pgWriteGroupCount(w pg.QueryResultWriter, counts []pilosa.GroupCount) error { - if len(counts) == 0 { +func pgWriteGroupCount(w pg.QueryResultWriter, counts *pilosa.GroupCounts) error { + groups := counts.Groups() + if len(groups) == 0 { // Not enough information is available to construct the header. // This is a significant flaw in the data type. return nil } + expectedLen := len(groups[0].Group) + 1 - headers := make([]pg.ColumnInfo, len(counts[0].Group)+2) - for i, g := range counts[0].Group { + agg := counts.AggregateColumn() + if agg != "" { + expectedLen++ + } + + headers := make([]pg.ColumnInfo, expectedLen) + for i, g := range groups[0].Group { headers[i] = pg.ColumnInfo{ Name: g.Field, Type: pg.TypeCharoid, } } - headers[len(headers)-2] = pg.ColumnInfo{ + next := len(groups[0].Group) + headers[next] = pg.ColumnInfo{ Name: "count", Type: pg.TypeCharoid, } - headers[len(headers)-1] = pg.ColumnInfo{ - Name: "sum", - Type: pg.TypeCharoid, + if agg != "" { + next++ + headers[next] = pg.ColumnInfo{ + Name: agg, + Type: pg.TypeCharoid, + } } err := w.WriteHeader(headers...) if err != nil { @@ -280,8 +291,10 @@ func pgWriteGroupCount(w pg.QueryResultWriter, counts []pilosa.GroupCount) error } vals := make([]string, len(headers)) - for _, gc := range counts { - for j, g := range gc.Group { + for _, gc := range groups { + var j int + var g pilosa.FieldRow + for j, g = range gc.Group { var v string switch { case g.Value != nil: @@ -293,8 +306,12 @@ func pgWriteGroupCount(w pg.QueryResultWriter, counts []pilosa.GroupCount) error } vals[j] = v } - vals[len(vals)-2] = strconv.FormatUint(gc.Count, 10) - vals[len(vals)-1] = strconv.FormatInt(gc.Sum, 10) + j++ + vals[j] = strconv.FormatUint(gc.Count, 10) + if agg != "" { + j++ + vals[j] = strconv.FormatInt(gc.Agg, 10) + } err := w.WriteRowText(vals...) if err != nil { @@ -370,6 +387,9 @@ func pgWriteResult(w pg.QueryResultWriter, result interface{}) error { case pilosa.ExtractedTable: return pgWriteExtractedTable(w, result) case []pilosa.GroupCount: + gc := pilosa.NewGroupCounts("", result...) + return pgWriteGroupCount(w, gc) + case *pilosa.GroupCounts: return pgWriteGroupCount(w, result) case pb.ToRowser: // we should avoid protobuf where we can... return pgWriteRowser(w, result) diff --git a/server/pg_test.go b/server/pg_test.go index 86d743ade..59768dfe9 100644 --- a/server/pg_test.go +++ b/server/pg_test.go @@ -169,11 +169,10 @@ func TestPostgresHandler(t *testing.T) { Columns: []pg.ColumnInfo{ {Name: "set", Type: pg.TypeCharoid}, {Name: "count", Type: pg.TypeCharoid}, - {Name: "sum", Type: pg.TypeCharoid}, }, Data: [][]string{ - {"4", "3", "0"}, - {"5", "3", "0"}, + {"4", "3"}, + {"5", "3"}, }, }, }, diff --git a/server/server_test.go b/server/server_test.go index 46e79daac..78fe7d24e 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -298,7 +298,7 @@ func TestMain_GroupBy(t *testing.T) { if res, err := m.QueryProtobuf("i", `GroupBy(Rows(generalk), Rows(subk))`); err != nil { t.Fatal(err) } else { - test.CheckGroupBy(t, expected, res.Results[0].([]pilosa.GroupCount)) + test.CheckGroupBy(t, expected, res.Results[0].(*pilosa.GroupCounts).Groups()) } } diff --git a/snapshotqueue.go b/snapshotqueue.go index 4fc3e4f79..e4fa67db5 100644 --- a/snapshotqueue.go +++ b/snapshotqueue.go @@ -445,7 +445,7 @@ func (s *prioritySnapshotQueueScanner) ProcessFragment(f *fragment) error { s.hits++ select { case s.queue <- snapshotRequest{frag: f, when: time.Now()}: - s.sq.logger.Debugf("found fragment needing snapshot: %s\n", f.path) + s.sq.logger.Debugf("found fragment needing snapshot: %s\n", f.path()) case <-s.ctx.Done(): return io.EOF } diff --git a/sql/select.go b/sql/select.go index 9ac7d5011..b54eee0b4 100644 --- a/sql/select.go +++ b/sql/select.go @@ -90,7 +90,7 @@ func (s *SelectHandler) execMappingResult(ctx context.Context, mr *MappingResult case pproto.ToRowser: result = res case []pilosa.GroupCount: - result = pilosa.GroupCounts(res) + result = pilosa.NewGroupCounts("", res...) case uint64: result = pproto.ConstRowser{ { diff --git a/test/cluster.go b/test/cluster.go index 6bdfc8153..006729ced 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -63,7 +63,7 @@ func (c *Cluster) QueryHTTP(t testing.TB, index, query string) (string, error) { if len(c.Nodes) == 0 { t.Fatal("must have at least one node in cluster to query") } - + return c.Nodes[0].Query(t, index, "", query) } @@ -189,6 +189,30 @@ func (c *Cluster) ImportIntKey(t testing.TB, index, field string, pairs []IntKey } } +type IntID struct { + Val int64 + ID uint64 +} + +// ImportIntID imports data into an int field in an unkeyed index. +func (c *Cluster) ImportIntID(t testing.TB, index, field string, pairs []IntID) { + t.Helper() + importRequest := &pilosa.ImportValueRequest{ + Index: index, + Field: field, + Shard: math.MaxUint64, + ColumnIDs: make([]uint64, len(pairs)), + Values: make([]int64, len(pairs)), + } + for i, pair := range pairs { + importRequest.Values[i] = pair.Val + importRequest.ColumnIDs[i] = pair.ID + } + if err := c.Nodes[0].API.ImportValue(context.Background(), nil, importRequest); err != nil { + t.Fatalf("importing IntID data: %v", err) + } +} + // KeyID represents a key and an ID for importing data into an index // and field where one uses string keys and the other does not. type KeyID struct {