From b435e9d7936a64da87be221b0542518688320f3a Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 17 Dec 2020 12:00:46 -0600 Subject: [PATCH 1/4] Add Distinct call as GroupBy aggregate --- executor.go | 45 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/executor.go b/executor.go index 1910d42e8..a12b34f82 100644 --- a/executor.go +++ b/executor.go @@ -468,7 +468,11 @@ func (e *executor) handlePreCallChildren(ctx context.Context, qcx *Qcx, index st return err } } - for _, val := range c.Args { + for key, val := range c.Args { + // Do not precompute GroupBy aggregates + if key == "aggregate" { + continue + } // Handle Call() operations which exist inside named arguments, too. if call, ok := val.(*pql.Call); ok { if err := ctx.Err(); err != nil { @@ -2703,6 +2707,43 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c results = results[:limit] } } + aggregate, _, err := c.CallArg("aggregate") + if err == nil && aggregate != nil && aggregate.Name == "Count" && len(aggregate.Children) > 0 && aggregate.Children[0].Name == "Distinct" { + for n, gc := range results { + intersectRows := make([]*pql.Call, 0, len(gc.Group)) + for _, fr := range gc.Group { + intersectRows = append(intersectRows, &pql.Call{Name: "Row", Args: map[string]interface{}{fr.Field: fr.RowID}}) + } + + countDistinctIntersect := &pql.Call{ + Name: "Count", + Children: []*pql.Call{ + &pql.Call{ + Name: "Distinct", + Children: []*pql.Call{ + &pql.Call{ + Name: "Intersect", + Children: intersectRows, + }, + }, + Args: aggregate.Children[0].Args, + Type: pql.PrecallGlobal, + }, + }, + } + + err = e.handlePreCallChildren(ctx, qcx, index, countDistinctIntersect, shards, opt) + if err != nil { + return nil, err + } + + aggregateCount, err := e.executeCount(ctx, qcx, index, countDistinctIntersect, shards, opt) + if err != nil { + return nil, err + } + results[n].Sum = int64(aggregateCount) + } + } return results, nil } @@ -7193,7 +7234,7 @@ func (gbi *groupByIterator) Next(ctx context.Context) (ret GroupCount, done bool if gbi.done { return ret, true, nil } - if gbi.aggregate == nil { + if gbi.aggregate == nil || gbi.aggregate.Name == "Count" { if len(gbi.rows) == 1 { ret.Count = gbi.rows[len(gbi.rows)-1].row.Count() } else { From f099a902643a20325de01deb107f2b5dcea84bb2 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 23 Dec 2020 17:44:15 -0600 Subject: [PATCH 2/4] Modify aggregate distinct logic and add tests Use execute instead of directly using executeCount Address code review feedback (add additional filters if provided) Add basic tests --- executor.go | 17 ++++++++++------- executor_test.go | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/executor.go b/executor.go index a12b34f82..aa5d545a4 100644 --- a/executor.go +++ b/executor.go @@ -2714,6 +2714,14 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c for _, fr := range gc.Group { intersectRows = append(intersectRows, &pql.Call{Name: "Row", Args: map[string]interface{}{fr.Field: fr.RowID}}) } + // apply any filter, if present + if filter != nil { + intersectRows = append(intersectRows, filter) + } + // also intersect with any children of Distinct + if len(aggregate.Children[0].Children) > 0 { + intersectRows = append(intersectRows, aggregate.Children[0].Children[0]) + } countDistinctIntersect := &pql.Call{ Name: "Count", @@ -2732,16 +2740,11 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c }, } - err = e.handlePreCallChildren(ctx, qcx, index, countDistinctIntersect, shards, opt) + aggregateCount, err := e.execute(ctx, qcx, index, &pql.Query{Calls: []*pql.Call{countDistinctIntersect}}, []uint64{}, opt) if err != nil { return nil, err } - - aggregateCount, err := e.executeCount(ctx, qcx, index, countDistinctIntersect, shards, opt) - if err != nil { - return nil, err - } - results[n].Sum = int64(aggregateCount) + results[n].Sum = int64(aggregateCount[0].(uint64)) } } return results, nil diff --git a/executor_test.go b/executor_test.go index a20d60495..54e30447d 100644 --- a/executor_test.go +++ b/executor_test.go @@ -5517,6 +5517,8 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { t.Fatal(err) } else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1, v=100)`}); err != nil { t.Fatal(err) + } else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1500000, v=100)`}); err != nil { + t.Fatal(err) } t.Run("No Field List Arguments", func(t *testing.T) { @@ -5581,6 +5583,39 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { test.CheckGroupBy(t, expected, results) }) + t.Run("AggregateCountDistinct", func(t *testing.T) { + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Count: 3, Sum: 2}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Count: 1, Sum: 1}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}, {Field: "sub", RowID: 110}}, Count: 1, Sum: 0}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 12}, {Field: "sub", RowID: 110}}, Count: 1, Sum: 0}, + } + + results := c.Query(t, "i", `GroupBy(Rows(general), Rows(sub), aggregate=Count(Distinct(field=v)))`).Results[0].([]pilosa.GroupCount) + test.CheckGroupBy(t, expected, results) + }) + + t.Run("AggregateCountDistinctFilter", func(t *testing.T) { + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Count: 1, Sum: 1}, + } + + results := c.Query(t, "i", `GroupBy(Rows(general), Rows(sub), filter=Row(v > 10), aggregate=Count(Distinct(field=v)))`).Results[0].([]pilosa.GroupCount) + test.CheckGroupBy(t, expected, results) + }) + + t.Run("AggregateCountDistinctFilterDistinct", func(t *testing.T) { + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Count: 3, Sum: 1}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Count: 1, Sum: 0}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}, {Field: "sub", RowID: 110}}, Count: 1, Sum: 0}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 12}, {Field: "sub", RowID: 110}}, Count: 1, Sum: 0}, + } + + results := c.Query(t, "i", `GroupBy(Rows(general), Rows(sub), aggregate=Count(Distinct(Row(v > 10), field=v)))`).Results[0].([]pilosa.GroupCount) + test.CheckGroupBy(t, expected, results) + }) + t.Run("check field offset no limit", func(t *testing.T) { expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Count: 2}, From fd7417a49bf5adc5c1c2c5670cbaf84a94dc1a05 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 23 Dec 2020 17:54:31 -0600 Subject: [PATCH 3/4] Use shardwidth instead of hardcoded value --- executor_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor_test.go b/executor_test.go index 54e30447d..0e34ca1a6 100644 --- a/executor_test.go +++ b/executor_test.go @@ -5517,7 +5517,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { t.Fatal(err) } else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1, v=100)`}); err != nil { t.Fatal(err) - } else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1500000, v=100)`}); err != nil { + } else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf(`Set(%d, v=100)`, ShardWidth+10)}); err != nil { // Workaround distinct bug where v must be set in every shard t.Fatal(err) } From 16fd6a7eddba2da3d2a0eb19fa57ed15648c7058 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 29 Dec 2020 13:27:24 -0600 Subject: [PATCH 4/4] add tests for GroupBy(Distinct), fix various problems We execute the aggregate Distinct calls after the GroupBy is complete, and we need these to act like non-remote calls in that they forward to all nodes, but like remote calls in that they bypass key translation. Added a "PreTranslated" flag to the QueryRequest to achieve this. Discovered an issue where a nil *Row in EmbeddedData would cause a panic in the protobuf serialization. Changed the encoding code we control to never pass a nil *Row. Got fed up with lack of context on errors and added wrapping to all calls under executor.executeCall as well as a few other places. Handled a situation where not having data on a shard for a particular field could cause a query to error instead of just treating that fragment as being empty. (see the switch in executeDistinctShardSet) Stopped GroupBy from executing the Count(Distinct) aggregate on Remote calls. Fixed a longstanding issue where errors retrieved from remote query calls had a garbage character at the front due to treating a protobuf payload as an error message instead of decoding it. (see http/client.go) --- api.go | 4 +- cluster.go | 2 +- encoding/proto/proto.go | 4 +- executor.go | 103 ++++++++++------ executor_test.go | 85 +++++++++++-- handler.go | 4 + http/client.go | 8 +- internal/public.pb.go | 256 +++++++++++++++++++++++----------------- internal/public.proto | 1 + rrtx.go | 4 +- 10 files changed, 315 insertions(+), 156 deletions(-) diff --git a/api.go b/api.go index 177f57c94..9140153a1 100644 --- a/api.go +++ b/api.go @@ -163,13 +163,15 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er if !req.Remote { defer api.tracker.Finish(api.tracker.Start(req.Query, api.server.nodeID, req.Index, start)) } + // TODO can we get rid of exec options and pass the QueryRequest directly to executor? execOpts := &execOptions{ Remote: req.Remote, Profile: req.Profile, ExcludeRowAttrs: req.ExcludeRowAttrs, // NOTE: Kept for Pilosa 1.x compat. ExcludeColumns: req.ExcludeColumns, // NOTE: Kept for Pilosa 1.x compat. ColumnAttrs: req.ColumnAttrs, // NOTE: Kept for Pilosa 1.x compat. - EmbeddedData: req.EmbeddedData, // precomputed values that needed to be passed with the request + PreTranslated: req.PreTranslated, + EmbeddedData: req.EmbeddedData, // precomputed values that needed to be passed with the request } resp, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts) if err != nil { diff --git a/cluster.go b/cluster.go index dbf2ad67b..3a9976e72 100644 --- a/cluster.go +++ b/cluster.go @@ -1047,7 +1047,7 @@ func (c *cluster) ShardNodes(index string, shard uint64) []*Node { return c.shardNodes(index, shard) } -// shardNodes returns a list of nodes that own a fragment. unprotected +// shardNodes returns a list of nodes that own a shard. unprotected func (c *cluster) shardNodes(index string, shard uint64) []*Node { return c.partitionNodes(c.shardToShardPartition(index, shard)) } diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index ec3e3c950..10a2f05a2 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -490,6 +490,7 @@ func (s Serializer) encodeQueryRequest(m *pilosa.QueryRequest) *internal.QueryRe Remote: m.Remote, ExcludeRowAttrs: m.ExcludeRowAttrs, ExcludeColumns: m.ExcludeColumns, + PreTranslated: m.PreTranslated, EmbeddedData: make([]*internal.Row, len(m.EmbeddedData)), } for i := range m.EmbeddedData { @@ -1210,6 +1211,7 @@ func (s Serializer) decodeQueryRequest(pb *internal.QueryRequest, m *pilosa.Quer m.ExcludeRowAttrs = pb.ExcludeRowAttrs m.ExcludeColumns = pb.ExcludeColumns m.EmbeddedData = make([]*pilosa.Row, len(pb.EmbeddedData)) + m.PreTranslated = pb.PreTranslated for i := range pb.EmbeddedData { m.EmbeddedData[i] = s.decodeRow(pb.EmbeddedData[i]) } @@ -1696,7 +1698,7 @@ func (s Serializer) encodeSignedRow(r pilosa.SignedRow) *internal.SignedRow { func (s Serializer) encodeRow(r *pilosa.Row) *internal.Row { if r == nil { - return nil + return &internal.Row{} // Generated proto code doesn't like a nil Row. } ir := &internal.Row{ diff --git a/executor.go b/executor.go index aa5d545a4..f4e5bdd45 100644 --- a/executor.go +++ b/executor.go @@ -532,7 +532,7 @@ func (e *executor) execute(ctx context.Context, qcx *Qcx, index string, q *pql.Q } // Apply call translation. - if !opt.Remote { + if !opt.Remote && !opt.PreTranslated { translated, err := e.translateCall(call, index, colTranslations, rowTranslations) if err != nil { return nil, errors.Wrap(err, "translating call") @@ -676,77 +676,101 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p switch c.Name { case "Sum": statFn() - return e.executeSum(ctx, qcx, index, c, shards, opt) + res, err := e.executeSum(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeSum %v", shards) case "Min": statFn() - return e.executeMin(ctx, qcx, index, c, shards, opt) + res, err := e.executeMin(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeMin %v", shards) case "Max": statFn() - return e.executeMax(ctx, qcx, index, c, shards, opt) + res, err := e.executeMax(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeMax %v", shards) case "MinRow": statFn() - return e.executeMinRow(ctx, qcx, index, c, shards, opt) + res, err := e.executeMinRow(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeMinRow %v", shards) case "MaxRow": statFn() - return e.executeMaxRow(ctx, qcx, index, c, shards, opt) + res, err := e.executeMaxRow(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeMaxRow %v", shards) case "Clear": statFn() - return e.executeClearBit(ctx, qcx, index, c, opt) + res, err := e.executeClearBit(ctx, qcx, index, c, opt) + return res, errors.Wrapf(err, "executeClearBit %v", shards) case "ClearRow": statFn() - return e.executeClearRow(ctx, qcx, index, c, shards, opt) + res, err := e.executeClearRow(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeClearRow %v", shards) case "Distinct": statFn() - return e.executeDistinct(ctx, qcx, index, c, shards, opt) + res, err := e.executeDistinct(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeDistinct %v", shards) case "Store": statFn() - return e.executeSetRow(ctx, qcx, index, c, shards, opt) + res, err := e.executeSetRow(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeSetRow %v", shards) case "Count": statFn() - return e.executeCount(ctx, qcx, index, c, shards, opt) + res, err := e.executeCount(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeCount %v", shards) case "Set": statFn() - return e.executeSet(ctx, qcx, index, c, opt) + res, err := e.executeSet(ctx, qcx, index, c, opt) + return res, errors.Wrapf(err, "executeSet %v", shards) case "SetRowAttrs": statFn() - return nil, e.executeSetRowAttrs(ctx, qcx, index, c, opt) + return nil, errors.Wrap(e.executeSetRowAttrs(ctx, qcx, index, c, opt), "executeSetRowAttrs") case "SetColumnAttrs": statFn() - return nil, e.executeSetColumnAttrs(ctx, qcx, index, c, opt) + return nil, errors.Wrap(e.executeSetColumnAttrs(ctx, qcx, index, c, opt), "executeSetColumnAttrs") case "TopK": statFn() - return e.executeTopK(ctx, qcx, index, c, shards, opt) + res, err := e.executeTopK(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeTopK %v", shards) case "TopN": statFn() - return e.executeTopN(ctx, qcx, index, c, shards, opt) + res, err := e.executeTopN(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeTopN %v", shards) case "Rows": statFn() - return e.executeRows(ctx, qcx, index, c, shards, opt) + res, err := e.executeRows(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeRows %v", shards) case "Extract": statFn() - return e.executeExtract(ctx, qcx, index, c, shards, opt) + res, err := e.executeExtract(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeExtract %v", shards) case "GroupBy": statFn() - return e.executeGroupBy(ctx, qcx, index, c, shards, opt) + res, err := e.executeGroupBy(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeGroupBy %v", shards) case "Options": statFn() - return e.executeOptionsCall(ctx, qcx, index, c, shards, opt) + res, err := e.executeOptionsCall(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeOptionsCall %v", shards) case "IncludesColumn": - return e.executeIncludesColumnCall(ctx, qcx, index, c, shards, opt) + res, err := e.executeIncludesColumnCall(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeIncludesColumnCall %v", shards) case "FieldValue": statFn() - return e.executeFieldValueCall(ctx, qcx, index, c, shards, opt) + res, err := e.executeFieldValueCall(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeFieldValueCall %v", shards) case "Precomputed": - return e.executePrecomputedCall(ctx, qcx, index, c, shards, opt) + res, err := e.executePrecomputedCall(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executePrecomputedCall %v", shards) case "UnionRows": - return e.executeUnionRows(ctx, qcx, index, c, shards, opt) + res, err := e.executeUnionRows(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeUnionRows %v", shards) case "ConstRow": - return e.executeConstRow(ctx, index, c) + res, err := e.executeConstRow(ctx, index, c) + return res, errors.Wrapf(err, "executeConstRow %v", shards) case "Limit": - return e.executeLimitCall(ctx, qcx, index, c, shards, opt) + res, err := e.executeLimitCall(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeLimitCall %v", shards) default: // e.g. "Row", "Union", "Intersect" or anything that returns a bitmap. statFn() - return e.executeBitmapCall(ctx, qcx, index, c, shards, opt) + res, err := e.executeBitmapCall(ctx, qcx, index, c, shards, opt) + return res, errors.Wrapf(err, "executeBitmapCall %v", shards) } } @@ -1111,6 +1135,11 @@ func (e *executor) executeDistinct(ctx context.Context, qcx *Qcx, index string, case SignedRow: return other.union(v.(SignedRow)) case *Row: + if other == nil { + return v + } else if v.(*Row) == nil { + return other + } return other.Union(v.(*Row)) case nil: return v @@ -1121,13 +1150,12 @@ func (e *executor) executeDistinct(ctx context.Context, qcx *Qcx, index string, result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { - return nil, err + return nil, errors.Wrap(err, "mapReduce") } if other, ok := result.(SignedRow); ok { other.field = field } - return result, nil } @@ -1470,10 +1498,15 @@ func executeDistinctShardSet(ctx context.Context, qcx *Qcx, idx *Index, fieldNam defer finisher(&err0) fragData, _, err := tx.ContainerIterator(index, fieldName, "standard", shard, 0) - if err != nil { + switch errors.Cause(err) { + case ViewNotFound, FragmentNotFound: + return nil, nil + case nil: + default: return nil, errors.Wrap(err, "getting fragment data") } defer fragData.Close() + // We can't grab the containers "for each row" from the set-type field, // because we don't know how many rows there are, and some of them // might be empty, so really, we're going to iterate through the @@ -2666,7 +2699,7 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c // Get full result set. other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "mapReduce shards: %v", shards) } results, _ := other.([]GroupCount) @@ -2708,7 +2741,7 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c } } aggregate, _, err := c.CallArg("aggregate") - if err == nil && aggregate != nil && aggregate.Name == "Count" && len(aggregate.Children) > 0 && aggregate.Children[0].Name == "Distinct" { + if err == nil && aggregate != nil && aggregate.Name == "Count" && len(aggregate.Children) > 0 && aggregate.Children[0].Name == "Distinct" && !opt.Remote { for n, gc := range results { intersectRows := make([]*pql.Call, 0, len(gc.Group)) for _, fr := range gc.Group { @@ -2740,6 +2773,7 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c }, } + opt.PreTranslated = true aggregateCount, err := e.execute(ctx, qcx, index, &pql.Query{Calls: []*pql.Call{countDistinctIntersect}}, []uint64{}, opt) if err != nil { return nil, err @@ -5146,7 +5180,7 @@ loop: continue loop } } - return nil, errShardUnavailable + return nil, errors.Wrapf(errShardUnavailable, "%s:%d:%v:%v", index, shard, shards, nodes) } return m, nil } @@ -5269,7 +5303,7 @@ func (e *executor) mapper(ctx context.Context, cancel context.CancelFunc, ch cha // Group shards together by nodes. m, err := e.shardsByNode(nodes, index, shards) if err != nil { - return errors.Wrap(err, "shards by node") + return errors.Wrapf(err, "shards by node %v", shards) } // Execute each node in a separate goroutine. @@ -6672,6 +6706,7 @@ type execOptions struct { ExcludeRowAttrs bool ExcludeColumns bool ColumnAttrs bool + PreTranslated bool EmbeddedData []*Row } diff --git a/executor_test.go b/executor_test.go index 0e34ca1a6..5d18bdaac 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1343,7 +1343,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Fatal(err) } else if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(0, 100)); err != nil { t.Fatal(err) - } else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err == nil || err.Error() != `executing: finding top results: cannot compute TopN() on integer field: "f"` { + } else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err == nil || !strings.Contains(err.Error(), `finding top results: cannot compute TopN() on integer field: "f"`) { t.Fatalf("unexpected error: %v", err) } }) @@ -1362,7 +1362,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { Set(0, f=1) `}); err != nil { t.Fatal(err) - } else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err == nil || err.Error() != `executing: finding top results: cannot compute TopN(), field has no cache: "f"` { + } else if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err == nil || !strings.Contains(err.Error(), `finding top results: cannot compute TopN(), field has no cache: "f"`) { t.Fatalf("unexpected error: %v", err) } }) @@ -6787,7 +6787,15 @@ func TestMissingKeyRegression(t *testing.T) { // (single and multi-node clusters, different endpoints for the // queries (HTTP, GRPC, Postgres), etc.). func TestVariousQueries(t *testing.T) { - c := test.MustRunCluster(t, 3) + for _, clusterSize := range []int{1, 3, 4, 7} { + t.Run(fmt.Sprintf("%d-node", clusterSize), func(t *testing.T) { + testVariousQueries(t, clusterSize) + }) + } +} + +func testVariousQueries(t *testing.T, clusterSize int) { + c := test.MustRunCluster(t, clusterSize) defer c.Close() // Create and populate "likenums" similar to "likes", but without keys on the field. @@ -6803,7 +6811,9 @@ func TestVariousQueries(t *testing.T) { {ID: 7, Key: "userB"}, {ID: 7, Key: "userC"}, {ID: 7, Key: "userD"}, - {ID: 7, Key: "userE"}, + // we intentionally leave user E out because then there is no + // data for userE's shard for this field, which triggered a + // "fragment not found" problem {ID: 7, Key: "userF"}, }) @@ -6834,6 +6844,16 @@ func TestVariousQueries(t *testing.T) { {Val: 0, Key: "userE"}, }) + c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "zip_code", pilosa.OptFieldTypeInt(0, 100000)) + c.ImportIntKey(t, "users", "zip_code", []test.IntKey{ + {Val: 78739, Key: "userA"}, + {Val: 78739, Key: "userB"}, + {Val: 19707, Key: "userC"}, + {Val: 19707, Key: "userD"}, + {Val: 86753, Key: "userE"}, + {Val: 78739, Key: "userG"}, + }) + tests := []struct { query string qrVerifier func(t *testing.T, resp pilosa.QueryResponse) @@ -6842,11 +6862,11 @@ func TestVariousQueries(t *testing.T) { { query: "Count(All())", qrVerifier: func(t *testing.T, resp pilosa.QueryResponse) { - if resp.Results[0].(uint64) != 6 { - t.Errorf("expected 6, got %+v", resp.Results[0]) + if resp.Results[0].(uint64) != 7 { + t.Errorf("expected 7, got %+v", resp.Results[0]) } }, - csvVerifier: "6\n", + csvVerifier: "7\n", }, { query: "Count(Distinct(field=likenums))", @@ -6977,6 +6997,57 @@ func TestVariousQueries(t *testing.T) { }, csvVerifier: "molecula\npilosa\npangolin\nzebra\ntoucan\ndog\nicecream\n", }, + { + query: "GroupBy(Rows(field=likes))", + csvVerifier: `molecula,1,0 +pilosa,1,0 +pangolin,1,0 +zebra,1,0 +toucan,1,0 +dog,1,0 +icecream,6,0 +`, + }, + { + query: "GroupBy(Rows(field=likes), filter=Row(affinity>-7))", + csvVerifier: `molecula,1,0 +pangolin,1,0 +zebra,1,0 +toucan,1,0 +icecream,4,0 +`, + }, + { + query: "GroupBy(Rows(field=likes), aggregate=Count(Distinct(field=zip_code)))", + csvVerifier: `molecula,1,1 +pilosa,1,1 +pangolin,1,1 +zebra,1,1 +toucan,1,1 +dog,1,0 +icecream,6,3 +`, + }, + { + query: "GroupBy(Rows(field=likes), filter=Row(affinity>-11), aggregate=Count(Distinct(field=zip_code)))", + csvVerifier: `molecula,1,1 +pilosa,1,1 +pangolin,1,1 +zebra,1,1 +toucan,1,1 +icecream,5,3 +`, + }, + { + query: "GroupBy(Rows(field=likes), filter=Row(affinity>-11), aggregate=Count(Distinct(Row(affinity>-7), field=zip_code)))", + csvVerifier: `molecula,1,1 +pilosa,1,0 +pangolin,1,1 +zebra,1,1 +toucan,1,1 +icecream,5,3 +`, + }, } for i, tst := range tests { diff --git a/handler.go b/handler.go index fd937e01d..d8e370d56 100644 --- a/handler.go +++ b/handler.go @@ -46,6 +46,10 @@ type QueryRequest struct { // If false, this request is on the originating node. Remote bool + // Query has already been translated. This is only used if Remote + // is false, Remote=true implies this. + PreTranslated bool + // Should we profile this query? Profile bool diff --git a/http/client.go b/http/client.go index 0f3cab4ec..4f37d7a36 100644 --- a/http/client.go +++ b/http/client.go @@ -286,7 +286,6 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index s } else if queryRequest.Query == "" { return nil, pilosa.ErrQueryRequired } - buf, err := c.serializer.Marshal(queryRequest) if err != nil { return nil, errors.Wrap(err, "marshaling queryRequest") @@ -308,7 +307,7 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pilosa.URI, index s // Execute request against the host. resp, err := c.executeRequest(req.WithContext(ctx)) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "'%s', shards %v", queryRequest.Query, queryRequest.Shards) } defer resp.Body.Close() @@ -1714,12 +1713,15 @@ func (c *InternalClient) executeRequest(req *http.Request, opts ...executeReques var msg string // try to decode a JSON response var sr successResponse + qr := &pilosa.QueryResponse{} if err = json.Unmarshal(buf, &sr); err == nil { msg = sr.Error.Error() + } else if err := c.serializer.Unmarshal(buf, qr); err == nil { + msg = qr.Err.Error() } else { msg = string(buf) } - return resp, errors.Errorf("server error %s: '%s'", resp.Status, msg) + return resp, errors.Errorf("against %s %s: '%s'", req.URL.String(), resp.Status, msg) } return resp, nil } diff --git a/internal/public.pb.go b/internal/public.pb.go index b334a48e3..30eddfcb8 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -1502,6 +1502,7 @@ type QueryRequest struct { ExcludeRowAttrs bool `protobuf:"varint,6,opt,name=ExcludeRowAttrs,proto3" json:"ExcludeRowAttrs,omitempty"` ExcludeColumns bool `protobuf:"varint,7,opt,name=ExcludeColumns,proto3" json:"ExcludeColumns,omitempty"` EmbeddedData []*Row `protobuf:"bytes,8,rep,name=EmbeddedData,proto3" json:"EmbeddedData,omitempty"` + PreTranslated bool `protobuf:"varint,9,opt,name=PreTranslated,proto3" json:"PreTranslated,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1589,6 +1590,13 @@ func (m *QueryRequest) GetEmbeddedData() []*Row { return nil } +func (m *QueryRequest) GetPreTranslated() bool { + if m != nil { + return m.PreTranslated + } + return false +} + type QueryResponse struct { Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` Results []*QueryResult `protobuf:"bytes,2,rep,name=Results,proto3" json:"Results,omitempty"` @@ -2683,113 +2691,114 @@ func init() { func init() { proto.RegisterFile("public.proto", fileDescriptor_413a91106d7bcce8) } var fileDescriptor_413a91106d7bcce8 = []byte{ - // 1694 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x4f, 0x73, 0x1b, 0x4b, - 0x11, 0xf7, 0x6a, 0x57, 0x96, 0xd4, 0x92, 0x1d, 0xbf, 0x89, 0xde, 0x63, 0x2b, 0x38, 0x7e, 0x62, - 0xcb, 0xf0, 0x04, 0x45, 0x39, 0xe5, 0x90, 0xa4, 0x72, 0x01, 0x62, 0x47, 0x0e, 0xde, 0x0a, 0x36, - 0x61, 0xe4, 0x72, 0x6e, 0x54, 0xad, 0xa5, 0xc1, 0xd9, 0x62, 0xa5, 0x15, 0xab, 0x55, 0x64, 0x5f, - 0xa8, 0xe2, 0x33, 0xe4, 0xc2, 0x8d, 0x2b, 0x27, 0x3e, 0x04, 0x17, 0x38, 0x72, 0xa4, 0x8a, 0x0b, - 0x15, 0xf8, 0x16, 0x5c, 0xa8, 0xee, 0xd9, 0xd9, 0x99, 0x5d, 0xad, 0x1d, 0x57, 0x8a, 0xdb, 0xf4, - 0x9f, 0xe9, 0xe9, 0xfe, 0x75, 0x4f, 0x4f, 0xef, 0x42, 0x67, 0xb6, 0xb8, 0x88, 0xc2, 0xd1, 0xde, - 0x2c, 0x89, 0xd3, 0x98, 0x35, 0xc3, 0x69, 0x2a, 0x92, 0x69, 0x10, 0x79, 0x7f, 0xb4, 0xc0, 0xe6, - 0xf1, 0x92, 0xb9, 0xd0, 0x78, 0x19, 0x47, 0x8b, 0xc9, 0x74, 0xee, 0x5a, 0x3d, 0xbb, 0xef, 0x70, - 0x45, 0x32, 0x06, 0xce, 0x6b, 0x71, 0x3d, 0x77, 0xed, 0x9e, 0xdd, 0x6f, 0x71, 0x5a, 0xb3, 0x5d, - 0xa8, 0x1f, 0xa4, 0x69, 0x32, 0x77, 0x6b, 0x3d, 0xbb, 0xdf, 0x7e, 0xbc, 0xb9, 0xa7, 0xec, 0xed, - 0x21, 0x9b, 0x4b, 0x21, 0xda, 0xe4, 0x71, 0x90, 0x84, 0xd3, 0x4b, 0xd7, 0xe9, 0x59, 0xfd, 0x0e, - 0x57, 0x24, 0xeb, 0x42, 0xdd, 0x9f, 0x8e, 0xc5, 0x95, 0x5b, 0xef, 0x59, 0xfd, 0x16, 0x97, 0x04, - 0x72, 0x5f, 0x85, 0x22, 0x1a, 0xbb, 0xeb, 0x92, 0x4b, 0x84, 0xb7, 0x07, 0x2d, 0x1e, 0x2f, 0x4f, - 0x82, 0x34, 0x09, 0xaf, 0xd8, 0x77, 0xc0, 0xe1, 0xf1, 0x52, 0xfa, 0xd8, 0x7e, 0xbc, 0xa1, 0xcf, - 0xe5, 0xf1, 0x92, 0x93, 0xc8, 0x3b, 0x81, 0xd6, 0x30, 0xbc, 0x9c, 0x8a, 0x31, 0x86, 0xf5, 0x35, - 0xd8, 0x6f, 0x62, 0x54, 0xb7, 0x56, 0xd5, 0x51, 0x82, 0x0a, 0xa7, 0xe2, 0xd2, 0xad, 0x55, 0x2a, - 0x9c, 0x8a, 0x4b, 0xef, 0x39, 0x6c, 0xf2, 0x78, 0xe9, 0x8f, 0xc5, 0x34, 0x0d, 0x7f, 0x1d, 0x8a, - 0x84, 0x00, 0xc9, 0x7d, 0x70, 0xe4, 0xa1, 0x39, 0x48, 0x35, 0x0d, 0x92, 0xf7, 0x00, 0xd6, 0xfd, - 0xc1, 0xcf, 0xc3, 0x79, 0xca, 0xb6, 0xc0, 0xf6, 0x07, 0x6a, 0x03, 0x2e, 0x3d, 0x1f, 0xbe, 0x38, - 0xba, 0x4a, 0x93, 0x60, 0x94, 0x8a, 0xb1, 0x3f, 0x90, 0x50, 0xb3, 0x4d, 0xa8, 0xf9, 0x03, 0xf2, - 0xd5, 0xe1, 0x35, 0x7f, 0xc0, 0x76, 0xc1, 0x39, 0x0f, 0x22, 0x05, 0xf2, 0x96, 0x76, 0x4e, 0x9a, - 0xe5, 0x24, 0xf5, 0x2e, 0x0a, 0xa6, 0x32, 0x9c, 0xbe, 0x82, 0x75, 0x42, 0x4f, 0x1e, 0xda, 0xe2, - 0x19, 0xc5, 0x9e, 0xea, 0x34, 0x4b, 0xab, 0xdf, 0xd6, 0x56, 0x57, 0x1c, 0xca, 0x6b, 0xc0, 0x7b, - 0x08, 0x8d, 0xd7, 0xe2, 0x9a, 0x62, 0x51, 0x91, 0x5a, 0x46, 0xa4, 0xff, 0xb4, 0xe0, 0x7e, 0xbe, - 0xfb, 0x2c, 0xb8, 0x88, 0xc4, 0x79, 0x10, 0x2d, 0x04, 0xdb, 0x55, 0x71, 0x5b, 0x55, 0xfe, 0x1f, - 0xaf, 0x11, 0x16, 0xec, 0x9b, 0x1c, 0x3b, 0x54, 0xfb, 0x42, 0xab, 0x65, 0x47, 0x1e, 0xaf, 0x65, - 0x55, 0xb7, 0x0d, 0xcd, 0xc3, 0xa1, 0x4f, 0xa6, 0x5d, 0xbb, 0x67, 0xf5, 0xed, 0xe3, 0x35, 0x9e, - 0x73, 0xd8, 0x03, 0x68, 0x9c, 0x2c, 0x52, 0x71, 0xe5, 0x0f, 0xa8, 0xda, 0x9c, 0xe3, 0x35, 0xae, - 0x18, 0xb8, 0x93, 0x96, 0xaf, 0xc5, 0xb5, 0x2c, 0x39, 0xdc, 0xa9, 0x38, 0xac, 0x0b, 0xce, 0x61, - 0x1c, 0x47, 0x54, 0x76, 0x4d, 0x3c, 0x0d, 0xa9, 0xc3, 0x06, 0xd4, 0xc9, 0xb0, 0xf7, 0x3b, 0xe8, - 0x16, 0x83, 0xcb, 0xd2, 0xc5, 0xc0, 0x46, 0x7b, 0x56, 0x66, 0x0f, 0x09, 0xb6, 0x45, 0x29, 0xac, - 0x65, 0xe7, 0x63, 0x12, 0x9f, 0xc2, 0x3a, 0x99, 0x91, 0x17, 0xa8, 0xfd, 0xf8, 0x61, 0x05, 0xe0, - 0x1a, 0x32, 0x9e, 0x29, 0x1f, 0xb6, 0x08, 0xf1, 0x5f, 0x24, 0xfe, 0xc0, 0xfb, 0x71, 0x19, 0x5c, - 0xca, 0x25, 0x26, 0xe2, 0x34, 0x98, 0x08, 0x79, 0x3e, 0xa7, 0x35, 0xf2, 0xce, 0xae, 0x67, 0x82, - 0x1c, 0x68, 0x71, 0x5a, 0x7b, 0xbf, 0xb7, 0x60, 0xb3, 0xb8, 0x1f, 0x7d, 0x32, 0xaa, 0xe3, 0x16, - 0x9f, 0x48, 0x2b, 0x2f, 0x9e, 0xe7, 0xe5, 0xe2, 0xd9, 0xb9, 0x69, 0x5f, 0xb9, 0x7e, 0x7e, 0x02, - 0xce, 0x9b, 0x20, 0x4c, 0x56, 0x2a, 0x7c, 0x4b, 0x42, 0x68, 0x93, 0xbb, 0xb6, 0xcc, 0x45, 0xfd, - 0x65, 0xbc, 0x98, 0xa6, 0x12, 0x43, 0x2e, 0x09, 0xef, 0x08, 0x5a, 0xb8, 0x5f, 0x06, 0xee, 0x49, - 0x63, 0x59, 0x59, 0x19, 0xbd, 0x07, 0xb9, 0x5c, 0x1e, 0x94, 0xb7, 0x92, 0x9a, 0xd9, 0x4a, 0x8e, - 0x01, 0x50, 0x3a, 0x97, 0x76, 0x76, 0xa1, 0x4e, 0x54, 0x06, 0x42, 0xd9, 0x90, 0x14, 0xde, 0x60, - 0xe9, 0x21, 0x36, 0xb0, 0xf4, 0xd9, 0x13, 0x14, 0xcb, 0x82, 0x44, 0x6f, 0x6c, 0x9e, 0x95, 0xcc, - 0x02, 0x9a, 0x12, 0xba, 0x78, 0xa9, 0x0d, 0x58, 0x86, 0x01, 0xe4, 0x62, 0x5b, 0x19, 0xa8, 0x38, - 0x89, 0xc0, 0x6b, 0xcb, 0xe3, 0xa5, 0x86, 0x24, 0xa3, 0xd8, 0x77, 0xd5, 0x29, 0x0e, 0xc5, 0x7c, - 0xcf, 0xb8, 0x4a, 0xe8, 0x85, 0x3a, 0xf6, 0x57, 0x00, 0x3f, 0x4b, 0xe2, 0xc5, 0x8c, 0x40, 0x63, - 0x7d, 0xa8, 0x13, 0x95, 0xc5, 0xc7, 0xf4, 0x26, 0xe5, 0x1b, 0x97, 0x0a, 0xd5, 0xa0, 0x63, 0x72, - 0x86, 0x8b, 0x89, 0xbc, 0x69, 0x1c, 0x97, 0x58, 0x4a, 0xcd, 0xf3, 0x20, 0xca, 0xc5, 0xe7, 0x41, - 0x94, 0xc5, 0x8d, 0xcb, 0xa2, 0x19, 0x5b, 0x99, 0x79, 0x00, 0xcd, 0x57, 0x51, 0x1c, 0xa4, 0xa8, - 0x8c, 0xb6, 0x2c, 0x9e, 0xd3, 0x6c, 0x1f, 0x60, 0x20, 0x46, 0xe1, 0x24, 0x88, 0x50, 0xea, 0x94, - 0x1b, 0x40, 0x26, 0xe3, 0x86, 0x92, 0xf7, 0x14, 0x1a, 0x19, 0x55, 0x8d, 0x3d, 0x72, 0x87, 0xa3, - 0x20, 0x12, 0xca, 0x0b, 0x22, 0xbc, 0xb7, 0xb0, 0x21, 0x8b, 0x11, 0x9f, 0xa6, 0xa1, 0x48, 0xef, - 0x50, 0x8a, 0x77, 0x7a, 0xe4, 0xbc, 0x3f, 0x59, 0xe0, 0xe0, 0x4a, 0x19, 0xb0, 0xb4, 0x01, 0xf3, - 0x36, 0x3a, 0xf2, 0x36, 0xb2, 0x1e, 0xb4, 0x87, 0x29, 0xbe, 0x81, 0xba, 0x8d, 0xb5, 0xb8, 0xc9, - 0x42, 0xbc, 0xfc, 0x69, 0xaa, 0xd3, 0x6d, 0xf3, 0x9c, 0x66, 0xdb, 0xd0, 0xc2, 0xde, 0x24, 0x85, - 0xd8, 0xc8, 0x9a, 0x5c, 0x33, 0xd8, 0x0e, 0x80, 0x42, 0x76, 0x21, 0xa8, 0x9b, 0x59, 0xdc, 0xe0, - 0x78, 0x8f, 0xa0, 0x81, 0x9e, 0x9e, 0x04, 0x33, 0x1d, 0x9b, 0x75, 0x5b, 0x6c, 0xff, 0xb5, 0xa0, - 0xf3, 0xcb, 0x85, 0x48, 0xae, 0xb9, 0xf8, 0xed, 0x42, 0xcc, 0x53, 0xc4, 0x96, 0x68, 0x55, 0xcb, - 0x44, 0x60, 0xd5, 0x0e, 0xdf, 0x05, 0xc9, 0x58, 0x22, 0xe5, 0xf0, 0x8c, 0xc2, 0x58, 0x35, 0xe6, - 0x73, 0x8a, 0xb5, 0xc9, 0x4d, 0x16, 0xd5, 0xbb, 0x98, 0xc4, 0xa9, 0x0a, 0x26, 0xa3, 0x58, 0x1f, - 0xee, 0x1d, 0x5d, 0x8d, 0xa2, 0xc5, 0x58, 0xf0, 0x78, 0x29, 0x77, 0x53, 0x73, 0xe6, 0x65, 0x36, - 0xfb, 0x1e, 0x36, 0x37, 0x62, 0xa9, 0xd6, 0xd4, 0x20, 0xc5, 0x12, 0x97, 0xed, 0x43, 0xe7, 0x68, - 0x72, 0x21, 0xc6, 0x63, 0x31, 0x1e, 0x04, 0x69, 0xe0, 0x36, 0xab, 0x06, 0x88, 0x82, 0x8a, 0xf7, - 0xc1, 0x82, 0x8d, 0x2c, 0xfa, 0xf9, 0x2c, 0x9e, 0xce, 0x05, 0xa6, 0xf8, 0x28, 0x49, 0x54, 0x8a, - 0x8f, 0x92, 0x84, 0x3d, 0x82, 0x06, 0x17, 0xf3, 0x45, 0x94, 0xaa, 0x2a, 0xf9, 0x52, 0x5b, 0x54, - 0x7b, 0x17, 0x51, 0xca, 0x95, 0x16, 0xfb, 0x29, 0x6c, 0x16, 0xea, 0x50, 0x3d, 0x0b, 0xdf, 0xd2, - 0xfb, 0x0a, 0x72, 0x5e, 0x52, 0xf7, 0xfe, 0x5c, 0x87, 0xb6, 0x61, 0x39, 0x2f, 0x32, 0xc4, 0x67, - 0x23, 0x2b, 0xb2, 0xaf, 0x69, 0xa6, 0xbb, 0x61, 0xea, 0xc1, 0x9e, 0xd4, 0x01, 0xeb, 0x34, 0x2b, - 0x4b, 0xeb, 0x54, 0x37, 0x42, 0xfb, 0xb6, 0x46, 0x88, 0x13, 0xe2, 0xbb, 0x60, 0x7a, 0x29, 0xc6, - 0x54, 0x96, 0x4d, 0xae, 0x48, 0xb6, 0xa7, 0xbb, 0x02, 0xe5, 0xb1, 0xd0, 0x6b, 0x94, 0x84, 0xeb, - 0xce, 0x21, 0xbb, 0x1c, 0x4e, 0x06, 0x0d, 0x59, 0x2f, 0x92, 0x62, 0xcf, 0xa0, 0xad, 0xdb, 0xd7, - 0x3c, 0x4b, 0x51, 0x57, 0x9b, 0xd2, 0x42, 0x6e, 0x2a, 0xb2, 0x17, 0xe5, 0x11, 0xcd, 0x6d, 0x91, - 0x17, 0x6e, 0x21, 0x72, 0x43, 0xce, 0xcb, 0x23, 0xdd, 0xbe, 0x31, 0x33, 0xba, 0x40, 0x9b, 0xef, - 0xeb, 0xcd, 0xb9, 0x88, 0x1b, 0x93, 0xe5, 0x13, 0xf3, 0x2d, 0x71, 0xdb, 0xb4, 0xa7, 0x5b, 0x44, - 0x4e, 0xca, 0xb8, 0xf9, 0xe6, 0xec, 0x1b, 0x0f, 0x99, 0xdb, 0x29, 0x1f, 0x94, 0x8b, 0xb8, 0xf1, - 0xdc, 0xf9, 0x15, 0xf3, 0x9d, 0xbb, 0x41, 0x5b, 0xab, 0x87, 0x37, 0xa9, 0xc2, 0x2b, 0xa6, 0xc2, - 0x17, 0xe5, 0x49, 0xc0, 0xdd, 0x2c, 0x03, 0x55, 0x94, 0xf3, 0xf2, 0xe4, 0xb0, 0x6f, 0x0c, 0xe3, - 0xee, 0xbd, 0xb2, 0xff, 0xb9, 0x88, 0x6b, 0x2d, 0xef, 0xaf, 0x35, 0xd8, 0xf0, 0x27, 0xb3, 0x38, - 0x49, 0x8d, 0x2e, 0x22, 0xa7, 0x7f, 0xab, 0x72, 0xfa, 0xaf, 0x95, 0xde, 0x49, 0xea, 0x26, 0xd4, - 0x3d, 0x1c, 0x2e, 0x09, 0xa3, 0x82, 0x9c, 0x42, 0x05, 0x6d, 0x43, 0x4b, 0x5e, 0x17, 0x14, 0xd5, - 0x49, 0xa4, 0x19, 0xf2, 0x7b, 0x64, 0x49, 0xb3, 0x66, 0x83, 0xa6, 0x57, 0x45, 0x62, 0xe7, 0x94, - 0x6a, 0x24, 0x6c, 0x92, 0xd0, 0xe0, 0xa0, 0xfc, 0x2c, 0x9c, 0x88, 0x79, 0x1a, 0x4c, 0x66, 0xd8, - 0x8a, 0xec, 0xbe, 0xcd, 0x0d, 0x0e, 0x76, 0x21, 0x0a, 0xe2, 0x65, 0x22, 0x82, 0x54, 0x8c, 0x0f, - 0x52, 0xaa, 0x40, 0x9b, 0x97, 0xb8, 0xa8, 0x47, 0x61, 0x69, 0x3d, 0x90, 0x7a, 0x45, 0x2e, 0xbd, - 0xa4, 0x91, 0x08, 0x12, 0xaa, 0xab, 0x26, 0x97, 0x84, 0xf7, 0x8f, 0x1a, 0x30, 0x89, 0xa4, 0x9c, - 0x15, 0xff, 0x6f, 0x70, 0xde, 0x0e, 0x5b, 0x11, 0x9c, 0xc6, 0x0a, 0x38, 0x5f, 0xe5, 0x13, 0xae, - 0x04, 0x26, 0xa3, 0xb0, 0xfd, 0xeb, 0xc7, 0x47, 0xa2, 0x6a, 0x71, 0x93, 0xc5, 0x3c, 0xe8, 0x18, - 0x2f, 0x1f, 0x5e, 0x5b, 0xb4, 0x5d, 0xe0, 0x55, 0x40, 0x0b, 0x77, 0x84, 0xb6, 0x7d, 0x3b, 0xb4, - 0x1d, 0x13, 0xda, 0x0f, 0x16, 0x74, 0x0e, 0xd2, 0x78, 0x12, 0x8e, 0xb8, 0x18, 0xc5, 0xc9, 0xf8, - 0x66, 0x50, 0x25, 0x7c, 0x35, 0x13, 0xbe, 0x3d, 0xb0, 0xfd, 0xf7, 0x49, 0xd6, 0x3d, 0xb7, 0x8d, - 0xd9, 0x6c, 0x25, 0x57, 0x1c, 0x15, 0xd9, 0x37, 0x50, 0xf3, 0x13, 0xaa, 0xdc, 0x42, 0xdf, 0x2f, - 0x5c, 0x12, 0x5e, 0xf3, 0x13, 0xef, 0x87, 0xd0, 0x95, 0x4e, 0x29, 0x51, 0xf6, 0x0e, 0x75, 0xa1, - 0x7e, 0x94, 0x24, 0xb1, 0x7a, 0x89, 0x24, 0xe1, 0x5d, 0x41, 0xf7, 0x2c, 0x09, 0xa6, 0xf3, 0x28, - 0x48, 0x05, 0x26, 0xe6, 0x73, 0xea, 0xa3, 0xea, 0x63, 0xbf, 0x07, 0xed, 0xd3, 0x38, 0x7d, 0x9b, - 0x84, 0x29, 0xb5, 0x0c, 0xd9, 0xfc, 0x4d, 0x96, 0xf7, 0x7d, 0xf8, 0xb2, 0x74, 0xb2, 0x7e, 0x30, - 0xb1, 0xa4, 0x6c, 0xfd, 0xe1, 0x3b, 0x84, 0xfb, 0xb9, 0xaa, 0x3f, 0xf8, 0x2c, 0x1f, 0x57, 0x8d, - 0xfe, 0xc0, 0x88, 0x9c, 0x8c, 0x66, 0xc7, 0x57, 0x44, 0xe3, 0x1d, 0x82, 0x9b, 0xa1, 0x29, 0xff, - 0x45, 0x64, 0x1e, 0x9c, 0x87, 0x62, 0x79, 0xd3, 0x27, 0x15, 0x0d, 0x0c, 0x35, 0xfa, 0x83, 0x41, - 0x6b, 0xef, 0x3f, 0x16, 0x74, 0xab, 0x8c, 0xe8, 0xe2, 0xb2, 0x8c, 0xe2, 0x62, 0xcf, 0xa1, 0xfe, - 0x3e, 0x14, 0x4b, 0x35, 0x22, 0x78, 0x2b, 0x29, 0x5f, 0xf1, 0x84, 0xcb, 0x0d, 0x78, 0xb5, 0x0e, - 0x46, 0x69, 0x18, 0x4f, 0xd5, 0xf7, 0x80, 0xa4, 0xf0, 0x9c, 0xc3, 0x28, 0x1e, 0xfd, 0x46, 0x7e, - 0xe9, 0x72, 0x49, 0x54, 0x5c, 0x95, 0xfa, 0x1d, 0xaf, 0xca, 0x7a, 0xd5, 0x55, 0xf1, 0xfe, 0x62, - 0x29, 0xac, 0x8c, 0x99, 0xed, 0x93, 0x19, 0xd3, 0x17, 0xc4, 0x56, 0x17, 0xc4, 0x95, 0x83, 0xa7, - 0x9e, 0xaf, 0x15, 0x89, 0xc3, 0x2e, 0x2e, 0xe9, 0x37, 0x87, 0x43, 0x59, 0xca, 0xe9, 0x4f, 0x74, - 0xa5, 0xd5, 0x60, 0xd7, 0xab, 0x82, 0x3d, 0xdc, 0xfa, 0xdb, 0xc7, 0x1d, 0xeb, 0xef, 0x1f, 0x77, - 0xac, 0x7f, 0x7d, 0xdc, 0xb1, 0xfe, 0xf0, 0xef, 0x9d, 0xb5, 0x8b, 0x75, 0xfa, 0x07, 0xf6, 0xa3, - 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0x2d, 0xb9, 0x97, 0xfb, 0x13, 0x13, 0x00, 0x00, + // 1709 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x4f, 0x6f, 0x23, 0x4b, + 0x11, 0xcf, 0x78, 0xc6, 0xb1, 0x5d, 0x76, 0xb2, 0x79, 0xbd, 0x7e, 0x8f, 0xd1, 0x92, 0xcd, 0x33, + 0xa3, 0xc0, 0x33, 0x08, 0xe5, 0x29, 0xe1, 0xed, 0x6a, 0x2f, 0xc0, 0x26, 0xeb, 0x2c, 0x19, 0x2d, + 0x09, 0x4b, 0x3b, 0xca, 0xde, 0x90, 0x26, 0x76, 0x93, 0x1d, 0x31, 0xf6, 0x98, 0xf1, 0x78, 0x9d, + 0x5c, 0x90, 0xf8, 0x0c, 0x7b, 0xe1, 0x86, 0xb8, 0x71, 0xe2, 0x43, 0x70, 0x81, 0x23, 0x47, 0x24, + 0x2e, 0x68, 0xe1, 0x83, 0xa0, 0xaa, 0x9e, 0x9e, 0xee, 0x19, 0x4f, 0xb2, 0xd1, 0x8a, 0x5b, 0xd7, + 0x9f, 0xae, 0xee, 0xfa, 0xf5, 0xaf, 0xab, 0x6b, 0x06, 0x3a, 0xb3, 0xc5, 0x65, 0x14, 0x8e, 0xf6, + 0x66, 0x49, 0x9c, 0xc6, 0xac, 0x19, 0x4e, 0x53, 0x91, 0x4c, 0x83, 0xc8, 0xfb, 0xa3, 0x05, 0x36, + 0x8f, 0x97, 0xcc, 0x85, 0xc6, 0x8b, 0x38, 0x5a, 0x4c, 0xa6, 0x73, 0xd7, 0xea, 0xd9, 0x7d, 0x87, + 0x2b, 0x91, 0x31, 0x70, 0x5e, 0x89, 0x9b, 0xb9, 0x6b, 0xf7, 0xec, 0x7e, 0x8b, 0xd3, 0x98, 0xed, + 0x42, 0xfd, 0x30, 0x4d, 0x93, 0xb9, 0x5b, 0xeb, 0xd9, 0xfd, 0xf6, 0xc1, 0xe6, 0x9e, 0x8a, 0xb7, + 0x87, 0x6a, 0x2e, 0x8d, 0x18, 0x93, 0xc7, 0x41, 0x12, 0x4e, 0xaf, 0x5c, 0xa7, 0x67, 0xf5, 0x3b, + 0x5c, 0x89, 0xac, 0x0b, 0x75, 0x7f, 0x3a, 0x16, 0xd7, 0x6e, 0xbd, 0x67, 0xf5, 0x5b, 0x5c, 0x0a, + 0xa8, 0x7d, 0x19, 0x8a, 0x68, 0xec, 0xae, 0x4b, 0x2d, 0x09, 0xde, 0x1e, 0xb4, 0x78, 0xbc, 0x3c, + 0x0d, 0xd2, 0x24, 0xbc, 0x66, 0xdf, 0x01, 0x87, 0xc7, 0x4b, 0xb9, 0xc7, 0xf6, 0xc1, 0x86, 0x5e, + 0x97, 0xc7, 0x4b, 0x4e, 0x26, 0xef, 0x14, 0x5a, 0xc3, 0xf0, 0x6a, 0x2a, 0xc6, 0x98, 0xd6, 0x97, + 0x60, 0xbf, 0x8e, 0xd1, 0xdd, 0x5a, 0x75, 0x47, 0x0b, 0x3a, 0x9c, 0x89, 0x2b, 0xb7, 0x56, 0xe9, + 0x70, 0x26, 0xae, 0xbc, 0x67, 0xb0, 0xc9, 0xe3, 0xa5, 0x3f, 0x16, 0xd3, 0x34, 0xfc, 0x75, 0x28, + 0x12, 0x02, 0x24, 0xdf, 0x83, 0x23, 0x17, 0xcd, 0x41, 0xaa, 0x69, 0x90, 0xbc, 0x47, 0xb0, 0xee, + 0x0f, 0x7e, 0x1e, 0xce, 0x53, 0xb6, 0x05, 0xb6, 0x3f, 0x50, 0x13, 0x70, 0xe8, 0xf9, 0xf0, 0xd9, + 0xf1, 0x75, 0x9a, 0x04, 0xa3, 0x54, 0x8c, 0xfd, 0x81, 0x84, 0x9a, 0x6d, 0x42, 0xcd, 0x1f, 0xd0, + 0x5e, 0x1d, 0x5e, 0xf3, 0x07, 0x6c, 0x17, 0x9c, 0x8b, 0x20, 0x52, 0x20, 0x6f, 0xe9, 0xcd, 0xc9, + 0xb0, 0x9c, 0xac, 0xde, 0x65, 0x21, 0x54, 0x86, 0xd3, 0x17, 0xb0, 0x4e, 0xe8, 0xc9, 0x45, 0x5b, + 0x3c, 0x93, 0xd8, 0x13, 0x7d, 0xcc, 0x32, 0xea, 0xb7, 0x75, 0xd4, 0x95, 0x0d, 0xe5, 0x1c, 0xf0, + 0x1e, 0x43, 0xe3, 0x95, 0xb8, 0xa1, 0x5c, 0x54, 0xa6, 0x96, 0x91, 0xe9, 0xbf, 0x2c, 0x78, 0x98, + 0xcf, 0x3e, 0x0f, 0x2e, 0x23, 0x71, 0x11, 0x44, 0x0b, 0xc1, 0x76, 0x55, 0xde, 0x56, 0xd5, 0xfe, + 0x4f, 0xd6, 0x08, 0x0b, 0xf6, 0x55, 0x8e, 0x1d, 0xba, 0x7d, 0xa6, 0xdd, 0xb2, 0x25, 0x4f, 0xd6, + 0x32, 0xd6, 0x6d, 0x43, 0xf3, 0x68, 0xe8, 0x53, 0x68, 0xd7, 0xee, 0x59, 0x7d, 0xfb, 0x64, 0x8d, + 0xe7, 0x1a, 0xf6, 0x08, 0x1a, 0xa7, 0x8b, 0x54, 0x5c, 0xfb, 0x03, 0x62, 0x9b, 0x73, 0xb2, 0xc6, + 0x95, 0x02, 0x67, 0xd2, 0xf0, 0x95, 0xb8, 0x91, 0x94, 0xc3, 0x99, 0x4a, 0xc3, 0xba, 0xe0, 0x1c, + 0xc5, 0x71, 0x44, 0xb4, 0x6b, 0xe2, 0x6a, 0x28, 0x1d, 0x35, 0xa0, 0x4e, 0x81, 0xbd, 0xdf, 0x41, + 0xb7, 0x98, 0x5c, 0x76, 0x5c, 0x0c, 0x6c, 0x8c, 0x67, 0x65, 0xf1, 0x50, 0x60, 0x5b, 0x74, 0x84, + 0xb5, 0x6c, 0x7d, 0x3c, 0xc4, 0x27, 0xb0, 0x4e, 0x61, 0xe4, 0x05, 0x6a, 0x1f, 0x3c, 0xae, 0x00, + 0x5c, 0x43, 0xc6, 0x33, 0xe7, 0xa3, 0x16, 0x21, 0xfe, 0x8b, 0xc4, 0x1f, 0x78, 0x3f, 0x2e, 0x83, + 0x4b, 0x67, 0x89, 0x07, 0x71, 0x16, 0x4c, 0x84, 0x5c, 0x9f, 0xd3, 0x18, 0x75, 0xe7, 0x37, 0x33, + 0x41, 0x1b, 0x68, 0x71, 0x1a, 0x7b, 0xbf, 0xb7, 0x60, 0xb3, 0x38, 0x1f, 0xf7, 0x64, 0xb0, 0xe3, + 0x8e, 0x3d, 0x91, 0x57, 0x4e, 0x9e, 0x67, 0x65, 0xf2, 0xec, 0xdc, 0x36, 0xaf, 0xcc, 0x9f, 0x9f, + 0x80, 0xf3, 0x3a, 0x08, 0x93, 0x15, 0x86, 0x6f, 0x49, 0x08, 0x6d, 0xda, 0xae, 0x2d, 0xcf, 0xa2, + 0xfe, 0x22, 0x5e, 0x4c, 0x53, 0x89, 0x21, 0x97, 0x82, 0x77, 0x0c, 0x2d, 0x9c, 0x2f, 0x13, 0xf7, + 0x64, 0xb0, 0x8c, 0x56, 0x46, 0xed, 0x41, 0x2d, 0x97, 0x0b, 0xe5, 0xa5, 0xa4, 0x66, 0x96, 0x92, + 0x13, 0x00, 0xb4, 0xce, 0x65, 0x9c, 0x5d, 0xa8, 0x93, 0x94, 0x81, 0x50, 0x0e, 0x24, 0x8d, 0xb7, + 0x44, 0x7a, 0x8c, 0x05, 0x2c, 0x7d, 0xfa, 0x0d, 0x9a, 0x25, 0x21, 0x71, 0x37, 0x36, 0xcf, 0x28, + 0xb3, 0x80, 0xa6, 0x84, 0x2e, 0x5e, 0xea, 0x00, 0x96, 0x11, 0x00, 0xb5, 0x58, 0x56, 0x06, 0x2a, + 0x4f, 0x12, 0xf0, 0xda, 0xf2, 0x78, 0xa9, 0x21, 0xc9, 0x24, 0xf6, 0x5d, 0xb5, 0x8a, 0x43, 0x39, + 0x3f, 0x30, 0xae, 0x12, 0xee, 0x42, 0x2d, 0xfb, 0x2b, 0x80, 0x9f, 0x25, 0xf1, 0x62, 0x46, 0xa0, + 0xb1, 0x3e, 0xd4, 0x49, 0xca, 0xf2, 0x63, 0x7a, 0x92, 0xda, 0x1b, 0x97, 0x0e, 0xd5, 0xa0, 0xe3, + 0xe1, 0x0c, 0x17, 0x13, 0x79, 0xd3, 0x38, 0x0e, 0x91, 0x4a, 0xcd, 0x8b, 0x20, 0xca, 0xcd, 0x17, + 0x41, 0x94, 0xe5, 0x8d, 0xc3, 0x62, 0x18, 0x5b, 0x85, 0x79, 0x04, 0xcd, 0x97, 0x51, 0x1c, 0xa4, + 0xe8, 0x8c, 0xb1, 0x2c, 0x9e, 0xcb, 0x6c, 0x1f, 0x60, 0x20, 0x46, 0xe1, 0x24, 0x88, 0xd0, 0xea, + 0x94, 0x0b, 0x40, 0x66, 0xe3, 0x86, 0x93, 0xf7, 0x04, 0x1a, 0x99, 0x54, 0x8d, 0x3d, 0x6a, 0x87, + 0xa3, 0x20, 0x12, 0x6a, 0x17, 0x24, 0x78, 0x6f, 0x60, 0x43, 0x92, 0x11, 0x9f, 0xa6, 0xa1, 0x48, + 0xef, 0x41, 0xc5, 0x7b, 0x3d, 0x72, 0xde, 0x9f, 0x2d, 0x70, 0x70, 0xa4, 0x02, 0x58, 0x3a, 0x80, + 0x79, 0x1b, 0x1d, 0x79, 0x1b, 0x59, 0x0f, 0xda, 0xc3, 0x14, 0xdf, 0x40, 0x5d, 0xc6, 0x5a, 0xdc, + 0x54, 0x21, 0x5e, 0xfe, 0x34, 0xd5, 0xc7, 0x6d, 0xf3, 0x5c, 0x66, 0xdb, 0xd0, 0xc2, 0xda, 0x24, + 0x8d, 0x58, 0xc8, 0x9a, 0x5c, 0x2b, 0xd8, 0x0e, 0x80, 0x42, 0x76, 0x21, 0xa8, 0x9a, 0x59, 0xdc, + 0xd0, 0x78, 0x5f, 0x43, 0x03, 0x77, 0x7a, 0x1a, 0xcc, 0x74, 0x6e, 0xd6, 0x5d, 0xb9, 0xfd, 0xa9, + 0x06, 0x9d, 0x5f, 0x2e, 0x44, 0x72, 0xc3, 0xc5, 0x6f, 0x17, 0x62, 0x9e, 0x22, 0xb6, 0x24, 0x2b, + 0x2e, 0x93, 0x80, 0xac, 0x1d, 0xbe, 0x0d, 0x92, 0xb1, 0x44, 0xca, 0xe1, 0x99, 0x84, 0xb9, 0x6a, + 0xcc, 0xe7, 0x94, 0x6b, 0x93, 0x9b, 0x2a, 0xe2, 0xbb, 0x98, 0xc4, 0xa9, 0x4a, 0x26, 0x93, 0x58, + 0x1f, 0x1e, 0x1c, 0x5f, 0x8f, 0xa2, 0xc5, 0x58, 0xf0, 0x78, 0x29, 0x67, 0x53, 0x71, 0xe6, 0x65, + 0x35, 0xfb, 0x1e, 0x16, 0x37, 0x52, 0xa9, 0xd2, 0xd4, 0x20, 0xc7, 0x92, 0x96, 0xed, 0x43, 0xe7, + 0x78, 0x72, 0x29, 0xc6, 0x63, 0x31, 0x1e, 0x04, 0x69, 0xe0, 0x36, 0xab, 0x1a, 0x88, 0x82, 0x0b, + 0xdb, 0x85, 0x8d, 0xd7, 0x89, 0x38, 0x4f, 0x82, 0xe9, 0x3c, 0x0a, 0x52, 0x31, 0x76, 0x5b, 0x14, + 0xb9, 0xa8, 0xf4, 0xde, 0x5b, 0xb0, 0x91, 0x61, 0x34, 0x9f, 0xc5, 0xd3, 0xb9, 0x40, 0x22, 0x1c, + 0x27, 0x89, 0x22, 0xc2, 0x71, 0x92, 0xb0, 0xaf, 0xa1, 0xc1, 0xc5, 0x7c, 0x11, 0xa5, 0x8a, 0x4b, + 0x9f, 0xeb, 0x75, 0xd5, 0xdc, 0x45, 0x94, 0x72, 0xe5, 0xc5, 0x7e, 0x0a, 0x9b, 0x05, 0xb6, 0xaa, + 0xc7, 0xe3, 0x5b, 0x7a, 0x5e, 0xc1, 0xce, 0x4b, 0xee, 0xde, 0x5f, 0xea, 0xd0, 0x36, 0x22, 0xe7, + 0x54, 0x44, 0x14, 0x37, 0x32, 0x2a, 0x7e, 0x49, 0x9d, 0xdf, 0x2d, 0xbd, 0x11, 0x56, 0xae, 0x0e, + 0x58, 0x67, 0x19, 0x79, 0xad, 0x33, 0x5d, 0x2e, 0xed, 0xbb, 0xca, 0x25, 0xf6, 0x91, 0x6f, 0x83, + 0xe9, 0x95, 0x18, 0x13, 0x79, 0x9b, 0x5c, 0x89, 0x6c, 0x4f, 0xd7, 0x0e, 0x3a, 0xed, 0x42, 0x45, + 0x52, 0x16, 0xae, 0xeb, 0x8b, 0xac, 0x85, 0xd8, 0x3f, 0x34, 0x24, 0xab, 0xa4, 0xc4, 0x9e, 0x42, + 0x5b, 0x17, 0xb9, 0x79, 0x76, 0x90, 0x5d, 0x1d, 0x4a, 0x1b, 0xb9, 0xe9, 0xc8, 0x9e, 0x97, 0x1b, + 0x39, 0x3a, 0xcf, 0xf6, 0x81, 0x5b, 0xc8, 0xdc, 0xb0, 0xf3, 0x72, 0xe3, 0xb7, 0x6f, 0x74, 0x96, + 0x2e, 0xd0, 0xe4, 0x87, 0x7a, 0x72, 0x6e, 0xe2, 0x46, 0xff, 0xf9, 0x8d, 0xf9, 0xe2, 0xb8, 0x6d, + 0x9a, 0xd3, 0x2d, 0x22, 0x27, 0x6d, 0xdc, 0x7c, 0x99, 0xf6, 0x8d, 0xe7, 0xce, 0xed, 0x94, 0x17, + 0xca, 0x4d, 0xdc, 0x78, 0x14, 0xfd, 0x8a, 0x2e, 0xd0, 0xdd, 0xa0, 0xa9, 0xd5, 0x2d, 0x9e, 0x74, + 0xe1, 0x15, 0xbd, 0xe3, 0xf3, 0x72, 0xbf, 0xe0, 0x6e, 0x96, 0x81, 0x2a, 0xda, 0x79, 0xb9, 0xbf, + 0xd8, 0x37, 0x5a, 0x76, 0xf7, 0x41, 0x79, 0xff, 0xb9, 0x89, 0x6b, 0x2f, 0xef, 0x6f, 0x35, 0xd8, + 0xf0, 0x27, 0xb3, 0x38, 0x49, 0x8d, 0x5a, 0x23, 0xbf, 0x11, 0xac, 0xca, 0x6f, 0x84, 0x5a, 0xe9, + 0x35, 0xa5, 0x9a, 0x43, 0x35, 0xc6, 0xe1, 0x52, 0x30, 0x18, 0xe4, 0x14, 0x18, 0xb4, 0x0d, 0x2d, + 0x79, 0x5d, 0xd0, 0x54, 0x27, 0x93, 0x56, 0xc8, 0xaf, 0x96, 0x25, 0x75, 0xa4, 0x0d, 0xea, 0x71, + 0x95, 0x88, 0xf5, 0x55, 0xba, 0x91, 0xb1, 0x49, 0x46, 0x43, 0x83, 0xf6, 0xf3, 0x70, 0x22, 0xe6, + 0x69, 0x30, 0x99, 0x61, 0xc1, 0xb2, 0xfb, 0x36, 0x37, 0x34, 0x58, 0xab, 0x28, 0x89, 0x17, 0x89, + 0xc0, 0xd2, 0x71, 0x98, 0x12, 0x03, 0x6d, 0x5e, 0xd2, 0xa2, 0x1f, 0xa5, 0xa5, 0xfd, 0x40, 0xfa, + 0x15, 0xb5, 0xf4, 0xde, 0x46, 0x22, 0x48, 0x88, 0x57, 0x4d, 0x2e, 0x05, 0xef, 0x9f, 0x35, 0x60, + 0x12, 0x49, 0xd9, 0x51, 0xfe, 0xdf, 0xe0, 0xbc, 0x1b, 0xb6, 0x22, 0x38, 0x8d, 0x15, 0x70, 0xbe, + 0xc8, 0xfb, 0x60, 0x09, 0x4c, 0x26, 0xe1, 0x23, 0xa1, 0x9f, 0x28, 0x89, 0xaa, 0xc5, 0x4d, 0x15, + 0xf3, 0xa0, 0x63, 0xbc, 0x8f, 0x78, 0x6d, 0x31, 0x76, 0x41, 0x57, 0x01, 0x2d, 0xdc, 0x13, 0xda, + 0xf6, 0xdd, 0xd0, 0x76, 0x4c, 0x68, 0xdf, 0x5b, 0xd0, 0x39, 0x4c, 0xe3, 0x49, 0x38, 0xe2, 0x62, + 0x14, 0x27, 0xe3, 0xdb, 0x41, 0x95, 0xf0, 0xd5, 0x4c, 0xf8, 0xf6, 0xc0, 0xf6, 0xdf, 0x25, 0x59, + 0xf5, 0xdc, 0x36, 0x3a, 0xb8, 0x95, 0xb3, 0xe2, 0xe8, 0xc8, 0xbe, 0x82, 0x9a, 0x9f, 0x10, 0x73, + 0x0b, 0x75, 0xbf, 0x70, 0x49, 0x78, 0xcd, 0x4f, 0xbc, 0x1f, 0x42, 0x57, 0x6e, 0x4a, 0x99, 0xb2, + 0x77, 0xa8, 0x0b, 0xf5, 0xe3, 0x24, 0x89, 0xd5, 0x4b, 0x24, 0x05, 0xef, 0x1a, 0xba, 0xf9, 0xeb, + 0x85, 0x07, 0xf3, 0x29, 0xfc, 0xa8, 0xfa, 0x25, 0xd0, 0x83, 0xf6, 0x59, 0x9c, 0xbe, 0x49, 0xc2, + 0x94, 0x4a, 0x86, 0x2c, 0xfe, 0xa6, 0xca, 0xfb, 0x3e, 0x7c, 0x5e, 0x5a, 0x59, 0x3f, 0x98, 0x48, + 0x29, 0x5b, 0x7f, 0x1e, 0x0f, 0xe1, 0x61, 0xee, 0xea, 0x0f, 0x3e, 0x69, 0x8f, 0xab, 0x41, 0x7f, + 0x60, 0x64, 0x4e, 0x41, 0xb3, 0xe5, 0x2b, 0xb2, 0xf1, 0x8e, 0xc0, 0xcd, 0xd0, 0x94, 0x7f, 0x2c, + 0xb2, 0x1d, 0x5c, 0x84, 0x62, 0x79, 0xdb, 0x87, 0x17, 0xb5, 0x15, 0x35, 0xfa, 0xcf, 0x41, 0x63, + 0xef, 0xbf, 0x16, 0x74, 0xab, 0x82, 0x68, 0x72, 0x59, 0x06, 0xb9, 0xd8, 0x33, 0xa8, 0xbf, 0x0b, + 0xc5, 0x52, 0xb5, 0x08, 0xde, 0xca, 0x91, 0xaf, 0xec, 0x84, 0xcb, 0x09, 0x78, 0xb5, 0x0e, 0x47, + 0x69, 0x18, 0x4f, 0xd5, 0x57, 0x83, 0x94, 0x70, 0x9d, 0xa3, 0x28, 0x1e, 0xfd, 0x46, 0x7e, 0x0f, + 0x73, 0x29, 0x54, 0x5c, 0x95, 0xfa, 0x3d, 0xaf, 0xca, 0x7a, 0xd5, 0x55, 0xf1, 0xfe, 0x6a, 0x29, + 0xac, 0x8c, 0xce, 0xee, 0xa3, 0x27, 0xa6, 0x2f, 0x88, 0xad, 0x2e, 0x88, 0x2b, 0xdb, 0x53, 0xdd, + 0x85, 0x2b, 0x11, 0x5b, 0x62, 0x1c, 0xd2, 0xcf, 0x10, 0x87, 0x4e, 0x29, 0x97, 0x3f, 0x52, 0x95, + 0x56, 0x93, 0x5d, 0xaf, 0x4a, 0xf6, 0x68, 0xeb, 0xef, 0x1f, 0x76, 0xac, 0x7f, 0x7c, 0xd8, 0xb1, + 0xfe, 0xfd, 0x61, 0xc7, 0xfa, 0xc3, 0x7f, 0x76, 0xd6, 0x2e, 0xd7, 0xe9, 0x4f, 0xd9, 0x8f, 0xfe, + 0x17, 0x00, 0x00, 0xff, 0xff, 0x97, 0x50, 0x77, 0xb2, 0x39, 0x13, 0x00, 0x00, } func (m *Row) Marshal() (dAtA []byte, err error) { @@ -4065,6 +4074,16 @@ func (m *QueryRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.PreTranslated { + i-- + if m.PreTranslated { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x48 + } if len(m.EmbeddedData) > 0 { for iNdEx := len(m.EmbeddedData) - 1; iNdEx >= 0; iNdEx-- { { @@ -5815,6 +5834,9 @@ func (m *QueryRequest) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } + if m.PreTranslated { + n += 2 + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -9634,6 +9656,26 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 9: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field PreTranslated", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.PreTranslated = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) diff --git a/internal/public.proto b/internal/public.proto index 9e3b4241e..3a4a251b5 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -144,6 +144,7 @@ message QueryRequest { bool ExcludeRowAttrs = 6; bool ExcludeColumns = 7; repeated Row EmbeddedData = 8; + bool PreTranslated = 9; } message QueryResponse { diff --git a/rrtx.go b/rrtx.go index 6113067c4..8c75a2924 100644 --- a/rrtx.go +++ b/rrtx.go @@ -250,7 +250,7 @@ func (tx *RoaringTx) Contains(index, field, view string, shard uint64, v uint64) func (tx *RoaringTx) ContainerIterator(index, field, view string, shard uint64, key uint64) (citer roaring.ContainerIterator, found bool, err error) { b, err := tx.bitmap(index, field, view, shard) if err != nil { - return nil, false, err + return nil, false, errors.Wrap(err, "getting bitmap") } //vv("b bitmap back from bitmap(index='%v', field='%v', view='%v', shard='%v')='%#v'", index, field, view, shard, b.Slice()) citer, found = b.Containers.Iterator(key) @@ -395,7 +395,7 @@ const FragmentNotFound = Error("fragment not found") func (tx *RoaringTx) bitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) { frag, err := tx.getFragment(index, field, view, shard) if err != nil { - return nil, err + return nil, errors.Wrap(err, "getFragment") } return frag.storage, nil }