From 16171b3e65be5211bd2e42d04621b589d1688c57 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 2 Dec 2019 20:51:56 -0600 Subject: [PATCH 01/20] return total match counts for either min or max --- executor.go | 12 +++++-- executor_test.go | 87 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) diff --git a/executor.go b/executor.go index 5d8c94c75..9a39d7697 100644 --- a/executor.go +++ b/executor.go @@ -3769,9 +3769,13 @@ func (vc *ValCount) smaller(other ValCount) ValCount { if vc.Count == 0 || (other.Val < vc.Val && other.Count > 0) { return other } + extra := int64(0) + if vc.Val == other.Val { + extra += other.Count + } return ValCount{ Val: vc.Val, - Count: vc.Count, + Count: vc.Count + extra, } } @@ -3780,9 +3784,13 @@ func (vc *ValCount) larger(other ValCount) ValCount { if vc.Count == 0 || (other.Val > vc.Val && other.Count > 0) { return other } + extra := int64(0) + if vc.Val == other.Val { + extra += other.Count + } return ValCount{ Val: vc.Val, - Count: vc.Count, + Count: vc.Count + extra, } } diff --git a/executor_test.go b/executor_test.go index 212b1eebf..8eae51f1c 100644 --- a/executor_test.go +++ b/executor_test.go @@ -4254,3 +4254,90 @@ func TestExecutor_Execute_IncludesColumn(t *testing.T) { }) }) } + +func TestExecutor_Execute_MinMaxCountEqual(t *testing.T) { + t.Run("ColumnID", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) + if err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("x", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-1100, 1000)); err != nil { + t.Fatal(err) + } + + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set(0, f=3) + Set(1, f=3) + Set(2, f=4) + Set(3, f=5) + Set(4, f=5) + Set(` + strconv.Itoa(ShardWidth+1) + `, f=3) + Set(` + strconv.Itoa(ShardWidth+2) + `, f=5) + Set(` + strconv.Itoa(ShardWidth+3) + `, f=5) + Set(` + strconv.Itoa(ShardWidth+4) + `, f=5) + Set(` + strconv.Itoa(ShardWidth+5) + `, f=4) + Set(` + strconv.Itoa(2*ShardWidth+1) + `, f=3) + Set(0, x=3) + Set(1, x=3) + + `}); err != nil { + t.Fatal(err) + } + + t.Run("Min", func(t *testing.T) { + tests := []struct { + filter string + exp int64 + cnt int64 + }{ + {filter: ``, exp: 3, cnt: 4}, + {filter: `Row(x=3)`, exp: 3, cnt: 2}, + } + for i, tt := range tests { + var pql string + if tt.filter == "" { + pql = `Min(field=f)` + } else { + pql = fmt.Sprintf(`Min(%s, field=f)`, tt.filter) + } + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { + t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) + } + } + }) + + t.Run("Max", func(t *testing.T) { + tests := []struct { + filter string + exp int64 + cnt int64 + }{ + {filter: ``, exp: 5, cnt: 5}, + } + for i, tt := range tests { + var pql string + if tt.filter == "" { + pql = `Max(field=f)` + } else { + pql = fmt.Sprintf(`Max(%s, field=f)`, tt.filter) + } + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { + t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) + } + } + }) + }) +} From 4523a4d6932a3c23f0f51fc0c37c3d347b025188 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 2 Dec 2019 21:46:41 -0600 Subject: [PATCH 02/20] removed uneeded test run --- executor_test.go | 118 +++++++++++++++++++++++------------------------ 1 file changed, 58 insertions(+), 60 deletions(-) diff --git a/executor_test.go b/executor_test.go index 8eae51f1c..cea26c8af 100644 --- a/executor_test.go +++ b/executor_test.go @@ -4256,25 +4256,24 @@ func TestExecutor_Execute_IncludesColumn(t *testing.T) { } func TestExecutor_Execute_MinMaxCountEqual(t *testing.T) { - t.Run("ColumnID", func(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} - idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) - if err != nil { - t.Fatal(err) - } + idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) + if err != nil { + t.Fatal(err) + } - if _, err := idx.CreateField("x", pilosa.OptFieldTypeDefault()); err != nil { - t.Fatal(err) - } + if _, err := idx.CreateField("x", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } - if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-1100, 1000)); err != nil { - t.Fatal(err) - } + if _, err := idx.CreateField("f", pilosa.OptFieldTypeInt(-1100, 1000)); err != nil { + t.Fatal(err) + } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(0, f=3) Set(1, f=3) Set(2, f=4) @@ -4290,54 +4289,53 @@ func TestExecutor_Execute_MinMaxCountEqual(t *testing.T) { Set(1, x=3) `}); err != nil { - t.Fatal(err) + t.Fatal(err) + } + + t.Run("Min", func(t *testing.T) { + tests := []struct { + filter string + exp int64 + cnt int64 + }{ + {filter: ``, exp: 3, cnt: 4}, + {filter: `Row(x=3)`, exp: 3, cnt: 2}, } + for i, tt := range tests { + var pql string + if tt.filter == "" { + pql = `Min(field=f)` + } else { + pql = fmt.Sprintf(`Min(%s, field=f)`, tt.filter) + } + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { + t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) + } + } + }) - t.Run("Min", func(t *testing.T) { - tests := []struct { - filter string - exp int64 - cnt int64 - }{ - {filter: ``, exp: 3, cnt: 4}, - {filter: `Row(x=3)`, exp: 3, cnt: 2}, + t.Run("Max", func(t *testing.T) { + tests := []struct { + filter string + exp int64 + cnt int64 + }{ + {filter: ``, exp: 5, cnt: 5}, + } + for i, tt := range tests { + var pql string + if tt.filter == "" { + pql = `Max(field=f)` + } else { + pql = fmt.Sprintf(`Max(%s, field=f)`, tt.filter) } - for i, tt := range tests { - var pql string - if tt.filter == "" { - pql = `Min(field=f)` - } else { - pql = fmt.Sprintf(`Min(%s, field=f)`, tt.filter) - } - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { - t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) - } + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { + t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) } - }) - - t.Run("Max", func(t *testing.T) { - tests := []struct { - filter string - exp int64 - cnt int64 - }{ - {filter: ``, exp: 5, cnt: 5}, - } - for i, tt := range tests { - var pql string - if tt.filter == "" { - pql = `Max(field=f)` - } else { - pql = fmt.Sprintf(`Max(%s, field=f)`, tt.filter) - } - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: pql}); err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(result.Results[0], pilosa.ValCount{Val: tt.exp, Count: tt.cnt}) { - t.Fatalf("unexpected result, test %d: %s", i, spew.Sdump(result)) - } - } - }) + } }) } From 5cb37834a0304b17f75ec4aab40048e4acf74d46 Mon Sep 17 00:00:00 2001 From: Travis Date: Tue, 10 Dec 2019 16:05:45 -0600 Subject: [PATCH 03/20] Wrap return types: RowIdentifiers, Pair, and []Pair This PR adds a field name (string) to the return types which represent the values from a specific field. For example, a TopN query on field `x` would be `TopN(x)` and have results like: ``` []Pair{ {ID: 14, Count: 10}, {ID: 3, Count: 8}, {ID: 7, Count: 3}, } ``` In order to know what field this result type refers to, we wrap `[]Pair` in a new struct called `PairsField` which contains an addition `Field` string where `x` is stored. This is useful for informing the gRPC server how to construct more appropriate headers for the result stream (in this case, the column headers can now be "x" and "count"). Similar logic was applied to `RowIdentifiers` and `Pair` as well. --- cache.go | 25 + encoding/proto/proto.go | 56 +++ executor.go | 161 ++++--- executor_test.go | 177 ++++--- http/client_test.go | 35 +- internal/private.pb.go | 72 +-- internal/public.pb.go | 862 ++++++++++++++++++++++++++++------- internal/public.proto | 13 +- server/grpc.go | 30 +- server/grpc_internal_test.go | 4 +- server/handler_test.go | 11 +- 11 files changed, 1086 insertions(+), 360 deletions(-) diff --git a/cache.go b/cache.go index 05572fa11..6ff6f954a 100644 --- a/cache.go +++ b/cache.go @@ -16,6 +16,7 @@ package pilosa import ( "bytes" + "encoding/json" "fmt" "io" "sort" @@ -322,6 +323,18 @@ type Pair struct { Count uint64 `json:"count"` } +// PairField +type PairField struct { + Pair Pair + Field string +} + +// MarshalJSON marshals PairField into a JSON-encoded byte slice, +// excluding `Field`. +func (p PairField) MarshalJSON() ([]byte, error) { + return json.Marshal(p.Pair) +} + // Pairs is a sortable slice of Pair objects. type Pairs []Pair @@ -397,6 +410,18 @@ func (p Pairs) String() string { return buf.String() } +// PairsField +type PairsField struct { + Pairs []Pair + Field string +} + +// MarshalJSON marshals PairsField into a JSON-encoded byte slice, +// excluding `Field`. +func (p PairsField) MarshalJSON() ([]byte, error) { + return json.Marshal(p.Pairs) +} + // uint64Slice represents a sortable slice of uint64 numbers. type uint64Slice []uint64 diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 322cca1d1..c1644ed58 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -450,6 +450,9 @@ func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse { case []pilosa.Pair: pb.Results[i].Type = queryResultTypePairs pb.Results[i].Pairs = encodePairs(result) + case *pilosa.PairsField: + pb.Results[i].Type = queryResultTypePairsField + pb.Results[i].PairsField = encodePairsField(result) case pilosa.ValCount: pb.Results[i].Type = queryResultTypeValCount pb.Results[i].ValCount = encodeValCount(result) @@ -471,6 +474,9 @@ func encodeQueryResponse(m *pilosa.QueryResponse) *internal.QueryResponse { case pilosa.Pair: pb.Results[i].Type = queryResultTypePair pb.Results[i].Pairs = []*internal.Pair{encodePair(result)} + case pilosa.PairField: + pb.Results[i].Type = queryResultTypePairField + pb.Results[i].Pairs = []*internal.Pair{encodePairField(result)} case nil: pb.Results[i].Type = queryResultTypeNil default: @@ -1101,6 +1107,7 @@ const ( queryResultTypeNil uint32 = iota queryResultTypeRow queryResultTypePairs + queryResultTypePairsField queryResultTypeValCount queryResultTypeUint64 queryResultTypeBool @@ -1108,6 +1115,7 @@ const ( queryResultTypeGroupCounts queryResultTypeRowIdentifiers queryResultTypePair + queryResultTypePairField queryResultTypeSignedRow ) @@ -1119,6 +1127,8 @@ func decodeQueryResult(pb *internal.QueryResult) interface{} { return decodeRow(pb.Row) case queryResultTypePairs: return decodePairs(pb.Pairs) + case queryResultTypePairsField: + return decodePairsField(pb.PairsField) case queryResultTypeValCount: return decodeValCount(pb.ValCount) case queryResultTypeUint64: @@ -1135,6 +1145,8 @@ func decodeQueryResult(pb *internal.QueryResult) interface{} { return decodeGroupCounts(pb.GroupCounts) case queryResultTypePair: return decodePair(pb.Pairs[0]) + case queryResultTypePairField: + return decodePairField(pb.Pairs[0]) } panic(fmt.Sprintf("unknown type: %d", pb.Type)) } @@ -1243,6 +1255,17 @@ func decodePairs(a []*internal.Pair) []pilosa.Pair { return other } +func decodePairsField(a *internal.PairsField) *pilosa.PairsField { + other := &pilosa.PairsField{ + Pairs: make([]pilosa.Pair, len(a.Pairs)), + } + for i := range a.Pairs { + other.Pairs[i] = decodePair(a.Pairs[i]) + } + other.Field = a.Field + return other +} + func decodePair(pb *internal.Pair) pilosa.Pair { return pilosa.Pair{ ID: pb.ID, @@ -1251,6 +1274,17 @@ func decodePair(pb *internal.Pair) pilosa.Pair { } } +func decodePairField(pb *internal.Pair) pilosa.PairField { + return pilosa.PairField{ + Pair: pilosa.Pair{ + ID: pb.ID, + Key: pb.Key, + Count: pb.Count, + }, + //Field: pb.Field, // TODO: in order to have this, we need PairField in QueryResponse. + } +} + func decodeValCount(pb *internal.ValCount) pilosa.ValCount { return pilosa.ValCount{ Val: pb.Val, @@ -1346,6 +1380,17 @@ func encodePairs(a pilosa.Pairs) []*internal.Pair { return other } +func encodePairsField(a *pilosa.PairsField) *internal.PairsField { + other := &internal.PairsField{ + Pairs: make([]*internal.Pair, len(a.Pairs)), + } + for i := range a.Pairs { + other.Pairs[i] = encodePair(a.Pairs[i]) + } + other.Field = a.Field + return other +} + func encodePair(p pilosa.Pair) *internal.Pair { return &internal.Pair{ ID: p.ID, @@ -1354,6 +1399,17 @@ func encodePair(p pilosa.Pair) *internal.Pair { } } +func encodePairField(p pilosa.PairField) *internal.Pair { + /* + // TODO: in order to have this, we need PairField in QueryResponse. + return &internal.Pair{ + Pair: encodePair(p.Pair), + Field: p.Field, + } + */ + return encodePair(p.Pair) +} + func encodeValCount(vc pilosa.ValCount) *internal.ValCount { return &internal.ValCount{ Val: vc.Val, diff --git a/executor.go b/executor.go index 9a39d7697..72c6da626 100644 --- a/executor.go +++ b/executor.go @@ -808,14 +808,19 @@ func (e *executor) executeMinRow(ctx context.Context, index string, c *pql.Call, reduceFn := func(prev, v interface{}) interface{} { // if minRowID exists, and if it is smaller than the other one return it. // otherwise return the minRowID of the one which exists. - prevp, _ := prev.(Pair) - vp, _ := v.(Pair) - if prevp.Count > 0 && vp.Count > 0 { - if prevp.ID < vp.ID { + if prev == nil { + return v + } else if v == nil { + return prev + } + prevp, _ := prev.(PairField) + vp, _ := v.(PairField) + if prevp.Pair.Count > 0 && vp.Pair.Count > 0 { + if prevp.Pair.ID < vp.Pair.ID { return prevp } return vp - } else if prevp.Count > 0 { + } else if prevp.Pair.Count > 0 { return prevp } return vp @@ -824,7 +829,7 @@ func (e *executor) executeMinRow(ctx context.Context, index string, c *pql.Call, return e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) } -// executeMinRow executes a MaxRow() call. +// executeMaxRow executes a MaxRow() call. func (e *executor) executeMaxRow(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (interface{}, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeMaxRow") defer span.Finish() @@ -842,14 +847,19 @@ func (e *executor) executeMaxRow(ctx context.Context, index string, c *pql.Call, reduceFn := func(prev, v interface{}) interface{} { // if minRowID exists, and if it is smaller than the other one return it. // otherwise return the minRowID of the one which exists. - prevp, _ := prev.(Pair) - vp, _ := v.(Pair) - if prevp.Count > 0 && vp.Count > 0 { - if prevp.ID > vp.ID { + if prev == nil { + return v + } else if v == nil { + return prev + } + prevp, _ := prev.(PairField) + vp, _ := v.(PairField) + if prevp.Pair.Count > 0 && vp.Pair.Count > 0 { + if prevp.Pair.ID > vp.Pair.ID { return prevp } return vp - } else if prevp.Count > 0 { + } else if prevp.Pair.Count > 0 { return prevp } return vp @@ -1175,12 +1185,12 @@ func (e *executor) executeMaxShard(ctx context.Context, index string, c *pql.Cal } // executeMinRowShard returns the minimum row ID for a shard. -func (e *executor) executeMinRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (Pair, error) { +func (e *executor) executeMinRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (PairField, error) { var filter *Row if len(c.Children) == 1 { row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) if err != nil { - return Pair{}, err + return PairField{}, err } filter = row } @@ -1188,28 +1198,31 @@ func (e *executor) executeMinRowShard(ctx context.Context, index string, c *pql. fieldName, _ := c.Args["field"].(string) field := e.Holder.Field(index, fieldName) if field == nil { - return Pair{}, nil + return PairField{}, nil } fragment := e.Holder.fragment(index, fieldName, viewStandard, shard) if fragment == nil { - return Pair{}, nil + return PairField{}, nil } minRowID, count := fragment.minRow(filter) - return Pair{ - ID: minRowID, - Count: count, + return PairField{ + Pair: Pair{ + ID: minRowID, + Count: count, + }, + Field: fieldName, }, nil } // executeMaxRowShard returns the maximum row ID for a shard. -func (e *executor) executeMaxRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (Pair, error) { +func (e *executor) executeMaxRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (PairField, error) { var filter *Row if len(c.Children) == 1 { row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard) if err != nil { - return Pair{}, err + return PairField{}, err } filter = row } @@ -1217,25 +1230,28 @@ func (e *executor) executeMaxRowShard(ctx context.Context, index string, c *pql. fieldName, _ := c.Args["field"].(string) field := e.Holder.Field(index, fieldName) if field == nil { - return Pair{}, nil + return PairField{}, nil } fragment := e.Holder.fragment(index, fieldName, viewStandard, shard) if fragment == nil { - return Pair{}, nil + return PairField{}, nil } maxRowID, count := fragment.maxRow(filter) - return Pair{ - ID: maxRowID, - Count: count, + return PairField{ + Pair: Pair{ + ID: maxRowID, + Count: count, + }, + Field: fieldName, }, nil } // executeTopN executes a TopN() call. // This first performs the TopN() to determine the top results and then // requeries to retrieve the full counts for each of the top results. -func (e *executor) executeTopN(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) ([]Pair, error) { +func (e *executor) executeTopN(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*PairsField, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeTopN") defer span.Finish() @@ -1243,6 +1259,8 @@ func (e *executor) executeTopN(ctx context.Context, index string, c *pql.Call, s if err != nil { return nil, fmt.Errorf("executeTopN: %v", err) } + + fieldName, _ := c.Args["_field"].(string) n, _, err := c.UintArg("n") if err != nil { return nil, fmt.Errorf("executeTopN: %v", err) @@ -1256,13 +1274,16 @@ func (e *executor) executeTopN(ctx context.Context, index string, c *pql.Call, s // If this call is against specific ids, or we didn't get results, // or we are part of a larger distributed query then don't refetch. - if len(pairs) == 0 || len(idsArg) > 0 || opt.Remote { - return pairs, nil + if len(pairs.Pairs) == 0 || len(idsArg) > 0 || opt.Remote { + return &PairsField{ + Pairs: pairs.Pairs, + Field: fieldName, + }, nil } // Only the original caller should refetch the full counts. other := c.Clone() - ids := Pairs(pairs).Keys() + ids := Pairs(pairs.Pairs).Keys() sort.Sort(uint64Slice(ids)) other.Args["ids"] = ids @@ -1271,13 +1292,17 @@ func (e *executor) executeTopN(ctx context.Context, index string, c *pql.Call, s return nil, errors.Wrap(err, "retrieving full counts") } - if n != 0 && int(n) < len(trimmedList) { - trimmedList = trimmedList[0:n] + if n != 0 && int(n) < len(trimmedList.Pairs) { + trimmedList.Pairs = trimmedList.Pairs[0:n] } - return trimmedList, nil + + return &PairsField{ + Pairs: trimmedList.Pairs, + Field: fieldName, + }, nil } -func (e *executor) executeTopNShards(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) ([]Pair, error) { +func (e *executor) executeTopNShards(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*PairsField, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeTopNShards") defer span.Finish() @@ -1288,24 +1313,31 @@ func (e *executor) executeTopNShards(ctx context.Context, index string, c *pql.C // Merge returned results at coordinating node. reduceFn := func(prev, v interface{}) interface{} { - other, _ := prev.([]Pair) - return Pairs(other).Add(v.([]Pair)) + other, _ := prev.(*PairsField) + vpf, _ := v.(*PairsField) + if other == nil { + return vpf + } else if vpf == nil { + return other + } + other.Pairs = Pairs(other.Pairs).Add(vpf.Pairs) + return other } other, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) if err != nil { return nil, err } - results, _ := other.([]Pair) + results, _ := other.(*PairsField) // Sort final merged results. - sort.Sort(Pairs(results)) + sort.Sort(Pairs(results.Pairs)) return results, nil } // executeTopNShard executes a TopN call for a single shard. -func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Call, shard uint64) ([]Pair, error) { +func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*PairsField, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeTopNShard") defer span.Finish() @@ -1351,7 +1383,7 @@ func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Ca f := e.Holder.fragment(index, fieldName, viewStandard, shard) if f == nil { - return nil, nil + return &PairsField{}, nil } else if f.CacheType == CacheTypeNone { return nil, fmt.Errorf("cannot compute TopN(), field has no cache: %q", fieldName) } @@ -1363,7 +1395,7 @@ func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Ca if tanimotoThreshold > 100 { return nil, errors.New("Tanimoto Threshold is from 1 to 100 only") } - return f.top(topOptions{ + pairs, err := f.top(topOptions{ N: int(n), Src: src, RowIDs: rowIDs, @@ -1372,6 +1404,13 @@ func (e *executor) executeTopNShard(ctx context.Context, index string, c *pql.Ca MinThreshold: minThreshold, TanimotoThreshold: tanimotoThreshold, }) + if err != nil { + return nil, errors.Wrap(err, "getting top") + } + + return &PairsField{ + Pairs: pairs, + }, nil } // executeDifferenceShard executes a difference() call for a local shard. @@ -1405,8 +1444,14 @@ func (e *executor) executeDifferenceShard(ctx context.Context, index string, c * // Row query which returns `Columns` and `Keys`. // TODO: Rename this to something better. Anything. type RowIdentifiers struct { - Rows []uint64 `json:"rows"` - Keys []string `json:"keys,omitempty"` + Rows []uint64 `json:"rows"` + Keys []string `json:"keys,omitempty"` + field string +} + +// Field returns the field name associated to the row. +func (r *RowIdentifiers) Field() string { + return r.field } // RowIDs is a query return type for just uint64 row ids. @@ -3535,41 +3580,47 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res return other, nil } - case Pair: + case PairField: if fieldName := callArgString(call, "field"); fieldName != "" { field := idx.Field(fieldName) if field == nil { return nil, fmt.Errorf("field %q not found", fieldName) } if field.keys() { - key, err := field.translateStore.TranslateID(result.ID) + key, err := field.translateStore.TranslateID(result.Pair.ID) if err != nil { return nil, err } if call.Name == "MinRow" || call.Name == "MaxRow" { - result.Key = key + result.Pair.Key = key return result, nil } - return Pair{Key: key, Count: result.Count}, nil + return PairField{ + Pair: Pair{Key: key, Count: result.Pair.Count}, + Field: fieldName, + }, nil } } - case []Pair: + case *PairsField: if fieldName := callArgString(call, "_field"); fieldName != "" { field := idx.Field(fieldName) if field == nil { return nil, fmt.Errorf("field %q not found", fieldName) } if field.keys() { - other := make([]Pair, len(result)) - for i := range result { - key, err := field.translateStore.TranslateID(result[i].ID) + other := make([]Pair, len(result.Pairs)) + for i := range result.Pairs { + key, err := field.translateStore.TranslateID(result.Pairs[i].ID) if err != nil { return nil, err } - other[i] = Pair{Key: key, Count: result[i].Count} + other[i] = Pair{Key: key, Count: result.Pairs[i].Count} } - return other, nil + return &PairsField{ + Pairs: other, + Field: fieldName, + }, nil } } @@ -3604,13 +3655,15 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res return other, nil case RowIDs: - other := RowIdentifiers{} - fieldName := callArgString(call, "_field") if fieldName == "" { return nil, ErrFieldNotFound } + other := RowIdentifiers{ + field: fieldName, + } + if field := idx.Field(fieldName); field == nil { return nil, ErrFieldNotFound } else if field.keys() { diff --git a/executor_test.go b/executor_test.go index cea26c8af..665ef9bba 100644 --- a/executor_test.go +++ b/executor_test.go @@ -945,9 +945,12 @@ func TestExecutor_Execute_TopN(t *testing.T) { if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result.Results[0], []pilosa.Pair{ - {ID: 0, Count: 5}, - {ID: 10, Count: 2}, + } else if !reflect.DeepEqual(result.Results[0], &pilosa.PairsField{ + Pairs: []pilosa.Pair{ + {ID: 0, Count: 5}, + {ID: 10, Count: 2}, + }, + Field: "f", }) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } @@ -986,9 +989,12 @@ func TestExecutor_Execute_TopN(t *testing.T) { if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result.Results[0], []pilosa.Pair{ - {ID: 0, Count: 5}, - {ID: 10, Count: 2}, + } else if !reflect.DeepEqual(result.Results[0], &pilosa.PairsField{ + Pairs: []pilosa.Pair{ + {ID: 0, Count: 5}, + {ID: 10, Count: 2}, + }, + Field: "f", }) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } @@ -1027,11 +1033,16 @@ func TestExecutor_Execute_TopN(t *testing.T) { if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result.Results[0], []pilosa.Pair{ - {Key: "zero", Count: 5}, - {Key: "ten", Count: 2}, - }) { - t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } else { + if !reflect.DeepEqual(result.Results[0], &pilosa.PairsField{ + Pairs: []pilosa.Pair{ + {Key: "zero", Count: 5}, + {Key: "ten", Count: 2}, + }, + Field: "f", + }) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } } }) @@ -1069,9 +1080,12 @@ func TestExecutor_Execute_TopN(t *testing.T) { if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=2)`}); err != nil { t.Fatal(err) } else if diff := cmp.Diff(result.Results, []interface{}{ - []pilosa.Pair{ - {Key: "foo", Count: 5}, - {Key: "bar", Count: 2}, + &pilosa.PairsField{ + Pairs: []pilosa.Pair{ + {Key: "foo", Count: 5}, + {Key: "bar", Count: 2}, + }, + Field: "f", }, }); diff != "" { t.Fatal(diff) @@ -1154,8 +1168,11 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { // Execute query. if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=1)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{ - {ID: 0, Count: 4}, + } else if !reflect.DeepEqual(result.Results, []interface{}{&pilosa.PairsField{ + Pairs: []pilosa.Pair{ + {ID: 0, Count: 4}, + }, + Field: "f", }}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } @@ -1188,8 +1205,11 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) { // Execute query. if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=1)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{ - {ID: 0, Count: 5}, + } else if !reflect.DeepEqual(result.Results, []interface{}{&pilosa.PairsField{ + Pairs: []pilosa.Pair{ + {ID: 0, Count: 5}, + }, + Field: "f", }}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } @@ -1224,10 +1244,13 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { // Execute query. if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, Row(other=100), n=3)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{ - {ID: 20, Count: 3}, - {ID: 10, Count: 2}, - {ID: 0, Count: 1}, + } else if !reflect.DeepEqual(result.Results, []interface{}{&pilosa.PairsField{ + Pairs: []pilosa.Pair{ + {ID: 20, Count: 3}, + {ID: 10, Count: 2}, + {ID: 0, Count: 1}, + }, + Field: "f", }}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } @@ -1247,8 +1270,11 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { } if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=1, attrName="category", attrValues=[123])`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{ - {ID: 10, Count: 1}, + } else if !reflect.DeepEqual(result.Results, []interface{}{&pilosa.PairsField{ + Pairs: []pilosa.Pair{ + {ID: 10, Count: 1}, + }, + Field: "f", }}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } @@ -1270,8 +1296,11 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { } if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, Row(f=10), n=1, attrName="category", attrValues=[123])`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(result.Results, []interface{}{[]pilosa.Pair{ - {ID: 10, Count: 1}, + } else if !reflect.DeepEqual(result.Results, []interface{}{&pilosa.PairsField{ + Pairs: []pilosa.Pair{ + {ID: 10, Count: 1}, + }, + Field: "f", }}) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } @@ -1465,7 +1494,10 @@ func TestExecutor_Execute_MinMaxRow(t *testing.T) { if err != nil { t.Fatal(err) } - target := pilosa.Pair{ID: 1, Count: 1} + target := pilosa.PairField{ + Pair: pilosa.Pair{ID: 1, Count: 1}, + Field: "f", + } if !reflect.DeepEqual(target, result.Results[0]) { t.Fatalf("unexpected result %v != %v", target, result.Results[0]) } @@ -1476,7 +1508,10 @@ func TestExecutor_Execute_MinMaxRow(t *testing.T) { if err != nil { t.Fatal(err) } - target := pilosa.Pair{ID: 10000, Count: 1} + target := pilosa.PairField{ + Pair: pilosa.Pair{ID: 10000, Count: 1}, + Field: "f", + } if !reflect.DeepEqual(target, result.Results[0]) { t.Fatalf("unexpected result %v != %v", target, result.Results[0]) } @@ -1512,7 +1547,10 @@ func TestExecutor_Execute_MinMaxRow(t *testing.T) { if err != nil { t.Fatal(err) } - target := pilosa.Pair{Key: "seven-thousand", ID: 1, Count: 1} + target := pilosa.PairField{ + Pair: pilosa.Pair{Key: "seven-thousand", ID: 1, Count: 1}, + Field: "f", + } if !reflect.DeepEqual(target, result.Results[0]) { t.Fatalf("unexpected result %v != %v", target, result.Results[0]) } @@ -1523,7 +1561,10 @@ func TestExecutor_Execute_MinMaxRow(t *testing.T) { if err != nil { t.Fatal(err) } - target := pilosa.Pair{Key: "five-thousand", ID: 5, Count: 1} + target := pilosa.PairField{ + Pair: pilosa.Pair{Key: "five-thousand", ID: 5, Count: 1}, + Field: "f", + } if !reflect.DeepEqual(target, result.Results[0]) { t.Fatalf("unexpected result %v != %v", target, result.Results[0]) } @@ -2420,7 +2461,6 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { if err != nil { t.Fatalf("creating field: %v", err) } - if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` Set(500001, fn=5) Set(1500001, fn=5) @@ -2432,11 +2472,11 @@ Set(3500003, fn=3) Set(500001, fn=4) Set(4500001, fn=4) `}); err != nil { - t.Fatalf("quuerying remote: %v", err) + t.Fatalf("querying remote: %v", err) } err := c[0].API.RecalculateCaches(context.Background()) if err != nil { - t.Fatalf("recalcing caches: %v", err) + t.Fatalf("recalculating caches: %v", err) } if res, err := c[1].API.Query(context.Background(), &pilosa.QueryRequest{ @@ -2444,10 +2484,13 @@ Set(4500001, fn=4) Query: `TopN(fn, n=3)`, }); err != nil { t.Fatalf("topn querying: %v", err) - } else if !reflect.DeepEqual(res.Results, []interface{}{[]pilosa.Pair{ - {ID: 5, Count: 4}, - {ID: 3, Count: 3}, - {ID: 4, Count: 2}, + } else if !reflect.DeepEqual(res.Results, []interface{}{&pilosa.PairsField{ + Pairs: []pilosa.Pair{ + {ID: 5, Count: 4}, + {ID: 3, Count: 3}, + {ID: 4, Count: 2}, + }, + Field: "fn", }}) { t.Fatalf("topn wrong results: %v", res.Results) } @@ -3039,10 +3082,13 @@ func TestExecutor_Execute_ClearRow(t *testing.T) { // Check the TopN results. if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=5)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(res.Results, []interface{}{[]pilosa.Pair{ - {ID: 1, Count: 7}, - {ID: 2, Count: 6}, - {ID: 3, Count: 5}, + } else if !reflect.DeepEqual(res.Results, []interface{}{&pilosa.PairsField{ + Pairs: []pilosa.Pair{ + {ID: 1, Count: 7}, + {ID: 2, Count: 6}, + {ID: 3, Count: 5}, + }, + Field: "f", }}) { t.Fatalf("topn wrong results: %v", res.Results) } @@ -3057,9 +3103,12 @@ func TestExecutor_Execute_ClearRow(t *testing.T) { // Ensure that the cleared row doesn't show up in TopN (i.e. it was removed from the cache). if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `TopN(f, n=5)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(res.Results, []interface{}{[]pilosa.Pair{ - {ID: 1, Count: 7}, - {ID: 3, Count: 5}, + } else if !reflect.DeepEqual(res.Results, []interface{}{&pilosa.PairsField{ + Pairs: []pilosa.Pair{ + {ID: 1, Count: 7}, + {ID: 3, Count: 5}, + }, + Field: "f", }}) { t.Fatalf("topn wrong results: %v", res.Results) } @@ -3280,30 +3329,40 @@ func TestExecutor_Execute_Rows(t *testing.T) { }) rows := c.Query(t, "i", `Rows(general)`).Results[0].(pilosa.RowIdentifiers) - if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{10, 11, 12, 13}}) { - t.Fatalf("unexpected rows: %+v", rows) + if !reflect.DeepEqual(rows.Rows, []uint64{10, 11, 12, 13}) { + t.Fatalf("unexpected rows: %+v", rows.Rows) + } else if rows.Keys != nil { + t.Fatalf("unexpected keys: %+v", rows.Keys) } // backwards compatibility // TODO: remove at Pilosa 2.0 rows = c.Query(t, "i", `Rows(field=general)`).Results[0].(pilosa.RowIdentifiers) - if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{10, 11, 12, 13}}) { - t.Fatalf("unexpected rows: %+v", rows) + if !reflect.DeepEqual(rows.Rows, []uint64{10, 11, 12, 13}) { + t.Fatalf("unexpected rows: %+v", rows.Rows) + } else if rows.Keys != nil { + t.Fatalf("unexpected keys: %+v", rows.Keys) } rows = c.Query(t, "i", `Rows(general, limit=2)`).Results[0].(pilosa.RowIdentifiers) - if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{10, 11}}) { - t.Fatalf("unexpected rows: %+v", rows) + if !reflect.DeepEqual(rows.Rows, []uint64{10, 11}) { + t.Fatalf("unexpected rows: %+v", rows.Rows) + } else if rows.Keys != nil { + t.Fatalf("unexpected keys: %+v", rows.Keys) } rows = c.Query(t, "i", `Rows(general, previous=10,limit=2)`).Results[0].(pilosa.RowIdentifiers) - if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) { - t.Fatalf("unexpected rows: %+v", rows) + if !reflect.DeepEqual(rows.Rows, []uint64{11, 12}) { + t.Fatalf("unexpected rows: %+v", rows.Rows) + } else if rows.Keys != nil { + t.Fatalf("unexpected keys: %+v", rows.Keys) } rows = c.Query(t, "i", `Rows(general, column=2)`).Results[0].(pilosa.RowIdentifiers) - if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) { - t.Fatalf("unexpected rows: %+v", rows) + if !reflect.DeepEqual(rows.Rows, []uint64{11, 12}) { + t.Fatalf("unexpected rows: %+v", rows.Rows) + } else if rows.Keys != nil { + t.Fatalf("unexpected keys: %+v", rows.Keys) } } @@ -3613,9 +3672,13 @@ func TestExecutor_Execute_Rows_Keys(t *testing.T) { t.Run(fmt.Sprintf("#%d_%s", i, test.q), func(t *testing.T) { if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.q}); err != nil { t.Fatal(err) - } else if rows := res.Results[0].(pilosa.RowIdentifiers); !reflect.DeepEqual( - rows, pilosa.RowIdentifiers{Keys: test.exp}) { - t.Fatalf("\ngot: %+v\nexp: %+v", rows, pilosa.RowIdentifiers{Keys: test.exp}) + } else { + rows := res.Results[0].(pilosa.RowIdentifiers) + if !reflect.DeepEqual(rows.Keys, test.exp) { + t.Fatalf("\ngot: %+v\nexp: %+v", rows.Keys, test.exp) + } else if rows.Rows != nil { + t.Fatalf("\ngot: %+v\nexp: nil", rows.Rows) + } } }) } diff --git a/http/client_test.go b/http/client_test.go index f8743555e..077a11ff4 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -148,7 +148,8 @@ func TestClient_MultiNode(t *testing.T) { } // Test must return exactly N results. - if len(result.Results[0].([]pilosa.Pair)) != topN { + pairsField := result.Results[0].(*pilosa.PairsField) + if len(pairsField.Pairs) != topN { t.Fatalf("unexpected number of TopN results: %s", spew.Sdump(result)) } p := []pilosa.Pair{ @@ -158,7 +159,7 @@ func TestClient_MultiNode(t *testing.T) { {ID: 99, Count: 7}} // Valdidate the Top 4 result counts. - if !reflect.DeepEqual(result.Results[0].([]pilosa.Pair), p) { + if !reflect.DeepEqual(pairsField.Pairs, p) { t.Fatalf("Invalid TopN result set: %s", spew.Sdump(result)) } @@ -605,14 +606,14 @@ func TestClient_ImportKeys(t *testing.T) { Index: "keyed", Query: "TopN(keyedf)", }) - if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok { + if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok { t.Fatalf("unexpected response type %T", resp.Results[0]) - } else if !reflect.DeepEqual(pairs, []pilosa.Pair{ + } else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{ {Key: "green", Count: 3}, {Key: "blue", Count: 2}, {Key: "purple", Count: 1}, }) { - t.Fatalf("unexpected topn result: %v", pairs) + t.Fatalf("unexpected topn result: %v", pairs.Pairs) } }) @@ -632,14 +633,14 @@ func TestClient_ImportKeys(t *testing.T) { Index: "keyed", Query: "TopN(unkeyedf)", }) - if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok { + if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok { t.Fatalf("unexpected response type %T", resp.Results[0]) - } else if !reflect.DeepEqual(pairs, []pilosa.Pair{ + } else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{ {ID: 1, Count: 3}, {ID: 2, Count: 2}, {ID: 3, Count: 1}, }) { - t.Fatalf("unexpected topn result: %v", pairs) + t.Fatalf("unexpected topn result: %v", pairs.Pairs) } }) @@ -659,14 +660,14 @@ func TestClient_ImportKeys(t *testing.T) { Index: "unkeyed", Query: "TopN(keyedf)", }) - if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok { + if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok { t.Fatalf("unexpected response type %T", resp.Results[0]) - } else if !reflect.DeepEqual(pairs, []pilosa.Pair{ + } else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{ {Key: "green", Count: 3}, {Key: "blue", Count: 2}, {Key: "purple", Count: 1}, }) { - t.Fatalf("unexpected topn result: %v", pairs) + t.Fatalf("unexpected topn result: %v", pairs.Pairs) } }) }) @@ -704,14 +705,14 @@ func TestClient_ImportKeys(t *testing.T) { Index: "keyed", Query: "TopN(keyedf0)", }) - if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok { + if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok { t.Fatalf("unexpected response type %T", resp.Results[0]) - } else if !reflect.DeepEqual(pairs, []pilosa.Pair{ + } else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{ {Key: "green", Count: 3}, {Key: "blue", Count: 2}, {Key: "purple", Count: 1}, }) { - t.Fatalf("unexpected topn result: %v", pairs) + t.Fatalf("unexpected topn result: %v", pairs.Pairs) } }) @@ -736,14 +737,14 @@ func TestClient_ImportKeys(t *testing.T) { Index: "keyed", Query: "TopN(keyedf1)", }) - if pairs, ok := resp.Results[0].([]pilosa.Pair); !ok { + if pairs, ok := resp.Results[0].(*pilosa.PairsField); !ok { t.Fatalf("unexpected response type %T", resp.Results[0]) - } else if !reflect.DeepEqual(pairs, []pilosa.Pair{ + } else if !reflect.DeepEqual(pairs.Pairs, []pilosa.Pair{ {Key: "green", Count: 3}, {Key: "blue", Count: 2}, {Key: "purple", Count: 1}, }) { - t.Fatalf("unexpected topn result: %#v", pairs) + t.Fatalf("unexpected topn result: %#v", pairs.Pairs) } }) }) diff --git a/internal/private.pb.go b/internal/private.pb.go index 755370e74..d42e410cf 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -32,7 +32,7 @@ func (m *IndexMeta) Reset() { *m = IndexMeta{} } func (m *IndexMeta) String() string { return proto.CompactTextString(m) } func (*IndexMeta) ProtoMessage() {} func (*IndexMeta) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{0} + return fileDescriptor_private_e6d12fddb5948a73, []int{0} } func (m *IndexMeta) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -96,7 +96,7 @@ func (m *FieldOptions) Reset() { *m = FieldOptions{} } func (m *FieldOptions) String() string { return proto.CompactTextString(m) } func (*FieldOptions) ProtoMessage() {} func (*FieldOptions) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{1} + return fileDescriptor_private_e6d12fddb5948a73, []int{1} } func (m *FieldOptions) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -213,7 +213,7 @@ func (m *ImportResponse) Reset() { *m = ImportResponse{} } func (m *ImportResponse) String() string { return proto.CompactTextString(m) } func (*ImportResponse) ProtoMessage() {} func (*ImportResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{2} + return fileDescriptor_private_e6d12fddb5948a73, []int{2} } func (m *ImportResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -264,7 +264,7 @@ func (m *BlockDataRequest) Reset() { *m = BlockDataRequest{} } func (m *BlockDataRequest) String() string { return proto.CompactTextString(m) } func (*BlockDataRequest) ProtoMessage() {} func (*BlockDataRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{3} + return fileDescriptor_private_e6d12fddb5948a73, []int{3} } func (m *BlockDataRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -340,7 +340,7 @@ func (m *BlockDataResponse) Reset() { *m = BlockDataResponse{} } func (m *BlockDataResponse) String() string { return proto.CompactTextString(m) } func (*BlockDataResponse) ProtoMessage() {} func (*BlockDataResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{4} + return fileDescriptor_private_e6d12fddb5948a73, []int{4} } func (m *BlockDataResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -394,7 +394,7 @@ func (m *Cache) Reset() { *m = Cache{} } func (m *Cache) String() string { return proto.CompactTextString(m) } func (*Cache) ProtoMessage() {} func (*Cache) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{5} + return fileDescriptor_private_e6d12fddb5948a73, []int{5} } func (m *Cache) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -441,7 +441,7 @@ func (m *MaxShards) Reset() { *m = MaxShards{} } func (m *MaxShards) String() string { return proto.CompactTextString(m) } func (*MaxShards) ProtoMessage() {} func (*MaxShards) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{6} + return fileDescriptor_private_e6d12fddb5948a73, []int{6} } func (m *MaxShards) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -490,7 +490,7 @@ func (m *CreateShardMessage) Reset() { *m = CreateShardMessage{} } func (m *CreateShardMessage) String() string { return proto.CompactTextString(m) } func (*CreateShardMessage) ProtoMessage() {} func (*CreateShardMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{7} + return fileDescriptor_private_e6d12fddb5948a73, []int{7} } func (m *CreateShardMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -551,7 +551,7 @@ func (m *DeleteIndexMessage) Reset() { *m = DeleteIndexMessage{} } func (m *DeleteIndexMessage) String() string { return proto.CompactTextString(m) } func (*DeleteIndexMessage) ProtoMessage() {} func (*DeleteIndexMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{8} + return fileDescriptor_private_e6d12fddb5948a73, []int{8} } func (m *DeleteIndexMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -599,7 +599,7 @@ func (m *CreateIndexMessage) Reset() { *m = CreateIndexMessage{} } func (m *CreateIndexMessage) String() string { return proto.CompactTextString(m) } func (*CreateIndexMessage) ProtoMessage() {} func (*CreateIndexMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{9} + return fileDescriptor_private_e6d12fddb5948a73, []int{9} } func (m *CreateIndexMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -655,7 +655,7 @@ func (m *CreateFieldMessage) Reset() { *m = CreateFieldMessage{} } func (m *CreateFieldMessage) String() string { return proto.CompactTextString(m) } func (*CreateFieldMessage) ProtoMessage() {} func (*CreateFieldMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{10} + return fileDescriptor_private_e6d12fddb5948a73, []int{10} } func (m *CreateFieldMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -717,7 +717,7 @@ func (m *DeleteFieldMessage) Reset() { *m = DeleteFieldMessage{} } func (m *DeleteFieldMessage) String() string { return proto.CompactTextString(m) } func (*DeleteFieldMessage) ProtoMessage() {} func (*DeleteFieldMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{11} + return fileDescriptor_private_e6d12fddb5948a73, []int{11} } func (m *DeleteFieldMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -773,7 +773,7 @@ func (m *DeleteAvailableShardMessage) Reset() { *m = DeleteAvailableShar func (m *DeleteAvailableShardMessage) String() string { return proto.CompactTextString(m) } func (*DeleteAvailableShardMessage) ProtoMessage() {} func (*DeleteAvailableShardMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{12} + return fileDescriptor_private_e6d12fddb5948a73, []int{12} } func (m *DeleteAvailableShardMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -836,7 +836,7 @@ func (m *Field) Reset() { *m = Field{} } func (m *Field) String() string { return proto.CompactTextString(m) } func (*Field) ProtoMessage() {} func (*Field) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{13} + return fileDescriptor_private_e6d12fddb5948a73, []int{13} } func (m *Field) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -897,7 +897,7 @@ func (m *Schema) Reset() { *m = Schema{} } func (m *Schema) String() string { return proto.CompactTextString(m) } func (*Schema) ProtoMessage() {} func (*Schema) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{14} + return fileDescriptor_private_e6d12fddb5948a73, []int{14} } func (m *Schema) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -945,7 +945,7 @@ func (m *Index) Reset() { *m = Index{} } func (m *Index) String() string { return proto.CompactTextString(m) } func (*Index) ProtoMessage() {} func (*Index) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{15} + return fileDescriptor_private_e6d12fddb5948a73, []int{15} } func (m *Index) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1001,7 +1001,7 @@ func (m *URI) Reset() { *m = URI{} } func (m *URI) String() string { return proto.CompactTextString(m) } func (*URI) ProtoMessage() {} func (*URI) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{16} + return fileDescriptor_private_e6d12fddb5948a73, []int{16} } func (m *URI) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1065,7 +1065,7 @@ func (m *Node) Reset() { *m = Node{} } func (m *Node) String() string { return proto.CompactTextString(m) } func (*Node) ProtoMessage() {} func (*Node) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{17} + return fileDescriptor_private_e6d12fddb5948a73, []int{17} } func (m *Node) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1134,7 +1134,7 @@ func (m *NodeStateMessage) Reset() { *m = NodeStateMessage{} } func (m *NodeStateMessage) String() string { return proto.CompactTextString(m) } func (*NodeStateMessage) ProtoMessage() {} func (*NodeStateMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{18} + return fileDescriptor_private_e6d12fddb5948a73, []int{18} } func (m *NodeStateMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1189,7 +1189,7 @@ func (m *NodeEventMessage) Reset() { *m = NodeEventMessage{} } func (m *NodeEventMessage) String() string { return proto.CompactTextString(m) } func (*NodeEventMessage) ProtoMessage() {} func (*NodeEventMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{19} + return fileDescriptor_private_e6d12fddb5948a73, []int{19} } func (m *NodeEventMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1245,7 +1245,7 @@ func (m *NodeStatus) Reset() { *m = NodeStatus{} } func (m *NodeStatus) String() string { return proto.CompactTextString(m) } func (*NodeStatus) ProtoMessage() {} func (*NodeStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{20} + return fileDescriptor_private_e6d12fddb5948a73, []int{20} } func (m *NodeStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1307,7 +1307,7 @@ func (m *IndexStatus) Reset() { *m = IndexStatus{} } func (m *IndexStatus) String() string { return proto.CompactTextString(m) } func (*IndexStatus) ProtoMessage() {} func (*IndexStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{21} + return fileDescriptor_private_e6d12fddb5948a73, []int{21} } func (m *IndexStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1362,7 +1362,7 @@ func (m *FieldStatus) Reset() { *m = FieldStatus{} } func (m *FieldStatus) String() string { return proto.CompactTextString(m) } func (*FieldStatus) ProtoMessage() {} func (*FieldStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{22} + return fileDescriptor_private_e6d12fddb5948a73, []int{22} } func (m *FieldStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1418,7 +1418,7 @@ func (m *ClusterStatus) Reset() { *m = ClusterStatus{} } func (m *ClusterStatus) String() string { return proto.CompactTextString(m) } func (*ClusterStatus) ProtoMessage() {} func (*ClusterStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{23} + return fileDescriptor_private_e6d12fddb5948a73, []int{23} } func (m *ClusterStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1482,7 +1482,7 @@ func (m *BSIGroup) Reset() { *m = BSIGroup{} } func (m *BSIGroup) String() string { return proto.CompactTextString(m) } func (*BSIGroup) ProtoMessage() {} func (*BSIGroup) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{24} + return fileDescriptor_private_e6d12fddb5948a73, []int{24} } func (m *BSIGroup) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1552,7 +1552,7 @@ func (m *CreateViewMessage) Reset() { *m = CreateViewMessage{} } func (m *CreateViewMessage) String() string { return proto.CompactTextString(m) } func (*CreateViewMessage) ProtoMessage() {} func (*CreateViewMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{25} + return fileDescriptor_private_e6d12fddb5948a73, []int{25} } func (m *CreateViewMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1615,7 +1615,7 @@ func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} } func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) } func (*DeleteViewMessage) ProtoMessage() {} func (*DeleteViewMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{26} + return fileDescriptor_private_e6d12fddb5948a73, []int{26} } func (m *DeleteViewMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1681,7 +1681,7 @@ func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} } func (m *ResizeInstruction) String() string { return proto.CompactTextString(m) } func (*ResizeInstruction) ProtoMessage() {} func (*ResizeInstruction) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{27} + return fileDescriptor_private_e6d12fddb5948a73, []int{27} } func (m *ResizeInstruction) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1767,7 +1767,7 @@ func (m *ResizeSource) Reset() { *m = ResizeSource{} } func (m *ResizeSource) String() string { return proto.CompactTextString(m) } func (*ResizeSource) ProtoMessage() {} func (*ResizeSource) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{28} + return fileDescriptor_private_e6d12fddb5948a73, []int{28} } func (m *ResizeSource) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1844,7 +1844,7 @@ func (m *ResizeInstructionComplete) Reset() { *m = ResizeInstructionComp func (m *ResizeInstructionComplete) String() string { return proto.CompactTextString(m) } func (*ResizeInstructionComplete) ProtoMessage() {} func (*ResizeInstructionComplete) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{29} + return fileDescriptor_private_e6d12fddb5948a73, []int{29} } func (m *ResizeInstructionComplete) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1905,7 +1905,7 @@ func (m *SetCoordinatorMessage) Reset() { *m = SetCoordinatorMessage{} } func (m *SetCoordinatorMessage) String() string { return proto.CompactTextString(m) } func (*SetCoordinatorMessage) ProtoMessage() {} func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{30} + return fileDescriptor_private_e6d12fddb5948a73, []int{30} } func (m *SetCoordinatorMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1952,7 +1952,7 @@ func (m *UpdateCoordinatorMessage) Reset() { *m = UpdateCoordinatorMessa func (m *UpdateCoordinatorMessage) String() string { return proto.CompactTextString(m) } func (*UpdateCoordinatorMessage) ProtoMessage() {} func (*UpdateCoordinatorMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{31} + return fileDescriptor_private_e6d12fddb5948a73, []int{31} } func (m *UpdateCoordinatorMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2000,7 +2000,7 @@ func (m *Topology) Reset() { *m = Topology{} } func (m *Topology) String() string { return proto.CompactTextString(m) } func (*Topology) ProtoMessage() {} func (*Topology) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{32} + return fileDescriptor_private_e6d12fddb5948a73, []int{32} } func (m *Topology) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2053,7 +2053,7 @@ func (m *RecalculateCaches) Reset() { *m = RecalculateCaches{} } func (m *RecalculateCaches) String() string { return proto.CompactTextString(m) } func (*RecalculateCaches) ProtoMessage() {} func (*RecalculateCaches) Descriptor() ([]byte, []int) { - return fileDescriptor_private_b229d027a4642df7, []int{33} + return fileDescriptor_private_e6d12fddb5948a73, []int{33} } func (m *RecalculateCaches) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -8982,9 +8982,9 @@ var ( ErrIntOverflowPrivate = fmt.Errorf("proto: integer overflow") ) -func init() { proto.RegisterFile("private.proto", fileDescriptor_private_b229d027a4642df7) } +func init() { proto.RegisterFile("private.proto", fileDescriptor_private_e6d12fddb5948a73) } -var fileDescriptor_private_b229d027a4642df7 = []byte{ +var fileDescriptor_private_e6d12fddb5948a73 = []byte{ // 1174 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdb, 0x6e, 0x1b, 0x45, 0x18, 0x66, 0x0f, 0x71, 0xec, 0xdf, 0x71, 0x0e, 0xdb, 0x36, 0x6c, 0x0b, 0x0a, 0x66, 0x54, 0x51, diff --git a/internal/public.pb.go b/internal/public.pb.go index 731ae14c0..08ed42318 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -36,7 +36,7 @@ func (m *Row) Reset() { *m = Row{} } func (m *Row) String() string { return proto.CompactTextString(m) } func (*Row) ProtoMessage() {} func (*Row) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{0} + return fileDescriptor_public_48374b395a722341, []int{0} } func (m *Row) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -105,7 +105,7 @@ func (m *SignedRow) Reset() { *m = SignedRow{} } func (m *SignedRow) String() string { return proto.CompactTextString(m) } func (*SignedRow) ProtoMessage() {} func (*SignedRow) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{1} + return fileDescriptor_public_48374b395a722341, []int{1} } func (m *SignedRow) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -160,7 +160,7 @@ func (m *RowIdentifiers) Reset() { *m = RowIdentifiers{} } func (m *RowIdentifiers) String() string { return proto.CompactTextString(m) } func (*RowIdentifiers) ProtoMessage() {} func (*RowIdentifiers) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{2} + return fileDescriptor_public_48374b395a722341, []int{2} } func (m *RowIdentifiers) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -216,7 +216,7 @@ func (m *Pair) Reset() { *m = Pair{} } func (m *Pair) String() string { return proto.CompactTextString(m) } func (*Pair) ProtoMessage() {} func (*Pair) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{3} + return fileDescriptor_public_48374b395a722341, []int{3} } func (m *Pair) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -266,6 +266,116 @@ func (m *Pair) GetCount() uint64 { return 0 } +type PairField struct { + Pair *Pair `protobuf:"bytes,1,opt,name=Pair" json:"Pair,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *PairField) Reset() { *m = PairField{} } +func (m *PairField) String() string { return proto.CompactTextString(m) } +func (*PairField) ProtoMessage() {} +func (*PairField) Descriptor() ([]byte, []int) { + return fileDescriptor_public_48374b395a722341, []int{4} +} +func (m *PairField) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *PairField) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_PairField.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (dst *PairField) XXX_Merge(src proto.Message) { + xxx_messageInfo_PairField.Merge(dst, src) +} +func (m *PairField) XXX_Size() int { + return m.Size() +} +func (m *PairField) XXX_DiscardUnknown() { + xxx_messageInfo_PairField.DiscardUnknown(m) +} + +var xxx_messageInfo_PairField proto.InternalMessageInfo + +func (m *PairField) GetPair() *Pair { + if m != nil { + return m.Pair + } + return nil +} + +func (m *PairField) GetField() string { + if m != nil { + return m.Field + } + return "" +} + +type PairsField struct { + Pairs []*Pair `protobuf:"bytes,1,rep,name=Pairs" json:"Pairs,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *PairsField) Reset() { *m = PairsField{} } +func (m *PairsField) String() string { return proto.CompactTextString(m) } +func (*PairsField) ProtoMessage() {} +func (*PairsField) Descriptor() ([]byte, []int) { + return fileDescriptor_public_48374b395a722341, []int{5} +} +func (m *PairsField) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *PairsField) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_PairsField.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (dst *PairsField) XXX_Merge(src proto.Message) { + xxx_messageInfo_PairsField.Merge(dst, src) +} +func (m *PairsField) XXX_Size() int { + return m.Size() +} +func (m *PairsField) XXX_DiscardUnknown() { + xxx_messageInfo_PairsField.DiscardUnknown(m) +} + +var xxx_messageInfo_PairsField proto.InternalMessageInfo + +func (m *PairsField) GetPairs() []*Pair { + if m != nil { + return m.Pairs + } + return nil +} + +func (m *PairsField) GetField() string { + if m != nil { + return m.Field + } + return "" +} + type FieldRow struct { Field string `protobuf:"bytes,1,opt,name=Field,proto3" json:"Field,omitempty"` RowID uint64 `protobuf:"varint,2,opt,name=RowID,proto3" json:"RowID,omitempty"` @@ -279,7 +389,7 @@ func (m *FieldRow) Reset() { *m = FieldRow{} } func (m *FieldRow) String() string { return proto.CompactTextString(m) } func (*FieldRow) ProtoMessage() {} func (*FieldRow) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{4} + return fileDescriptor_public_48374b395a722341, []int{6} } func (m *FieldRow) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -342,7 +452,7 @@ func (m *GroupCount) Reset() { *m = GroupCount{} } func (m *GroupCount) String() string { return proto.CompactTextString(m) } func (*GroupCount) ProtoMessage() {} func (*GroupCount) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{5} + return fileDescriptor_public_48374b395a722341, []int{7} } func (m *GroupCount) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -404,7 +514,7 @@ func (m *ValCount) Reset() { *m = ValCount{} } func (m *ValCount) String() string { return proto.CompactTextString(m) } func (*ValCount) ProtoMessage() {} func (*ValCount) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{6} + return fileDescriptor_public_48374b395a722341, []int{8} } func (m *ValCount) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -460,7 +570,7 @@ func (m *ColumnAttrSet) Reset() { *m = ColumnAttrSet{} } func (m *ColumnAttrSet) String() string { return proto.CompactTextString(m) } func (*ColumnAttrSet) ProtoMessage() {} func (*ColumnAttrSet) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{7} + return fileDescriptor_public_48374b395a722341, []int{9} } func (m *ColumnAttrSet) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -526,7 +636,7 @@ func (m *Attr) Reset() { *m = Attr{} } func (m *Attr) String() string { return proto.CompactTextString(m) } func (*Attr) ProtoMessage() {} func (*Attr) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{8} + return fileDescriptor_public_48374b395a722341, []int{10} } func (m *Attr) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -608,7 +718,7 @@ func (m *AttrMap) Reset() { *m = AttrMap{} } func (m *AttrMap) String() string { return proto.CompactTextString(m) } func (*AttrMap) ProtoMessage() {} func (*AttrMap) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{9} + return fileDescriptor_public_48374b395a722341, []int{11} } func (m *AttrMap) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -661,7 +771,7 @@ func (m *QueryRequest) Reset() { *m = QueryRequest{} } func (m *QueryRequest) String() string { return proto.CompactTextString(m) } func (*QueryRequest) ProtoMessage() {} func (*QueryRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{10} + return fileDescriptor_public_48374b395a722341, []int{12} } func (m *QueryRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -752,7 +862,7 @@ func (m *QueryResponse) Reset() { *m = QueryResponse{} } func (m *QueryResponse) String() string { return proto.CompactTextString(m) } func (*QueryResponse) ProtoMessage() {} func (*QueryResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{11} + return fileDescriptor_public_48374b395a722341, []int{13} } func (m *QueryResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -813,6 +923,7 @@ type QueryResult struct { GroupCounts []*GroupCount `protobuf:"bytes,8,rep,name=GroupCounts" json:"GroupCounts,omitempty"` RowIdentifiers *RowIdentifiers `protobuf:"bytes,9,opt,name=RowIdentifiers" json:"RowIdentifiers,omitempty"` SignedRow *SignedRow `protobuf:"bytes,10,opt,name=SignedRow" json:"SignedRow,omitempty"` + PairsField *PairsField `protobuf:"bytes,11,opt,name=PairsField" json:"PairsField,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -822,7 +933,7 @@ func (m *QueryResult) Reset() { *m = QueryResult{} } func (m *QueryResult) String() string { return proto.CompactTextString(m) } func (*QueryResult) ProtoMessage() {} func (*QueryResult) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{12} + return fileDescriptor_public_48374b395a722341, []int{14} } func (m *QueryResult) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -921,6 +1032,13 @@ func (m *QueryResult) GetSignedRow() *SignedRow { return nil } +func (m *QueryResult) GetPairsField() *PairsField { + if m != nil { + return m.PairsField + } + return nil +} + type ImportRequest struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` @@ -939,7 +1057,7 @@ func (m *ImportRequest) Reset() { *m = ImportRequest{} } func (m *ImportRequest) String() string { return proto.CompactTextString(m) } func (*ImportRequest) ProtoMessage() {} func (*ImportRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{13} + return fileDescriptor_public_48374b395a722341, []int{15} } func (m *ImportRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1041,7 +1159,7 @@ func (m *ImportValueRequest) Reset() { *m = ImportValueRequest{} } func (m *ImportValueRequest) String() string { return proto.CompactTextString(m) } func (*ImportValueRequest) ProtoMessage() {} func (*ImportValueRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{14} + return fileDescriptor_public_48374b395a722341, []int{16} } func (m *ImportValueRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1132,7 +1250,7 @@ func (m *TranslateKeysRequest) Reset() { *m = TranslateKeysRequest{} } func (m *TranslateKeysRequest) String() string { return proto.CompactTextString(m) } func (*TranslateKeysRequest) ProtoMessage() {} func (*TranslateKeysRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{15} + return fileDescriptor_public_48374b395a722341, []int{17} } func (m *TranslateKeysRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1193,7 +1311,7 @@ func (m *TranslateKeysResponse) Reset() { *m = TranslateKeysResponse{} } func (m *TranslateKeysResponse) String() string { return proto.CompactTextString(m) } func (*TranslateKeysResponse) ProtoMessage() {} func (*TranslateKeysResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{16} + return fileDescriptor_public_48374b395a722341, []int{18} } func (m *TranslateKeysResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1241,7 +1359,7 @@ func (m *ImportRoaringRequestView) Reset() { *m = ImportRoaringRequestVi func (m *ImportRoaringRequestView) String() string { return proto.CompactTextString(m) } func (*ImportRoaringRequestView) ProtoMessage() {} func (*ImportRoaringRequestView) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{17} + return fileDescriptor_public_48374b395a722341, []int{19} } func (m *ImportRoaringRequestView) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1296,7 +1414,7 @@ func (m *ImportRoaringRequest) Reset() { *m = ImportRoaringRequest{} } func (m *ImportRoaringRequest) String() string { return proto.CompactTextString(m) } func (*ImportRoaringRequest) ProtoMessage() {} func (*ImportRoaringRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{18} + return fileDescriptor_public_48374b395a722341, []int{20} } func (m *ImportRoaringRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1354,7 +1472,7 @@ func (m *ImportColumnAttrsRequest) Reset() { *m = ImportColumnAttrsReque func (m *ImportColumnAttrsRequest) String() string { return proto.CompactTextString(m) } func (*ImportColumnAttrsRequest) ProtoMessage() {} func (*ImportColumnAttrsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_568b1fcbeadcdcca, []int{19} + return fileDescriptor_public_48374b395a722341, []int{21} } func (m *ImportColumnAttrsRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1423,6 +1541,8 @@ func init() { proto.RegisterType((*SignedRow)(nil), "internal.SignedRow") proto.RegisterType((*RowIdentifiers)(nil), "internal.RowIdentifiers") proto.RegisterType((*Pair)(nil), "internal.Pair") + proto.RegisterType((*PairField)(nil), "internal.PairField") + proto.RegisterType((*PairsField)(nil), "internal.PairsField") proto.RegisterType((*FieldRow)(nil), "internal.FieldRow") proto.RegisterType((*GroupCount)(nil), "internal.GroupCount") proto.RegisterType((*ValCount)(nil), "internal.ValCount") @@ -1642,6 +1762,82 @@ func (m *Pair) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *PairField) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PairField) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Pair != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.Pair.Size())) + n7, err := m.Pair.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n7 + } + if len(m.Field) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *PairsField) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PairsField) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Pairs) > 0 { + for _, msg := range m.Pairs { + dAtA[i] = 0xa + i++ + i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if len(m.Field) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + func (m *FieldRow) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -1912,21 +2108,21 @@ func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], m.Query) } if len(m.Shards) > 0 { - dAtA8 := make([]byte, len(m.Shards)*10) - var j7 int + dAtA9 := make([]byte, len(m.Shards)*10) + var j8 int for _, num := range m.Shards { for num >= 1<<7 { - dAtA8[j7] = uint8(uint64(num)&0x7f | 0x80) + dAtA9[j8] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j7++ + j8++ } - dAtA8[j7] = uint8(num) - j7++ + dAtA9[j8] = uint8(num) + j8++ } dAtA[i] = 0x12 i++ - i = encodeVarintPublic(dAtA, i, uint64(j7)) - i += copy(dAtA[i:], dAtA8[:j7]) + i = encodeVarintPublic(dAtA, i, uint64(j8)) + i += copy(dAtA[i:], dAtA9[:j8]) } if m.ColumnAttrs { dAtA[i] = 0x18 @@ -2056,11 +2252,11 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintPublic(dAtA, i, uint64(m.Row.Size())) - n9, err := m.Row.MarshalTo(dAtA[i:]) + n10, err := m.Row.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n9 + i += n10 } if m.N != 0 { dAtA[i] = 0x10 @@ -2093,11 +2289,11 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x2a i++ i = encodeVarintPublic(dAtA, i, uint64(m.ValCount.Size())) - n10, err := m.ValCount.MarshalTo(dAtA[i:]) + n11, err := m.ValCount.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n10 + i += n11 } if m.Type != 0 { dAtA[i] = 0x30 @@ -2105,21 +2301,21 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(m.Type)) } if len(m.RowIDs) > 0 { - dAtA12 := make([]byte, len(m.RowIDs)*10) - var j11 int + dAtA13 := make([]byte, len(m.RowIDs)*10) + var j12 int for _, num := range m.RowIDs { for num >= 1<<7 { - dAtA12[j11] = uint8(uint64(num)&0x7f | 0x80) + dAtA13[j12] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j11++ + j12++ } - dAtA12[j11] = uint8(num) - j11++ + dAtA13[j12] = uint8(num) + j12++ } dAtA[i] = 0x3a i++ - i = encodeVarintPublic(dAtA, i, uint64(j11)) - i += copy(dAtA[i:], dAtA12[:j11]) + i = encodeVarintPublic(dAtA, i, uint64(j12)) + i += copy(dAtA[i:], dAtA13[:j12]) } if len(m.GroupCounts) > 0 { for _, msg := range m.GroupCounts { @@ -2137,21 +2333,31 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x4a i++ i = encodeVarintPublic(dAtA, i, uint64(m.RowIdentifiers.Size())) - n13, err := m.RowIdentifiers.MarshalTo(dAtA[i:]) + n14, err := m.RowIdentifiers.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n13 + i += n14 } if m.SignedRow != nil { dAtA[i] = 0x52 i++ i = encodeVarintPublic(dAtA, i, uint64(m.SignedRow.Size())) - n14, err := m.SignedRow.MarshalTo(dAtA[i:]) + n15, err := m.SignedRow.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n14 + i += n15 + } + if m.PairsField != nil { + dAtA[i] = 0x5a + i++ + i = encodeVarintPublic(dAtA, i, uint64(m.PairsField.Size())) + n16, err := m.PairsField.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n16 } if m.XXX_unrecognized != nil { i += copy(dAtA[i:], m.XXX_unrecognized) @@ -2192,26 +2398,9 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(m.Shard)) } if len(m.RowIDs) > 0 { - dAtA16 := make([]byte, len(m.RowIDs)*10) - var j15 int - for _, num := range m.RowIDs { - for num >= 1<<7 { - dAtA16[j15] = uint8(uint64(num)&0x7f | 0x80) - num >>= 7 - j15++ - } - dAtA16[j15] = uint8(num) - j15++ - } - dAtA[i] = 0x22 - i++ - i = encodeVarintPublic(dAtA, i, uint64(j15)) - i += copy(dAtA[i:], dAtA16[:j15]) - } - if len(m.ColumnIDs) > 0 { - dAtA18 := make([]byte, len(m.ColumnIDs)*10) + dAtA18 := make([]byte, len(m.RowIDs)*10) var j17 int - for _, num := range m.ColumnIDs { + for _, num := range m.RowIDs { for num >= 1<<7 { dAtA18[j17] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -2220,16 +2409,15 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { dAtA18[j17] = uint8(num) j17++ } - dAtA[i] = 0x2a + dAtA[i] = 0x22 i++ i = encodeVarintPublic(dAtA, i, uint64(j17)) i += copy(dAtA[i:], dAtA18[:j17]) } - if len(m.Timestamps) > 0 { - dAtA20 := make([]byte, len(m.Timestamps)*10) + if len(m.ColumnIDs) > 0 { + dAtA20 := make([]byte, len(m.ColumnIDs)*10) var j19 int - for _, num1 := range m.Timestamps { - num := uint64(num1) + for _, num := range m.ColumnIDs { for num >= 1<<7 { dAtA20[j19] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -2238,11 +2426,29 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { dAtA20[j19] = uint8(num) j19++ } - dAtA[i] = 0x32 + dAtA[i] = 0x2a i++ i = encodeVarintPublic(dAtA, i, uint64(j19)) i += copy(dAtA[i:], dAtA20[:j19]) } + if len(m.Timestamps) > 0 { + dAtA22 := make([]byte, len(m.Timestamps)*10) + var j21 int + for _, num1 := range m.Timestamps { + num := uint64(num1) + for num >= 1<<7 { + dAtA22[j21] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j21++ + } + dAtA22[j21] = uint8(num) + j21++ + } + dAtA[i] = 0x32 + i++ + i = encodeVarintPublic(dAtA, i, uint64(j21)) + i += copy(dAtA[i:], dAtA22[:j21]) + } if len(m.RowKeys) > 0 { for _, s := range m.RowKeys { dAtA[i] = 0x3a @@ -2312,27 +2518,9 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(m.Shard)) } if len(m.ColumnIDs) > 0 { - dAtA22 := make([]byte, len(m.ColumnIDs)*10) - var j21 int - for _, num := range m.ColumnIDs { - for num >= 1<<7 { - dAtA22[j21] = uint8(uint64(num)&0x7f | 0x80) - num >>= 7 - j21++ - } - dAtA22[j21] = uint8(num) - j21++ - } - dAtA[i] = 0x2a - i++ - i = encodeVarintPublic(dAtA, i, uint64(j21)) - i += copy(dAtA[i:], dAtA22[:j21]) - } - if len(m.Values) > 0 { - dAtA24 := make([]byte, len(m.Values)*10) + dAtA24 := make([]byte, len(m.ColumnIDs)*10) var j23 int - for _, num1 := range m.Values { - num := uint64(num1) + for _, num := range m.ColumnIDs { for num >= 1<<7 { dAtA24[j23] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -2341,11 +2529,29 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { dAtA24[j23] = uint8(num) j23++ } - dAtA[i] = 0x32 + dAtA[i] = 0x2a i++ i = encodeVarintPublic(dAtA, i, uint64(j23)) i += copy(dAtA[i:], dAtA24[:j23]) } + if len(m.Values) > 0 { + dAtA26 := make([]byte, len(m.Values)*10) + var j25 int + for _, num1 := range m.Values { + num := uint64(num1) + for num >= 1<<7 { + dAtA26[j25] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j25++ + } + dAtA26[j25] = uint8(num) + j25++ + } + dAtA[i] = 0x32 + i++ + i = encodeVarintPublic(dAtA, i, uint64(j25)) + i += copy(dAtA[i:], dAtA26[:j25]) + } if len(m.ColumnKeys) > 0 { for _, s := range m.ColumnKeys { dAtA[i] = 0x3a @@ -2366,8 +2572,8 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPublic(dAtA, i, uint64(len(m.FloatValues)*8)) for _, num := range m.FloatValues { - f25 := math.Float64bits(float64(num)) - encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(f25)) + f27 := math.Float64bits(float64(num)) + encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(f27)) i += 8 } } @@ -2441,21 +2647,21 @@ func (m *TranslateKeysResponse) MarshalTo(dAtA []byte) (int, error) { var l int _ = l if len(m.IDs) > 0 { - dAtA27 := make([]byte, len(m.IDs)*10) - var j26 int + dAtA29 := make([]byte, len(m.IDs)*10) + var j28 int for _, num := range m.IDs { for num >= 1<<7 { - dAtA27[j26] = uint8(uint64(num)&0x7f | 0x80) + dAtA29[j28] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j26++ + j28++ } - dAtA27[j26] = uint8(num) - j26++ + dAtA29[j28] = uint8(num) + j28++ } dAtA[i] = 0x1a i++ - i = encodeVarintPublic(dAtA, i, uint64(j26)) - i += copy(dAtA[i:], dAtA27[:j26]) + i = encodeVarintPublic(dAtA, i, uint64(j28)) + i += copy(dAtA[i:], dAtA29[:j28]) } if m.XXX_unrecognized != nil { i += copy(dAtA[i:], m.XXX_unrecognized) @@ -2587,21 +2793,21 @@ func (m *ImportColumnAttrsRequest) MarshalTo(dAtA []byte) (int, error) { } } if len(m.ColumnIDs) > 0 { - dAtA29 := make([]byte, len(m.ColumnIDs)*10) - var j28 int + dAtA31 := make([]byte, len(m.ColumnIDs)*10) + var j30 int for _, num := range m.ColumnIDs { for num >= 1<<7 { - dAtA29[j28] = uint8(uint64(num)&0x7f | 0x80) + dAtA31[j30] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j28++ + j30++ } - dAtA29[j28] = uint8(num) - j28++ + dAtA31[j30] = uint8(num) + j30++ } dAtA[i] = 0x2a i++ - i = encodeVarintPublic(dAtA, i, uint64(j28)) - i += copy(dAtA[i:], dAtA29[:j28]) + i = encodeVarintPublic(dAtA, i, uint64(j30)) + i += copy(dAtA[i:], dAtA31[:j30]) } if m.XXX_unrecognized != nil { i += copy(dAtA[i:], m.XXX_unrecognized) @@ -2720,6 +2926,48 @@ func (m *Pair) Size() (n int) { return n } +func (m *PairField) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Pair != nil { + l = m.Pair.Size() + n += 1 + l + sovPublic(uint64(l)) + } + l = len(m.Field) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *PairsField) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Pairs) > 0 { + for _, e := range m.Pairs { + l = e.Size() + n += 1 + l + sovPublic(uint64(l)) + } + } + l = len(m.Field) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func (m *FieldRow) Size() (n int) { if m == nil { return 0 @@ -2979,6 +3227,10 @@ func (m *QueryResult) Size() (n int) { l = m.SignedRow.Size() n += 1 + l + sovPublic(uint64(l)) } + if m.PairsField != nil { + l = m.PairsField.Size() + n += 1 + l + sovPublic(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -3825,6 +4077,230 @@ func (m *Pair) Unmarshal(dAtA []byte) error { } return nil } +func (m *PairField) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PairField: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PairField: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Pair", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Pair == nil { + m.Pair = &Pair{} + } + if err := m.Pair.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Field = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPublic(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPublic + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *PairsField) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PairsField: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PairsField: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Pairs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Pairs = append(m.Pairs, &Pair{}) + if err := m.Pairs[len(m.Pairs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Field = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPublic(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPublic + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *FieldRow) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -5312,6 +5788,39 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 11: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PairsField", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.PairsField == nil { + m.PairsField = &PairsField{} + } + if err := m.PairsField.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -6903,72 +7412,75 @@ var ( ErrIntOverflowPublic = fmt.Errorf("proto: integer overflow") ) -func init() { proto.RegisterFile("public.proto", fileDescriptor_public_568b1fcbeadcdcca) } +func init() { proto.RegisterFile("public.proto", fileDescriptor_public_48374b395a722341) } -var fileDescriptor_public_568b1fcbeadcdcca = []byte{ - // 1016 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x6e, 0x1b, 0x45, - 0x14, 0x66, 0xbc, 0xeb, 0x78, 0x7d, 0x9c, 0x84, 0x6a, 0x48, 0xcb, 0x0a, 0x55, 0xc1, 0x1a, 0x21, - 0xb4, 0xdc, 0xa4, 0x6a, 0x90, 0x50, 0xaf, 0xf8, 0x49, 0x93, 0x22, 0xab, 0xaa, 0x55, 0x8e, 0x23, - 0x73, 0x87, 0xb4, 0xa9, 0xa7, 0xee, 0x4a, 0xeb, 0x1d, 0xb3, 0x3f, 0x6c, 0xf3, 0x00, 0x3c, 0x01, - 0x37, 0x88, 0x27, 0xe0, 0x51, 0xb8, 0x42, 0x3c, 0x02, 0x84, 0xc7, 0xe0, 0x06, 0xcd, 0x99, 0x1d, - 0xcf, 0x7a, 0x9b, 0x04, 0x84, 0xb8, 0x3b, 0xdf, 0x39, 0x33, 0x67, 0xce, 0x37, 0xe7, 0x67, 0x06, - 0x76, 0xd7, 0xd5, 0x45, 0x9a, 0xbc, 0x38, 0x5a, 0xe7, 0xaa, 0x54, 0x3c, 0x48, 0xb2, 0x52, 0xe6, - 0x59, 0x9c, 0x8a, 0x02, 0x3c, 0x54, 0x35, 0x0f, 0x61, 0xf0, 0x58, 0xa5, 0xd5, 0x2a, 0x2b, 0x42, - 0x36, 0xf6, 0x22, 0x1f, 0x2d, 0xe4, 0x1f, 0x40, 0xff, 0x8b, 0xb2, 0xcc, 0x8b, 0xb0, 0x37, 0xf6, - 0xa2, 0xd1, 0xf1, 0xfe, 0x91, 0xdd, 0x7a, 0xa4, 0xd5, 0x68, 0x8c, 0x9c, 0x83, 0xff, 0x54, 0x5e, - 0x16, 0xa1, 0x37, 0xf6, 0xa2, 0x21, 0x92, 0xac, 0x7d, 0xa2, 0x8a, 0xf3, 0x24, 0x5b, 0x86, 0xfe, - 0x98, 0x45, 0xbb, 0x68, 0xa1, 0x78, 0x06, 0xc3, 0x59, 0xb2, 0xcc, 0xe4, 0x42, 0x1f, 0xfd, 0x3e, - 0x78, 0xcf, 0x95, 0x3e, 0x96, 0x45, 0xa3, 0xe3, 0x3d, 0xe7, 0x1e, 0x55, 0x8d, 0xda, 0xa2, 0x17, - 0x4c, 0xe5, 0x32, 0xec, 0x5d, 0xbb, 0x60, 0x2a, 0x97, 0xe2, 0x11, 0xec, 0xa3, 0xaa, 0x27, 0x0b, - 0x99, 0x95, 0xc9, 0xcb, 0x44, 0x9a, 0x70, 0x50, 0xd5, 0x96, 0x0b, 0xc9, 0x9b, 0x10, 0x7b, 0x2e, - 0x44, 0xf1, 0x29, 0xf8, 0xcf, 0xe3, 0x24, 0xe7, 0xfb, 0xd0, 0x9b, 0x9c, 0x52, 0x08, 0x3e, 0xf6, - 0x26, 0xa7, 0xfc, 0x00, 0xfa, 0x8f, 0x55, 0x95, 0x95, 0x74, 0xa8, 0x8f, 0x06, 0xf0, 0x3b, 0xe0, - 0x3d, 0x95, 0x97, 0xa1, 0x37, 0x66, 0xd1, 0x10, 0xb5, 0x28, 0xa6, 0x10, 0x3c, 0x49, 0x64, 0x4a, - 0x3c, 0x0e, 0xa0, 0x4f, 0x32, 0xb9, 0x19, 0xa2, 0x01, 0x5a, 0xab, 0x63, 0x3b, 0xb5, 0x9e, 0x08, - 0xf0, 0x7b, 0xb0, 0x83, 0xaa, 0x76, 0xce, 0x1a, 0x24, 0xbe, 0x01, 0xf8, 0x32, 0x57, 0xd5, 0xda, - 0x9c, 0x17, 0x41, 0x9f, 0x10, 0xd1, 0x18, 0x1d, 0x73, 0x47, 0xdd, 0x1e, 0x8a, 0x66, 0xc1, 0xcd, - 0xf1, 0xce, 0xaa, 0x15, 0x1d, 0xe1, 0xa1, 0x16, 0xc5, 0x31, 0x04, 0xf3, 0x38, 0xdd, 0x58, 0xe7, - 0x71, 0x4a, 0xd1, 0x7a, 0xa8, 0xc5, 0x6d, 0x2f, 0x5e, 0xe3, 0x45, 0x7c, 0x0d, 0x7b, 0xa6, 0x16, - 0x74, 0xa6, 0x67, 0xb2, 0x7c, 0xe3, 0xb2, 0xfe, 0x5d, 0x85, 0xbc, 0x79, 0x79, 0x3f, 0x33, 0xf0, - 0xb5, 0xcd, 0x9a, 0xd8, 0xc6, 0xa4, 0x73, 0x75, 0x7e, 0xb9, 0x96, 0x0d, 0x1d, 0x92, 0xf9, 0x18, - 0x46, 0xb3, 0x52, 0x97, 0xcf, 0x3c, 0x4e, 0x2b, 0xd9, 0x38, 0x6a, 0xab, 0xf8, 0x7b, 0x10, 0x4c, - 0xb2, 0xd2, 0x98, 0x7d, 0xa2, 0xb0, 0xc1, 0xfc, 0x3e, 0x0c, 0x4f, 0x94, 0x4a, 0x8d, 0xb1, 0x3f, - 0x66, 0x51, 0x80, 0x4e, 0xc1, 0x0f, 0x01, 0x9e, 0xa4, 0x2a, 0x6e, 0xf6, 0xee, 0x8c, 0x59, 0xc4, - 0xb0, 0xa5, 0x11, 0x0f, 0x60, 0xa0, 0x23, 0x7d, 0x16, 0xaf, 0x1d, 0x5b, 0x76, 0x0b, 0x5b, 0xf1, - 0x17, 0x83, 0xdd, 0xaf, 0x2a, 0x99, 0x5f, 0xa2, 0xfc, 0xb6, 0x92, 0x45, 0xa9, 0xef, 0x96, 0xb0, - 0xad, 0x0e, 0x02, 0xba, 0x0e, 0x66, 0xaf, 0xe2, 0x7c, 0x61, 0xee, 0xce, 0xc7, 0x06, 0x69, 0xae, - 0xee, 0xce, 0x0b, 0xe2, 0x1a, 0x60, 0x5b, 0x45, 0x15, 0x24, 0x57, 0xaa, 0xb4, 0x64, 0x1a, 0xc4, - 0x23, 0x78, 0xfb, 0xec, 0xf5, 0x8b, 0xb4, 0x5a, 0x48, 0x54, 0xb5, 0xd9, 0xbd, 0x43, 0x0b, 0xba, - 0x6a, 0xfe, 0x21, 0xec, 0x37, 0x2a, 0xdb, 0xf9, 0x03, 0x5a, 0xd8, 0xd1, 0xf2, 0x87, 0xb0, 0x7b, - 0xb6, 0xba, 0x90, 0x8b, 0x85, 0x5c, 0x9c, 0xc6, 0x65, 0x1c, 0x06, 0xc4, 0xbb, 0xd3, 0x87, 0x5b, - 0x4b, 0xc4, 0x0f, 0x0c, 0xf6, 0x1a, 0xf6, 0xc5, 0x5a, 0x65, 0x85, 0xd4, 0x29, 0x3e, 0xcb, 0x73, - 0x9b, 0xe2, 0xb3, 0x3c, 0xe7, 0x0f, 0x60, 0x80, 0xb2, 0xa8, 0xd2, 0xd2, 0xd6, 0xcd, 0x5d, 0xe7, - 0xd1, 0xee, 0xad, 0xd2, 0x12, 0xed, 0x2a, 0xfe, 0x19, 0xec, 0x6f, 0xd5, 0xa1, 0x19, 0x36, 0xa3, - 0xe3, 0x77, 0xdd, 0xbe, 0x2d, 0x3b, 0x76, 0x96, 0x8b, 0xef, 0x3d, 0x18, 0xb5, 0x3c, 0xeb, 0xb9, - 0x82, 0xaa, 0xbe, 0x61, 0xf0, 0xe8, 0x8e, 0xde, 0x05, 0x36, 0x6d, 0x4a, 0x90, 0x4d, 0x75, 0xe2, - 0xf5, 0xac, 0xb0, 0xc7, 0xb6, 0x12, 0xaf, 0xd5, 0x68, 0x8c, 0x34, 0x48, 0x5f, 0xc5, 0xd9, 0x52, - 0x2e, 0xa8, 0x04, 0x03, 0xb4, 0x90, 0x1f, 0xb9, 0xde, 0xa3, 0x9c, 0x6d, 0x35, 0xb4, 0xb5, 0xa0, - 0xeb, 0x4f, 0xdb, 0x03, 0x3a, 0x7d, 0x7b, 0x4d, 0x0f, 0x98, 0xb9, 0x31, 0x39, 0xd5, 0xb9, 0xa2, - 0x7a, 0x31, 0x88, 0x7f, 0x02, 0x23, 0x37, 0x37, 0x8a, 0x26, 0x45, 0x07, 0xce, 0xbd, 0x33, 0x62, - 0x7b, 0x21, 0xff, 0xbc, 0x3b, 0x39, 0xc3, 0x21, 0x45, 0x16, 0x6e, 0xdd, 0x46, 0xcb, 0x8e, 0xdd, - 0x49, 0xfb, 0xb0, 0x35, 0xca, 0x43, 0xa0, 0xcd, 0xef, 0xb8, 0xcd, 0x1b, 0x13, 0xba, 0x55, 0xe2, - 0x0f, 0x06, 0x7b, 0x93, 0xd5, 0x5a, 0xe5, 0x65, 0xab, 0x39, 0x26, 0xd9, 0x42, 0xbe, 0xb6, 0xcd, - 0x41, 0xc0, 0x0d, 0xd4, 0x5e, 0x67, 0xa0, 0x52, 0x93, 0x50, 0x53, 0xf8, 0x68, 0x40, 0xeb, 0x62, - 0xfc, 0xad, 0x8b, 0xb9, 0x0f, 0x43, 0x53, 0x05, 0xda, 0xd4, 0x27, 0x93, 0x53, 0xe8, 0xb6, 0x3f, - 0x4f, 0x56, 0xb2, 0x28, 0xe3, 0xd5, 0x5a, 0xf7, 0x89, 0x17, 0x79, 0xd8, 0xd2, 0x98, 0x17, 0xac, - 0xa6, 0x57, 0x63, 0x40, 0xaf, 0x86, 0x85, 0x7a, 0xa7, 0x71, 0x43, 0xc6, 0x80, 0x8c, 0x2d, 0x8d, - 0xf8, 0x95, 0x01, 0x37, 0x1c, 0x69, 0x80, 0xfc, 0x7f, 0x44, 0x6f, 0x27, 0x74, 0x0f, 0x76, 0xe8, - 0x3c, 0x4b, 0xa6, 0x41, 0x9d, 0x70, 0x07, 0xdd, 0x70, 0xf5, 0xbc, 0x71, 0xd3, 0xce, 0xf0, 0x61, - 0xd8, 0x56, 0x89, 0x39, 0x1c, 0x9c, 0xe7, 0x71, 0x56, 0xa4, 0x71, 0x29, 0xf5, 0x96, 0xff, 0xc2, - 0xe8, 0x9a, 0x4f, 0x82, 0xf8, 0x08, 0xee, 0x76, 0xfc, 0xba, 0x89, 0xa1, 0x29, 0x7a, 0x44, 0x51, - 0x8b, 0xe2, 0x04, 0xc2, 0xa6, 0x6c, 0xcc, 0x37, 0xa2, 0x09, 0x61, 0x9e, 0xc8, 0x5a, 0xbb, 0x9e, - 0xc6, 0x2b, 0xd9, 0x44, 0x41, 0xb2, 0xd6, 0xd1, 0xc0, 0xea, 0xd1, 0xe7, 0x83, 0x64, 0xf1, 0x12, - 0x0e, 0xae, 0xf3, 0x41, 0x4f, 0x5f, 0x2a, 0x63, 0x33, 0xa1, 0x02, 0x34, 0x80, 0x3f, 0x82, 0xfe, - 0x77, 0x89, 0xac, 0xed, 0x84, 0x12, 0xae, 0xb0, 0x6f, 0x0a, 0x04, 0xcd, 0x06, 0xf1, 0x13, 0xb3, - 0xc1, 0xb6, 0x86, 0xf6, 0x3f, 0xde, 0x99, 0xc9, 0x77, 0xf3, 0xfa, 0x9a, 0x7c, 0x87, 0xe6, 0xe5, - 0x71, 0x4f, 0xa7, 0x85, 0xfa, 0xb5, 0xd3, 0xe2, 0x3c, 0x4e, 0x4d, 0xd1, 0x0f, 0x71, 0x83, 0x6f, - 0xaf, 0x92, 0x93, 0x3b, 0xbf, 0x5c, 0x1d, 0xb2, 0xdf, 0xae, 0x0e, 0xd9, 0xef, 0x57, 0x87, 0xec, - 0xc7, 0x3f, 0x0f, 0xdf, 0xba, 0xd8, 0xa1, 0x6f, 0xe1, 0xc7, 0x7f, 0x07, 0x00, 0x00, 0xff, 0xff, - 0x9d, 0x83, 0xa0, 0x41, 0x26, 0x0a, 0x00, 0x00, +var fileDescriptor_public_48374b395a722341 = []byte{ + // 1065 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdb, 0x6e, 0x1c, 0x45, + 0x13, 0xfe, 0x7b, 0x67, 0xd6, 0xbb, 0x5b, 0x6b, 0xfb, 0x8f, 0x9a, 0x4d, 0x18, 0xa1, 0xc8, 0xac, + 0x5a, 0x11, 0x1a, 0x6e, 0x1c, 0xc5, 0x20, 0x94, 0x2b, 0x0e, 0x8e, 0x1d, 0x58, 0x45, 0x59, 0x85, + 0xb2, 0xb5, 0xdc, 0x21, 0x8d, 0xb3, 0x1d, 0x67, 0xa4, 0xd9, 0x99, 0x65, 0x0e, 0x4c, 0xfc, 0x1c, + 0xdc, 0x20, 0x9e, 0x80, 0x77, 0xe0, 0x05, 0xb8, 0x42, 0x3c, 0x02, 0x98, 0xc7, 0xe0, 0x06, 0x55, + 0xf5, 0xf4, 0xf6, 0xec, 0xfa, 0x00, 0x42, 0xdc, 0xf5, 0x57, 0xa7, 0xa9, 0xaa, 0xae, 0xfa, 0x7a, + 0x60, 0x7b, 0x59, 0x9d, 0x25, 0xf1, 0xcb, 0xfd, 0x65, 0x9e, 0x95, 0x99, 0xec, 0xc7, 0x69, 0xa9, + 0xf3, 0x34, 0x4a, 0x54, 0x01, 0x1e, 0x66, 0xb5, 0x0c, 0xa0, 0xf7, 0x24, 0x4b, 0xaa, 0x45, 0x5a, + 0x04, 0x62, 0xec, 0x85, 0x3e, 0x5a, 0x28, 0x1f, 0x40, 0xf7, 0xb3, 0xb2, 0xcc, 0x8b, 0xa0, 0x33, + 0xf6, 0xc2, 0xe1, 0xc1, 0xee, 0xbe, 0x75, 0xdd, 0x27, 0x31, 0x1a, 0xa5, 0x94, 0xe0, 0x3f, 0xd3, + 0x17, 0x45, 0xe0, 0x8d, 0xbd, 0x70, 0x80, 0x7c, 0xa6, 0x98, 0x98, 0x45, 0x79, 0x9c, 0x9e, 0x07, + 0xfe, 0x58, 0x84, 0xdb, 0x68, 0xa1, 0x7a, 0x0e, 0x83, 0x93, 0xf8, 0x3c, 0xd5, 0x73, 0xfa, 0xf4, + 0xbb, 0xe0, 0xbd, 0xc8, 0xe8, 0xb3, 0x22, 0x1c, 0x1e, 0xec, 0xb8, 0xf0, 0x98, 0xd5, 0x48, 0x1a, + 0x32, 0x98, 0xea, 0xf3, 0xa0, 0x73, 0xad, 0xc1, 0x54, 0x9f, 0xab, 0xc7, 0xb0, 0x8b, 0x59, 0x3d, + 0x99, 0xeb, 0xb4, 0x8c, 0x5f, 0xc5, 0xda, 0xa4, 0x83, 0x59, 0x6d, 0x6b, 0xe1, 0xf3, 0x2a, 0xc5, + 0x8e, 0x4b, 0x51, 0x7d, 0x0c, 0xfe, 0x8b, 0x28, 0xce, 0xe5, 0x2e, 0x74, 0x26, 0x47, 0x9c, 0x82, + 0x8f, 0x9d, 0xc9, 0x91, 0x1c, 0x41, 0xf7, 0x49, 0x56, 0xa5, 0x25, 0x7f, 0xd4, 0x47, 0x03, 0xe4, + 0x1d, 0xf0, 0x9e, 0xe9, 0x8b, 0xc0, 0x1b, 0x8b, 0x70, 0x80, 0x74, 0x54, 0xc7, 0x30, 0x20, 0xff, + 0xa7, 0xb1, 0x4e, 0xe6, 0x52, 0x99, 0x60, 0x4d, 0x25, 0xad, 0x46, 0x91, 0x14, 0xcd, 0x87, 0x46, + 0xd0, 0x65, 0x63, 0x0e, 0x3c, 0x40, 0x03, 0xd4, 0x17, 0x00, 0xa4, 0x2d, 0x4c, 0x9c, 0x07, 0xd0, + 0x65, 0xc4, 0xd9, 0x5f, 0x0d, 0x64, 0x94, 0x37, 0x44, 0x9a, 0x42, 0x9f, 0x0f, 0xd4, 0xd8, 0x95, + 0x85, 0x68, 0x59, 0x90, 0x94, 0x9a, 0x75, 0x64, 0x4b, 0x63, 0x20, 0xef, 0xc1, 0x16, 0x66, 0xb5, + 0xab, 0xae, 0x41, 0xea, 0x6b, 0x80, 0xcf, 0xf3, 0xac, 0x5a, 0x9a, 0x06, 0x84, 0xd0, 0x65, 0xd4, + 0x64, 0x26, 0x5d, 0x66, 0xf6, 0xa3, 0x68, 0x0c, 0x6e, 0x6e, 0xe0, 0x49, 0xb5, 0xe0, 0x4f, 0x78, + 0x48, 0x47, 0x75, 0x00, 0xfd, 0x59, 0x94, 0xac, 0xb4, 0xb3, 0x28, 0xe1, 0x6c, 0x3d, 0xa4, 0xe3, + 0x7a, 0x14, 0xaf, 0x89, 0xa2, 0xbe, 0x82, 0x1d, 0x33, 0x9c, 0x34, 0x7a, 0x27, 0xba, 0xbc, 0x72, + 0x7b, 0xff, 0x6c, 0x64, 0xaf, 0xde, 0xe6, 0x8f, 0x02, 0x7c, 0xd2, 0x59, 0x95, 0x58, 0xa9, 0x68, + 0x78, 0x4e, 0x2f, 0x96, 0xba, 0x29, 0x87, 0xcf, 0x72, 0x0c, 0xc3, 0x93, 0x92, 0xe6, 0x79, 0x16, + 0x25, 0x95, 0x6e, 0x02, 0xb5, 0x45, 0xf2, 0x1d, 0xe8, 0x4f, 0xd2, 0xd2, 0xa8, 0x7d, 0x2e, 0x61, + 0x85, 0xe5, 0x7d, 0x18, 0x1c, 0x66, 0x59, 0x62, 0x94, 0xdd, 0xb1, 0x08, 0xfb, 0xe8, 0x04, 0x72, + 0x0f, 0xe0, 0x69, 0x92, 0x45, 0x8d, 0xef, 0xd6, 0x58, 0x84, 0x02, 0x5b, 0x12, 0xf5, 0x10, 0x7a, + 0x94, 0xe9, 0xf3, 0x68, 0xe9, 0xaa, 0x15, 0xb7, 0x54, 0xab, 0xfe, 0x14, 0xb0, 0xfd, 0x65, 0xa5, + 0xf3, 0x0b, 0xd4, 0xdf, 0x54, 0xba, 0x28, 0xa9, 0xb7, 0x8c, 0xed, 0x74, 0x30, 0xa0, 0x39, 0x38, + 0x79, 0x1d, 0xe5, 0x73, 0xd3, 0x3b, 0x1f, 0x1b, 0x44, 0xb5, 0xba, 0x9e, 0x17, 0x5c, 0x6b, 0x1f, + 0xdb, 0x22, 0x9e, 0x20, 0xbd, 0xc8, 0x4a, 0x5b, 0x4c, 0x83, 0x64, 0x08, 0xff, 0x3f, 0x7e, 0xf3, + 0x32, 0xa9, 0xe6, 0x1a, 0xb3, 0xda, 0x78, 0x6f, 0xb1, 0xc1, 0xa6, 0x58, 0xbe, 0x07, 0xbb, 0x8d, + 0xc8, 0x52, 0x51, 0x8f, 0x0d, 0x37, 0xa4, 0xf2, 0x11, 0x6c, 0x1f, 0x2f, 0xce, 0xf4, 0x7c, 0xae, + 0xe7, 0x47, 0x51, 0x19, 0x05, 0x7d, 0xae, 0x7b, 0x83, 0x18, 0xd6, 0x4c, 0xd4, 0x77, 0x02, 0x76, + 0x9a, 0xea, 0x8b, 0x65, 0x96, 0x16, 0x9a, 0xae, 0xf8, 0x38, 0xcf, 0xed, 0x15, 0x1f, 0xe7, 0xb9, + 0x7c, 0x08, 0x3d, 0xd4, 0x45, 0x95, 0x94, 0x76, 0x6e, 0xee, 0xba, 0x88, 0xd6, 0xb7, 0x4a, 0x4a, + 0xb4, 0x56, 0xf2, 0x13, 0xd8, 0x5d, 0x9b, 0x43, 0xc3, 0x7e, 0xc3, 0x83, 0xb7, 0x9d, 0xdf, 0x9a, + 0x1e, 0x37, 0xcc, 0xd5, 0x4f, 0x1e, 0x0c, 0x5b, 0x91, 0x89, 0xe8, 0x30, 0xab, 0x6f, 0x60, 0x42, + 0xda, 0xe8, 0x6d, 0x10, 0xd3, 0x66, 0x04, 0xc5, 0xd4, 0xf1, 0x84, 0x77, 0x1b, 0x4f, 0x10, 0xb3, + 0xbf, 0x8e, 0xd2, 0x73, 0x3d, 0xe7, 0x11, 0xec, 0xa3, 0x85, 0x72, 0xdf, 0xed, 0x1e, 0xdf, 0xd9, + 0xda, 0x42, 0x5b, 0x0d, 0xba, 0xfd, 0xb4, 0x3b, 0x40, 0xd7, 0xb7, 0xd3, 0xec, 0x80, 0xe1, 0x8d, + 0xc9, 0x11, 0xdd, 0x15, 0xcf, 0x8b, 0x41, 0xf2, 0x23, 0x18, 0x3a, 0xde, 0x28, 0x9a, 0x2b, 0x1a, + 0xb9, 0xf0, 0x4e, 0x89, 0x6d, 0x43, 0xf9, 0xe9, 0x26, 0x95, 0x07, 0x03, 0xce, 0x2c, 0x58, 0xeb, + 0x46, 0x4b, 0x8f, 0x9b, 0xd4, 0xff, 0xa8, 0xf5, 0xb6, 0x04, 0xc0, 0xce, 0x6f, 0x39, 0xe7, 0x95, + 0x0a, 0x5b, 0x2f, 0xd0, 0x87, 0x6d, 0xfa, 0x0d, 0x86, 0xec, 0x33, 0x5a, 0xef, 0xa6, 0xd1, 0x61, + 0xcb, 0x4e, 0xfd, 0x2e, 0x60, 0x67, 0xb2, 0x58, 0x66, 0x79, 0xd9, 0x5a, 0xa9, 0x49, 0x3a, 0xd7, + 0x6f, 0xec, 0x4a, 0x31, 0xb8, 0x9e, 0xa8, 0x49, 0xca, 0xab, 0xc5, 0xab, 0xe4, 0xa3, 0x01, 0xad, + 0x76, 0xfa, 0x6b, 0xed, 0xbc, 0x0f, 0x03, 0x33, 0x3b, 0xa4, 0xea, 0xb2, 0xca, 0x09, 0x88, 0x2c, + 0x4e, 0xe3, 0x85, 0x2e, 0xca, 0x68, 0xb1, 0xa4, 0xed, 0xf2, 0x42, 0x0f, 0x5b, 0x12, 0xf3, 0x10, + 0xd7, 0xfc, 0xf8, 0xf5, 0xf8, 0xf1, 0xb3, 0x90, 0x3c, 0x4d, 0x18, 0x56, 0xf6, 0x59, 0xd9, 0x92, + 0xa8, 0x5f, 0x04, 0x48, 0x53, 0x23, 0xd3, 0xce, 0x7f, 0x57, 0xe8, 0xed, 0x05, 0xdd, 0x83, 0x2d, + 0xfe, 0x9e, 0x2d, 0xa6, 0x41, 0x1b, 0xe9, 0xf6, 0x36, 0xd3, 0x25, 0x96, 0x72, 0x1c, 0x69, 0xea, + 0x11, 0xd8, 0x16, 0xa9, 0x19, 0x8c, 0x4e, 0xf3, 0x28, 0x2d, 0x92, 0xa8, 0xd4, 0xe4, 0xf2, 0x6f, + 0x2a, 0xba, 0xe6, 0x5f, 0x47, 0xbd, 0x0f, 0x77, 0x37, 0xe2, 0x3a, 0x9e, 0xa1, 0x12, 0x3d, 0x2e, + 0x91, 0x8e, 0xea, 0x10, 0x82, 0x66, 0x6c, 0xcc, 0xdf, 0x50, 0x93, 0xc2, 0x2c, 0xd6, 0x35, 0x85, + 0x9e, 0x46, 0x0b, 0xdd, 0x64, 0xc1, 0x67, 0x92, 0x31, 0xcd, 0x75, 0xf8, 0x1f, 0x8a, 0xcf, 0xea, + 0x15, 0x8c, 0xae, 0x8b, 0xc1, 0x0f, 0x66, 0xa2, 0x23, 0xc3, 0x6b, 0x7d, 0x34, 0x40, 0x3e, 0x86, + 0xee, 0xb7, 0xb1, 0xae, 0x2d, 0xaf, 0x29, 0x37, 0xda, 0x37, 0x25, 0x82, 0xc6, 0x41, 0xfd, 0x20, + 0x6c, 0xb2, 0x2d, 0xaa, 0xff, 0xdb, 0x9e, 0x99, 0xfb, 0x6e, 0xde, 0x6c, 0x73, 0xdf, 0x81, 0x79, + 0xaf, 0xdc, 0x83, 0x6b, 0x21, 0xbd, 0x91, 0x74, 0x9c, 0x45, 0x89, 0x19, 0xfa, 0x01, 0xae, 0xf0, + 0xed, 0x53, 0x72, 0x78, 0xe7, 0xe7, 0xcb, 0x3d, 0xf1, 0xeb, 0xe5, 0x9e, 0xf8, 0xed, 0x72, 0x4f, + 0x7c, 0xff, 0xc7, 0xde, 0xff, 0xce, 0xb6, 0xf8, 0xef, 0xf6, 0x83, 0xbf, 0x02, 0x00, 0x00, 0xff, + 0xff, 0xdd, 0x9e, 0xfe, 0xf7, 0xed, 0x0a, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index b8d569f85..a58c75e96 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -25,6 +25,16 @@ message Pair { uint64 Count = 2; } +message PairField { + Pair Pair = 1; + string Field = 2; +} + +message PairsField { + repeated Pair Pairs = 1; + string Field = 2; +} + message FieldRow{ string Field = 1; uint64 RowID = 2; @@ -88,6 +98,7 @@ message QueryResult { repeated GroupCount GroupCounts = 8; RowIdentifiers RowIdentifiers = 9; SignedRow SignedRow = 10; + PairsField PairsField = 11; } message ImportRequest { @@ -137,4 +148,4 @@ message ImportColumnAttrsRequest { string AttrKey = 3; repeated string AttrVals = 4; repeated uint64 ColumnIDs = 5; -} \ No newline at end of file +} diff --git a/server/grpc.go b/server/grpc.go index c27527af0..81060b6a3 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -433,35 +433,35 @@ func makeRows(resp pilosa.QueryResponse, logger logger.Logger) chan *pb.RowRespo } */ } - case pilosa.Pair: - if r.Key != "" { + case pilosa.PairField: + if r.Pair.Key != "" { results <- &pb.RowResponse{ Headers: []*pb.ColumnInfo{ - {Name: "_id", Datatype: "string"}, + {Name: r.Field, Datatype: "string"}, {Name: "count", Datatype: "uint64"}, }, Columns: []*pb.ColumnResponse{ - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: r.Key}}, - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: r.Count}}, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{StringVal: r.Pair.Key}}, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: r.Pair.Count}}, }, } } else { results <- &pb.RowResponse{ Headers: []*pb.ColumnInfo{ - {Name: "_id", Datatype: "uint64"}, + {Name: r.Field, Datatype: "uint64"}, {Name: "count", Datatype: "uint64"}, }, Columns: []*pb.ColumnResponse{ - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: r.ID}}, - &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: r.Count}}, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: r.Pair.ID}}, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{Uint64Val: r.Pair.Count}}, }, } } - case []pilosa.Pair: + case *pilosa.PairsField: // Determine if the ID has string keys. var stringKeys bool - if len(r) > 0 { - if r[0].Key != "" { + if len(r.Pairs) > 0 { + if r.Pairs[0].Key != "" { stringKeys = true } } @@ -471,10 +471,10 @@ func makeRows(resp pilosa.QueryResponse, logger logger.Logger) chan *pb.RowRespo dtype = "string" } ci := []*pb.ColumnInfo{ - {Name: "_id", Datatype: dtype}, + {Name: r.Field, Datatype: dtype}, {Name: "count", Datatype: "uint64"}, } - for _, pair := range r { + for _, pair := range r.Pairs { if stringKeys { results <- &pb.RowResponse{ Headers: ci, @@ -528,7 +528,7 @@ func makeRows(resp pilosa.QueryResponse, logger logger.Logger) chan *pb.RowRespo } case pilosa.RowIdentifiers: if len(r.Keys) > 0 { - ci := []*pb.ColumnInfo{{Name: "_id", Datatype: "string"}} + ci := []*pb.ColumnInfo{{Name: r.Field(), Datatype: "string"}} for _, key := range r.Keys { results <- &pb.RowResponse{ Headers: ci, @@ -538,7 +538,7 @@ func makeRows(resp pilosa.QueryResponse, logger logger.Logger) chan *pb.RowRespo ci = nil } } else { - ci := []*pb.ColumnInfo{{Name: "_id", Datatype: "uint64"}} + ci := []*pb.ColumnInfo{{Name: r.Field(), Datatype: "uint64"}} for _, id := range r.Rows { results <- &pb.RowResponse{ Headers: ci, diff --git a/server/grpc_internal_test.go b/server/grpc_internal_test.go index 285f35eb1..c0701be3e 100644 --- a/server/grpc_internal_test.go +++ b/server/grpc_internal_test.go @@ -175,7 +175,7 @@ func TestGRPC(t *testing.T) { Rows: []uint64{10, 11, 12}, }, []expHeader{ - {"_id", "uint64"}, + {"", "uint64"}, // This is blank because we don't expose RowIdentifiers.field, so we have no way to set it for tests. }, [][]expColumn{ {uint64(10)}, @@ -189,7 +189,7 @@ func TestGRPC(t *testing.T) { Keys: []string{"ten", "eleven", "twelve"}, }, []expHeader{ - {"_id", "string"}, + {"", "string"}, // This is blank because we don't expose RowIdentifiers.field, so we have no way to set it for tests. }, [][]expColumn{ {"ten"}, diff --git a/server/handler_test.go b/server/handler_test.go index c67d98657..2f0af0345 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -224,7 +224,12 @@ func TestHandler_Endpoints(t *testing.T) { if err != nil { t.Fatalf("querying: %v", err) } - if !reflect.DeepEqual(resp.Results[0], []pilosa.Pair{{Count: 12, ID: 0}}) { + if !reflect.DeepEqual(resp.Results[0], &pilosa.PairsField{ + Pairs: []pilosa.Pair{ + {Count: 12, ID: 0}, + }, + Field: "f1", + }) { t.Fatalf("Unexpected result %v", resp.Results[0]) } @@ -504,8 +509,8 @@ func TestHandler_Endpoints(t *testing.T) { var resp pilosa.QueryResponse if err := cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if a := resp.Results[0].([]pilosa.Pair); len(a) != 2 { - t.Fatalf("unexpected pair length: %d", len(a)) + } else if a := resp.Results[0].(*pilosa.PairsField); len(a.Pairs) != 2 { + t.Fatalf("unexpected pair length: %d", len(a.Pairs)) } }) From 3b7b54094a0799080ef5f980a76efa9a0991cf72 Mon Sep 17 00:00:00 2001 From: Travis Date: Fri, 13 Dec 2019 18:45:43 -0600 Subject: [PATCH 04/20] update clustertests to use v2 (and go 1.13) --- Dockerfile-clustertests | 2 +- internal/clustertests/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile-clustertests b/Dockerfile-clustertests index 53621b9ab..69fbd9a54 100644 --- a/Dockerfile-clustertests +++ b/Dockerfile-clustertests @@ -1,7 +1,7 @@ # This Dockerfile is used for cluster testing - it produces a much larger image # and includes all of Go as well as some utilities. -FROM golang:1.11 +FROM golang:1.13 LABEL maintainer "dev@pilosa.com" diff --git a/internal/clustertests/docker-compose.yml b/internal/clustertests/docker-compose.yml index 94a96ad5a..1546a5b13 100644 --- a/internal/clustertests/docker-compose.yml +++ b/internal/clustertests/docker-compose.yml @@ -51,6 +51,6 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock command: - - "cd /go/src/github.com/pilosa/pilosa/ && go test -v -count=1 github.com/pilosa/pilosa/internal/clustertests" + - "cd /go/src/github.com/pilosa/pilosa/ && go test -v -count=1 github.com/pilosa/pilosa/v2/internal/clustertests" networks: pilosanet: From 4f7f4f58b1193040e0586daec9eccb2a99a5d3ab Mon Sep 17 00:00:00 2001 From: Travis Date: Sat, 14 Dec 2019 15:56:06 -0600 Subject: [PATCH 05/20] add field to SignedRow, and implement its grpc response --- executor.go | 16 ++++++++++++---- server/grpc.go | 21 +++++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/executor.go b/executor.go index 72c6da626..25dfbf4c1 100644 --- a/executor.go +++ b/executor.go @@ -694,7 +694,8 @@ func (e *executor) executeGenericField(ctx context.Context, index string, c *pql span.LogKV("name", c.Name) defer span.Finish() - if field := c.Args["field"]; field == "" { + field := c.Args["field"] + if field == "" { return SignedRow{}, fmt.Errorf("plugin operation %s(): field required", c.Name) } @@ -714,6 +715,7 @@ func (e *executor) executeGenericField(ctx context.Context, index string, c *pql return SignedRow{}, err } other, _ := result.(SignedRow) + other.field = field.(string) return other, nil } @@ -3776,12 +3778,18 @@ func needsShards(calls []*pql.Call) bool { // SignedRow represents a signed *Row with two (neg/pos) *Rows. type SignedRow struct { - Neg *Row `json:"neg"` - Pos *Row `json:"pos"` + Neg *Row `json:"neg"` + Pos *Row `json:"pos"` + field string +} + +// Field returns the field name associated to the signed row. +func (s *SignedRow) Field() string { + return s.field } func (sr *SignedRow) union(other SignedRow) SignedRow { - ret := SignedRow{&Row{}, &Row{}} + ret := SignedRow{&Row{}, &Row{}, ""} // merge in sr if sr != nil { diff --git a/server/grpc.go b/server/grpc.go index 81060b6a3..cb1856b9f 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -573,6 +573,27 @@ func makeRows(resp pilosa.QueryResponse, logger logger.Logger) chan *pb.RowRespo &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: r.Val}}, &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: r.Count}}, }} + case pilosa.SignedRow: + // TODO: address the overflow issue with values outside the int64 range + ci := []*pb.ColumnInfo{{Name: r.Field(), Datatype: "int64"}} + negs := r.Neg.Columns() + for i := len(negs) - 1; i >= 0; i-- { + results <- &pb.RowResponse{ + Headers: ci, + Columns: []*pb.ColumnResponse{ + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: -1 * int64(negs[i])}}, + }} + ci = nil + } + for _, id := range r.Pos.Columns() { + results <- &pb.RowResponse{ + Headers: ci, + Columns: []*pb.ColumnResponse{ + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{Int64Val: int64(id)}}, + }} + ci = nil + } + default: logger.Printf("unhandled %T\n", r) breakLoop = true From 532caa0fbfeb4034b64b134dc31419150e918106 Mon Sep 17 00:00:00 2001 From: Travis Date: Mon, 16 Dec 2019 21:56:57 -0600 Subject: [PATCH 06/20] fix an impossible code path raised by the linter --- api.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/api.go b/api.go index 151de44a9..af468914f 100644 --- a/api.go +++ b/api.go @@ -1188,13 +1188,10 @@ func (api *API) ImportColumnAttrs(ctx context.Context, req *ImportColumnAttrsReq bulkAttrs[uint64(req.ColumnIDs[n])] = map[string]interface{}{req.AttrKey: req.AttrVals[n]} } if err := index.ColumnAttrStore().SetBulkAttrs(bulkAttrs); err != nil { - return err - } - - if err != nil { api.server.logger.Printf("import error: index=%s, shard=%d, len(columns)=%d, err=%s", req.Index, req.Shard, len(req.ColumnIDs), err) + return errors.Wrap(err, "importing column attrs") } - return errors.Wrap(err, "importing column attrs") + return nil } func importExistenceColumns(index *Index, columnIDs []uint64) error { From 83aa50567362acfd12e32b3cd31491003f95adc5 Mon Sep 17 00:00:00 2001 From: Seebs Date: Tue, 17 Dec 2019 14:58:32 -0600 Subject: [PATCH 07/20] Simplify unionArrayArray Also short-circuit it in some cases. --- roaring/roaring.go | 119 +++++++++++++++++++++++++-------------------- 1 file changed, 65 insertions(+), 54 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 884889a90..bb0479f8f 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -3688,41 +3688,43 @@ func union(a, b *Container) *Container { func unionArrayArray(a, b *Container) *Container { statsHit("union/ArrayArray") - aa, ab := a.array(), b.array() - na, nb := len(aa), len(ab) - output := make([]uint16, na+nb) - n := 0 - for i, j := 0, 0; ; { - if i >= na && j >= nb { - break - } else if i < na && j >= nb { - output[n] = aa[i] - n++ - i++ - continue - } else if i >= na && j < nb { - output[n] = ab[j] - n++ - j++ - continue - } - - va, vb := aa[i], ab[j] + if a.N() == 0 { + return b + } + if b.N() == 0 { + return a + } + s1, s2 := a.array(), b.array() + n1, n2 := len(s1), len(s2) + output := make([]uint16, 0, n1+n2) + i, j := 0, 0 + for { + va, vb := s1[i], s2[j] if va < vb { - output[n] = va - n++ + output = append(output, va) i++ } else if va > vb { - output[n] = vb - n++ + output = append(output, vb) j++ } else { - output[n] = va - n++ - i, j = i+1, j+1 + output = append(output, va) + i++ + j++ + } + // It's possible we hit the ends at the same time, + // in which case the append will copy 0 items. This + // is cheaper than performing a separate conditional + // check every time... + if j >= n2 { + output = append(output, s1[i:]...) + break + } + if i >= n1 { + output = append(output, s2[j:]...) + break } } - return NewContainerArray(output[:n]) + return NewContainerArray(output) } // unionArrayArrayInPlace does what it sounds like -- tries to combine @@ -3730,47 +3732,56 @@ func unionArrayArray(a, b *Container) *Container { // of a good array size, so it could be up to twice that size, temporarily. func unionArrayArrayInPlace(a, b *Container) *Container { statsHit("union/ArrayArrayInPlace") - aa, ab := a.array(), b.array() - na, nb := len(aa), len(ab) - output := make([]uint16, na+nb) - outN := 0 - for i, j := 0, 0; ; { - if i >= na && j >= nb { - break - } else if i < na && j >= nb { - copy(output[outN:], aa[i:]) - outN += na - i - break - } else if i >= na && j < nb { - copy(output[outN:], ab[j:]) - outN += nb - j - break + if a.N() == 0 { + if b.N() != 0 { + // for InPlace, we actually want to ensure that + // we update a, as long as it's not frozen. + a = a.Thaw() + a.setArray(b.array()) + return a.optimize() } - - va, vb := aa[i], ab[j] + return a + } + if b.N() == 0 { + return a + } + s1, s2 := a.array(), b.array() + n1, n2 := len(s1), len(s2) + output := make([]uint16, 0, n1+n2) + i, j := 0, 0 + for { + va, vb := s1[i], s2[j] if va < vb { - output[outN] = va - outN++ + output = append(output, va) i++ } else if va > vb { - output[outN] = vb - outN++ + output = append(output, vb) j++ } else { - output[outN] = va - outN++ + output = append(output, va) i++ j++ } + // It's possible we hit the ends at the same time, + // in which case the append will copy 0 items. This + // is cheaper than performing a separate conditional + // check every time... + if j >= n2 { + output = append(output, s1[i:]...) + break + } + if i >= n1 { + output = append(output, s2[j:]...) + break + } } // a union can't omit anything that was previously in a, so if // the output is the same length, nothing changed. if len(output) != int(a.N()) { a = a.Thaw() - a.setArray(output[:outN]) - a = a.optimize() + a.setArray(output) } - return a + return a.optimize() } // unionArrayRun optimistically assumes that the result will be a run container, From 361e51cb41ce83bc56af744a5c08e432e61bc2cb Mon Sep 17 00:00:00 2001 From: Travis Date: Mon, 16 Dec 2019 18:35:55 -0600 Subject: [PATCH 08/20] Add All() support to PQL, including limit and offset This PR is meant to get all columns from an index based on the TrackExistence row. `All()` is a PQL function that can be used as a typical row object. Optional arguments are `limit` and `offset`. --- api/client/grpc.go | 5 +- executor.go | 138 +++++++++++++++++- executor_test.go | 143 +++++++++++++++++++ go.mod | 4 +- go.sum | 4 + pql/ast.go | 15 +- proto/pilosa.pb.go | 343 +++++++++++++++++++++++++++++++++++++-------- proto/pilosa.proto | 2 + server/grpc.go | 106 +++++++++++++- 9 files changed, 689 insertions(+), 71 deletions(-) diff --git a/api/client/grpc.go b/api/client/grpc.go index 2d99ea76e..fa9f33b68 100644 --- a/api/client/grpc.go +++ b/api/client/grpc.go @@ -80,7 +80,7 @@ func (c *GRPCClient) Query(ctx context.Context, index string, pql string) (pb.St // Inspect returns a stream of RowResponse for the given index, columns, and filters. // It is intended to mimic something like "select [fields] from table where recordID IN (...)". -func (c *GRPCClient) Inspect(ctx context.Context, index string, columnIDs []uint64, columnKeys []string, fieldFilters []string) (pb.StreamClient, error) { +func (c *GRPCClient) Inspect(ctx context.Context, index string, columnIDs []uint64, columnKeys []string, fieldFilters []string, limit, offset uint64) (pb.StreamClient, error) { if c.conn == nil { return nil, errors.New("client has not established a grpc connection") } @@ -103,7 +103,10 @@ func (c *GRPCClient) Inspect(ctx context.Context, index string, columnIDs []uint Index: index, Columns: idsOrKeys, FilterFields: fieldFilters, + Limit: limit, + Offset: offset, }) + if err != nil { return nil, errors.Wrap(err, "getting stream") } else if stream == nil { diff --git a/executor.go b/executor.go index 25dfbf4c1..47c08043b 100644 --- a/executor.go +++ b/executor.go @@ -527,6 +527,8 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s return e.executeOptionsCall(ctx, index, c, shards, opt) case "IncludesColumn": return e.executeIncludesColumnCall(ctx, index, c, shards, opt) + case "All": + return e.executeAllCall(ctx, index, c, shards, opt) case "Precomputed": return e.executePrecomputedCall(ctx, index, c, shards, opt) default: @@ -635,6 +637,112 @@ func (e *executor) executeIncludesColumnCall(ctx context.Context, index string, return result.(bool), nil } +// executeAllCall executes an All() call. +func (e *executor) executeAllCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*Row, error) { + rslt := NewRow() + + var limit uint64 + var offset uint64 + + if lim, hasLimit, err := c.UintArg("limit"); err != nil { + return nil, errors.Wrap(err, "getting limit") + } else if hasLimit && lim > 0 { + limit = uint64(lim) + } + if off, hasOffset, err := c.UintArg("offset"); err != nil { + return nil, errors.Wrap(err, "getting offset") + } else if hasOffset && off > 0 { + offset = uint64(off) + } + + if limit == 0 { + limit = math.MaxUint64 + } + + // skip tracks the number of records left to be skipped + // in support of getting to the offset. + var skip uint64 = offset + + // got tracks the number of records gotten to that point. + var got uint64 + + for _, shard := range shards { + row, err := e.executeAllCallMapReduce(ctx, index, c, shard, opt) + if err != nil { + return nil, errors.Wrap(err, "executing map reduce on shard") + } + + segCnt := row.Count() + + // If this segment doesn't reach the offset, skip it. + if segCnt <= skip { + skip -= segCnt + continue + } + + // This segment doesn't have enough to finish fulfilling the limit + // (or it has exactly enough). + if segCnt-skip <= limit-got { + if skip == 0 { + rslt.Merge(row) + } else { + cols := row.Columns() + partialRow := NewRow() + for _, bit := range cols[skip:] { + partialRow.SetBit(bit) + } + rslt.Merge(partialRow) + } + got += segCnt - skip + // In the case where this segment exactly fulfills the limit, break. + if got == limit { + break + } + skip = 0 + continue + } + + // This segment has more records than the remaining limit requires. + cols := row.Columns() + partialRow := NewRow() + for _, bit := range cols[skip : skip+limit-got] { + partialRow.SetBit(bit) + } + rslt.Merge(partialRow) + break + } + + return rslt, nil +} + +// executeAllCallMapReduce executes a single shard of the All() call +// using the executor.mapReduce() method. +func (e *executor) executeAllCallMapReduce(ctx context.Context, index string, c *pql.Call, shard uint64, opt *execOptions) (*Row, error) { + // Execute calls in bulk on each remote node and merge. + mapFn := func(shard uint64) (interface{}, error) { + return e.executeAllShard(ctx, index, c, shard) + } + + // Merge returned results at coordinating node. + reduceFn := func(prev, v interface{}) interface{} { + other, _ := prev.(*Row) + if other == nil { + other = NewRow() + } + other.Merge(v.(*Row)) + return other + } + + result, err := e.mapReduce(ctx, index, []uint64{shard}, c, opt, mapFn, reduceFn) + if err != nil { + return nil, errors.Wrap(err, "map reduce") + } + + row, _ := result.(*Row) + + return row, nil +} + // executeIncludesColumnCallShard func (e *executor) executeIncludesColumnCallShard(ctx context.Context, index string, c *pql.Call, shard uint64, column uint64) (bool, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeIncludesColumnCallShard") @@ -2381,7 +2489,7 @@ func (e *executor) executePrecomputedCallShard(ctx context.Context, index string return nil, fmt.Errorf("per-shard: missing precomputed values for shard %d", shard) } -// executeNotShard executes a not() call for a local shard. +// executeNotShard executes a Not() call for a local shard. func (e *executor) executeNotShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeNotShard") defer span.Finish() @@ -2416,6 +2524,34 @@ func (e *executor) executeNotShard(ctx context.Context, index string, c *pql.Cal return existenceRow.Difference(row), nil } +// executeAllShard executes an All() call for a local shard. +func (e *executor) executeAllShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeAllShard") + defer span.Finish() + + if len(c.Children) > 0 { + return nil, errors.New("All() does not accept an input row") + } + + // Make sure the index supports existence tracking. + idx := e.Holder.Index(index) + if idx == nil { + return nil, ErrIndexNotFound + } else if idx.existenceField() == nil { + return nil, errors.Errorf("index does not support existence tracking: %s", index) + } + + var existenceRow *Row + existenceFrag := e.Holder.fragment(index, existenceFieldName, viewStandard, shard) + if existenceFrag == nil { + existenceRow = NewRow() + } else { + existenceRow = existenceFrag.row(0) + } + + return existenceRow, nil +} + // executeShiftShard executes a shift() call for a local shard. func (e *executor) executeShiftShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { n, _, err := c.IntArg("n") diff --git a/executor_test.go b/executor_test.go index 665ef9bba..3750a04ad 100644 --- a/executor_test.go +++ b/executor_test.go @@ -2898,6 +2898,149 @@ func TestExecutor_Execute_Not(t *testing.T) { }) } +// Ensure an all query can be executed. +func TestExecutor_Execute_All(t *testing.T) { + t.Run("ColumnID", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + fld, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) + } + + // Create an import request that sets a full shard, + // plus a couple bits set on either side of it, and + // a final bit set in a fourth shard. + // + // shard0 shard1 shard2 shard3 + // |----------|----------|----------|----------| + // | **|**********|** | * + // + bitCount := ShardWidth + 5 + req := &pilosa.ImportRequest{ + Index: index.Name(), + Field: fld.Name(), + Shard: 0, + RowIDs: make([]uint64, bitCount), + ColumnIDs: make([]uint64, bitCount), + } + for i := 0; i < bitCount-1; i++ { + req.RowIDs[i] = 10 + req.ColumnIDs[i] = uint64(i + ShardWidth - 2) + } + req.RowIDs[bitCount-1] = 10 + req.ColumnIDs[bitCount-1] = uint64((3 * ShardWidth) + 2) + + if err := c[0].API.Import(context.Background(), req); err != nil { + t.Fatal(err) + } + + tests := []struct { + qry string + expCols []uint64 + expCnt uint64 + }{ + {qry: "All()", expCols: req.ColumnIDs, expCnt: uint64(bitCount)}, + {qry: "All(limit=1)", expCols: req.ColumnIDs[:1], expCnt: 1}, + {qry: "All(limit=4)", expCols: req.ColumnIDs[:4], expCnt: 4}, + {qry: "All(limit=4, offset=4)", expCols: req.ColumnIDs[4:8], expCnt: 4}, + {qry: fmt.Sprintf("All(limit=4, offset=%d)", bitCount-5), expCols: req.ColumnIDs[bitCount-5 : bitCount-1], expCnt: 4}, + {qry: fmt.Sprintf("All(limit=1, offset=%d)", bitCount-2), expCols: req.ColumnIDs[bitCount-2 : bitCount-1], expCnt: 1}, + {qry: fmt.Sprintf("All(limit=1, offset=%d)", bitCount-2), expCols: req.ColumnIDs[bitCount-2 : bitCount-1], expCnt: 1}, + {qry: fmt.Sprintf("All(limit=4, offset=%d)", bitCount-2), expCols: req.ColumnIDs[bitCount-2:], expCnt: 2}, + {qry: fmt.Sprintf("All(limit=4, offset=%d)", bitCount+1), expCols: []uint64{}, expCnt: 0}, + {qry: fmt.Sprintf("All(limit=2, offset=%d)", bitCount-3), expCols: req.ColumnIDs[bitCount-3 : bitCount-1], expCnt: 2}, + {qry: fmt.Sprintf("All(limit=2, offset=%d)", bitCount-5), expCols: req.ColumnIDs[bitCount-5 : bitCount-3], expCnt: 2}, + {qry: "All(limit=2, offset=2)", expCols: req.ColumnIDs[2:4], expCnt: 2}, + {qry: "All(limit=1, offset=1)", expCols: req.ColumnIDs[1:2], expCnt: 1}, + {qry: fmt.Sprintf("All(limit=%d, offset=2)", ShardWidth), expCols: req.ColumnIDs[2 : bitCount-3], expCnt: ShardWidth}, + } + for i, test := range tests { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.qry}); err != nil { + t.Fatal(err) + } else if cnt := res.Results[0].(*pilosa.Row).Count(); cnt != test.expCnt { + t.Fatalf("test %d, unexpected count, got: %d, but expected: %d", i, cnt, test.expCnt) + } else if cols := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(cols, test.expCols) { + // If the error results are too large, just show the count. + if len(cols) > 1000 || len(test.expCols) > 1000 { + t.Fatalf("test %d, unexpected columns, got: len(%d), but expected: len(%d)", i, len(cols), len(test.expCols)) + } else { + t.Fatalf("test %d, unexpected columns, got: %v, but expected: %v", i, cols, test.expCols) + } + } + } + }) + + t.Run("ColumnKey", func(t *testing.T) { + c := test.MustRunCluster(t, 1, []server.CommandOption{ + server.OptCommandServerOptions( + pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), + pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + ), + }) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true, Keys: true}) + fld, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) + } + + // Create an import request that sets key columns + // + // shard0 + // |----------| + // |**** | + // + bitCount := 4 + req := &pilosa.ImportRequest{ + Index: index.Name(), + Field: fld.Name(), + Shard: 0, + RowIDs: make([]uint64, bitCount), + ColumnKeys: make([]string, bitCount), + } + for i := 0; i < bitCount; i++ { + req.RowIDs[i] = 10 + req.ColumnKeys[i] = fmt.Sprintf("c%d", i) + } + + if err := c[0].API.Import(context.Background(), req); err != nil { + t.Fatal(err) + } + + tests := []struct { + qry string + expCols []string + expCnt uint64 + }{ + {qry: "All()", expCols: req.ColumnKeys, expCnt: uint64(bitCount)}, + {qry: "All(limit=1)", expCols: req.ColumnKeys[:1], expCnt: 1}, + {qry: "All(limit=4)", expCols: req.ColumnKeys, expCnt: 4}, + {qry: "All(limit=5)", expCols: req.ColumnKeys, expCnt: 4}, + {qry: "All(limit=1, offset=1)", expCols: req.ColumnKeys[1:2], expCnt: 1}, + {qry: "All(limit=4, offset=1)", expCols: req.ColumnKeys[1:], expCnt: 3}, + {qry: "All(limit=4, offset=5)", expCols: nil, expCnt: 0}, + } + for i, test := range tests { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.qry}); err != nil { + t.Fatal(err) + } else if cnt := len(res.Results[0].(*pilosa.Row).Keys); uint64(cnt) != test.expCnt { + t.Fatalf("test %d, unexpected count, got: %d, but expected: %d", i, cnt, test.expCnt) + } else if cols := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(cols, test.expCols) { + // If the error results are too large, just show the count. + if len(cols) > 1000 || len(test.expCols) > 1000 { + t.Fatalf("test %d, unexpected columns, got: len(%d), but expected: len(%d)", i, len(cols), len(test.expCols)) + } else { + t.Fatalf("test %d, unexpected columns, got: %T, but expected: %T", i, cols, test.expCols) + } + } + } + }) +} + // Ensure a row can be cleared. func TestExecutor_Execute_ClearRow(t *testing.T) { // Set and Mutex tests use the same data and queries diff --git a/go.mod b/go.mod index eb38d9288..f47f54dc1 100644 --- a/go.mod +++ b/go.mod @@ -34,15 +34,17 @@ require ( github.com/uber-go/atomic v1.4.0 // indirect github.com/uber/jaeger-client-go v2.16.0+incompatible github.com/uber/jaeger-lib v2.2.0+incompatible // indirect + github.com/youtube/vitess v2.1.1+incompatible // indirect go.uber.org/atomic v1.4.0 // indirect golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect - golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 // indirect + golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 golang.org/x/sync v0.0.0-20190423024810-112230192c58 golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 // indirect golang.org/x/text v0.3.2 // indirect google.golang.org/grpc v1.24.0 modernc.org/mathutil v1.0.0 modernc.org/strutil v1.0.0 + vitess.io/vitess v2.1.1+incompatible // indirect ) go 1.13 diff --git a/go.sum b/go.sum index 6607cf98d..177cd241f 100644 --- a/go.sum +++ b/go.sum @@ -153,6 +153,8 @@ github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/ github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/youtube/vitess v2.1.1+incompatible h1:SE+P7DNX/jw5RHFs5CHRhZQjq402EJFCD33JhzQMdDw= +github.com/youtube/vitess v2.1.1+incompatible/go.mod h1:hpMim5/30F1r+0P8GGtB29d0gWHr0IZ5unS+CG0zMx8= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -210,3 +212,5 @@ modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +vitess.io/vitess v2.1.1+incompatible h1:nuuGHiWYWpudD3gOCLeGzol2EJ25e/u5Wer2wV1O130= +vitess.io/vitess v2.1.1+incompatible/go.mod h1:h4qvkyNYTOC0xI+vcidSWoka0gQAZc9ZPHbkHo48gP0= diff --git a/pql/ast.go b/pql/ast.go index 7749c02dd..61b4b6540 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -347,10 +347,17 @@ var callInfoByFunc = map[string]callInfo{ "Difference": {allowUnknown: false}, "Intersect": {allowUnknown: false}, "Not": {allowUnknown: false}, - "ClearRow": {allowUnknown: true}, - "Store": {allowUnknown: true}, - "MinRow": allowField, - "MaxRow": allowField, + "All": { + allowUnknown: false, + prototypes: map[string]interface{}{ + "limit": int64(0), + "offset": int64(0), + }, + }, + "ClearRow": {allowUnknown: true}, + "Store": {allowUnknown: true}, + "MinRow": allowField, + "MaxRow": allowField, "Rows": { allowUnknown: false, prototypes: map[string]interface{}{ diff --git a/proto/pilosa.pb.go b/proto/pilosa.pb.go index b81722e06..1a5bfca49 100644 --- a/proto/pilosa.pb.go +++ b/proto/pilosa.pb.go @@ -4,15 +4,16 @@ package pilosa import ( - context "context" fmt "fmt" proto "github.com/golang/protobuf/proto" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" math "math" ) +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal var _ = fmt.Errorf @@ -22,7 +23,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type QueryPQLRequest struct { Index string `protobuf:"bytes,1,opt,name=index,proto3" json:"index,omitempty"` @@ -321,9 +322,9 @@ func (m *ColumnResponse) GetFloat64Val() float64 { return 0 } -// XXX_OneofWrappers is for the internal use of the proto package. -func (*ColumnResponse) XXX_OneofWrappers() []interface{} { - return []interface{}{ +// XXX_OneofFuncs is for the internal use of the proto package. +func (*ColumnResponse) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { + return _ColumnResponse_OneofMarshaler, _ColumnResponse_OneofUnmarshaler, _ColumnResponse_OneofSizer, []interface{}{ (*ColumnResponse_StringVal)(nil), (*ColumnResponse_Uint64Val)(nil), (*ColumnResponse_Int64Val)(nil), @@ -335,10 +336,162 @@ func (*ColumnResponse) XXX_OneofWrappers() []interface{} { } } +func _ColumnResponse_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*ColumnResponse) + // columnVal + switch x := m.ColumnVal.(type) { + case *ColumnResponse_StringVal: + b.EncodeVarint(1<<3 | proto.WireBytes) + b.EncodeStringBytes(x.StringVal) + case *ColumnResponse_Uint64Val: + b.EncodeVarint(2<<3 | proto.WireVarint) + b.EncodeVarint(uint64(x.Uint64Val)) + case *ColumnResponse_Int64Val: + b.EncodeVarint(3<<3 | proto.WireVarint) + b.EncodeVarint(uint64(x.Int64Val)) + case *ColumnResponse_BoolVal: + t := uint64(0) + if x.BoolVal { + t = 1 + } + b.EncodeVarint(4<<3 | proto.WireVarint) + b.EncodeVarint(t) + case *ColumnResponse_BlobVal: + b.EncodeVarint(5<<3 | proto.WireBytes) + b.EncodeRawBytes(x.BlobVal) + case *ColumnResponse_Uint64ArrayVal: + b.EncodeVarint(6<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Uint64ArrayVal); err != nil { + return err + } + case *ColumnResponse_StringArrayVal: + b.EncodeVarint(7<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.StringArrayVal); err != nil { + return err + } + case *ColumnResponse_Float64Val: + b.EncodeVarint(8<<3 | proto.WireFixed64) + b.EncodeFixed64(math.Float64bits(x.Float64Val)) + case nil: + default: + return fmt.Errorf("ColumnResponse.ColumnVal has unexpected type %T", x) + } + return nil +} + +func _ColumnResponse_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*ColumnResponse) + switch tag { + case 1: // columnVal.stringVal + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeStringBytes() + m.ColumnVal = &ColumnResponse_StringVal{x} + return true, err + case 2: // columnVal.uint64Val + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.ColumnVal = &ColumnResponse_Uint64Val{x} + return true, err + case 3: // columnVal.int64Val + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.ColumnVal = &ColumnResponse_Int64Val{int64(x)} + return true, err + case 4: // columnVal.boolVal + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.ColumnVal = &ColumnResponse_BoolVal{x != 0} + return true, err + case 5: // columnVal.blobVal + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeRawBytes(true) + m.ColumnVal = &ColumnResponse_BlobVal{x} + return true, err + case 6: // columnVal.uint64ArrayVal + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(Uint64Array) + err := b.DecodeMessage(msg) + m.ColumnVal = &ColumnResponse_Uint64ArrayVal{msg} + return true, err + case 7: // columnVal.stringArrayVal + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(StringArray) + err := b.DecodeMessage(msg) + m.ColumnVal = &ColumnResponse_StringArrayVal{msg} + return true, err + case 8: // columnVal.float64Val + if wire != proto.WireFixed64 { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeFixed64() + m.ColumnVal = &ColumnResponse_Float64Val{math.Float64frombits(x)} + return true, err + default: + return false, nil + } +} + +func _ColumnResponse_OneofSizer(msg proto.Message) (n int) { + m := msg.(*ColumnResponse) + // columnVal + switch x := m.ColumnVal.(type) { + case *ColumnResponse_StringVal: + n += 1 // tag and wire + n += proto.SizeVarint(uint64(len(x.StringVal))) + n += len(x.StringVal) + case *ColumnResponse_Uint64Val: + n += 1 // tag and wire + n += proto.SizeVarint(uint64(x.Uint64Val)) + case *ColumnResponse_Int64Val: + n += 1 // tag and wire + n += proto.SizeVarint(uint64(x.Int64Val)) + case *ColumnResponse_BoolVal: + n += 1 // tag and wire + n += 1 + case *ColumnResponse_BlobVal: + n += 1 // tag and wire + n += proto.SizeVarint(uint64(len(x.BlobVal))) + n += len(x.BlobVal) + case *ColumnResponse_Uint64ArrayVal: + s := proto.Size(x.Uint64ArrayVal) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *ColumnResponse_StringArrayVal: + s := proto.Size(x.StringArrayVal) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *ColumnResponse_Float64Val: + n += 1 // tag and wire + n += 8 + case nil: + default: + panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) + } + return n +} + type InspectRequest struct { Index string `protobuf:"bytes,1,opt,name=index,proto3" json:"index,omitempty"` Columns *IdsOrKeys `protobuf:"bytes,2,opt,name=columns,proto3" json:"columns,omitempty"` FilterFields []string `protobuf:"bytes,3,rep,name=filterFields,proto3" json:"filterFields,omitempty"` + Limit uint64 `protobuf:"varint,4,opt,name=limit,proto3" json:"limit,omitempty"` + Offset uint64 `protobuf:"varint,5,opt,name=offset,proto3" json:"offset,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -390,6 +543,20 @@ func (m *InspectRequest) GetFilterFields() []string { return nil } +func (m *InspectRequest) GetLimit() uint64 { + if m != nil { + return m.Limit + } + return 0 +} + +func (m *InspectRequest) GetOffset() uint64 { + if m != nil { + return m.Offset + } + return 0 +} + type Uint64Array struct { Vals []uint64 `protobuf:"varint,1,rep,packed,name=vals,proto3" json:"vals,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -540,14 +707,80 @@ func (m *IdsOrKeys) GetKeys() *StringArray { return nil } -// XXX_OneofWrappers is for the internal use of the proto package. -func (*IdsOrKeys) XXX_OneofWrappers() []interface{} { - return []interface{}{ +// XXX_OneofFuncs is for the internal use of the proto package. +func (*IdsOrKeys) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { + return _IdsOrKeys_OneofMarshaler, _IdsOrKeys_OneofUnmarshaler, _IdsOrKeys_OneofSizer, []interface{}{ (*IdsOrKeys_Ids)(nil), (*IdsOrKeys_Keys)(nil), } } +func _IdsOrKeys_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*IdsOrKeys) + // type + switch x := m.Type.(type) { + case *IdsOrKeys_Ids: + b.EncodeVarint(1<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Ids); err != nil { + return err + } + case *IdsOrKeys_Keys: + b.EncodeVarint(2<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Keys); err != nil { + return err + } + case nil: + default: + return fmt.Errorf("IdsOrKeys.Type has unexpected type %T", x) + } + return nil +} + +func _IdsOrKeys_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*IdsOrKeys) + switch tag { + case 1: // type.ids + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(Uint64Array) + err := b.DecodeMessage(msg) + m.Type = &IdsOrKeys_Ids{msg} + return true, err + case 2: // type.keys + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(StringArray) + err := b.DecodeMessage(msg) + m.Type = &IdsOrKeys_Keys{msg} + return true, err + default: + return false, nil + } +} + +func _IdsOrKeys_OneofSizer(msg proto.Message) (n int) { + m := msg.(*IdsOrKeys) + // type + switch x := m.Type.(type) { + case *IdsOrKeys_Ids: + s := proto.Size(x.Ids) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *IdsOrKeys_Keys: + s := proto.Size(x.Keys) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case nil: + default: + panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) + } + return n +} + func init() { proto.RegisterType((*QueryPQLRequest)(nil), "pilosa.QueryPQLRequest") proto.RegisterType((*RowResponse)(nil), "pilosa.RowResponse") @@ -559,44 +792,6 @@ func init() { proto.RegisterType((*IdsOrKeys)(nil), "pilosa.IdsOrKeys") } -func init() { proto.RegisterFile("pilosa.proto", fileDescriptor_ef0691a44d1e275c) } - -var fileDescriptor_ef0691a44d1e275c = []byte{ - // 497 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x53, 0x5d, 0x6b, 0xd4, 0x40, - 0x14, 0xcd, 0x34, 0xe9, 0xee, 0xe6, 0x66, 0x59, 0xf5, 0x2a, 0x1a, 0x16, 0x91, 0x98, 0x17, 0x23, - 0x4a, 0x29, 0xab, 0x08, 0x4a, 0x7d, 0xb0, 0x82, 0x64, 0x51, 0xb0, 0x1d, 0xb1, 0xef, 0xb3, 0xcd, - 0x6c, 0x0d, 0xce, 0x66, 0xd2, 0x4c, 0xd6, 0x9a, 0x57, 0xff, 0xa2, 0x7f, 0x48, 0x66, 0xf2, 0xb1, - 0x49, 0x61, 0x7d, 0x9b, 0x39, 0xe7, 0xdc, 0xef, 0x7b, 0x61, 0x9a, 0xa7, 0x42, 0x2a, 0x76, 0x94, - 0x17, 0xb2, 0x94, 0x38, 0xaa, 0x7f, 0xe1, 0x5b, 0xb8, 0x73, 0xbe, 0xe5, 0x45, 0x75, 0x76, 0xfe, - 0x85, 0xf2, 0xeb, 0x2d, 0x57, 0x25, 0x3e, 0x80, 0xc3, 0x34, 0x4b, 0xf8, 0x6f, 0x9f, 0x04, 0x24, - 0x72, 0x69, 0xfd, 0xc1, 0xbb, 0x60, 0xe7, 0xd7, 0xc2, 0x3f, 0x30, 0x98, 0x7e, 0x86, 0x1b, 0xf0, - 0xa8, 0xbc, 0xa1, 0x5c, 0xe5, 0x32, 0x53, 0x1c, 0x5f, 0xc2, 0xf8, 0x07, 0x67, 0x09, 0x2f, 0x94, - 0x4f, 0x02, 0x3b, 0xf2, 0x16, 0x78, 0xd4, 0x44, 0xfc, 0x28, 0xc5, 0x76, 0x93, 0x2d, 0xb3, 0xb5, - 0xa4, 0xad, 0x04, 0x8f, 0x61, 0x7c, 0x69, 0x60, 0xe5, 0x1f, 0x18, 0xf5, 0xc3, 0xa1, 0xba, 0x75, - 0x4b, 0x5b, 0x59, 0x78, 0x02, 0xb0, 0x73, 0x84, 0x08, 0x4e, 0xc6, 0x36, 0xbc, 0xc9, 0xd1, 0xbc, - 0x71, 0x0e, 0x93, 0x84, 0x95, 0xac, 0xac, 0x72, 0xde, 0xe4, 0xd9, 0xfd, 0xc3, 0xbf, 0x07, 0x30, - 0x1b, 0x7a, 0xc6, 0x27, 0xe0, 0xaa, 0xb2, 0x48, 0xb3, 0xab, 0x0b, 0x26, 0x6a, 0x3f, 0xb1, 0x45, - 0x77, 0x90, 0xe6, 0xb7, 0x69, 0x56, 0xbe, 0x79, 0xad, 0x79, 0xed, 0xcf, 0xd1, 0x7c, 0x07, 0xe1, - 0x63, 0x98, 0x74, 0xb4, 0x1d, 0x90, 0xc8, 0x8e, 0x2d, 0xda, 0x21, 0x38, 0x87, 0xf1, 0x4a, 0x4a, - 0xa1, 0x49, 0x27, 0x20, 0xd1, 0x24, 0xb6, 0x68, 0x0b, 0x18, 0x4e, 0xc8, 0x95, 0xe6, 0x0e, 0x03, - 0x12, 0x4d, 0x0d, 0x57, 0x03, 0xf8, 0x1e, 0x66, 0x75, 0x88, 0x0f, 0x45, 0xc1, 0x2a, 0x2d, 0x19, - 0x05, 0x24, 0xf2, 0x16, 0xf7, 0xdb, 0xfe, 0x7c, 0xdf, 0xb1, 0xb1, 0x45, 0x6f, 0x89, 0xb5, 0x79, - 0x5d, 0x41, 0x67, 0x3e, 0x1e, 0x9a, 0x7f, 0xdb, 0xb1, 0xda, 0x7c, 0x28, 0xc6, 0x00, 0x60, 0x2d, - 0x24, 0x6b, 0xaa, 0x9a, 0x04, 0x24, 0x22, 0xb1, 0x45, 0x7b, 0xd8, 0xa9, 0x07, 0x6e, 0x3d, 0x91, - 0x0b, 0x26, 0xc2, 0x1b, 0x98, 0x2d, 0x33, 0x95, 0xf3, 0xcb, 0xf2, 0xff, 0xcb, 0xf3, 0xa2, 0x3f, - 0x6d, 0x9d, 0xce, 0xbd, 0x36, 0x9d, 0x65, 0xa2, 0xbe, 0x16, 0x9f, 0x79, 0xa5, 0xba, 0x41, 0x63, - 0x08, 0xd3, 0x75, 0x2a, 0x4a, 0x5e, 0x7c, 0x4a, 0xb9, 0x48, 0x94, 0x6f, 0x07, 0x76, 0xe4, 0xd2, - 0x01, 0x16, 0x3e, 0x05, 0xaf, 0xd7, 0x07, 0xbd, 0x0d, 0xbf, 0x98, 0xa8, 0x17, 0xcf, 0xa1, 0xe6, - 0xad, 0x25, 0xbd, 0x5a, 0x07, 0x12, 0xb7, 0x91, 0x5c, 0x81, 0xdb, 0xc5, 0xc7, 0x67, 0x60, 0xa7, - 0x89, 0x32, 0x79, 0xef, 0xed, 0xb6, 0x56, 0xe0, 0x73, 0x70, 0x7e, 0xf2, 0xaa, 0xad, 0x64, 0x4f, - 0x63, 0x8d, 0xe4, 0x74, 0x04, 0x8e, 0xde, 0xbe, 0xc5, 0x1f, 0x02, 0xa3, 0x33, 0x23, 0xc3, 0x13, - 0x98, 0xb4, 0x07, 0x87, 0x8f, 0x5a, 0xdb, 0x5b, 0x27, 0x38, 0xef, 0x9c, 0xf6, 0x0e, 0x2c, 0xb4, - 0x8e, 0x09, 0xbe, 0x83, 0x71, 0xd3, 0x70, 0xec, 0x0e, 0x66, 0x38, 0x81, 0xbd, 0xb6, 0xab, 0x91, - 0xb9, 0xfc, 0x57, 0xff, 0x02, 0x00, 0x00, 0xff, 0xff, 0xca, 0x68, 0x70, 0x08, 0x09, 0x04, 0x00, - 0x00, -} - // Reference imports to suppress errors if they are not otherwise used. var _ context.Context var _ grpc.ClientConn @@ -691,17 +886,6 @@ type PilosaServer interface { Inspect(*InspectRequest, Pilosa_InspectServer) error } -// UnimplementedPilosaServer can be embedded to have forward compatible implementations. -type UnimplementedPilosaServer struct { -} - -func (*UnimplementedPilosaServer) QueryPQL(req *QueryPQLRequest, srv Pilosa_QueryPQLServer) error { - return status.Errorf(codes.Unimplemented, "method QueryPQL not implemented") -} -func (*UnimplementedPilosaServer) Inspect(req *InspectRequest, srv Pilosa_InspectServer) error { - return status.Errorf(codes.Unimplemented, "method Inspect not implemented") -} - func RegisterPilosaServer(s *grpc.Server, srv PilosaServer) { s.RegisterService(&_Pilosa_serviceDesc, srv) } @@ -766,3 +950,42 @@ var _Pilosa_serviceDesc = grpc.ServiceDesc{ }, Metadata: "pilosa.proto", } + +func init() { proto.RegisterFile("pilosa.proto", fileDescriptor_ef0691a44d1e275c) } + +var fileDescriptor_ef0691a44d1e275c = []byte{ + // 524 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x54, 0xdd, 0x8a, 0xd3, 0x40, + 0x14, 0xce, 0x34, 0xd9, 0xb4, 0x39, 0x2d, 0x55, 0x8f, 0xb2, 0x96, 0x22, 0x12, 0x73, 0x63, 0x44, + 0x59, 0x96, 0x2a, 0x82, 0xb2, 0x5e, 0xb8, 0x82, 0xb4, 0x28, 0xb8, 0x3b, 0xe2, 0xde, 0x4f, 0x37, + 0xd3, 0x35, 0x38, 0xcd, 0x64, 0x33, 0x53, 0xb5, 0xb7, 0xbe, 0x8b, 0x4f, 0xe4, 0x0b, 0xc9, 0x4c, + 0x7e, 0x9a, 0x2c, 0x74, 0xef, 0x72, 0xbe, 0xef, 0x3b, 0x67, 0xce, 0x6f, 0x60, 0x94, 0xa7, 0x42, + 0x2a, 0x76, 0x94, 0x17, 0x52, 0x4b, 0xf4, 0x4b, 0x2b, 0x7a, 0x03, 0x77, 0xce, 0x37, 0xbc, 0xd8, + 0x9e, 0x9d, 0x7f, 0xa6, 0xfc, 0x7a, 0xc3, 0x95, 0xc6, 0x07, 0x70, 0x90, 0x66, 0x09, 0xff, 0x3d, + 0x21, 0x21, 0x89, 0x03, 0x5a, 0x1a, 0x78, 0x17, 0xdc, 0xfc, 0x5a, 0x4c, 0x7a, 0x16, 0x33, 0x9f, + 0xd1, 0x1a, 0x86, 0x54, 0xfe, 0xa2, 0x5c, 0xe5, 0x32, 0x53, 0x1c, 0x5f, 0x40, 0xff, 0x3b, 0x67, + 0x09, 0x2f, 0xd4, 0x84, 0x84, 0x6e, 0x3c, 0x9c, 0xe1, 0x51, 0xf5, 0xe2, 0x07, 0x29, 0x36, 0xeb, + 0x6c, 0x91, 0xad, 0x24, 0xad, 0x25, 0x78, 0x0c, 0xfd, 0x4b, 0x0b, 0xab, 0x49, 0xcf, 0xaa, 0x0f, + 0xbb, 0xea, 0x3a, 0x2c, 0xad, 0x65, 0xd1, 0x09, 0xc0, 0x2e, 0x10, 0x22, 0x78, 0x19, 0x5b, 0xf3, + 0x2a, 0x47, 0xfb, 0x8d, 0x53, 0x18, 0x24, 0x4c, 0x33, 0xbd, 0xcd, 0x79, 0x95, 0x67, 0x63, 0x47, + 0xff, 0x7a, 0x30, 0xee, 0x46, 0xc6, 0xc7, 0x10, 0x28, 0x5d, 0xa4, 0xd9, 0xd5, 0x05, 0x13, 0x65, + 0x9c, 0xb9, 0x43, 0x77, 0x90, 0xe1, 0x37, 0x69, 0xa6, 0x5f, 0xbf, 0x32, 0xbc, 0x89, 0xe7, 0x19, + 0xbe, 0x81, 0xf0, 0x11, 0x0c, 0x1a, 0xda, 0x0d, 0x49, 0xec, 0xce, 0x1d, 0xda, 0x20, 0x38, 0x85, + 0xfe, 0x52, 0x4a, 0x61, 0x48, 0x2f, 0x24, 0xf1, 0x60, 0xee, 0xd0, 0x1a, 0xb0, 0x9c, 0x90, 0x4b, + 0xc3, 0x1d, 0x84, 0x24, 0x1e, 0x59, 0xae, 0x04, 0xf0, 0x1d, 0x8c, 0xcb, 0x27, 0xde, 0x17, 0x05, + 0xdb, 0x1a, 0x89, 0x1f, 0x92, 0x78, 0x38, 0xbb, 0x5f, 0xf7, 0xe7, 0xdb, 0x8e, 0x9d, 0x3b, 0xf4, + 0x86, 0xd8, 0xb8, 0x97, 0x15, 0x34, 0xee, 0xfd, 0xae, 0xfb, 0xd7, 0x1d, 0x6b, 0xdc, 0xbb, 0x62, + 0x0c, 0x01, 0x56, 0x42, 0xb2, 0xaa, 0xaa, 0x41, 0x48, 0x62, 0x32, 0x77, 0x68, 0x0b, 0x3b, 0x1d, + 0x42, 0x50, 0x4e, 0xe4, 0x82, 0x89, 0xe8, 0x2f, 0x81, 0xf1, 0x22, 0x53, 0x39, 0xbf, 0xd4, 0xb7, + 0x6f, 0xcf, 0xf3, 0xf6, 0xb8, 0x4d, 0x3e, 0xf7, 0xea, 0x7c, 0x16, 0x89, 0xfa, 0x52, 0x7c, 0xe2, + 0x5b, 0xd5, 0x4c, 0x1a, 0x23, 0x18, 0xad, 0x52, 0xa1, 0x79, 0xf1, 0x31, 0xe5, 0x22, 0x51, 0x13, + 0x37, 0x74, 0xe3, 0x80, 0x76, 0x30, 0xf3, 0x8c, 0x48, 0xd7, 0xa9, 0xb6, 0xcd, 0xf5, 0x68, 0x69, + 0xe0, 0x21, 0xf8, 0x72, 0xb5, 0x52, 0x5c, 0xdb, 0xbe, 0x7a, 0xb4, 0xb2, 0xa2, 0x27, 0x30, 0x6c, + 0xb5, 0xcd, 0x2c, 0xcf, 0x4f, 0x26, 0xca, 0x3d, 0xf5, 0xa8, 0xfd, 0x36, 0x92, 0x56, 0x6b, 0x3a, + 0x92, 0xa0, 0x92, 0x5c, 0x41, 0xd0, 0x64, 0x8b, 0x4f, 0xc1, 0x4d, 0x13, 0x65, 0xab, 0xdc, 0x3b, + 0x1c, 0xa3, 0xc0, 0x67, 0xe0, 0xfd, 0xe0, 0xdb, 0xba, 0xee, 0x3d, 0x73, 0xb0, 0x92, 0x53, 0x1f, + 0x3c, 0xb3, 0xac, 0xb3, 0x3f, 0x04, 0xfc, 0x33, 0x2b, 0xc3, 0x13, 0x18, 0xd4, 0xf7, 0x89, 0x0f, + 0x6b, 0xdf, 0x1b, 0x17, 0x3b, 0x6d, 0x82, 0xb6, 0xee, 0x31, 0x72, 0x8e, 0x09, 0xbe, 0x85, 0x7e, + 0x35, 0x1e, 0x6c, 0xee, 0xab, 0x3b, 0xaf, 0xbd, 0xbe, 0x4b, 0xdf, 0xfe, 0x28, 0x5e, 0xfe, 0x0f, + 0x00, 0x00, 0xff, 0xff, 0x0a, 0x57, 0xf8, 0xfb, 0x38, 0x04, 0x00, 0x00, +} diff --git a/proto/pilosa.proto b/proto/pilosa.proto index a8822ec46..9a5828624 100644 --- a/proto/pilosa.proto +++ b/proto/pilosa.proto @@ -34,6 +34,8 @@ message InspectRequest { string index = 1; IdsOrKeys columns = 2; repeated string filterFields = 3; + uint64 limit = 4; + uint64 offset = 5; } message Uint64Array { diff --git a/server/grpc.go b/server/grpc.go index cb1856b9f..a0ef90cbe 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -71,6 +71,8 @@ func (h grpcHandler) QueryPQL(req *pb.QueryPQLRequest, stream pb.Pilosa_QueryPQL // Inspect handles the inspect request and sends an InspectResponse to the stream. func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectServer) error { + const defaultLimit = 100000 + index, err := h.api.Index(context.Background(), req.Index) if err != nil { return errors.Wrap(err, "getting index") @@ -96,7 +98,17 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer return nil } - if ints, ok := req.Columns.Type.(*pb.IdsOrKeys_Ids); ok { + limit := req.Limit + if limit == 0 { + limit = defaultLimit + } + offset := req.Offset + + if !index.Options().Keys { + ints, ok := req.Columns.Type.(*pb.IdsOrKeys_Ids) + if !ok { + return errors.New("invalid int columns") + } ci := []*pb.ColumnInfo{ {Name: "_id", Datatype: "uint64"}, } @@ -104,7 +116,46 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer ci = append(ci, &pb.ColumnInfo{Name: field.Name(), Datatype: field.Type()}) // TODO: field.Type likely doesn't align with supported datatypes } - for _, col := range ints.Ids.Vals { + // If Columns is empty, then get the _exists list (via All()), + // from the index and loop over that instead. + cols := ints.Ids.Vals + if len(cols) > 0 { + // Apply limit/offset to the provided columns. + if int(offset) >= len(cols) { + return nil + } + end := limit + offset + if int(end) > len(cols) { + end = uint64(len(cols)) + } + cols = cols[offset:end] + } else { + // Prevent getting too many records by forcing a limit. + pql := fmt.Sprintf("All(limit=%d, offset=%d)", limit, offset) + query := pilosa.QueryRequest{ + Index: req.Index, + Query: pql, + } + resp, err := h.api.Query(context.Background(), &query) + if err != nil { + return errors.Wrapf(err, "querying for all: %s", pql) + } + + ids, ok := resp.Results[0].(*pilosa.Row) + if !ok { + return errors.Wrap(err, "getting results as a row") + } + + limitedCols := ids.Columns() + if len(limitedCols) == 0 { + // If cols is still empty after the limit/offset, then + // return with no results. + return nil + } + cols = limitedCols + } + + for _, col := range cols { rowResp := &pb.RowResponse{ Headers: ci, Columns: []*pb.ColumnResponse{ @@ -178,6 +229,7 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: nil}) } + case "decimal": value, exists, err := field.FloatValue(col) if err != nil { @@ -189,6 +241,7 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: nil}) } + case "bool": pql := fmt.Sprintf("Rows(%s, column=%d)", field.Name(), col) query := pilosa.QueryRequest{ @@ -216,6 +269,7 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: nil}) } + case "time": rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: nil}) @@ -227,7 +281,11 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer } } - } else if keys, ok := req.Columns.Type.(*pb.IdsOrKeys_Keys); ok { + } else { + keys, ok := req.Columns.Type.(*pb.IdsOrKeys_Keys) + if !ok { + return errToStatusError(errors.New("invalid key columns")) + } ci := []*pb.ColumnInfo{ {Name: "_id", Datatype: "string"}, } @@ -235,7 +293,47 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer ci = append(ci, &pb.ColumnInfo{Name: field.Name(), Datatype: field.Type()}) // TODO: field.Type likely doesn't align with supported datatypes } - for _, col := range keys.Keys.Vals { + // If Columns is empty, then get the _exists list (via All()), + // from the index and loop over that instead. + cols := keys.Keys.Vals + if len(cols) > 0 { + // Apply limit/offset to the provided columns. + if int(offset) >= len(cols) { + return nil + } + end := limit + offset + if int(end) > len(cols) { + end = uint64(len(cols)) + } + cols = cols[offset:end] + } else { + // Prevent getting too many records by forcing a limit. + pql := fmt.Sprintf("All(limit=%d, offset=%d)", limit, offset) + query := pilosa.QueryRequest{ + Index: req.Index, + Query: pql, + } + resp, err := h.api.Query(context.Background(), &query) + if err != nil { + fmt.Println("GOT ERROR trying to get ALL():", err) + return errors.Wrapf(err, "querying for all: %s", pql) + } + + ids, ok := resp.Results[0].(*pilosa.Row) + if !ok { + return errors.Wrap(err, "getting results as a row") + } + + limitedCols := ids.Keys + if len(limitedCols) == 0 { + // If cols is still empty after the limit/offset, then + // return with no results. + return nil + } + cols = limitedCols + } + + for _, col := range cols { rowResp := &pb.RowResponse{ Headers: ci, Columns: []*pb.ColumnResponse{ From f51c2dbc42628d73ea2fc0aed52941fc78515f08 Mon Sep 17 00:00:00 2001 From: Seebs Date: Mon, 2 Dec 2019 13:24:53 -0600 Subject: [PATCH 09/20] use extensions through build tags --- Makefile | 1 + ext/ext.go | 25 +++++++++++++ ext/extensions/distinct.go | 21 +++++++++++ ext/extensions/dummy.go | 18 +++++++++ server.go | 75 +++++++------------------------------- 5 files changed, 78 insertions(+), 62 deletions(-) create mode 100644 ext/extensions/distinct.go create mode 100644 ext/extensions/dummy.go diff --git a/Makefile b/Makefile index bee802120..ac204d3ca 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,7 @@ RELEASE_ENABLED = $(subst 0,,$(RELEASE)) BUILD_TAGS += $(if $(ENTERPRISE_ENABLED),enterprise) BUILD_TAGS += $(if $(RELEASE_ENABLED),release) BUILD_TAGS += shardwidth$(SHARD_WIDTH) +BUILD_TAGS += $(foreach p,$(PLUGINS),plugin$(p)) define LICENSE_HASH_CODE head -13 $1 | sed -e 's/Copyright 20[0-9][0-9]/Copyright 20XX/g' | shasum | cut -f 1 -d " " endef diff --git a/ext/ext.go b/ext/ext.go index b6daa3ef0..762ae44f8 100644 --- a/ext/ext.go +++ b/ext/ext.go @@ -38,6 +38,8 @@ // case, no ops are registered. package ext +import "sync" + // The Bitmap type represents a Pilosa bitmap, and is used for bitmap // operations. type Bitmap interface { @@ -236,3 +238,26 @@ type ExtensionInfo struct { License string // License info. BitmapOps []BitmapOp // List of provided ops. } + +var extMu sync.Mutex + +var knownExtensions []*ExtensionInfo +var newExtensions []*ExtensionInfo + +// RegisterExtension tells the extension system about a new extension. +func RegisterExtension(ext *ExtensionInfo) { + extMu.Lock() + defer extMu.Unlock() + newExtensions = append(newExtensions, ext) +} + +// NewExtentsions returns extensions that have been registered, but not previously +// returned by Newextensions. +func NewExtensions() []*ExtensionInfo { + extMu.Lock() + defer extMu.Unlock() + knownExtensions = append(knownExtensions, newExtensions...) + ret := newExtensions + newExtensions = nil + return ret +} diff --git a/ext/extensions/distinct.go b/ext/extensions/distinct.go new file mode 100644 index 000000000..42f11c90f --- /dev/null +++ b/ext/extensions/distinct.go @@ -0,0 +1,21 @@ +// Copyright 2019 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build plugindistinct + +package extensions + +import ( + _ "github.com/molecula/extensions/distinct" +) diff --git a/ext/extensions/dummy.go b/ext/extensions/dummy.go new file mode 100644 index 000000000..cf418edb5 --- /dev/null +++ b/ext/extensions/dummy.go @@ -0,0 +1,18 @@ +// Copyright 2019 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This package contains only things which are conditional on build +// tags. + +package extensions diff --git a/server.go b/server.go index ddf43ed50..645418898 100644 --- a/server.go +++ b/server.go @@ -17,12 +17,10 @@ package pilosa import ( "context" "fmt" - "io" "log" "os" "os/exec" "path/filepath" - "plugin" "runtime" "strconv" "strings" @@ -30,6 +28,8 @@ import ( "time" "github.com/pilosa/pilosa/v2/ext" + // extensions pulls in some extensions depending on build tags + _ "github.com/pilosa/pilosa/v2/ext/extensions" "github.com/pilosa/pilosa/v2/logger" "github.com/pilosa/pilosa/v2/pql" "github.com/pilosa/pilosa/v2/roaring" @@ -341,8 +341,6 @@ func NewServer(opts ...ServerOption) (*Server, error) { if err != nil { return nil, err } - s.extensionPath = filepath.Join(path, ".extensions") - s.holder.Path = path // s.holder.translateFile.Path = filepath.Join(path, ".keys") s.holder.Logger = s.logger @@ -383,7 +381,7 @@ func NewServer(opts ...ServerOption) (*Server, error) { s.cluster.broadcaster = s s.cluster.maxWritesPerRequest = s.maxWritesPerRequest s.holder.broadcaster = s - err = s.loadPlugins() + err = s.loadExtensions() if err != nil { s.logger.Printf("not all plugins loaded successfully") } @@ -420,67 +418,20 @@ func (s *Server) InternalClient() InternalClient { return s.defaultClient } -func (s *Server) loadPlugins() error { - var anyError error - dir, err := os.Open(s.extensionPath) - if err != nil { - // don't complain about it not existing, that's fine. - if os.IsNotExist(err) { - s.logger.Printf("extension interface v0: no extensions directory.") - return nil - } - return errors.Wrap(err, "opening extension path:") - } - defer dir.Close() - for files, err := dir.Readdir(64); err != io.EOF; files, err = dir.Readdir(64) { - if err != nil { - return errors.Wrap(err, "searching extension directory:") - } - for _, file := range files { - name := file.Name() - // only .so files are likely plugins. - if !strings.HasSuffix(name, ".so") { - continue - } - // only regular files are candidates for loading. - mode := file.Mode() - if !mode.IsRegular() { - s.logger.Printf("extension file '%s' is not a regular file", name) - continue - } - err = s.loadPlugin(name) - if err != nil { - s.logger.Printf("loading extension %s: %v", name, err) - anyError = err - } +func (s *Server) loadExtensions() error { + exts := ext.NewExtensions() + var lastError error + for _, extension := range exts { + if err := s.loadExtension(extension); err != nil { + lastError = err } } - return anyError + return lastError } -func (s *Server) loadPlugin(name string) error { - path := filepath.Join(s.extensionPath, name) - p, err := plugin.Open(path) - if err != nil { - return err - } - pluginExtInfo, err := p.Lookup("ExtensionInfo") - if err != nil { - return fmt.Errorf("%s: no ExtensionInfo found", name) - } - extInfoFunc, ok := pluginExtInfo.(func(string) (*ext.ExtensionInfo, error)) - if !ok { - return fmt.Errorf("%s: unexpected %T instead of ExtensionInfo object", name, pluginExtInfo) - } - extInfo, err := extInfoFunc("v0") - if err != nil { - return errors.Wrap(err, name) - } - if extInfo == nil { - return fmt.Errorf("%s: nil ExtensionInfo", name) - } +func (s *Server) loadExtension(extInfo *ext.ExtensionInfo) error { if extInfo.ExtensionAPI != "v0" { - return fmt.Errorf("%s: unsupported extension API %s", name, extInfo.ExtensionAPI) + return fmt.Errorf("%s: unsupported extension API %s", extInfo.Name, extInfo.ExtensionAPI) } s.extensions = append(s.extensions, extInfo) bitmapOps := extInfo.BitmapOps @@ -500,7 +451,7 @@ func (s *Server) loadPlugin(name string) error { unknownOps++ } } - err = s.executor.registerOps(bitmapOps) + err := s.executor.registerOps(bitmapOps) if err != nil { s.logger.Printf("warning: extension registration failed: %v", err) } else { From 0eba050054aa260cb780d3dab7f67c5f89c77c5a Mon Sep 17 00:00:00 2001 From: Seebs Date: Mon, 2 Dec 2019 13:42:51 -0600 Subject: [PATCH 10/20] stop using pkg/plugin, start using build tags After a few experiments with pkg/plugin, I'm ready to concede that the people warning me it was unsuitable for production use were in fact correct. In the brave new world, the "ext" package is moved to its own module outside pilosa. This means that importing it doesn't imply any need to version-check against pilosa; we can just use versioned copies of the ext package, which can be public because it doesn't contain anything we need to care about keeping proprietary. Then we can, conditional on build tags, import modules from a neighboring repo which contains the actual implementations, and if they're imported, their init functions register them. --- executor.go | 2 +- ext/ext.go | 263 --------------------- ext/samples/.gitignore | 1 - ext/samples/some/some.go | 125 ---------- extension.go | 2 +- {ext/extensions => extensions}/distinct.go | 0 {ext/extensions => extensions}/dummy.go | 0 pql/ast.go | 2 +- row.go | 2 +- server.go | 4 +- 10 files changed, 6 insertions(+), 395 deletions(-) delete mode 100644 ext/ext.go delete mode 100644 ext/samples/.gitignore delete mode 100644 ext/samples/some/some.go rename {ext/extensions => extensions}/distinct.go (100%) rename {ext/extensions => extensions}/dummy.go (100%) diff --git a/executor.go b/executor.go index 47c08043b..f9586f958 100644 --- a/executor.go +++ b/executor.go @@ -23,7 +23,7 @@ import ( "sync" "time" - "github.com/pilosa/pilosa/v2/ext" + "github.com/molecula/ext" "github.com/pilosa/pilosa/v2/pql" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/shardwidth" diff --git a/ext/ext.go b/ext/ext.go deleted file mode 100644 index 762ae44f8..000000000 --- a/ext/ext.go +++ /dev/null @@ -1,263 +0,0 @@ -// Copyright 2019 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package ext provides an EXPERIMENTAL AND TEMPORARY interface to use for -// plugin extensions to Pilosa. DO NOT DEVELOP NEW PLUGINS WITH THIS. The -// replacement design is already in process, but it needs more refinement -// to address issues. This one has those issues, and more. -// -// In the current design, plugins will be loaded at runtime using the -// go `plugin` package, so they should be built as a main package using -// the plugin build mode. -// -// Plugins should not import other packages from Pilosa. -// -// To advertise their functionality, plugins define one or more of a -// handful of symbols which will be checked for at plugin load and used -// to register their functionality. -// -// The plugin interface will check for the following function(s). If the -// functions exist, they must have the given signatures. If they return -// a non-nil error, no ops are registered, and the error message will -// be reported in the Pilosa server's logs. -// -// BitmapOps() ([]BitmapOp, error) -// -// These functions may be absent, and may return nil slices; in either -// case, no ops are registered. -package ext - -import "sync" - -// The Bitmap type represents a Pilosa bitmap, and is used for bitmap -// operations. -type Bitmap interface { - // AddN and RemoveN can be used to add or remove values from a bitmap. - AddN(a ...uint64) (int, error) - RemoveN(a ...uint64) (int, error) - - // Lookups - Max() uint64 - Min() (uint64, bool) - Count() uint64 - Any() bool - Contains(uint64) bool - Slice() []uint64 - SliceRange(uint64, uint64) []uint64 - // ContainerBits stores the next 1<<16 bits, starting at the provided - // bit index. It may use a provided []uint64 to store them, or may - // provide its own. Don't write to those bits. Offset must be a multiple - // of 1<<16. - ContainerBits(uint64, []uint64) []uint64 - - // These operators provide existing implemented binary ops. - Intersect(Bitmap) Bitmap - Union(Bitmap) Bitmap - IntersectionCount(Bitmap) uint64 - Difference(Bitmap) Bitmap - Xor(Bitmap) Bitmap - Shift(int) (Bitmap, error) - Flip(uint64, uint64) Bitmap - - // New() is an atrocity: it creates a new bitmap, unrelated to the - // existing bitmap. This lets you create a new bitmap without having - // imported any of the packages that have bitmap creation tools, because - // the bitmap wrapper type has to give you one. - New() Bitmap -} - -// SignedBitmap represents a bitmap that can contain both positive and negative -// values. -type SignedBitmap struct { - Pos, Neg Bitmap -} - -// A BitmapOp represents a new bitmap operation that should be exposed -// in PQL. - -type BitmapOpInput byte -type BitmapOpOutput byte -type BitmapOpArity byte -type BitmapOpPrecall byte -type BitmapOpType struct { - Input BitmapOpInput - Arity BitmapOpArity - Output BitmapOpOutput - Precall BitmapOpPrecall -} - -const ( - OpArityUnary = BitmapOpArity(iota) - OpArityBinary - OpArityNary -) - -const ( - // Unary: Exactly one bitmap. - OpInputBitmap = BitmapOpInput(iota) - // The really weird special case used for BSI, where we end up - // needing to do BSI computations. Arguments will be a - // single BitmapBSI, and a []Bitmap for other operands if any. - OpInputNaryBSI -) - -const ( - OpOutputCount = BitmapOpOutput(iota) - OpOutputBitmap - OpOutputSignedBitmap -) - -const ( - OpPrecallNone = BitmapOpPrecall(iota) - OpPrecallGlobal - OpPrecallLocal // unimplemented -) - -// Regardless of arity, non-BSI functions should always take []Bitmap. -type BitmapOpFunc interface { - BitmapOpType() BitmapOpType -} - -// BitmapOpBitmap should actually always be func([]Bitmap) Bitmap, but -// might be different kinds. -type BitmapOpBitmap interface { - BitmapOpArity() BitmapOpArity - BitmapOpFunc() GenericBitmapOpBitmap -} - -// the common underlying type of the other BitmapOpBitmap functions -type GenericBitmapOpBitmap func([]Bitmap, map[string]interface{}) Bitmap - -// BitmapBSI represents the way a single BSI field is passed into a function -// which takes a BSI field. -type BitmapBSI struct { - FieldData Bitmap - ShardWidth uint64 - Offset int64 - Depth uint -} - -type BitmapOpBSIBitmap func(BitmapBSI, []Bitmap, map[string]interface{}) SignedBitmap - -func (b BitmapOpBSIBitmap) BitmapOpType() BitmapOpType { - return BitmapOpType{Input: OpInputNaryBSI, Arity: OpArityNary, Output: OpOutputSignedBitmap} -} - -type BitmapOpBSIBitmapPrecall func(BitmapBSI, []Bitmap, map[string]interface{}) SignedBitmap - -func (b BitmapOpBSIBitmapPrecall) BitmapOpType() BitmapOpType { - return BitmapOpType{Input: OpInputNaryBSI, Arity: OpArityNary, Precall: OpPrecallGlobal, Output: OpOutputSignedBitmap} -} - -type BitmapOpUnaryCount func([]Bitmap, map[string]interface{}) int64 - -func (b BitmapOpUnaryCount) BitmapOpType() BitmapOpType { - return BitmapOpType{Input: OpInputBitmap, Arity: OpArityUnary, Output: OpOutputCount} -} - -type BitmapOpUnaryBitmap func([]Bitmap, map[string]interface{}) Bitmap - -func (b BitmapOpUnaryBitmap) BitmapOpType() BitmapOpType { - return BitmapOpType{Input: OpInputBitmap, Arity: OpArityUnary, Output: OpOutputBitmap} -} - -func (b BitmapOpUnaryBitmap) BitmapOpArity() BitmapOpArity { - return OpArityUnary -} - -func (b BitmapOpUnaryBitmap) BitmapOpFunc() GenericBitmapOpBitmap { - return GenericBitmapOpBitmap(b) -} - -type BitmapOpBinaryBitmap func([]Bitmap, map[string]interface{}) Bitmap - -func (b BitmapOpBinaryBitmap) BitmapOpType() BitmapOpType { - return BitmapOpType{Input: OpInputBitmap, Arity: OpArityBinary, Output: OpOutputBitmap} -} - -func (b BitmapOpBinaryBitmap) BitmapOpArity() BitmapOpArity { - return OpArityBinary -} - -func (b BitmapOpBinaryBitmap) BitmapOpFunc() GenericBitmapOpBitmap { - return GenericBitmapOpBitmap(b) -} - -type BitmapOpNaryBitmap func([]Bitmap, map[string]interface{}) Bitmap - -func (b BitmapOpNaryBitmap) BitmapOpType() BitmapOpType { - return BitmapOpType{Input: OpInputBitmap, Arity: OpArityNary, Output: OpOutputBitmap} -} - -func (b BitmapOpNaryBitmap) BitmapOpArity() BitmapOpArity { - return OpArityNary -} - -func (b BitmapOpNaryBitmap) BitmapOpFunc() GenericBitmapOpBitmap { - return GenericBitmapOpBitmap(b) -} - -// BitmapOp represents an operation to be supported in PQL. Operations -// on bitmaps should always take []Bitmap. Operations on InputNaryBSI should -// take a []Bitmap, plus a Bitmap/shard-width/offset/depth. -// -// Reserved is a list of words to treat as reserved words in a prototype. -// This is not currently used but might be later, and I want to have the -// concept handy now. -type BitmapOp struct { - Name string - Func BitmapOpFunc - Reserved []string -} - -// ExtensionInfo tells us about the extension. The ExtensionAPI string -// should be "v0". The version is a human-readable version, use something -// that seems meaningful. Name and Description are reasonably self-explanatory, -// I hope. -// -// Extensions should define a function: -// func ExtensionInfo(extensionAPI string) (*ExtensionInfo, error) -// which reports their extension info if they think they can coexist with that -// API string. -type ExtensionInfo struct { - Name string // Extension name. - Description string // Short description. - Version string // Human-readable version info for extension. - ExtensionAPI string // Extension API version. Should be v0 for now. - License string // License info. - BitmapOps []BitmapOp // List of provided ops. -} - -var extMu sync.Mutex - -var knownExtensions []*ExtensionInfo -var newExtensions []*ExtensionInfo - -// RegisterExtension tells the extension system about a new extension. -func RegisterExtension(ext *ExtensionInfo) { - extMu.Lock() - defer extMu.Unlock() - newExtensions = append(newExtensions, ext) -} - -// NewExtentsions returns extensions that have been registered, but not previously -// returned by Newextensions. -func NewExtensions() []*ExtensionInfo { - extMu.Lock() - defer extMu.Unlock() - knownExtensions = append(knownExtensions, newExtensions...) - ret := newExtensions - newExtensions = nil - return ret -} diff --git a/ext/samples/.gitignore b/ext/samples/.gitignore deleted file mode 100644 index a63fa2c94..000000000 --- a/ext/samples/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*/*.so diff --git a/ext/samples/some/some.go b/ext/samples/some/some.go deleted file mode 100644 index 07088d255..000000000 --- a/ext/samples/some/some.go +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2019 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "fmt" - "math/bits" - - "github.com/molecula/apophenia" - "github.com/pilosa/pilosa/v2/ext" -) - -// This could be dynamically generated, but for now it's not. -// nolint:unused,deadcode -var extInfoTemplate = &ext.ExtensionInfo{ - Name: "some", - Description: "some of the bits/all of the bits/none of the bits", - Version: "0.01", - ExtensionAPI: "v0", - License: "unreleased", - BitmapOps: []ext.BitmapOp{ - {Name: "Some", Func: ext.BitmapOpUnaryBitmap(Some), Reserved: []string{"p", "seed"}}, - }, -} - -// ExtensionInfo is the entry point used by the plugin code. -func ExtensionInfo(api string) (*ext.ExtensionInfo, error) { // nolint:unused,deadcode - return extInfoTemplate, nil -} - -const batchSize = 1024 - -// Some returns some of the bits from its first input bitmap. Takes seed (int) -// and p (float) values. Seed defaults to 0. -func Some(inputs []ext.Bitmap, args map[string]interface{}) ext.Bitmap { - if len(inputs) == 0 || inputs[0] == nil { - return nil - } - input := inputs[0] - min, ok := input.Min() - // no bits found? - if !ok { - return nil - } - // start at multiple of 128 not greater than min. - min &^= 127 - max := input.Max() - p, ok := args["p"].(float64) - if !ok { - return nil - } - // no bits or impossible probability range - if p <= 0 || p > 1 { - return nil - } - // every bit - if p == 1 { - return inputs[0] - } - // On failure, we default to 0. - seed, _ := args["seed"].(int64) - densityScale := uint64(256) - density := uint64(p * float64(densityScale)) - for density == 0 { - densityScale <<= 1 - density = uint64(p * float64(densityScale)) - // too small - if densityScale > (1 << 32) { - return nil - } - } - w, err := apophenia.NewWeighted(apophenia.NewSequence(seed)) - if err != nil { - return nil - } - someBits := input.New() - toAdd := make([]uint64, batchSize) - toAddN := 0 - offset := apophenia.OffsetFor(apophenia.SequenceWeighted, 0, 0, 0) - for i := min; i < max; i += 128 { - offset.Lo = i - randomBits := w.Bits(offset, density, densityScale) - bit := uint64(0) - for randomBits.Lo != 0 { - next := uint64(bits.TrailingZeros64(randomBits.Lo) + 1) - randomBits.Lo >>= next - toAdd[toAddN] = next + bit + i - toAddN++ - bit += next - } - bit = 64 - for randomBits.Hi != 0 { - next := uint64(bits.TrailingZeros64(randomBits.Hi) + 1) - randomBits.Hi >>= next - toAdd[toAddN] = next + bit + i - toAddN++ - bit += next - } - if toAddN > (batchSize - 128) { - // ignore error - _, _ = someBits.AddN(toAdd[:toAddN]...) - toAddN = 0 - } - } - if toAddN > 0 { - _, _ = someBits.AddN(toAdd[:toAddN]...) - } - return input.Intersect(someBits) -} - -func main() { - fmt.Printf("this is a plugin module only.\n") -} diff --git a/extension.go b/extension.go index d449f3f0c..1a492cce3 100644 --- a/extension.go +++ b/extension.go @@ -17,7 +17,7 @@ package pilosa import ( "fmt" - "github.com/pilosa/pilosa/v2/ext" + "github.com/molecula/ext" "github.com/pilosa/pilosa/v2/roaring" ) diff --git a/ext/extensions/distinct.go b/extensions/distinct.go similarity index 100% rename from ext/extensions/distinct.go rename to extensions/distinct.go diff --git a/ext/extensions/dummy.go b/extensions/dummy.go similarity index 100% rename from ext/extensions/dummy.go rename to extensions/dummy.go diff --git a/pql/ast.go b/pql/ast.go index 61b4b6540..738601100 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -23,7 +23,7 @@ import ( "strings" "time" - "github.com/pilosa/pilosa/v2/ext" + "github.com/molecula/ext" ) // Query represents a PQL query. diff --git a/row.go b/row.go index d9a4f9cc9..82a67cf8f 100644 --- a/row.go +++ b/row.go @@ -18,7 +18,7 @@ import ( "encoding/json" "sort" - "github.com/pilosa/pilosa/v2/ext" + "github.com/molecula/ext" "github.com/pilosa/pilosa/v2/roaring" "github.com/pkg/errors" ) diff --git a/server.go b/server.go index 645418898..44e27b1a5 100644 --- a/server.go +++ b/server.go @@ -27,9 +27,9 @@ import ( "sync" "time" - "github.com/pilosa/pilosa/v2/ext" + "github.com/molecula/ext" // extensions pulls in some extensions depending on build tags - _ "github.com/pilosa/pilosa/v2/ext/extensions" + _ "github.com/pilosa/pilosa/v2/extensions" "github.com/pilosa/pilosa/v2/logger" "github.com/pilosa/pilosa/v2/pql" "github.com/pilosa/pilosa/v2/roaring" From 7e1fd8392fa8cf8bd530b00170c88d4e3e84dc33 Mon Sep 17 00:00:00 2001 From: Seebs Date: Mon, 2 Dec 2019 13:58:36 -0600 Subject: [PATCH 11/20] go.mod/go.sum changes for using molecula/ext This pins us to the initial external release of molecula/ext, which with any luck will be the only one. (Narrator: It was not to be the only one.) We also use GOPRIVATE so we don't need a replace directive. --- Makefile | 1 + go.mod | 3 ++- go.sum | 4 ++++ server.go | 1 - 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index ac204d3ca..55356bd59 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,7 @@ endef LICENSE_HASH=$(shell $(call LICENSE_HASH_CODE, pilosa.go)) export GO111MODULE=on +export GOPRIVATE=github.com/molecula # Run tests and compile Pilosa default: test build diff --git a/go.mod b/go.mod index f47f54dc1..9366e18cd 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,8 @@ require ( github.com/gorilla/mux v1.7.0 github.com/hashicorp/memberlist v0.1.3 github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b + github.com/molecula/ext v0.0.0-20191202195653-240f38a75171 + github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4 github.com/opentracing/opentracing-go v1.1.0 github.com/pelletier/go-toml v1.2.0 github.com/pkg/errors v0.8.1 diff --git a/go.sum b/go.sum index 177cd241f..1b1b7af2f 100644 --- a/go.sum +++ b/go.sum @@ -89,6 +89,10 @@ github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQz github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b h1:cZADDaNYM7xn/nklO3g198JerGQjadFuA0ofxBJgK0Y= github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b/go.mod h1:uXd1BiH7xLmgkhVmspdJLENv6uGWrTL/MQX2TN7Yz9s= +github.com/molecula/ext v0.0.0-20191202195653-240f38a75171 h1:4VK7u/RM+54Yaz8aRB9vIaDSnbKi3M0NQYg5tsZvOT4= +github.com/molecula/ext v0.0.0-20191202195653-240f38a75171/go.mod h1:r6EIj0GH8dx5xxFLW6Voi1/mX3wXOUkJu6AoEE/xvGQ= +github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4 h1:mDB/dicofRVFuRYcCVPk+JBiVKXlfbzMahuqHvrYqu4= +github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4/go.mod h1:QQgN5OFjuBAi4Q2UYVMzfvi4k9yvg/qqC+MNFB4I9JI= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= diff --git a/server.go b/server.go index 44e27b1a5..96a4d5cde 100644 --- a/server.go +++ b/server.go @@ -61,7 +61,6 @@ type Server struct { // nolint: maligned hosts []string clusterDisabled bool serializer Serializer - extensionPath string extensions []*ext.ExtensionInfo // External From 49b2029656d882f3a72b7d947ae9f67f965f063a Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 19 Dec 2019 10:24:15 -0600 Subject: [PATCH 12/20] Run "go mod vendor" outside of Docker so authenticated modules may use system credentials --- Dockerfile-clustertests | 5 +---- Makefile | 4 ++-- internal/clustertests/docker-compose.yml | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/Dockerfile-clustertests b/Dockerfile-clustertests index 69fbd9a54..6241ae70d 100644 --- a/Dockerfile-clustertests +++ b/Dockerfile-clustertests @@ -8,10 +8,7 @@ LABEL maintainer "dev@pilosa.com" COPY . /go/src/github.com/pilosa/pilosa/ RUN cd /go/src/github.com/pilosa/pilosa \ - && GO111MODULE=on make vendor - -RUN cd /go/src/github.com/pilosa/pilosa \ - && CGO_ENABLED=0 make install FLAGS="-a" + && CGO_ENABLED=0 make install FLAGS="-a -mod=vendor" # download pumba for fault injection ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba diff --git a/Makefile b/Makefile index 55356bd59..20f00ac03 100644 --- a/Makefile +++ b/Makefile @@ -87,14 +87,14 @@ DOCKER_COMPOSE=internal/clustertests/docker-compose.yml # running. This will catch changes to internal/clustertests/*.go, but if you # make changes to Pilosa, you'll want to run clustertests-build to rebuild the # pilosa image. -clustertests: +clustertests: vendor docker-compose -f $(DOCKER_COMPOSE) down docker-compose -f $(DOCKER_COMPOSE) build client1 docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 # Like clustertests, but rebuilds all images. -clustertests-build: +clustertests-build: vendor docker-compose -f $(DOCKER_COMPOSE) down docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build diff --git a/internal/clustertests/docker-compose.yml b/internal/clustertests/docker-compose.yml index 1546a5b13..36b418921 100644 --- a/internal/clustertests/docker-compose.yml +++ b/internal/clustertests/docker-compose.yml @@ -51,6 +51,6 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock command: - - "cd /go/src/github.com/pilosa/pilosa/ && go test -v -count=1 github.com/pilosa/pilosa/v2/internal/clustertests" + - "cd /go/src/github.com/pilosa/pilosa/ && go test -mod=vendor -v -count=1 github.com/pilosa/pilosa/v2/internal/clustertests" networks: pilosanet: From fe0f57651ed24e4b2e30b0b491ef14fa411475a0 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 20 Dec 2019 12:19:57 -0600 Subject: [PATCH 13/20] build with distinct by default --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 20f00ac03..9d24ae75f 100644 --- a/Makefile +++ b/Makefile @@ -22,8 +22,10 @@ define LICENSE_HASH_CODE endef LICENSE_HASH=$(shell $(call LICENSE_HASH_CODE, pilosa.go)) +PLUGINS=distinct export GO111MODULE=on export GOPRIVATE=github.com/molecula +export PLUGINS # Run tests and compile Pilosa default: test build From d4117f3137e9259d2e18ca8b3c19f5d0599dabb1 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 20 Dec 2019 16:05:18 -0600 Subject: [PATCH 14/20] Vendor modules before building docker image so private modules can be downloaded --- Dockerfile | 2 +- Makefile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1bcabbdab..01f3a0341 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ ARG MAKE_FLAGS COPY . pilosa -RUN cd pilosa && CGO_ENABLED=0 make install FLAGS="-a ${BUILD_FLAGS}" ${MAKE_FLAGS} +RUN cd pilosa && CGO_ENABLED=0 make install FLAGS="-a -mod=vendor ${BUILD_FLAGS}" ${MAKE_FLAGS} FROM alpine:3.9.4 diff --git a/Makefile b/Makefile index 9d24ae75f..a4726b575 100644 --- a/Makefile +++ b/Makefile @@ -131,12 +131,12 @@ generate-proto-grpc: require-protoc require-protoc-gen-gofast generate: generate-protoc generate-stringer generate-pql # Create Docker image from Dockerfile -docker: +docker: vendor docker build --build-arg BUILD_FLAGS="${FLAGS}" -t "pilosa:$(VERSION)" . @echo Created docker image: pilosa:$(VERSION) # Create Docker image from Dockerfile (enterprise) -docker-enterprise: +docker-enterprise: vendor docker build --build-arg MAKE_FLAGS="ENTERPRISE=1" -t "pilosa-enterprise:$(VERSION)" . @echo Created docker image: pilosa-enterprise:$(VERSION) From 586a13e9423a3d79ef03ff597d0bc4e910e75352 Mon Sep 17 00:00:00 2001 From: Travis Date: Fri, 20 Dec 2019 22:43:50 -0600 Subject: [PATCH 15/20] Allow All() to be called at the shard level --- executor.go | 10 ++++++---- executor_test.go | 26 ++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/executor.go b/executor.go index f9586f958..d49247adf 100644 --- a/executor.go +++ b/executor.go @@ -720,7 +720,7 @@ func (e *executor) executeAllCall(ctx context.Context, index string, c *pql.Call func (e *executor) executeAllCallMapReduce(ctx context.Context, index string, c *pql.Call, shard uint64, opt *execOptions) (*Row, error) { // Execute calls in bulk on each remote node and merge. mapFn := func(shard uint64) (interface{}, error) { - return e.executeAllShard(ctx, index, c, shard) + return e.executeAllCallShard(ctx, index, c, shard) } // Merge returned results at coordinating node. @@ -1113,6 +1113,8 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c * return e.executeNotShard(ctx, index, c, shard) case "Shift": return e.executeShiftShard(ctx, index, c, shard) + case "All": // Allow a shard computation to use All() (note, limit/offset not applied) + return e.executeAllCallShard(ctx, index, c, shard) case "Precomputed": return e.executePrecomputedCallShard(ctx, index, c, shard) default: @@ -2524,9 +2526,9 @@ func (e *executor) executeNotShard(ctx context.Context, index string, c *pql.Cal return existenceRow.Difference(row), nil } -// executeAllShard executes an All() call for a local shard. -func (e *executor) executeAllShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { - span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeAllShard") +// executeAllCallShard executes an All() call for a local shard. +func (e *executor) executeAllCallShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeAllCallShard") defer span.Finish() if len(c.Children) > 0 { diff --git a/executor_test.go b/executor_test.go index 3750a04ad..ec8865c8a 100644 --- a/executor_test.go +++ b/executor_test.go @@ -3039,6 +3039,32 @@ func TestExecutor_Execute_All(t *testing.T) { } } }) + + // Ensure that a query which uses All() at the shard level can call it. + t.Run("AllShard", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) + } + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` +Set(3001, f=3) +Set(5001, f=5) +Set(5002, f=5) +`}); err != nil { + t.Fatalf("querying remote: %v", err) + } + + expCols := []uint64{5001, 5002} + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Intersect(All(), Row(f=5))"}); err != nil { + t.Fatal(err) + } else if cols := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(cols, expCols) { + t.Fatalf("unexpected columns, got: %v, but expected: %v", cols, expCols) + } + }) } // Ensure a row can be cleared. From d34e38f1342a1fef6c01591b8c045764bef69ed3 Mon Sep 17 00:00:00 2001 From: Travis Date: Thu, 26 Dec 2019 22:27:46 -0600 Subject: [PATCH 16/20] Remove empty field check in Inspect() The check for field existence is not necessary; since we add the `_id` field to every response then at the very least that field will be returned. This check was preventin a query like `select _id from ...` from returning any results. --- server/grpc.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/server/grpc.go b/server/grpc.go index a0ef90cbe..606732d6f 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -93,11 +93,6 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer } } - // If there are no matching fields, then don't return any records. - if len(fields) == 0 { - return nil - } - limit := req.Limit if limit == 0 { limit = defaultLimit From 73090e05c8ebe0cdf50d3233ced3eb04c791f411 Mon Sep 17 00:00:00 2001 From: Travis Date: Tue, 24 Dec 2019 11:25:08 -0600 Subject: [PATCH 17/20] Add StatusError to RowResponse for better error handling. This PR adds a `StatusError` to the `pproto.RowResponse` type, which allows a stream to pass an error on the stream (encoded into the `RowResponse.StatusError`). This can be checked downstream for matching `EOF` or `err != nil` and handled appropriately. This is helpful mainly with the `RowResponse` reducers which run in goroutines. Instead of trying to manage a separate channel of errors from those goroutines, we just follow the grpc model and send the error with the stream. --- proto/interface.go | 69 ++++++++++++++++++++++ proto/pilosa.pb.go | 141 ++++++++++++++++++++++++++++++++------------- proto/pilosa.proto | 5 ++ server/grpc.go | 9 ++- 4 files changed, 181 insertions(+), 43 deletions(-) diff --git a/proto/interface.go b/proto/interface.go index 0a3913921..f2157efe9 100644 --- a/proto/interface.go +++ b/proto/interface.go @@ -14,10 +14,79 @@ package pilosa +import ( + "fmt" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// StreamClient is an interface for a stream +// which can return a RowResponse sent to a +// stream via Send(). type StreamClient interface { Recv() (*RowResponse, error) } +// StreamServer is an interface for a stream +// which can accept a RowResponse to be later +// returned by the stream via Recv(). type StreamServer interface { Send(*RowResponse) error } + +// EOF acts as an io.EOF encoded into a RowResponse. +var EOF *RowResponse = &RowResponse{ + StatusError: &StatusError{ + Code: 0, + Message: "EOF", + }, +} + +// Error is a helper function to create a RowResponse +// based on an error message. If the error is a grpc +// Status, then the status code is passed through. +func Error(err error) *RowResponse { + status, _ := status.FromError(err) + return &RowResponse{ + StatusError: &StatusError{ + Code: uint32(status.Code()), + Message: status.Err().Error(), + }, + } +} + +// ErrorWrap prepends a message to the existing status +// error message. +func ErrorWrap(err error, message string) *RowResponse { + status, _ := status.FromError(err) + return &RowResponse{ + StatusError: &StatusError{ + Code: uint32(status.Code()), + Message: message + ": " + status.Err().Error(), + }, + } +} + +// ErrorWrapf prepends a message to the existing status +// error message with the format specifier. +func ErrorWrapf(err error, format string, args ...interface{}) *RowResponse { + status, _ := status.FromError(err) + return &RowResponse{ + StatusError: &StatusError{ + Code: uint32(status.Code()), + Message: fmt.Sprintf(format, args...) + ": " + status.Err().Error(), + }, + } +} + +// ErrorCode is a helper function to create a RowResponse +// based on a grpc status code and an error message. +func ErrorCode(err error, c codes.Code) *RowResponse { + return &RowResponse{ + StatusError: &StatusError{ + Code: uint32(c), + Message: err.Error(), + }, + } +} diff --git a/proto/pilosa.pb.go b/proto/pilosa.pb.go index 1a5bfca49..2cec94582 100644 --- a/proto/pilosa.pb.go +++ b/proto/pilosa.pb.go @@ -72,9 +72,57 @@ func (m *QueryPQLRequest) GetPql() string { return "" } +type StatusError struct { + Code uint32 `protobuf:"varint,1,opt,name=Code,proto3" json:"Code,omitempty"` + Message string `protobuf:"bytes,2,opt,name=Message,proto3" json:"Message,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *StatusError) Reset() { *m = StatusError{} } +func (m *StatusError) String() string { return proto.CompactTextString(m) } +func (*StatusError) ProtoMessage() {} +func (*StatusError) Descriptor() ([]byte, []int) { + return fileDescriptor_ef0691a44d1e275c, []int{1} +} + +func (m *StatusError) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_StatusError.Unmarshal(m, b) +} +func (m *StatusError) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_StatusError.Marshal(b, m, deterministic) +} +func (m *StatusError) XXX_Merge(src proto.Message) { + xxx_messageInfo_StatusError.Merge(m, src) +} +func (m *StatusError) XXX_Size() int { + return xxx_messageInfo_StatusError.Size(m) +} +func (m *StatusError) XXX_DiscardUnknown() { + xxx_messageInfo_StatusError.DiscardUnknown(m) +} + +var xxx_messageInfo_StatusError proto.InternalMessageInfo + +func (m *StatusError) GetCode() uint32 { + if m != nil { + return m.Code + } + return 0 +} + +func (m *StatusError) GetMessage() string { + if m != nil { + return m.Message + } + return "" +} + type RowResponse struct { Headers []*ColumnInfo `protobuf:"bytes,1,rep,name=headers,proto3" json:"headers,omitempty"` Columns []*ColumnResponse `protobuf:"bytes,2,rep,name=columns,proto3" json:"columns,omitempty"` + StatusError *StatusError `protobuf:"bytes,3,opt,name=StatusError,proto3" json:"StatusError,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -84,7 +132,7 @@ func (m *RowResponse) Reset() { *m = RowResponse{} } func (m *RowResponse) String() string { return proto.CompactTextString(m) } func (*RowResponse) ProtoMessage() {} func (*RowResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{1} + return fileDescriptor_ef0691a44d1e275c, []int{2} } func (m *RowResponse) XXX_Unmarshal(b []byte) error { @@ -119,6 +167,13 @@ func (m *RowResponse) GetColumns() []*ColumnResponse { return nil } +func (m *RowResponse) GetStatusError() *StatusError { + if m != nil { + return m.StatusError + } + return nil +} + type ColumnInfo struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` Datatype string `protobuf:"bytes,2,opt,name=datatype,proto3" json:"datatype,omitempty"` @@ -131,7 +186,7 @@ func (m *ColumnInfo) Reset() { *m = ColumnInfo{} } func (m *ColumnInfo) String() string { return proto.CompactTextString(m) } func (*ColumnInfo) ProtoMessage() {} func (*ColumnInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{2} + return fileDescriptor_ef0691a44d1e275c, []int{3} } func (m *ColumnInfo) XXX_Unmarshal(b []byte) error { @@ -186,7 +241,7 @@ func (m *ColumnResponse) Reset() { *m = ColumnResponse{} } func (m *ColumnResponse) String() string { return proto.CompactTextString(m) } func (*ColumnResponse) ProtoMessage() {} func (*ColumnResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{3} + return fileDescriptor_ef0691a44d1e275c, []int{4} } func (m *ColumnResponse) XXX_Unmarshal(b []byte) error { @@ -501,7 +556,7 @@ func (m *InspectRequest) Reset() { *m = InspectRequest{} } func (m *InspectRequest) String() string { return proto.CompactTextString(m) } func (*InspectRequest) ProtoMessage() {} func (*InspectRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{4} + return fileDescriptor_ef0691a44d1e275c, []int{5} } func (m *InspectRequest) XXX_Unmarshal(b []byte) error { @@ -568,7 +623,7 @@ func (m *Uint64Array) Reset() { *m = Uint64Array{} } func (m *Uint64Array) String() string { return proto.CompactTextString(m) } func (*Uint64Array) ProtoMessage() {} func (*Uint64Array) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{5} + return fileDescriptor_ef0691a44d1e275c, []int{6} } func (m *Uint64Array) XXX_Unmarshal(b []byte) error { @@ -607,7 +662,7 @@ func (m *StringArray) Reset() { *m = StringArray{} } func (m *StringArray) String() string { return proto.CompactTextString(m) } func (*StringArray) ProtoMessage() {} func (*StringArray) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{6} + return fileDescriptor_ef0691a44d1e275c, []int{7} } func (m *StringArray) XXX_Unmarshal(b []byte) error { @@ -649,7 +704,7 @@ func (m *IdsOrKeys) Reset() { *m = IdsOrKeys{} } func (m *IdsOrKeys) String() string { return proto.CompactTextString(m) } func (*IdsOrKeys) ProtoMessage() {} func (*IdsOrKeys) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{7} + return fileDescriptor_ef0691a44d1e275c, []int{8} } func (m *IdsOrKeys) XXX_Unmarshal(b []byte) error { @@ -783,6 +838,7 @@ func _IdsOrKeys_OneofSizer(msg proto.Message) (n int) { func init() { proto.RegisterType((*QueryPQLRequest)(nil), "pilosa.QueryPQLRequest") + proto.RegisterType((*StatusError)(nil), "pilosa.StatusError") proto.RegisterType((*RowResponse)(nil), "pilosa.RowResponse") proto.RegisterType((*ColumnInfo)(nil), "pilosa.ColumnInfo") proto.RegisterType((*ColumnResponse)(nil), "pilosa.ColumnResponse") @@ -954,38 +1010,41 @@ var _Pilosa_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("pilosa.proto", fileDescriptor_ef0691a44d1e275c) } var fileDescriptor_ef0691a44d1e275c = []byte{ - // 524 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x54, 0xdd, 0x8a, 0xd3, 0x40, - 0x14, 0xce, 0x34, 0xd9, 0xb4, 0x39, 0x2d, 0x55, 0x8f, 0xb2, 0x96, 0x22, 0x12, 0x73, 0x63, 0x44, - 0x59, 0x96, 0x2a, 0x82, 0xb2, 0x5e, 0xb8, 0x82, 0xb4, 0x28, 0xb8, 0x3b, 0xe2, 0xde, 0x4f, 0x37, - 0xd3, 0x35, 0x38, 0xcd, 0x64, 0x33, 0x53, 0xb5, 0xb7, 0xbe, 0x8b, 0x4f, 0xe4, 0x0b, 0xc9, 0x4c, - 0x7e, 0x9a, 0x2c, 0x74, 0xef, 0x72, 0xbe, 0xef, 0x3b, 0x67, 0xce, 0x6f, 0x60, 0x94, 0xa7, 0x42, - 0x2a, 0x76, 0x94, 0x17, 0x52, 0x4b, 0xf4, 0x4b, 0x2b, 0x7a, 0x03, 0x77, 0xce, 0x37, 0xbc, 0xd8, - 0x9e, 0x9d, 0x7f, 0xa6, 0xfc, 0x7a, 0xc3, 0x95, 0xc6, 0x07, 0x70, 0x90, 0x66, 0x09, 0xff, 0x3d, - 0x21, 0x21, 0x89, 0x03, 0x5a, 0x1a, 0x78, 0x17, 0xdc, 0xfc, 0x5a, 0x4c, 0x7a, 0x16, 0x33, 0x9f, - 0xd1, 0x1a, 0x86, 0x54, 0xfe, 0xa2, 0x5c, 0xe5, 0x32, 0x53, 0x1c, 0x5f, 0x40, 0xff, 0x3b, 0x67, - 0x09, 0x2f, 0xd4, 0x84, 0x84, 0x6e, 0x3c, 0x9c, 0xe1, 0x51, 0xf5, 0xe2, 0x07, 0x29, 0x36, 0xeb, - 0x6c, 0x91, 0xad, 0x24, 0xad, 0x25, 0x78, 0x0c, 0xfd, 0x4b, 0x0b, 0xab, 0x49, 0xcf, 0xaa, 0x0f, - 0xbb, 0xea, 0x3a, 0x2c, 0xad, 0x65, 0xd1, 0x09, 0xc0, 0x2e, 0x10, 0x22, 0x78, 0x19, 0x5b, 0xf3, - 0x2a, 0x47, 0xfb, 0x8d, 0x53, 0x18, 0x24, 0x4c, 0x33, 0xbd, 0xcd, 0x79, 0x95, 0x67, 0x63, 0x47, - 0xff, 0x7a, 0x30, 0xee, 0x46, 0xc6, 0xc7, 0x10, 0x28, 0x5d, 0xa4, 0xd9, 0xd5, 0x05, 0x13, 0x65, - 0x9c, 0xb9, 0x43, 0x77, 0x90, 0xe1, 0x37, 0x69, 0xa6, 0x5f, 0xbf, 0x32, 0xbc, 0x89, 0xe7, 0x19, - 0xbe, 0x81, 0xf0, 0x11, 0x0c, 0x1a, 0xda, 0x0d, 0x49, 0xec, 0xce, 0x1d, 0xda, 0x20, 0x38, 0x85, - 0xfe, 0x52, 0x4a, 0x61, 0x48, 0x2f, 0x24, 0xf1, 0x60, 0xee, 0xd0, 0x1a, 0xb0, 0x9c, 0x90, 0x4b, - 0xc3, 0x1d, 0x84, 0x24, 0x1e, 0x59, 0xae, 0x04, 0xf0, 0x1d, 0x8c, 0xcb, 0x27, 0xde, 0x17, 0x05, - 0xdb, 0x1a, 0x89, 0x1f, 0x92, 0x78, 0x38, 0xbb, 0x5f, 0xf7, 0xe7, 0xdb, 0x8e, 0x9d, 0x3b, 0xf4, - 0x86, 0xd8, 0xb8, 0x97, 0x15, 0x34, 0xee, 0xfd, 0xae, 0xfb, 0xd7, 0x1d, 0x6b, 0xdc, 0xbb, 0x62, - 0x0c, 0x01, 0x56, 0x42, 0xb2, 0xaa, 0xaa, 0x41, 0x48, 0x62, 0x32, 0x77, 0x68, 0x0b, 0x3b, 0x1d, - 0x42, 0x50, 0x4e, 0xe4, 0x82, 0x89, 0xe8, 0x2f, 0x81, 0xf1, 0x22, 0x53, 0x39, 0xbf, 0xd4, 0xb7, - 0x6f, 0xcf, 0xf3, 0xf6, 0xb8, 0x4d, 0x3e, 0xf7, 0xea, 0x7c, 0x16, 0x89, 0xfa, 0x52, 0x7c, 0xe2, - 0x5b, 0xd5, 0x4c, 0x1a, 0x23, 0x18, 0xad, 0x52, 0xa1, 0x79, 0xf1, 0x31, 0xe5, 0x22, 0x51, 0x13, - 0x37, 0x74, 0xe3, 0x80, 0x76, 0x30, 0xf3, 0x8c, 0x48, 0xd7, 0xa9, 0xb6, 0xcd, 0xf5, 0x68, 0x69, - 0xe0, 0x21, 0xf8, 0x72, 0xb5, 0x52, 0x5c, 0xdb, 0xbe, 0x7a, 0xb4, 0xb2, 0xa2, 0x27, 0x30, 0x6c, - 0xb5, 0xcd, 0x2c, 0xcf, 0x4f, 0x26, 0xca, 0x3d, 0xf5, 0xa8, 0xfd, 0x36, 0x92, 0x56, 0x6b, 0x3a, - 0x92, 0xa0, 0x92, 0x5c, 0x41, 0xd0, 0x64, 0x8b, 0x4f, 0xc1, 0x4d, 0x13, 0x65, 0xab, 0xdc, 0x3b, - 0x1c, 0xa3, 0xc0, 0x67, 0xe0, 0xfd, 0xe0, 0xdb, 0xba, 0xee, 0x3d, 0x73, 0xb0, 0x92, 0x53, 0x1f, - 0x3c, 0xb3, 0xac, 0xb3, 0x3f, 0x04, 0xfc, 0x33, 0x2b, 0xc3, 0x13, 0x18, 0xd4, 0xf7, 0x89, 0x0f, - 0x6b, 0xdf, 0x1b, 0x17, 0x3b, 0x6d, 0x82, 0xb6, 0xee, 0x31, 0x72, 0x8e, 0x09, 0xbe, 0x85, 0x7e, - 0x35, 0x1e, 0x6c, 0xee, 0xab, 0x3b, 0xaf, 0xbd, 0xbe, 0x4b, 0xdf, 0xfe, 0x28, 0x5e, 0xfe, 0x0f, - 0x00, 0x00, 0xff, 0xff, 0x0a, 0x57, 0xf8, 0xfb, 0x38, 0x04, 0x00, 0x00, + // 568 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x54, 0xdd, 0x6e, 0xd3, 0x30, + 0x14, 0x8e, 0x97, 0x2c, 0x69, 0x4e, 0xc6, 0x00, 0x83, 0x46, 0x54, 0x21, 0x14, 0x72, 0x43, 0x10, + 0x68, 0x9a, 0xca, 0x8f, 0x04, 0x8c, 0x0b, 0x36, 0x81, 0x5a, 0x01, 0x62, 0x33, 0x62, 0xf7, 0xee, + 0xe2, 0x96, 0x08, 0x37, 0xce, 0x62, 0x17, 0xe8, 0x2d, 0xcf, 0x02, 0x4f, 0xc4, 0x0b, 0x21, 0x3b, + 0x3f, 0x4d, 0x2a, 0x95, 0x3b, 0x9f, 0xef, 0xfb, 0xce, 0xf1, 0xf9, 0xb3, 0x61, 0xaf, 0xc8, 0xb8, + 0x90, 0xf4, 0xb0, 0x28, 0x85, 0x12, 0xd8, 0xad, 0xac, 0xf8, 0x05, 0x5c, 0x3f, 0x5f, 0xb2, 0x72, + 0x75, 0x76, 0xfe, 0x81, 0xb0, 0xab, 0x25, 0x93, 0x0a, 0xdf, 0x86, 0xdd, 0x2c, 0x4f, 0xd9, 0xcf, + 0x10, 0x45, 0x28, 0xf1, 0x49, 0x65, 0xe0, 0x1b, 0x60, 0x17, 0x57, 0x3c, 0xdc, 0x31, 0x98, 0x3e, + 0xc6, 0xaf, 0x20, 0xf8, 0xac, 0xa8, 0x5a, 0xca, 0xb7, 0x65, 0x29, 0x4a, 0x8c, 0xc1, 0x39, 0x15, + 0x29, 0x33, 0x5e, 0xd7, 0x88, 0x39, 0xe3, 0x10, 0xbc, 0x8f, 0x4c, 0x4a, 0x3a, 0x67, 0xb5, 0x63, + 0x63, 0xc6, 0xbf, 0x11, 0x04, 0x44, 0xfc, 0x20, 0x4c, 0x16, 0x22, 0x97, 0x0c, 0x3f, 0x06, 0xef, + 0x2b, 0xa3, 0x29, 0x2b, 0x65, 0x88, 0x22, 0x3b, 0x09, 0x46, 0xf8, 0xb0, 0xce, 0xf7, 0x54, 0xf0, + 0xe5, 0x22, 0x9f, 0xe4, 0x33, 0x41, 0x1a, 0x09, 0x3e, 0x02, 0xef, 0xd2, 0xc0, 0x32, 0xdc, 0x31, + 0xea, 0x83, 0xbe, 0xba, 0x09, 0x4b, 0x1a, 0x19, 0x7e, 0xd6, 0x4b, 0x36, 0xb4, 0x23, 0x94, 0x04, + 0xa3, 0x5b, 0x8d, 0x57, 0x87, 0x22, 0x5d, 0x5d, 0x7c, 0x0c, 0xb0, 0xbe, 0x5f, 0x97, 0x98, 0xd3, + 0x05, 0xab, 0x1b, 0x63, 0xce, 0x78, 0x08, 0x83, 0x94, 0x2a, 0xaa, 0x56, 0x45, 0x53, 0x63, 0x6b, + 0xc7, 0x7f, 0x77, 0x60, 0xbf, 0x9f, 0x10, 0xbe, 0x07, 0xbe, 0x54, 0x65, 0x96, 0xcf, 0x2f, 0x28, + 0xaf, 0xe2, 0x8c, 0x2d, 0xb2, 0x86, 0x34, 0xbf, 0xcc, 0x72, 0xf5, 0xfc, 0xa9, 0xe6, 0x75, 0x3c, + 0x47, 0xf3, 0x2d, 0x84, 0xef, 0xc2, 0xa0, 0xa5, 0x75, 0x11, 0xf6, 0xd8, 0x22, 0x2d, 0x82, 0x87, + 0xe0, 0x4d, 0x85, 0xe0, 0x9a, 0x74, 0x22, 0x94, 0x0c, 0xc6, 0x16, 0x69, 0x00, 0xc3, 0x71, 0x31, + 0xd5, 0xdc, 0x6e, 0x84, 0x92, 0x3d, 0xc3, 0x55, 0x00, 0x7e, 0x0d, 0xfb, 0xd5, 0x15, 0x6f, 0xca, + 0x92, 0xae, 0xb4, 0xc4, 0xed, 0x37, 0xe8, 0xcb, 0x9a, 0x1d, 0x5b, 0x64, 0x43, 0xac, 0xdd, 0xab, + 0x0a, 0x5a, 0x77, 0x6f, 0xb3, 0xbf, 0x2d, 0xab, 0xdd, 0xfb, 0x62, 0x1c, 0x01, 0xcc, 0xb8, 0xa0, + 0x75, 0x55, 0x83, 0x08, 0x25, 0x68, 0x6c, 0x91, 0x0e, 0x76, 0x12, 0x80, 0x5f, 0x0d, 0xf2, 0x82, + 0xf2, 0xf8, 0x0f, 0x82, 0xfd, 0x49, 0x2e, 0x0b, 0x76, 0xa9, 0xfe, 0xbf, 0xb2, 0x8f, 0xba, 0x5b, + 0xa2, 0xf3, 0xb9, 0xd9, 0xe4, 0x33, 0x49, 0xe5, 0xa7, 0xf2, 0x3d, 0x5b, 0xc9, 0xf5, 0x82, 0xc4, + 0xb0, 0x37, 0xcb, 0xb8, 0x62, 0xe5, 0xbb, 0x8c, 0xf1, 0x54, 0x86, 0x76, 0x64, 0x27, 0x3e, 0xe9, + 0x61, 0xfa, 0x1a, 0x9e, 0x2d, 0x32, 0x65, 0x9a, 0xeb, 0x90, 0xca, 0xc0, 0x07, 0xe0, 0x8a, 0xd9, + 0x4c, 0x32, 0x65, 0xfa, 0xea, 0x90, 0xda, 0x8a, 0xef, 0x43, 0xd0, 0x69, 0x9b, 0x5e, 0x9e, 0xef, + 0x94, 0x57, 0xeb, 0xed, 0x10, 0x73, 0xd6, 0x92, 0x4e, 0x6b, 0x7a, 0x12, 0xbf, 0x96, 0xcc, 0xc1, + 0x6f, 0xb3, 0xc5, 0x0f, 0xc0, 0xce, 0x52, 0x69, 0xaa, 0xdc, 0x3a, 0x1c, 0xad, 0xc0, 0x0f, 0xc1, + 0xf9, 0xc6, 0x56, 0x4d, 0xdd, 0x5b, 0xe6, 0x60, 0x24, 0x27, 0x2e, 0x38, 0x7a, 0x59, 0x47, 0xbf, + 0x10, 0xb8, 0x67, 0x46, 0x86, 0x8f, 0x61, 0xd0, 0x7c, 0x0a, 0xf8, 0x4e, 0xe3, 0xbb, 0xf1, 0x4d, + 0x0c, 0xdb, 0xa0, 0x9d, 0x67, 0x1c, 0x5b, 0x47, 0x08, 0xbf, 0x04, 0xaf, 0x1e, 0x0f, 0x6e, 0x9f, + 0x65, 0x7f, 0x5e, 0x5b, 0x7d, 0xa7, 0xae, 0xf9, 0x9d, 0x9e, 0xfc, 0x0b, 0x00, 0x00, 0xff, 0xff, + 0x9e, 0xd3, 0x7d, 0xab, 0xad, 0x04, 0x00, 0x00, } diff --git a/proto/pilosa.proto b/proto/pilosa.proto index 9a5828624..b7b30d521 100644 --- a/proto/pilosa.proto +++ b/proto/pilosa.proto @@ -6,10 +6,15 @@ message QueryPQLRequest { string pql = 2; } +message StatusError{ + uint32 Code = 1; + string Message = 2; +} message RowResponse{ repeated ColumnInfo headers = 1; repeated ColumnResponse columns = 2; + StatusError StatusError = 3; } message ColumnInfo { diff --git a/server/grpc.go b/server/grpc.go index a0ef90cbe..f4f0648e8 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -42,10 +42,16 @@ type grpcHandler struct { // to the error (returning it as a status.Error). It is // assumed that the input err is non-nil. func errToStatusError(err error) error { + // Check error string. switch errors.Cause(err) { case pilosa.ErrIndexNotFound, pilosa.ErrFieldNotFound: return status.Error(codes.NotFound, err.Error()) } + // Check error type. + switch errors.Cause(err).(type) { + case pilosa.NotFoundError: + return status.Error(codes.NotFound, err.Error()) + } return status.Error(codes.Unknown, err.Error()) } @@ -75,7 +81,7 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer index, err := h.api.Index(context.Background(), req.Index) if err != nil { - return errors.Wrap(err, "getting index") + return errToStatusError(err) } var fields []*pilosa.Field @@ -315,7 +321,6 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer } resp, err := h.api.Query(context.Background(), &query) if err != nil { - fmt.Println("GOT ERROR trying to get ALL():", err) return errors.Wrapf(err, "querying for all: %s", pql) } From 3be414138264b17ebd2a43e7bdb96f4de80eb9ff Mon Sep 17 00:00:00 2001 From: Travis Date: Thu, 26 Dec 2019 12:36:49 -0600 Subject: [PATCH 18/20] Return the correct data type label in grpc header Based on the pilosa field type, return the correct data type label in the gRPC column header. --- server/grpc.go | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/server/grpc.go b/server/grpc.go index f4f0648e8..fb4a3e0f3 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -75,6 +75,29 @@ func (h grpcHandler) QueryPQL(req *pb.QueryPQLRequest, stream pb.Pilosa_QueryPQL return nil } +// fieldDataType returns a useful data type (string, +// uint64, bool, etc.) based on the Pilosa field type. +func fieldDataType(f *pilosa.Field) string { + switch f.Type() { + case "set", "mutex": + if f.Options().Keys { + return "[]string" + } else { + return "[]uint64" + } + case "int": + return "int64" + case "decimal": + return "float64" + case "bool": + return "bool" + case "time": + return "int64" // TODO: this is a placeholder + default: + panic(fmt.Sprintf("unimplemented fieldDataType: %s", f.Type())) + } +} + // Inspect handles the inspect request and sends an InspectResponse to the stream. func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectServer) error { const defaultLimit = 100000 @@ -119,7 +142,7 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer {Name: "_id", Datatype: "uint64"}, } for _, field := range fields { - ci = append(ci, &pb.ColumnInfo{Name: field.Name(), Datatype: field.Type()}) // TODO: field.Type likely doesn't align with supported datatypes + ci = append(ci, &pb.ColumnInfo{Name: field.Name(), Datatype: fieldDataType(field)}) } // If Columns is empty, then get the _exists list (via All()), @@ -296,7 +319,7 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer {Name: "_id", Datatype: "string"}, } for _, field := range fields { - ci = append(ci, &pb.ColumnInfo{Name: field.Name(), Datatype: field.Type()}) // TODO: field.Type likely doesn't align with supported datatypes + ci = append(ci, &pb.ColumnInfo{Name: field.Name(), Datatype: fieldDataType(field)}) } // If Columns is empty, then get the _exists list (via All()), @@ -745,7 +768,7 @@ func (s *grpcServer) Serve(tlsConfig *tls.Config) error { if err != nil { return errors.Wrap(err, "creating listener") } - s.logger.Printf("enabled grpc listening on %s", s.hostPort) + s.logger.Printf("enabled grpc listening on %s", lis.Addr()) opts := make([]grpc.ServerOption, 0) if tlsConfig != nil { From 0fffd9a0cbb3912cf64965c95bcd0353d38a4e39 Mon Sep 17 00:00:00 2001 From: Travis Date: Tue, 31 Dec 2019 11:52:01 -0600 Subject: [PATCH 19/20] RowResponseSorter for sorting a list of RowResponse based on sort paraters --- proto/interface.go | 175 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) diff --git a/proto/interface.go b/proto/interface.go index f2157efe9..61a5dcd49 100644 --- a/proto/interface.go +++ b/proto/interface.go @@ -15,7 +15,9 @@ package pilosa import ( + "errors" "fmt" + "strings" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -90,3 +92,176 @@ func ErrorCode(err error, c codes.Code) *RowResponse { }, } } + +// RowResponseSorter implements the sort interface for a +// provided []RowResponse based on the column index, type, +// and sort direction. +type RowResponseSorter struct { + colIdx []int + colDescending []bool + colType []string + + rrs []*RowResponse +} + +// NewRowResponseSorter return a new RowResponseSorter. It +// does input validation and returns an error if the inputs +// aren't compatible. +func NewRowResponseSorter(idxs []int, dirs []bool, typs []string, rrs []*RowResponse) (*RowResponseSorter, error) { + // Ensure the input slices are non-empty and equal size. + if len(idxs) == 0 { + return nil, errors.New("index list cannot be empty") + } + if len(dirs) != len(idxs) || len(typs) != len(idxs) { + return nil, errors.New("index, direction, and type lists must be the same size") + } + + // Ensure the provided data types are supported by the sorter. + for i := range typs { + switch typs[i] { + case "[]uint64", "[]string", "bool", "float64", "int64", "string", "uint64": + // pass + default: + return nil, fmt.Errorf("unsupported data type: %s", typs[i]) + } + } + + // Ensure max(colIdx) is within size of rr.Columns. + if len(rrs) > 0 { + var maxColIdx int + for i := range idxs { + if idxs[i] > maxColIdx { + maxColIdx = idxs[i] + } + } + if maxColIdx >= len(rrs[0].Columns) { + return nil, fmt.Errorf("column index is out of range: %d", maxColIdx) + } + } + + return &RowResponseSorter{ + colIdx: idxs, + colDescending: dirs, + colType: typs, + rrs: rrs, + }, nil + +} + +func (r RowResponseSorter) Len() int { return len(r.rrs) } +func (r RowResponseSorter) Swap(i, j int) { r.rrs[i], r.rrs[j] = r.rrs[j], r.rrs[i] } +func (r RowResponseSorter) Less(i, j int) bool { + ri := r.rrs[i] + rj := r.rrs[j] + + for i, idx := range r.colIdx { + coli := ri.Columns[idx] + colj := rj.Columns[idx] + var comp int + switch r.colType[i] { + case "[]uint64": + ai := coli.GetUint64ArrayVal().Vals + aj := colj.GetUint64ArrayVal().Vals + comp = func() int { + for ii := 0; ii < len(ai); ii++ { + if len(aj) == ii { + return 1 + } + piv := ai[ii] + pjv := aj[ii] + if piv == pjv { + continue + } else if piv < pjv { + return -1 + } else { + return 1 + } + } + if len(aj) > len(ai) { + return -1 + } + return 0 + }() + case "[]string": + ai := coli.GetStringArrayVal().Vals + aj := colj.GetStringArrayVal().Vals + comp = func() int { + for ii := 0; ii < len(ai); ii++ { + if len(aj) == ii { + return 1 + } + sComp := strings.Compare(ai[ii], aj[ii]) + if sComp == 0 { + continue + } else { + return sComp + } + } + if len(aj) > len(ai) { + return -1 + } + return 0 + }() + case "bool": + bi := coli.GetBoolVal() + bj := colj.GetBoolVal() + if bi == bj { + comp = 0 + } else if !bi && bj { + comp = -1 + } else { + comp = 1 + } + case "float64": + fi := coli.GetFloat64Val() + fj := colj.GetFloat64Val() + if fi == fj { + comp = 0 + } else if fi < fj { + comp = -1 + } else { + comp = 1 + } + case "int64": + ni := coli.GetInt64Val() + nj := colj.GetInt64Val() + if ni == nj { + comp = 0 + } else if ni < nj { + comp = -1 + } else { + comp = 1 + } + case "string": + comp = strings.Compare(coli.GetStringVal(), colj.GetStringVal()) + case "uint64": + ni := coli.GetUint64Val() + nj := colj.GetUint64Val() + if ni == nj { + comp = 0 + } else if ni < nj { + comp = -1 + } else { + comp = 1 + } + } + + isDescending := r.colDescending[i] + + switch comp { + case 0: + continue + case -1: + if isDescending { + return false + } + return true + case 1: + if isDescending { + return true + } + return false + } + } + return false +} From 8d32005ede0cda3d83277b8c3e44a86766fa3dbe Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 2 Jan 2020 15:43:34 -0600 Subject: [PATCH 20/20] Initialize expvar lazily to prevent panic if importing both Pilosa v1 and v2. --- stats/stats.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/stats/stats.go b/stats/stats.go index 9c6944d4e..c360baab0 100644 --- a/stats/stats.go +++ b/stats/stats.go @@ -25,7 +25,7 @@ import ( ) // Expvar global expvar map. -var Expvar = expvar.NewMap("index") +var Expvar *expvar.Map // StatsClient represents a client to a stats server. type StatsClient interface { @@ -90,6 +90,9 @@ type expvarStatsClient struct { // NewExpvarStatsClient returns a new instance of ExpvarStatsClient. // This client points at the root of the expvar index map. func NewExpvarStatsClient() *expvarStatsClient { + if Expvar == nil { + Expvar = expvar.NewMap("index") + } return &expvarStatsClient{ m: Expvar, }