diff --git a/api/client/grpc.go b/api/client/grpc.go index 2d99ea76e..fa9f33b68 100644 --- a/api/client/grpc.go +++ b/api/client/grpc.go @@ -80,7 +80,7 @@ func (c *GRPCClient) Query(ctx context.Context, index string, pql string) (pb.St // Inspect returns a stream of RowResponse for the given index, columns, and filters. // It is intended to mimic something like "select [fields] from table where recordID IN (...)". -func (c *GRPCClient) Inspect(ctx context.Context, index string, columnIDs []uint64, columnKeys []string, fieldFilters []string) (pb.StreamClient, error) { +func (c *GRPCClient) Inspect(ctx context.Context, index string, columnIDs []uint64, columnKeys []string, fieldFilters []string, limit, offset uint64) (pb.StreamClient, error) { if c.conn == nil { return nil, errors.New("client has not established a grpc connection") } @@ -103,7 +103,10 @@ func (c *GRPCClient) Inspect(ctx context.Context, index string, columnIDs []uint Index: index, Columns: idsOrKeys, FilterFields: fieldFilters, + Limit: limit, + Offset: offset, }) + if err != nil { return nil, errors.Wrap(err, "getting stream") } else if stream == nil { diff --git a/executor.go b/executor.go index 25dfbf4c1..47c08043b 100644 --- a/executor.go +++ b/executor.go @@ -527,6 +527,8 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s return e.executeOptionsCall(ctx, index, c, shards, opt) case "IncludesColumn": return e.executeIncludesColumnCall(ctx, index, c, shards, opt) + case "All": + return e.executeAllCall(ctx, index, c, shards, opt) case "Precomputed": return e.executePrecomputedCall(ctx, index, c, shards, opt) default: @@ -635,6 +637,112 @@ func (e *executor) executeIncludesColumnCall(ctx context.Context, index string, return result.(bool), nil } +// executeAllCall executes an All() call. +func (e *executor) executeAllCall(ctx context.Context, index string, c *pql.Call, shards []uint64, opt *execOptions) (*Row, error) { + rslt := NewRow() + + var limit uint64 + var offset uint64 + + if lim, hasLimit, err := c.UintArg("limit"); err != nil { + return nil, errors.Wrap(err, "getting limit") + } else if hasLimit && lim > 0 { + limit = uint64(lim) + } + if off, hasOffset, err := c.UintArg("offset"); err != nil { + return nil, errors.Wrap(err, "getting offset") + } else if hasOffset && off > 0 { + offset = uint64(off) + } + + if limit == 0 { + limit = math.MaxUint64 + } + + // skip tracks the number of records left to be skipped + // in support of getting to the offset. + var skip uint64 = offset + + // got tracks the number of records gotten to that point. + var got uint64 + + for _, shard := range shards { + row, err := e.executeAllCallMapReduce(ctx, index, c, shard, opt) + if err != nil { + return nil, errors.Wrap(err, "executing map reduce on shard") + } + + segCnt := row.Count() + + // If this segment doesn't reach the offset, skip it. + if segCnt <= skip { + skip -= segCnt + continue + } + + // This segment doesn't have enough to finish fulfilling the limit + // (or it has exactly enough). + if segCnt-skip <= limit-got { + if skip == 0 { + rslt.Merge(row) + } else { + cols := row.Columns() + partialRow := NewRow() + for _, bit := range cols[skip:] { + partialRow.SetBit(bit) + } + rslt.Merge(partialRow) + } + got += segCnt - skip + // In the case where this segment exactly fulfills the limit, break. + if got == limit { + break + } + skip = 0 + continue + } + + // This segment has more records than the remaining limit requires. + cols := row.Columns() + partialRow := NewRow() + for _, bit := range cols[skip : skip+limit-got] { + partialRow.SetBit(bit) + } + rslt.Merge(partialRow) + break + } + + return rslt, nil +} + +// executeAllCallMapReduce executes a single shard of the All() call +// using the executor.mapReduce() method. +func (e *executor) executeAllCallMapReduce(ctx context.Context, index string, c *pql.Call, shard uint64, opt *execOptions) (*Row, error) { + // Execute calls in bulk on each remote node and merge. + mapFn := func(shard uint64) (interface{}, error) { + return e.executeAllShard(ctx, index, c, shard) + } + + // Merge returned results at coordinating node. + reduceFn := func(prev, v interface{}) interface{} { + other, _ := prev.(*Row) + if other == nil { + other = NewRow() + } + other.Merge(v.(*Row)) + return other + } + + result, err := e.mapReduce(ctx, index, []uint64{shard}, c, opt, mapFn, reduceFn) + if err != nil { + return nil, errors.Wrap(err, "map reduce") + } + + row, _ := result.(*Row) + + return row, nil +} + // executeIncludesColumnCallShard func (e *executor) executeIncludesColumnCallShard(ctx context.Context, index string, c *pql.Call, shard uint64, column uint64) (bool, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeIncludesColumnCallShard") @@ -2381,7 +2489,7 @@ func (e *executor) executePrecomputedCallShard(ctx context.Context, index string return nil, fmt.Errorf("per-shard: missing precomputed values for shard %d", shard) } -// executeNotShard executes a not() call for a local shard. +// executeNotShard executes a Not() call for a local shard. func (e *executor) executeNotShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeNotShard") defer span.Finish() @@ -2416,6 +2524,34 @@ func (e *executor) executeNotShard(ctx context.Context, index string, c *pql.Cal return existenceRow.Difference(row), nil } +// executeAllShard executes an All() call for a local shard. +func (e *executor) executeAllShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeAllShard") + defer span.Finish() + + if len(c.Children) > 0 { + return nil, errors.New("All() does not accept an input row") + } + + // Make sure the index supports existence tracking. + idx := e.Holder.Index(index) + if idx == nil { + return nil, ErrIndexNotFound + } else if idx.existenceField() == nil { + return nil, errors.Errorf("index does not support existence tracking: %s", index) + } + + var existenceRow *Row + existenceFrag := e.Holder.fragment(index, existenceFieldName, viewStandard, shard) + if existenceFrag == nil { + existenceRow = NewRow() + } else { + existenceRow = existenceFrag.row(0) + } + + return existenceRow, nil +} + // executeShiftShard executes a shift() call for a local shard. func (e *executor) executeShiftShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { n, _, err := c.IntArg("n") diff --git a/executor_test.go b/executor_test.go index 665ef9bba..3750a04ad 100644 --- a/executor_test.go +++ b/executor_test.go @@ -2898,6 +2898,149 @@ func TestExecutor_Execute_Not(t *testing.T) { }) } +// Ensure an all query can be executed. +func TestExecutor_Execute_All(t *testing.T) { + t.Run("ColumnID", func(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + fld, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) + } + + // Create an import request that sets a full shard, + // plus a couple bits set on either side of it, and + // a final bit set in a fourth shard. + // + // shard0 shard1 shard2 shard3 + // |----------|----------|----------|----------| + // | **|**********|** | * + // + bitCount := ShardWidth + 5 + req := &pilosa.ImportRequest{ + Index: index.Name(), + Field: fld.Name(), + Shard: 0, + RowIDs: make([]uint64, bitCount), + ColumnIDs: make([]uint64, bitCount), + } + for i := 0; i < bitCount-1; i++ { + req.RowIDs[i] = 10 + req.ColumnIDs[i] = uint64(i + ShardWidth - 2) + } + req.RowIDs[bitCount-1] = 10 + req.ColumnIDs[bitCount-1] = uint64((3 * ShardWidth) + 2) + + if err := c[0].API.Import(context.Background(), req); err != nil { + t.Fatal(err) + } + + tests := []struct { + qry string + expCols []uint64 + expCnt uint64 + }{ + {qry: "All()", expCols: req.ColumnIDs, expCnt: uint64(bitCount)}, + {qry: "All(limit=1)", expCols: req.ColumnIDs[:1], expCnt: 1}, + {qry: "All(limit=4)", expCols: req.ColumnIDs[:4], expCnt: 4}, + {qry: "All(limit=4, offset=4)", expCols: req.ColumnIDs[4:8], expCnt: 4}, + {qry: fmt.Sprintf("All(limit=4, offset=%d)", bitCount-5), expCols: req.ColumnIDs[bitCount-5 : bitCount-1], expCnt: 4}, + {qry: fmt.Sprintf("All(limit=1, offset=%d)", bitCount-2), expCols: req.ColumnIDs[bitCount-2 : bitCount-1], expCnt: 1}, + {qry: fmt.Sprintf("All(limit=1, offset=%d)", bitCount-2), expCols: req.ColumnIDs[bitCount-2 : bitCount-1], expCnt: 1}, + {qry: fmt.Sprintf("All(limit=4, offset=%d)", bitCount-2), expCols: req.ColumnIDs[bitCount-2:], expCnt: 2}, + {qry: fmt.Sprintf("All(limit=4, offset=%d)", bitCount+1), expCols: []uint64{}, expCnt: 0}, + {qry: fmt.Sprintf("All(limit=2, offset=%d)", bitCount-3), expCols: req.ColumnIDs[bitCount-3 : bitCount-1], expCnt: 2}, + {qry: fmt.Sprintf("All(limit=2, offset=%d)", bitCount-5), expCols: req.ColumnIDs[bitCount-5 : bitCount-3], expCnt: 2}, + {qry: "All(limit=2, offset=2)", expCols: req.ColumnIDs[2:4], expCnt: 2}, + {qry: "All(limit=1, offset=1)", expCols: req.ColumnIDs[1:2], expCnt: 1}, + {qry: fmt.Sprintf("All(limit=%d, offset=2)", ShardWidth), expCols: req.ColumnIDs[2 : bitCount-3], expCnt: ShardWidth}, + } + for i, test := range tests { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.qry}); err != nil { + t.Fatal(err) + } else if cnt := res.Results[0].(*pilosa.Row).Count(); cnt != test.expCnt { + t.Fatalf("test %d, unexpected count, got: %d, but expected: %d", i, cnt, test.expCnt) + } else if cols := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(cols, test.expCols) { + // If the error results are too large, just show the count. + if len(cols) > 1000 || len(test.expCols) > 1000 { + t.Fatalf("test %d, unexpected columns, got: len(%d), but expected: len(%d)", i, len(cols), len(test.expCols)) + } else { + t.Fatalf("test %d, unexpected columns, got: %v, but expected: %v", i, cols, test.expCols) + } + } + } + }) + + t.Run("ColumnKey", func(t *testing.T) { + c := test.MustRunCluster(t, 1, []server.CommandOption{ + server.OptCommandServerOptions( + pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), + pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + ), + }) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true, Keys: true}) + fld, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) + if err != nil { + t.Fatal(err) + } + + // Create an import request that sets key columns + // + // shard0 + // |----------| + // |**** | + // + bitCount := 4 + req := &pilosa.ImportRequest{ + Index: index.Name(), + Field: fld.Name(), + Shard: 0, + RowIDs: make([]uint64, bitCount), + ColumnKeys: make([]string, bitCount), + } + for i := 0; i < bitCount; i++ { + req.RowIDs[i] = 10 + req.ColumnKeys[i] = fmt.Sprintf("c%d", i) + } + + if err := c[0].API.Import(context.Background(), req); err != nil { + t.Fatal(err) + } + + tests := []struct { + qry string + expCols []string + expCnt uint64 + }{ + {qry: "All()", expCols: req.ColumnKeys, expCnt: uint64(bitCount)}, + {qry: "All(limit=1)", expCols: req.ColumnKeys[:1], expCnt: 1}, + {qry: "All(limit=4)", expCols: req.ColumnKeys, expCnt: 4}, + {qry: "All(limit=5)", expCols: req.ColumnKeys, expCnt: 4}, + {qry: "All(limit=1, offset=1)", expCols: req.ColumnKeys[1:2], expCnt: 1}, + {qry: "All(limit=4, offset=1)", expCols: req.ColumnKeys[1:], expCnt: 3}, + {qry: "All(limit=4, offset=5)", expCols: nil, expCnt: 0}, + } + for i, test := range tests { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.qry}); err != nil { + t.Fatal(err) + } else if cnt := len(res.Results[0].(*pilosa.Row).Keys); uint64(cnt) != test.expCnt { + t.Fatalf("test %d, unexpected count, got: %d, but expected: %d", i, cnt, test.expCnt) + } else if cols := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(cols, test.expCols) { + // If the error results are too large, just show the count. + if len(cols) > 1000 || len(test.expCols) > 1000 { + t.Fatalf("test %d, unexpected columns, got: len(%d), but expected: len(%d)", i, len(cols), len(test.expCols)) + } else { + t.Fatalf("test %d, unexpected columns, got: %T, but expected: %T", i, cols, test.expCols) + } + } + } + }) +} + // Ensure a row can be cleared. func TestExecutor_Execute_ClearRow(t *testing.T) { // Set and Mutex tests use the same data and queries diff --git a/go.mod b/go.mod index eb38d9288..f47f54dc1 100644 --- a/go.mod +++ b/go.mod @@ -34,15 +34,17 @@ require ( github.com/uber-go/atomic v1.4.0 // indirect github.com/uber/jaeger-client-go v2.16.0+incompatible github.com/uber/jaeger-lib v2.2.0+incompatible // indirect + github.com/youtube/vitess v2.1.1+incompatible // indirect go.uber.org/atomic v1.4.0 // indirect golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect - golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 // indirect + golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 golang.org/x/sync v0.0.0-20190423024810-112230192c58 golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 // indirect golang.org/x/text v0.3.2 // indirect google.golang.org/grpc v1.24.0 modernc.org/mathutil v1.0.0 modernc.org/strutil v1.0.0 + vitess.io/vitess v2.1.1+incompatible // indirect ) go 1.13 diff --git a/go.sum b/go.sum index 6607cf98d..177cd241f 100644 --- a/go.sum +++ b/go.sum @@ -153,6 +153,8 @@ github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/ github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/youtube/vitess v2.1.1+incompatible h1:SE+P7DNX/jw5RHFs5CHRhZQjq402EJFCD33JhzQMdDw= +github.com/youtube/vitess v2.1.1+incompatible/go.mod h1:hpMim5/30F1r+0P8GGtB29d0gWHr0IZ5unS+CG0zMx8= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -210,3 +212,5 @@ modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +vitess.io/vitess v2.1.1+incompatible h1:nuuGHiWYWpudD3gOCLeGzol2EJ25e/u5Wer2wV1O130= +vitess.io/vitess v2.1.1+incompatible/go.mod h1:h4qvkyNYTOC0xI+vcidSWoka0gQAZc9ZPHbkHo48gP0= diff --git a/pql/ast.go b/pql/ast.go index 7749c02dd..61b4b6540 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -347,10 +347,17 @@ var callInfoByFunc = map[string]callInfo{ "Difference": {allowUnknown: false}, "Intersect": {allowUnknown: false}, "Not": {allowUnknown: false}, - "ClearRow": {allowUnknown: true}, - "Store": {allowUnknown: true}, - "MinRow": allowField, - "MaxRow": allowField, + "All": { + allowUnknown: false, + prototypes: map[string]interface{}{ + "limit": int64(0), + "offset": int64(0), + }, + }, + "ClearRow": {allowUnknown: true}, + "Store": {allowUnknown: true}, + "MinRow": allowField, + "MaxRow": allowField, "Rows": { allowUnknown: false, prototypes: map[string]interface{}{ diff --git a/proto/pilosa.pb.go b/proto/pilosa.pb.go index b81722e06..1a5bfca49 100644 --- a/proto/pilosa.pb.go +++ b/proto/pilosa.pb.go @@ -4,15 +4,16 @@ package pilosa import ( - context "context" fmt "fmt" proto "github.com/golang/protobuf/proto" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" math "math" ) +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal var _ = fmt.Errorf @@ -22,7 +23,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type QueryPQLRequest struct { Index string `protobuf:"bytes,1,opt,name=index,proto3" json:"index,omitempty"` @@ -321,9 +322,9 @@ func (m *ColumnResponse) GetFloat64Val() float64 { return 0 } -// XXX_OneofWrappers is for the internal use of the proto package. -func (*ColumnResponse) XXX_OneofWrappers() []interface{} { - return []interface{}{ +// XXX_OneofFuncs is for the internal use of the proto package. +func (*ColumnResponse) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { + return _ColumnResponse_OneofMarshaler, _ColumnResponse_OneofUnmarshaler, _ColumnResponse_OneofSizer, []interface{}{ (*ColumnResponse_StringVal)(nil), (*ColumnResponse_Uint64Val)(nil), (*ColumnResponse_Int64Val)(nil), @@ -335,10 +336,162 @@ func (*ColumnResponse) XXX_OneofWrappers() []interface{} { } } +func _ColumnResponse_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*ColumnResponse) + // columnVal + switch x := m.ColumnVal.(type) { + case *ColumnResponse_StringVal: + b.EncodeVarint(1<<3 | proto.WireBytes) + b.EncodeStringBytes(x.StringVal) + case *ColumnResponse_Uint64Val: + b.EncodeVarint(2<<3 | proto.WireVarint) + b.EncodeVarint(uint64(x.Uint64Val)) + case *ColumnResponse_Int64Val: + b.EncodeVarint(3<<3 | proto.WireVarint) + b.EncodeVarint(uint64(x.Int64Val)) + case *ColumnResponse_BoolVal: + t := uint64(0) + if x.BoolVal { + t = 1 + } + b.EncodeVarint(4<<3 | proto.WireVarint) + b.EncodeVarint(t) + case *ColumnResponse_BlobVal: + b.EncodeVarint(5<<3 | proto.WireBytes) + b.EncodeRawBytes(x.BlobVal) + case *ColumnResponse_Uint64ArrayVal: + b.EncodeVarint(6<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Uint64ArrayVal); err != nil { + return err + } + case *ColumnResponse_StringArrayVal: + b.EncodeVarint(7<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.StringArrayVal); err != nil { + return err + } + case *ColumnResponse_Float64Val: + b.EncodeVarint(8<<3 | proto.WireFixed64) + b.EncodeFixed64(math.Float64bits(x.Float64Val)) + case nil: + default: + return fmt.Errorf("ColumnResponse.ColumnVal has unexpected type %T", x) + } + return nil +} + +func _ColumnResponse_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*ColumnResponse) + switch tag { + case 1: // columnVal.stringVal + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeStringBytes() + m.ColumnVal = &ColumnResponse_StringVal{x} + return true, err + case 2: // columnVal.uint64Val + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.ColumnVal = &ColumnResponse_Uint64Val{x} + return true, err + case 3: // columnVal.int64Val + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.ColumnVal = &ColumnResponse_Int64Val{int64(x)} + return true, err + case 4: // columnVal.boolVal + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.ColumnVal = &ColumnResponse_BoolVal{x != 0} + return true, err + case 5: // columnVal.blobVal + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeRawBytes(true) + m.ColumnVal = &ColumnResponse_BlobVal{x} + return true, err + case 6: // columnVal.uint64ArrayVal + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(Uint64Array) + err := b.DecodeMessage(msg) + m.ColumnVal = &ColumnResponse_Uint64ArrayVal{msg} + return true, err + case 7: // columnVal.stringArrayVal + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(StringArray) + err := b.DecodeMessage(msg) + m.ColumnVal = &ColumnResponse_StringArrayVal{msg} + return true, err + case 8: // columnVal.float64Val + if wire != proto.WireFixed64 { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeFixed64() + m.ColumnVal = &ColumnResponse_Float64Val{math.Float64frombits(x)} + return true, err + default: + return false, nil + } +} + +func _ColumnResponse_OneofSizer(msg proto.Message) (n int) { + m := msg.(*ColumnResponse) + // columnVal + switch x := m.ColumnVal.(type) { + case *ColumnResponse_StringVal: + n += 1 // tag and wire + n += proto.SizeVarint(uint64(len(x.StringVal))) + n += len(x.StringVal) + case *ColumnResponse_Uint64Val: + n += 1 // tag and wire + n += proto.SizeVarint(uint64(x.Uint64Val)) + case *ColumnResponse_Int64Val: + n += 1 // tag and wire + n += proto.SizeVarint(uint64(x.Int64Val)) + case *ColumnResponse_BoolVal: + n += 1 // tag and wire + n += 1 + case *ColumnResponse_BlobVal: + n += 1 // tag and wire + n += proto.SizeVarint(uint64(len(x.BlobVal))) + n += len(x.BlobVal) + case *ColumnResponse_Uint64ArrayVal: + s := proto.Size(x.Uint64ArrayVal) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *ColumnResponse_StringArrayVal: + s := proto.Size(x.StringArrayVal) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *ColumnResponse_Float64Val: + n += 1 // tag and wire + n += 8 + case nil: + default: + panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) + } + return n +} + type InspectRequest struct { Index string `protobuf:"bytes,1,opt,name=index,proto3" json:"index,omitempty"` Columns *IdsOrKeys `protobuf:"bytes,2,opt,name=columns,proto3" json:"columns,omitempty"` FilterFields []string `protobuf:"bytes,3,rep,name=filterFields,proto3" json:"filterFields,omitempty"` + Limit uint64 `protobuf:"varint,4,opt,name=limit,proto3" json:"limit,omitempty"` + Offset uint64 `protobuf:"varint,5,opt,name=offset,proto3" json:"offset,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -390,6 +543,20 @@ func (m *InspectRequest) GetFilterFields() []string { return nil } +func (m *InspectRequest) GetLimit() uint64 { + if m != nil { + return m.Limit + } + return 0 +} + +func (m *InspectRequest) GetOffset() uint64 { + if m != nil { + return m.Offset + } + return 0 +} + type Uint64Array struct { Vals []uint64 `protobuf:"varint,1,rep,packed,name=vals,proto3" json:"vals,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -540,14 +707,80 @@ func (m *IdsOrKeys) GetKeys() *StringArray { return nil } -// XXX_OneofWrappers is for the internal use of the proto package. -func (*IdsOrKeys) XXX_OneofWrappers() []interface{} { - return []interface{}{ +// XXX_OneofFuncs is for the internal use of the proto package. +func (*IdsOrKeys) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { + return _IdsOrKeys_OneofMarshaler, _IdsOrKeys_OneofUnmarshaler, _IdsOrKeys_OneofSizer, []interface{}{ (*IdsOrKeys_Ids)(nil), (*IdsOrKeys_Keys)(nil), } } +func _IdsOrKeys_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*IdsOrKeys) + // type + switch x := m.Type.(type) { + case *IdsOrKeys_Ids: + b.EncodeVarint(1<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Ids); err != nil { + return err + } + case *IdsOrKeys_Keys: + b.EncodeVarint(2<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Keys); err != nil { + return err + } + case nil: + default: + return fmt.Errorf("IdsOrKeys.Type has unexpected type %T", x) + } + return nil +} + +func _IdsOrKeys_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*IdsOrKeys) + switch tag { + case 1: // type.ids + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(Uint64Array) + err := b.DecodeMessage(msg) + m.Type = &IdsOrKeys_Ids{msg} + return true, err + case 2: // type.keys + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(StringArray) + err := b.DecodeMessage(msg) + m.Type = &IdsOrKeys_Keys{msg} + return true, err + default: + return false, nil + } +} + +func _IdsOrKeys_OneofSizer(msg proto.Message) (n int) { + m := msg.(*IdsOrKeys) + // type + switch x := m.Type.(type) { + case *IdsOrKeys_Ids: + s := proto.Size(x.Ids) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *IdsOrKeys_Keys: + s := proto.Size(x.Keys) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case nil: + default: + panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) + } + return n +} + func init() { proto.RegisterType((*QueryPQLRequest)(nil), "pilosa.QueryPQLRequest") proto.RegisterType((*RowResponse)(nil), "pilosa.RowResponse") @@ -559,44 +792,6 @@ func init() { proto.RegisterType((*IdsOrKeys)(nil), "pilosa.IdsOrKeys") } -func init() { proto.RegisterFile("pilosa.proto", fileDescriptor_ef0691a44d1e275c) } - -var fileDescriptor_ef0691a44d1e275c = []byte{ - // 497 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x53, 0x5d, 0x6b, 0xd4, 0x40, - 0x14, 0xcd, 0x34, 0xe9, 0xee, 0xe6, 0x66, 0x59, 0xf5, 0x2a, 0x1a, 0x16, 0x91, 0x98, 0x17, 0x23, - 0x4a, 0x29, 0xab, 0x08, 0x4a, 0x7d, 0xb0, 0x82, 0x64, 0x51, 0xb0, 0x1d, 0xb1, 0xef, 0xb3, 0xcd, - 0x6c, 0x0d, 0xce, 0x66, 0xd2, 0x4c, 0xd6, 0x9a, 0x57, 0xff, 0xa2, 0x7f, 0x48, 0x66, 0xf2, 0xb1, - 0x49, 0x61, 0x7d, 0x9b, 0x39, 0xe7, 0xdc, 0xef, 0x7b, 0x61, 0x9a, 0xa7, 0x42, 0x2a, 0x76, 0x94, - 0x17, 0xb2, 0x94, 0x38, 0xaa, 0x7f, 0xe1, 0x5b, 0xb8, 0x73, 0xbe, 0xe5, 0x45, 0x75, 0x76, 0xfe, - 0x85, 0xf2, 0xeb, 0x2d, 0x57, 0x25, 0x3e, 0x80, 0xc3, 0x34, 0x4b, 0xf8, 0x6f, 0x9f, 0x04, 0x24, - 0x72, 0x69, 0xfd, 0xc1, 0xbb, 0x60, 0xe7, 0xd7, 0xc2, 0x3f, 0x30, 0x98, 0x7e, 0x86, 0x1b, 0xf0, - 0xa8, 0xbc, 0xa1, 0x5c, 0xe5, 0x32, 0x53, 0x1c, 0x5f, 0xc2, 0xf8, 0x07, 0x67, 0x09, 0x2f, 0x94, - 0x4f, 0x02, 0x3b, 0xf2, 0x16, 0x78, 0xd4, 0x44, 0xfc, 0x28, 0xc5, 0x76, 0x93, 0x2d, 0xb3, 0xb5, - 0xa4, 0xad, 0x04, 0x8f, 0x61, 0x7c, 0x69, 0x60, 0xe5, 0x1f, 0x18, 0xf5, 0xc3, 0xa1, 0xba, 0x75, - 0x4b, 0x5b, 0x59, 0x78, 0x02, 0xb0, 0x73, 0x84, 0x08, 0x4e, 0xc6, 0x36, 0xbc, 0xc9, 0xd1, 0xbc, - 0x71, 0x0e, 0x93, 0x84, 0x95, 0xac, 0xac, 0x72, 0xde, 0xe4, 0xd9, 0xfd, 0xc3, 0xbf, 0x07, 0x30, - 0x1b, 0x7a, 0xc6, 0x27, 0xe0, 0xaa, 0xb2, 0x48, 0xb3, 0xab, 0x0b, 0x26, 0x6a, 0x3f, 0xb1, 0x45, - 0x77, 0x90, 0xe6, 0xb7, 0x69, 0x56, 0xbe, 0x79, 0xad, 0x79, 0xed, 0xcf, 0xd1, 0x7c, 0x07, 0xe1, - 0x63, 0x98, 0x74, 0xb4, 0x1d, 0x90, 0xc8, 0x8e, 0x2d, 0xda, 0x21, 0x38, 0x87, 0xf1, 0x4a, 0x4a, - 0xa1, 0x49, 0x27, 0x20, 0xd1, 0x24, 0xb6, 0x68, 0x0b, 0x18, 0x4e, 0xc8, 0x95, 0xe6, 0x0e, 0x03, - 0x12, 0x4d, 0x0d, 0x57, 0x03, 0xf8, 0x1e, 0x66, 0x75, 0x88, 0x0f, 0x45, 0xc1, 0x2a, 0x2d, 0x19, - 0x05, 0x24, 0xf2, 0x16, 0xf7, 0xdb, 0xfe, 0x7c, 0xdf, 0xb1, 0xb1, 0x45, 0x6f, 0x89, 0xb5, 0x79, - 0x5d, 0x41, 0x67, 0x3e, 0x1e, 0x9a, 0x7f, 0xdb, 0xb1, 0xda, 0x7c, 0x28, 0xc6, 0x00, 0x60, 0x2d, - 0x24, 0x6b, 0xaa, 0x9a, 0x04, 0x24, 0x22, 0xb1, 0x45, 0x7b, 0xd8, 0xa9, 0x07, 0x6e, 0x3d, 0x91, - 0x0b, 0x26, 0xc2, 0x1b, 0x98, 0x2d, 0x33, 0x95, 0xf3, 0xcb, 0xf2, 0xff, 0xcb, 0xf3, 0xa2, 0x3f, - 0x6d, 0x9d, 0xce, 0xbd, 0x36, 0x9d, 0x65, 0xa2, 0xbe, 0x16, 0x9f, 0x79, 0xa5, 0xba, 0x41, 0x63, - 0x08, 0xd3, 0x75, 0x2a, 0x4a, 0x5e, 0x7c, 0x4a, 0xb9, 0x48, 0x94, 0x6f, 0x07, 0x76, 0xe4, 0xd2, - 0x01, 0x16, 0x3e, 0x05, 0xaf, 0xd7, 0x07, 0xbd, 0x0d, 0xbf, 0x98, 0xa8, 0x17, 0xcf, 0xa1, 0xe6, - 0xad, 0x25, 0xbd, 0x5a, 0x07, 0x12, 0xb7, 0x91, 0x5c, 0x81, 0xdb, 0xc5, 0xc7, 0x67, 0x60, 0xa7, - 0x89, 0x32, 0x79, 0xef, 0xed, 0xb6, 0x56, 0xe0, 0x73, 0x70, 0x7e, 0xf2, 0xaa, 0xad, 0x64, 0x4f, - 0x63, 0x8d, 0xe4, 0x74, 0x04, 0x8e, 0xde, 0xbe, 0xc5, 0x1f, 0x02, 0xa3, 0x33, 0x23, 0xc3, 0x13, - 0x98, 0xb4, 0x07, 0x87, 0x8f, 0x5a, 0xdb, 0x5b, 0x27, 0x38, 0xef, 0x9c, 0xf6, 0x0e, 0x2c, 0xb4, - 0x8e, 0x09, 0xbe, 0x83, 0x71, 0xd3, 0x70, 0xec, 0x0e, 0x66, 0x38, 0x81, 0xbd, 0xb6, 0xab, 0x91, - 0xb9, 0xfc, 0x57, 0xff, 0x02, 0x00, 0x00, 0xff, 0xff, 0xca, 0x68, 0x70, 0x08, 0x09, 0x04, 0x00, - 0x00, -} - // Reference imports to suppress errors if they are not otherwise used. var _ context.Context var _ grpc.ClientConn @@ -691,17 +886,6 @@ type PilosaServer interface { Inspect(*InspectRequest, Pilosa_InspectServer) error } -// UnimplementedPilosaServer can be embedded to have forward compatible implementations. -type UnimplementedPilosaServer struct { -} - -func (*UnimplementedPilosaServer) QueryPQL(req *QueryPQLRequest, srv Pilosa_QueryPQLServer) error { - return status.Errorf(codes.Unimplemented, "method QueryPQL not implemented") -} -func (*UnimplementedPilosaServer) Inspect(req *InspectRequest, srv Pilosa_InspectServer) error { - return status.Errorf(codes.Unimplemented, "method Inspect not implemented") -} - func RegisterPilosaServer(s *grpc.Server, srv PilosaServer) { s.RegisterService(&_Pilosa_serviceDesc, srv) } @@ -766,3 +950,42 @@ var _Pilosa_serviceDesc = grpc.ServiceDesc{ }, Metadata: "pilosa.proto", } + +func init() { proto.RegisterFile("pilosa.proto", fileDescriptor_ef0691a44d1e275c) } + +var fileDescriptor_ef0691a44d1e275c = []byte{ + // 524 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x54, 0xdd, 0x8a, 0xd3, 0x40, + 0x14, 0xce, 0x34, 0xd9, 0xb4, 0x39, 0x2d, 0x55, 0x8f, 0xb2, 0x96, 0x22, 0x12, 0x73, 0x63, 0x44, + 0x59, 0x96, 0x2a, 0x82, 0xb2, 0x5e, 0xb8, 0x82, 0xb4, 0x28, 0xb8, 0x3b, 0xe2, 0xde, 0x4f, 0x37, + 0xd3, 0x35, 0x38, 0xcd, 0x64, 0x33, 0x53, 0xb5, 0xb7, 0xbe, 0x8b, 0x4f, 0xe4, 0x0b, 0xc9, 0x4c, + 0x7e, 0x9a, 0x2c, 0x74, 0xef, 0x72, 0xbe, 0xef, 0x3b, 0x67, 0xce, 0x6f, 0x60, 0x94, 0xa7, 0x42, + 0x2a, 0x76, 0x94, 0x17, 0x52, 0x4b, 0xf4, 0x4b, 0x2b, 0x7a, 0x03, 0x77, 0xce, 0x37, 0xbc, 0xd8, + 0x9e, 0x9d, 0x7f, 0xa6, 0xfc, 0x7a, 0xc3, 0x95, 0xc6, 0x07, 0x70, 0x90, 0x66, 0x09, 0xff, 0x3d, + 0x21, 0x21, 0x89, 0x03, 0x5a, 0x1a, 0x78, 0x17, 0xdc, 0xfc, 0x5a, 0x4c, 0x7a, 0x16, 0x33, 0x9f, + 0xd1, 0x1a, 0x86, 0x54, 0xfe, 0xa2, 0x5c, 0xe5, 0x32, 0x53, 0x1c, 0x5f, 0x40, 0xff, 0x3b, 0x67, + 0x09, 0x2f, 0xd4, 0x84, 0x84, 0x6e, 0x3c, 0x9c, 0xe1, 0x51, 0xf5, 0xe2, 0x07, 0x29, 0x36, 0xeb, + 0x6c, 0x91, 0xad, 0x24, 0xad, 0x25, 0x78, 0x0c, 0xfd, 0x4b, 0x0b, 0xab, 0x49, 0xcf, 0xaa, 0x0f, + 0xbb, 0xea, 0x3a, 0x2c, 0xad, 0x65, 0xd1, 0x09, 0xc0, 0x2e, 0x10, 0x22, 0x78, 0x19, 0x5b, 0xf3, + 0x2a, 0x47, 0xfb, 0x8d, 0x53, 0x18, 0x24, 0x4c, 0x33, 0xbd, 0xcd, 0x79, 0x95, 0x67, 0x63, 0x47, + 0xff, 0x7a, 0x30, 0xee, 0x46, 0xc6, 0xc7, 0x10, 0x28, 0x5d, 0xa4, 0xd9, 0xd5, 0x05, 0x13, 0x65, + 0x9c, 0xb9, 0x43, 0x77, 0x90, 0xe1, 0x37, 0x69, 0xa6, 0x5f, 0xbf, 0x32, 0xbc, 0x89, 0xe7, 0x19, + 0xbe, 0x81, 0xf0, 0x11, 0x0c, 0x1a, 0xda, 0x0d, 0x49, 0xec, 0xce, 0x1d, 0xda, 0x20, 0x38, 0x85, + 0xfe, 0x52, 0x4a, 0x61, 0x48, 0x2f, 0x24, 0xf1, 0x60, 0xee, 0xd0, 0x1a, 0xb0, 0x9c, 0x90, 0x4b, + 0xc3, 0x1d, 0x84, 0x24, 0x1e, 0x59, 0xae, 0x04, 0xf0, 0x1d, 0x8c, 0xcb, 0x27, 0xde, 0x17, 0x05, + 0xdb, 0x1a, 0x89, 0x1f, 0x92, 0x78, 0x38, 0xbb, 0x5f, 0xf7, 0xe7, 0xdb, 0x8e, 0x9d, 0x3b, 0xf4, + 0x86, 0xd8, 0xb8, 0x97, 0x15, 0x34, 0xee, 0xfd, 0xae, 0xfb, 0xd7, 0x1d, 0x6b, 0xdc, 0xbb, 0x62, + 0x0c, 0x01, 0x56, 0x42, 0xb2, 0xaa, 0xaa, 0x41, 0x48, 0x62, 0x32, 0x77, 0x68, 0x0b, 0x3b, 0x1d, + 0x42, 0x50, 0x4e, 0xe4, 0x82, 0x89, 0xe8, 0x2f, 0x81, 0xf1, 0x22, 0x53, 0x39, 0xbf, 0xd4, 0xb7, + 0x6f, 0xcf, 0xf3, 0xf6, 0xb8, 0x4d, 0x3e, 0xf7, 0xea, 0x7c, 0x16, 0x89, 0xfa, 0x52, 0x7c, 0xe2, + 0x5b, 0xd5, 0x4c, 0x1a, 0x23, 0x18, 0xad, 0x52, 0xa1, 0x79, 0xf1, 0x31, 0xe5, 0x22, 0x51, 0x13, + 0x37, 0x74, 0xe3, 0x80, 0x76, 0x30, 0xf3, 0x8c, 0x48, 0xd7, 0xa9, 0xb6, 0xcd, 0xf5, 0x68, 0x69, + 0xe0, 0x21, 0xf8, 0x72, 0xb5, 0x52, 0x5c, 0xdb, 0xbe, 0x7a, 0xb4, 0xb2, 0xa2, 0x27, 0x30, 0x6c, + 0xb5, 0xcd, 0x2c, 0xcf, 0x4f, 0x26, 0xca, 0x3d, 0xf5, 0xa8, 0xfd, 0x36, 0x92, 0x56, 0x6b, 0x3a, + 0x92, 0xa0, 0x92, 0x5c, 0x41, 0xd0, 0x64, 0x8b, 0x4f, 0xc1, 0x4d, 0x13, 0x65, 0xab, 0xdc, 0x3b, + 0x1c, 0xa3, 0xc0, 0x67, 0xe0, 0xfd, 0xe0, 0xdb, 0xba, 0xee, 0x3d, 0x73, 0xb0, 0x92, 0x53, 0x1f, + 0x3c, 0xb3, 0xac, 0xb3, 0x3f, 0x04, 0xfc, 0x33, 0x2b, 0xc3, 0x13, 0x18, 0xd4, 0xf7, 0x89, 0x0f, + 0x6b, 0xdf, 0x1b, 0x17, 0x3b, 0x6d, 0x82, 0xb6, 0xee, 0x31, 0x72, 0x8e, 0x09, 0xbe, 0x85, 0x7e, + 0x35, 0x1e, 0x6c, 0xee, 0xab, 0x3b, 0xaf, 0xbd, 0xbe, 0x4b, 0xdf, 0xfe, 0x28, 0x5e, 0xfe, 0x0f, + 0x00, 0x00, 0xff, 0xff, 0x0a, 0x57, 0xf8, 0xfb, 0x38, 0x04, 0x00, 0x00, +} diff --git a/proto/pilosa.proto b/proto/pilosa.proto index a8822ec46..9a5828624 100644 --- a/proto/pilosa.proto +++ b/proto/pilosa.proto @@ -34,6 +34,8 @@ message InspectRequest { string index = 1; IdsOrKeys columns = 2; repeated string filterFields = 3; + uint64 limit = 4; + uint64 offset = 5; } message Uint64Array { diff --git a/server/grpc.go b/server/grpc.go index cb1856b9f..a0ef90cbe 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -71,6 +71,8 @@ func (h grpcHandler) QueryPQL(req *pb.QueryPQLRequest, stream pb.Pilosa_QueryPQL // Inspect handles the inspect request and sends an InspectResponse to the stream. func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectServer) error { + const defaultLimit = 100000 + index, err := h.api.Index(context.Background(), req.Index) if err != nil { return errors.Wrap(err, "getting index") @@ -96,7 +98,17 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer return nil } - if ints, ok := req.Columns.Type.(*pb.IdsOrKeys_Ids); ok { + limit := req.Limit + if limit == 0 { + limit = defaultLimit + } + offset := req.Offset + + if !index.Options().Keys { + ints, ok := req.Columns.Type.(*pb.IdsOrKeys_Ids) + if !ok { + return errors.New("invalid int columns") + } ci := []*pb.ColumnInfo{ {Name: "_id", Datatype: "uint64"}, } @@ -104,7 +116,46 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer ci = append(ci, &pb.ColumnInfo{Name: field.Name(), Datatype: field.Type()}) // TODO: field.Type likely doesn't align with supported datatypes } - for _, col := range ints.Ids.Vals { + // If Columns is empty, then get the _exists list (via All()), + // from the index and loop over that instead. + cols := ints.Ids.Vals + if len(cols) > 0 { + // Apply limit/offset to the provided columns. + if int(offset) >= len(cols) { + return nil + } + end := limit + offset + if int(end) > len(cols) { + end = uint64(len(cols)) + } + cols = cols[offset:end] + } else { + // Prevent getting too many records by forcing a limit. + pql := fmt.Sprintf("All(limit=%d, offset=%d)", limit, offset) + query := pilosa.QueryRequest{ + Index: req.Index, + Query: pql, + } + resp, err := h.api.Query(context.Background(), &query) + if err != nil { + return errors.Wrapf(err, "querying for all: %s", pql) + } + + ids, ok := resp.Results[0].(*pilosa.Row) + if !ok { + return errors.Wrap(err, "getting results as a row") + } + + limitedCols := ids.Columns() + if len(limitedCols) == 0 { + // If cols is still empty after the limit/offset, then + // return with no results. + return nil + } + cols = limitedCols + } + + for _, col := range cols { rowResp := &pb.RowResponse{ Headers: ci, Columns: []*pb.ColumnResponse{ @@ -178,6 +229,7 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: nil}) } + case "decimal": value, exists, err := field.FloatValue(col) if err != nil { @@ -189,6 +241,7 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: nil}) } + case "bool": pql := fmt.Sprintf("Rows(%s, column=%d)", field.Name(), col) query := pilosa.QueryRequest{ @@ -216,6 +269,7 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: nil}) } + case "time": rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: nil}) @@ -227,7 +281,11 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer } } - } else if keys, ok := req.Columns.Type.(*pb.IdsOrKeys_Keys); ok { + } else { + keys, ok := req.Columns.Type.(*pb.IdsOrKeys_Keys) + if !ok { + return errToStatusError(errors.New("invalid key columns")) + } ci := []*pb.ColumnInfo{ {Name: "_id", Datatype: "string"}, } @@ -235,7 +293,47 @@ func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSer ci = append(ci, &pb.ColumnInfo{Name: field.Name(), Datatype: field.Type()}) // TODO: field.Type likely doesn't align with supported datatypes } - for _, col := range keys.Keys.Vals { + // If Columns is empty, then get the _exists list (via All()), + // from the index and loop over that instead. + cols := keys.Keys.Vals + if len(cols) > 0 { + // Apply limit/offset to the provided columns. + if int(offset) >= len(cols) { + return nil + } + end := limit + offset + if int(end) > len(cols) { + end = uint64(len(cols)) + } + cols = cols[offset:end] + } else { + // Prevent getting too many records by forcing a limit. + pql := fmt.Sprintf("All(limit=%d, offset=%d)", limit, offset) + query := pilosa.QueryRequest{ + Index: req.Index, + Query: pql, + } + resp, err := h.api.Query(context.Background(), &query) + if err != nil { + fmt.Println("GOT ERROR trying to get ALL():", err) + return errors.Wrapf(err, "querying for all: %s", pql) + } + + ids, ok := resp.Results[0].(*pilosa.Row) + if !ok { + return errors.Wrap(err, "getting results as a row") + } + + limitedCols := ids.Keys + if len(limitedCols) == 0 { + // If cols is still empty after the limit/offset, then + // return with no results. + return nil + } + cols = limitedCols + } + + for _, col := range cols { rowResp := &pb.RowResponse{ Headers: ci, Columns: []*pb.ColumnResponse{