diff --git a/api/client/grpc.go b/api/client/grpc.go new file mode 100644 index 000000000..6d8247d8f --- /dev/null +++ b/api/client/grpc.go @@ -0,0 +1,106 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "context" + + pb "github.com/pilosa/pilosa/v2/proto" + "github.com/pkg/errors" + "google.golang.org/grpc" +) + +// GRPCClient is a client for working with the gRPC server. +type GRPCClient struct { + conn *grpc.ClientConn +} + +// NewGRPCClient returns a new instance of GRPCClient. +func NewGRPCClient(dialTarget string) (*GRPCClient, error) { + var opts []grpc.DialOption + opts = append(opts, grpc.WithInsecure()) // TODO: consider implementing WithTransportCredentials() + gconn, err := grpc.Dial(dialTarget, opts...) + if err != nil { + return nil, errors.Wrap(err, "creating new grpc client") + } + + return &GRPCClient{ + conn: gconn, + }, nil +} + +// Close closes any connections the client has opened. +func (c *GRPCClient) Close() error { + if c.conn != nil { + return c.conn.Close() + } + return nil +} + +// Query returns a stream of RowResponse for the given index and PQL string. +func (c *GRPCClient) Query(ctx context.Context, index string, pql string) (grpc.ClientStream, error) { + if c.conn == nil { + return nil, errors.New("client has not established a grpc connection") + } + + grpcClient := pb.NewPilosaClient(c.conn) + + stream, err := grpcClient.QueryPQL(ctx, &pb.QueryPQLRequest{ + Index: index, + Pql: pql, + }) + if err != nil { + return nil, errors.Wrap(err, "getting stream") + } else if stream == nil { + return nil, errors.New("could not create stream") + } + + return stream, err +} + +// Inspect returns a stream of RowResponse for the given index, columns, and filters. +// It is inteded to mimic something like "select [fields] from table where recordID IN (...)". +func (c *GRPCClient) Inspect(ctx context.Context, index string, columnIDs []uint64, columnKeys []string, fieldFilters []string) (grpc.ClientStream, error) { + if c.conn == nil { + return nil, errors.New("client has not established a grpc connection") + } + + if len(columnIDs) > 0 && len(columnKeys) > 0 { + return nil, errors.New("only provide column ids or keys, not both") + } + + // Convert columns to proto type IdsOrKeys. + idsOrKeys := &pb.IdsOrKeys{} + if len(columnKeys) > 0 { + idsOrKeys.Type = &pb.IdsOrKeys_Keys{Keys: &pb.StringArray{Vals: columnKeys}} + } else { + idsOrKeys.Type = &pb.IdsOrKeys_Ids{Ids: &pb.Uint64Array{Vals: columnIDs}} + } + + grpcClient := pb.NewPilosaClient(c.conn) + + stream, err := grpcClient.Inspect(ctx, &pb.InspectRequest{ + Index: index, + Columns: idsOrKeys, + FilterFields: fieldFilters, + }) + if err != nil { + return nil, errors.Wrap(err, "getting stream") + } else if stream == nil { + return nil, errors.New("could not create stream") + } + + return stream, err +} diff --git a/cmd/server_test.go b/cmd/server_test.go index 32ab17d3b..67f6888ed 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -42,7 +42,7 @@ func TestServerConfig(t *testing.T) { tests := []commandTest{ // TEST 0 { - args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:42454,localhost:10110", "--bind", "localhost:42454", "--translation.map-size", "100000"}, + args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:42454,localhost:10110", "--bind", "localhost:42454", "--bind-grpc", "localhost:20111", "--translation.map-size", "100000"}, env: map[string]string{ "PILOSA_DATA_DIR": "/tmp/myEnvDatadir", "PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s", @@ -53,6 +53,7 @@ func TestServerConfig(t *testing.T) { cfgFileContent: ` data-dir = "/tmp/myFileDatadir" bind = "localhost:0" + bind-grpc = "localhost:0" max-writes-per-request = 3000 [cluster] @@ -96,6 +97,7 @@ func TestServerConfig(t *testing.T) { }, cfgFileContent: ` bind = "localhost:0" + bind-grpc = "localhost:0" data-dir = "` + actualDataDir + `" [cluster] disabled = true @@ -122,6 +124,7 @@ func TestServerConfig(t *testing.T) { env: map[string]string{}, cfgFileContent: ` bind = "localhost:19444" + bind-grpc = "localhost:29444" data-dir = "` + actualDataDir + `" [cluster] hosts = [ diff --git a/ctl/server.go b/ctl/server.go index 464323fc8..ee7f9368c 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -26,6 +26,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags := cmd.Flags() flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", srv.Config.DataDir, "Directory to store pilosa data files.") flags.StringVarP(&srv.Config.Bind, "bind", "b", srv.Config.Bind, "Default URI on which pilosa should listen.") + flags.StringVar(&srv.Config.BindGRPC, "bind-grpc", srv.Config.BindGRPC, "URI on which pilosa should listen for gRPC requests.") flags.StringVar(&srv.Config.Advertise, "advertise", srv.Config.Advertise, "Address to advertise externally.") flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.") flags.StringVar(&srv.Config.LogPath, "log-path", srv.Config.LogPath, "Log path") diff --git a/go.mod b/go.mod index ed96bdf4e..e546f530e 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/pilosa/pilosa/v2 replace github.com/hashicorp/memberlist => github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 require ( - github.com/BurntSushi/toml v0.3.1 // indirect github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect @@ -13,7 +12,7 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/go-ole/go-ole v1.2.4 // indirect github.com/gogo/protobuf v1.2.0 - github.com/golang/protobuf v1.3.1 + github.com/golang/protobuf v1.3.2 github.com/google/go-cmp v0.2.0 github.com/gorilla/handlers v1.3.0 github.com/gorilla/mux v1.7.0 @@ -36,10 +35,11 @@ require ( github.com/uber/jaeger-lib v2.2.0+incompatible // indirect go.uber.org/atomic v1.4.0 // indirect golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect - golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 // indirect + golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 golang.org/x/sync v0.0.0-20190423024810-112230192c58 golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 // indirect golang.org/x/text v0.3.2 // indirect + google.golang.org/grpc v1.24.0 modernc.org/mathutil v1.0.0 modernc.org/strutil v1.0.0 ) diff --git a/go.sum b/go.sum index e36fda5d3..68186bbec 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d h1:n0G4ckjMEj7bWuGYUX0i8YlBeBBJuZ+HEHvHfyBDZtI= @@ -20,6 +21,7 @@ github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -39,10 +41,14 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= @@ -153,13 +159,16 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 h1:p/H982KKEjUnLJkM3tt/LemDnOc1GiZL5FCVlORJ5zo= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519 h1:x6rhz8Y9CjbgQkccRGmELH6K+LJj7tOoh3XWeC1yaQM= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 h1:FP8hkuE6yUEaJnK7O2eTuejKWwW+Rhfj80dQ2JcKxCU= golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -180,12 +189,20 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s= +google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE= diff --git a/proto/pilosa.pb.go b/proto/pilosa.pb.go new file mode 100644 index 000000000..8a4ac9950 --- /dev/null +++ b/proto/pilosa.pb.go @@ -0,0 +1,945 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: pilosa.proto + +package proto + +import ( + fmt "fmt" + proto "github.com/golang/protobuf/proto" + math "math" +) + +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type QueryPQLRequest struct { + Index string `protobuf:"bytes,1,opt,name=index,proto3" json:"index,omitempty"` + Pql string `protobuf:"bytes,2,opt,name=pql,proto3" json:"pql,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *QueryPQLRequest) Reset() { *m = QueryPQLRequest{} } +func (m *QueryPQLRequest) String() string { return proto.CompactTextString(m) } +func (*QueryPQLRequest) ProtoMessage() {} +func (*QueryPQLRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_ef0691a44d1e275c, []int{0} +} + +func (m *QueryPQLRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_QueryPQLRequest.Unmarshal(m, b) +} +func (m *QueryPQLRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_QueryPQLRequest.Marshal(b, m, deterministic) +} +func (m *QueryPQLRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_QueryPQLRequest.Merge(m, src) +} +func (m *QueryPQLRequest) XXX_Size() int { + return xxx_messageInfo_QueryPQLRequest.Size(m) +} +func (m *QueryPQLRequest) XXX_DiscardUnknown() { + xxx_messageInfo_QueryPQLRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_QueryPQLRequest proto.InternalMessageInfo + +func (m *QueryPQLRequest) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *QueryPQLRequest) GetPql() string { + if m != nil { + return m.Pql + } + return "" +} + +type RowResponse struct { + Headers []*ColumnInfo `protobuf:"bytes,1,rep,name=headers,proto3" json:"headers,omitempty"` + Columns []*ColumnResponse `protobuf:"bytes,2,rep,name=columns,proto3" json:"columns,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *RowResponse) Reset() { *m = RowResponse{} } +func (m *RowResponse) String() string { return proto.CompactTextString(m) } +func (*RowResponse) ProtoMessage() {} +func (*RowResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_ef0691a44d1e275c, []int{1} +} + +func (m *RowResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_RowResponse.Unmarshal(m, b) +} +func (m *RowResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_RowResponse.Marshal(b, m, deterministic) +} +func (m *RowResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_RowResponse.Merge(m, src) +} +func (m *RowResponse) XXX_Size() int { + return xxx_messageInfo_RowResponse.Size(m) +} +func (m *RowResponse) XXX_DiscardUnknown() { + xxx_messageInfo_RowResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_RowResponse proto.InternalMessageInfo + +func (m *RowResponse) GetHeaders() []*ColumnInfo { + if m != nil { + return m.Headers + } + return nil +} + +func (m *RowResponse) GetColumns() []*ColumnResponse { + if m != nil { + return m.Columns + } + return nil +} + +type ColumnInfo struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Datatype string `protobuf:"bytes,2,opt,name=datatype,proto3" json:"datatype,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ColumnInfo) Reset() { *m = ColumnInfo{} } +func (m *ColumnInfo) String() string { return proto.CompactTextString(m) } +func (*ColumnInfo) ProtoMessage() {} +func (*ColumnInfo) Descriptor() ([]byte, []int) { + return fileDescriptor_ef0691a44d1e275c, []int{2} +} + +func (m *ColumnInfo) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ColumnInfo.Unmarshal(m, b) +} +func (m *ColumnInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ColumnInfo.Marshal(b, m, deterministic) +} +func (m *ColumnInfo) XXX_Merge(src proto.Message) { + xxx_messageInfo_ColumnInfo.Merge(m, src) +} +func (m *ColumnInfo) XXX_Size() int { + return xxx_messageInfo_ColumnInfo.Size(m) +} +func (m *ColumnInfo) XXX_DiscardUnknown() { + xxx_messageInfo_ColumnInfo.DiscardUnknown(m) +} + +var xxx_messageInfo_ColumnInfo proto.InternalMessageInfo + +func (m *ColumnInfo) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *ColumnInfo) GetDatatype() string { + if m != nil { + return m.Datatype + } + return "" +} + +type ColumnResponse struct { + // Types that are valid to be assigned to ColumnVal: + // *ColumnResponse_StringVal + // *ColumnResponse_Uint64Val + // *ColumnResponse_Int64Val + // *ColumnResponse_BoolVal + // *ColumnResponse_BlobVal + // *ColumnResponse_Uint64ArrayVal + // *ColumnResponse_StringArrayVal + ColumnVal isColumnResponse_ColumnVal `protobuf_oneof:"columnVal"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ColumnResponse) Reset() { *m = ColumnResponse{} } +func (m *ColumnResponse) String() string { return proto.CompactTextString(m) } +func (*ColumnResponse) ProtoMessage() {} +func (*ColumnResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_ef0691a44d1e275c, []int{3} +} + +func (m *ColumnResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ColumnResponse.Unmarshal(m, b) +} +func (m *ColumnResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ColumnResponse.Marshal(b, m, deterministic) +} +func (m *ColumnResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ColumnResponse.Merge(m, src) +} +func (m *ColumnResponse) XXX_Size() int { + return xxx_messageInfo_ColumnResponse.Size(m) +} +func (m *ColumnResponse) XXX_DiscardUnknown() { + xxx_messageInfo_ColumnResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_ColumnResponse proto.InternalMessageInfo + +type isColumnResponse_ColumnVal interface { + isColumnResponse_ColumnVal() +} + +type ColumnResponse_StringVal struct { + StringVal string `protobuf:"bytes,1,opt,name=stringVal,proto3,oneof"` +} + +type ColumnResponse_Uint64Val struct { + Uint64Val uint64 `protobuf:"varint,2,opt,name=uint64Val,proto3,oneof"` +} + +type ColumnResponse_Int64Val struct { + Int64Val int64 `protobuf:"varint,3,opt,name=int64Val,proto3,oneof"` +} + +type ColumnResponse_BoolVal struct { + BoolVal bool `protobuf:"varint,4,opt,name=boolVal,proto3,oneof"` +} + +type ColumnResponse_BlobVal struct { + BlobVal []byte `protobuf:"bytes,5,opt,name=blobVal,proto3,oneof"` +} + +type ColumnResponse_Uint64ArrayVal struct { + Uint64ArrayVal *Uint64Array `protobuf:"bytes,6,opt,name=uint64ArrayVal,proto3,oneof"` +} + +type ColumnResponse_StringArrayVal struct { + StringArrayVal *StringArray `protobuf:"bytes,7,opt,name=stringArrayVal,proto3,oneof"` +} + +func (*ColumnResponse_StringVal) isColumnResponse_ColumnVal() {} + +func (*ColumnResponse_Uint64Val) isColumnResponse_ColumnVal() {} + +func (*ColumnResponse_Int64Val) isColumnResponse_ColumnVal() {} + +func (*ColumnResponse_BoolVal) isColumnResponse_ColumnVal() {} + +func (*ColumnResponse_BlobVal) isColumnResponse_ColumnVal() {} + +func (*ColumnResponse_Uint64ArrayVal) isColumnResponse_ColumnVal() {} + +func (*ColumnResponse_StringArrayVal) isColumnResponse_ColumnVal() {} + +func (m *ColumnResponse) GetColumnVal() isColumnResponse_ColumnVal { + if m != nil { + return m.ColumnVal + } + return nil +} + +func (m *ColumnResponse) GetStringVal() string { + if x, ok := m.GetColumnVal().(*ColumnResponse_StringVal); ok { + return x.StringVal + } + return "" +} + +func (m *ColumnResponse) GetUint64Val() uint64 { + if x, ok := m.GetColumnVal().(*ColumnResponse_Uint64Val); ok { + return x.Uint64Val + } + return 0 +} + +func (m *ColumnResponse) GetInt64Val() int64 { + if x, ok := m.GetColumnVal().(*ColumnResponse_Int64Val); ok { + return x.Int64Val + } + return 0 +} + +func (m *ColumnResponse) GetBoolVal() bool { + if x, ok := m.GetColumnVal().(*ColumnResponse_BoolVal); ok { + return x.BoolVal + } + return false +} + +func (m *ColumnResponse) GetBlobVal() []byte { + if x, ok := m.GetColumnVal().(*ColumnResponse_BlobVal); ok { + return x.BlobVal + } + return nil +} + +func (m *ColumnResponse) GetUint64ArrayVal() *Uint64Array { + if x, ok := m.GetColumnVal().(*ColumnResponse_Uint64ArrayVal); ok { + return x.Uint64ArrayVal + } + return nil +} + +func (m *ColumnResponse) GetStringArrayVal() *StringArray { + if x, ok := m.GetColumnVal().(*ColumnResponse_StringArrayVal); ok { + return x.StringArrayVal + } + return nil +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*ColumnResponse) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { + return _ColumnResponse_OneofMarshaler, _ColumnResponse_OneofUnmarshaler, _ColumnResponse_OneofSizer, []interface{}{ + (*ColumnResponse_StringVal)(nil), + (*ColumnResponse_Uint64Val)(nil), + (*ColumnResponse_Int64Val)(nil), + (*ColumnResponse_BoolVal)(nil), + (*ColumnResponse_BlobVal)(nil), + (*ColumnResponse_Uint64ArrayVal)(nil), + (*ColumnResponse_StringArrayVal)(nil), + } +} + +func _ColumnResponse_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*ColumnResponse) + // columnVal + switch x := m.ColumnVal.(type) { + case *ColumnResponse_StringVal: + b.EncodeVarint(1<<3 | proto.WireBytes) + b.EncodeStringBytes(x.StringVal) + case *ColumnResponse_Uint64Val: + b.EncodeVarint(2<<3 | proto.WireVarint) + b.EncodeVarint(uint64(x.Uint64Val)) + case *ColumnResponse_Int64Val: + b.EncodeVarint(3<<3 | proto.WireVarint) + b.EncodeVarint(uint64(x.Int64Val)) + case *ColumnResponse_BoolVal: + t := uint64(0) + if x.BoolVal { + t = 1 + } + b.EncodeVarint(4<<3 | proto.WireVarint) + b.EncodeVarint(t) + case *ColumnResponse_BlobVal: + b.EncodeVarint(5<<3 | proto.WireBytes) + b.EncodeRawBytes(x.BlobVal) + case *ColumnResponse_Uint64ArrayVal: + b.EncodeVarint(6<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Uint64ArrayVal); err != nil { + return err + } + case *ColumnResponse_StringArrayVal: + b.EncodeVarint(7<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.StringArrayVal); err != nil { + return err + } + case nil: + default: + return fmt.Errorf("ColumnResponse.ColumnVal has unexpected type %T", x) + } + return nil +} + +func _ColumnResponse_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*ColumnResponse) + switch tag { + case 1: // columnVal.stringVal + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeStringBytes() + m.ColumnVal = &ColumnResponse_StringVal{x} + return true, err + case 2: // columnVal.uint64Val + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.ColumnVal = &ColumnResponse_Uint64Val{x} + return true, err + case 3: // columnVal.int64Val + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.ColumnVal = &ColumnResponse_Int64Val{int64(x)} + return true, err + case 4: // columnVal.boolVal + if wire != proto.WireVarint { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeVarint() + m.ColumnVal = &ColumnResponse_BoolVal{x != 0} + return true, err + case 5: // columnVal.blobVal + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + x, err := b.DecodeRawBytes(true) + m.ColumnVal = &ColumnResponse_BlobVal{x} + return true, err + case 6: // columnVal.uint64ArrayVal + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(Uint64Array) + err := b.DecodeMessage(msg) + m.ColumnVal = &ColumnResponse_Uint64ArrayVal{msg} + return true, err + case 7: // columnVal.stringArrayVal + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(StringArray) + err := b.DecodeMessage(msg) + m.ColumnVal = &ColumnResponse_StringArrayVal{msg} + return true, err + default: + return false, nil + } +} + +func _ColumnResponse_OneofSizer(msg proto.Message) (n int) { + m := msg.(*ColumnResponse) + // columnVal + switch x := m.ColumnVal.(type) { + case *ColumnResponse_StringVal: + n += 1 // tag and wire + n += proto.SizeVarint(uint64(len(x.StringVal))) + n += len(x.StringVal) + case *ColumnResponse_Uint64Val: + n += 1 // tag and wire + n += proto.SizeVarint(uint64(x.Uint64Val)) + case *ColumnResponse_Int64Val: + n += 1 // tag and wire + n += proto.SizeVarint(uint64(x.Int64Val)) + case *ColumnResponse_BoolVal: + n += 1 // tag and wire + n += 1 + case *ColumnResponse_BlobVal: + n += 1 // tag and wire + n += proto.SizeVarint(uint64(len(x.BlobVal))) + n += len(x.BlobVal) + case *ColumnResponse_Uint64ArrayVal: + s := proto.Size(x.Uint64ArrayVal) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *ColumnResponse_StringArrayVal: + s := proto.Size(x.StringArrayVal) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case nil: + default: + panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) + } + return n +} + +type InspectRequest struct { + Index string `protobuf:"bytes,1,opt,name=index,proto3" json:"index,omitempty"` + Columns *IdsOrKeys `protobuf:"bytes,2,opt,name=columns,proto3" json:"columns,omitempty"` + FilterFields []string `protobuf:"bytes,3,rep,name=filterFields,proto3" json:"filterFields,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *InspectRequest) Reset() { *m = InspectRequest{} } +func (m *InspectRequest) String() string { return proto.CompactTextString(m) } +func (*InspectRequest) ProtoMessage() {} +func (*InspectRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_ef0691a44d1e275c, []int{4} +} + +func (m *InspectRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_InspectRequest.Unmarshal(m, b) +} +func (m *InspectRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_InspectRequest.Marshal(b, m, deterministic) +} +func (m *InspectRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_InspectRequest.Merge(m, src) +} +func (m *InspectRequest) XXX_Size() int { + return xxx_messageInfo_InspectRequest.Size(m) +} +func (m *InspectRequest) XXX_DiscardUnknown() { + xxx_messageInfo_InspectRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_InspectRequest proto.InternalMessageInfo + +func (m *InspectRequest) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *InspectRequest) GetColumns() *IdsOrKeys { + if m != nil { + return m.Columns + } + return nil +} + +func (m *InspectRequest) GetFilterFields() []string { + if m != nil { + return m.FilterFields + } + return nil +} + +type Uint64Array struct { + Vals []uint64 `protobuf:"varint,1,rep,packed,name=vals,proto3" json:"vals,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Uint64Array) Reset() { *m = Uint64Array{} } +func (m *Uint64Array) String() string { return proto.CompactTextString(m) } +func (*Uint64Array) ProtoMessage() {} +func (*Uint64Array) Descriptor() ([]byte, []int) { + return fileDescriptor_ef0691a44d1e275c, []int{5} +} + +func (m *Uint64Array) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Uint64Array.Unmarshal(m, b) +} +func (m *Uint64Array) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Uint64Array.Marshal(b, m, deterministic) +} +func (m *Uint64Array) XXX_Merge(src proto.Message) { + xxx_messageInfo_Uint64Array.Merge(m, src) +} +func (m *Uint64Array) XXX_Size() int { + return xxx_messageInfo_Uint64Array.Size(m) +} +func (m *Uint64Array) XXX_DiscardUnknown() { + xxx_messageInfo_Uint64Array.DiscardUnknown(m) +} + +var xxx_messageInfo_Uint64Array proto.InternalMessageInfo + +func (m *Uint64Array) GetVals() []uint64 { + if m != nil { + return m.Vals + } + return nil +} + +type StringArray struct { + Vals []string `protobuf:"bytes,1,rep,name=vals,proto3" json:"vals,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *StringArray) Reset() { *m = StringArray{} } +func (m *StringArray) String() string { return proto.CompactTextString(m) } +func (*StringArray) ProtoMessage() {} +func (*StringArray) Descriptor() ([]byte, []int) { + return fileDescriptor_ef0691a44d1e275c, []int{6} +} + +func (m *StringArray) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_StringArray.Unmarshal(m, b) +} +func (m *StringArray) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_StringArray.Marshal(b, m, deterministic) +} +func (m *StringArray) XXX_Merge(src proto.Message) { + xxx_messageInfo_StringArray.Merge(m, src) +} +func (m *StringArray) XXX_Size() int { + return xxx_messageInfo_StringArray.Size(m) +} +func (m *StringArray) XXX_DiscardUnknown() { + xxx_messageInfo_StringArray.DiscardUnknown(m) +} + +var xxx_messageInfo_StringArray proto.InternalMessageInfo + +func (m *StringArray) GetVals() []string { + if m != nil { + return m.Vals + } + return nil +} + +type IdsOrKeys struct { + // Types that are valid to be assigned to Type: + // *IdsOrKeys_Ids + // *IdsOrKeys_Keys + Type isIdsOrKeys_Type `protobuf_oneof:"type"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *IdsOrKeys) Reset() { *m = IdsOrKeys{} } +func (m *IdsOrKeys) String() string { return proto.CompactTextString(m) } +func (*IdsOrKeys) ProtoMessage() {} +func (*IdsOrKeys) Descriptor() ([]byte, []int) { + return fileDescriptor_ef0691a44d1e275c, []int{7} +} + +func (m *IdsOrKeys) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_IdsOrKeys.Unmarshal(m, b) +} +func (m *IdsOrKeys) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_IdsOrKeys.Marshal(b, m, deterministic) +} +func (m *IdsOrKeys) XXX_Merge(src proto.Message) { + xxx_messageInfo_IdsOrKeys.Merge(m, src) +} +func (m *IdsOrKeys) XXX_Size() int { + return xxx_messageInfo_IdsOrKeys.Size(m) +} +func (m *IdsOrKeys) XXX_DiscardUnknown() { + xxx_messageInfo_IdsOrKeys.DiscardUnknown(m) +} + +var xxx_messageInfo_IdsOrKeys proto.InternalMessageInfo + +type isIdsOrKeys_Type interface { + isIdsOrKeys_Type() +} + +type IdsOrKeys_Ids struct { + Ids *Uint64Array `protobuf:"bytes,1,opt,name=ids,proto3,oneof"` +} + +type IdsOrKeys_Keys struct { + Keys *StringArray `protobuf:"bytes,2,opt,name=keys,proto3,oneof"` +} + +func (*IdsOrKeys_Ids) isIdsOrKeys_Type() {} + +func (*IdsOrKeys_Keys) isIdsOrKeys_Type() {} + +func (m *IdsOrKeys) GetType() isIdsOrKeys_Type { + if m != nil { + return m.Type + } + return nil +} + +func (m *IdsOrKeys) GetIds() *Uint64Array { + if x, ok := m.GetType().(*IdsOrKeys_Ids); ok { + return x.Ids + } + return nil +} + +func (m *IdsOrKeys) GetKeys() *StringArray { + if x, ok := m.GetType().(*IdsOrKeys_Keys); ok { + return x.Keys + } + return nil +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*IdsOrKeys) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { + return _IdsOrKeys_OneofMarshaler, _IdsOrKeys_OneofUnmarshaler, _IdsOrKeys_OneofSizer, []interface{}{ + (*IdsOrKeys_Ids)(nil), + (*IdsOrKeys_Keys)(nil), + } +} + +func _IdsOrKeys_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*IdsOrKeys) + // type + switch x := m.Type.(type) { + case *IdsOrKeys_Ids: + b.EncodeVarint(1<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Ids); err != nil { + return err + } + case *IdsOrKeys_Keys: + b.EncodeVarint(2<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Keys); err != nil { + return err + } + case nil: + default: + return fmt.Errorf("IdsOrKeys.Type has unexpected type %T", x) + } + return nil +} + +func _IdsOrKeys_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*IdsOrKeys) + switch tag { + case 1: // type.ids + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(Uint64Array) + err := b.DecodeMessage(msg) + m.Type = &IdsOrKeys_Ids{msg} + return true, err + case 2: // type.keys + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(StringArray) + err := b.DecodeMessage(msg) + m.Type = &IdsOrKeys_Keys{msg} + return true, err + default: + return false, nil + } +} + +func _IdsOrKeys_OneofSizer(msg proto.Message) (n int) { + m := msg.(*IdsOrKeys) + // type + switch x := m.Type.(type) { + case *IdsOrKeys_Ids: + s := proto.Size(x.Ids) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *IdsOrKeys_Keys: + s := proto.Size(x.Keys) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case nil: + default: + panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) + } + return n +} + +func init() { + proto.RegisterType((*QueryPQLRequest)(nil), "proto.QueryPQLRequest") + proto.RegisterType((*RowResponse)(nil), "proto.RowResponse") + proto.RegisterType((*ColumnInfo)(nil), "proto.ColumnInfo") + proto.RegisterType((*ColumnResponse)(nil), "proto.ColumnResponse") + proto.RegisterType((*InspectRequest)(nil), "proto.InspectRequest") + proto.RegisterType((*Uint64Array)(nil), "proto.Uint64Array") + proto.RegisterType((*StringArray)(nil), "proto.StringArray") + proto.RegisterType((*IdsOrKeys)(nil), "proto.IdsOrKeys") +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// PilosaClient is the client API for Pilosa service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type PilosaClient interface { + QueryPQL(ctx context.Context, in *QueryPQLRequest, opts ...grpc.CallOption) (Pilosa_QueryPQLClient, error) + Inspect(ctx context.Context, in *InspectRequest, opts ...grpc.CallOption) (Pilosa_InspectClient, error) +} + +type pilosaClient struct { + cc *grpc.ClientConn +} + +func NewPilosaClient(cc *grpc.ClientConn) PilosaClient { + return &pilosaClient{cc} +} + +func (c *pilosaClient) QueryPQL(ctx context.Context, in *QueryPQLRequest, opts ...grpc.CallOption) (Pilosa_QueryPQLClient, error) { + stream, err := c.cc.NewStream(ctx, &_Pilosa_serviceDesc.Streams[0], "/proto.Pilosa/QueryPQL", opts...) + if err != nil { + return nil, err + } + x := &pilosaQueryPQLClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type Pilosa_QueryPQLClient interface { + Recv() (*RowResponse, error) + grpc.ClientStream +} + +type pilosaQueryPQLClient struct { + grpc.ClientStream +} + +func (x *pilosaQueryPQLClient) Recv() (*RowResponse, error) { + m := new(RowResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *pilosaClient) Inspect(ctx context.Context, in *InspectRequest, opts ...grpc.CallOption) (Pilosa_InspectClient, error) { + stream, err := c.cc.NewStream(ctx, &_Pilosa_serviceDesc.Streams[1], "/proto.Pilosa/Inspect", opts...) + if err != nil { + return nil, err + } + x := &pilosaInspectClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type Pilosa_InspectClient interface { + Recv() (*RowResponse, error) + grpc.ClientStream +} + +type pilosaInspectClient struct { + grpc.ClientStream +} + +func (x *pilosaInspectClient) Recv() (*RowResponse, error) { + m := new(RowResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// PilosaServer is the server API for Pilosa service. +type PilosaServer interface { + QueryPQL(*QueryPQLRequest, Pilosa_QueryPQLServer) error + Inspect(*InspectRequest, Pilosa_InspectServer) error +} + +func RegisterPilosaServer(s *grpc.Server, srv PilosaServer) { + s.RegisterService(&_Pilosa_serviceDesc, srv) +} + +func _Pilosa_QueryPQL_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(QueryPQLRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(PilosaServer).QueryPQL(m, &pilosaQueryPQLServer{stream}) +} + +type Pilosa_QueryPQLServer interface { + Send(*RowResponse) error + grpc.ServerStream +} + +type pilosaQueryPQLServer struct { + grpc.ServerStream +} + +func (x *pilosaQueryPQLServer) Send(m *RowResponse) error { + return x.ServerStream.SendMsg(m) +} + +func _Pilosa_Inspect_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(InspectRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(PilosaServer).Inspect(m, &pilosaInspectServer{stream}) +} + +type Pilosa_InspectServer interface { + Send(*RowResponse) error + grpc.ServerStream +} + +type pilosaInspectServer struct { + grpc.ServerStream +} + +func (x *pilosaInspectServer) Send(m *RowResponse) error { + return x.ServerStream.SendMsg(m) +} + +var _Pilosa_serviceDesc = grpc.ServiceDesc{ + ServiceName: "proto.Pilosa", + HandlerType: (*PilosaServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "QueryPQL", + Handler: _Pilosa_QueryPQL_Handler, + ServerStreams: true, + }, + { + StreamName: "Inspect", + Handler: _Pilosa_Inspect_Handler, + ServerStreams: true, + }, + }, + Metadata: "pilosa.proto", +} + +func init() { proto.RegisterFile("pilosa.proto", fileDescriptor_ef0691a44d1e275c) } + +var fileDescriptor_ef0691a44d1e275c = []byte{ + // 483 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x53, 0x51, 0x6f, 0xd3, 0x30, + 0x10, 0x4e, 0x9a, 0xb4, 0x69, 0x2e, 0x55, 0x19, 0x27, 0x40, 0x51, 0x85, 0x50, 0xc8, 0x03, 0x8a, + 0x40, 0x1a, 0xa8, 0x20, 0x04, 0x68, 0x2f, 0x0c, 0x09, 0xb5, 0x02, 0x89, 0xcd, 0x88, 0xbd, 0xbb, + 0x8b, 0x07, 0xd1, 0xbc, 0x38, 0x8b, 0xd3, 0x41, 0x5e, 0xf8, 0x4b, 0xfc, 0x45, 0x64, 0x27, 0x4e, + 0x93, 0x89, 0xf2, 0x14, 0xe7, 0xfb, 0xbe, 0xbb, 0xf3, 0x7d, 0xbe, 0x83, 0x59, 0x91, 0x71, 0x21, + 0xe9, 0x61, 0x51, 0x8a, 0x4a, 0xe0, 0x58, 0x7f, 0xe2, 0xb7, 0x70, 0xe7, 0x74, 0xcb, 0xca, 0xfa, + 0xe4, 0xf4, 0x33, 0x61, 0xd7, 0x5b, 0x26, 0x2b, 0xbc, 0x07, 0xe3, 0x2c, 0x4f, 0xd9, 0xaf, 0xd0, + 0x8e, 0xec, 0xc4, 0x27, 0xcd, 0x0f, 0x1e, 0x80, 0x53, 0x5c, 0xf3, 0x70, 0xa4, 0x31, 0x75, 0x8c, + 0x2f, 0x21, 0x20, 0xe2, 0x27, 0x61, 0xb2, 0x10, 0xb9, 0x64, 0xf8, 0x0c, 0xbc, 0x1f, 0x8c, 0xa6, + 0xac, 0x94, 0xa1, 0x1d, 0x39, 0x49, 0xb0, 0xbc, 0xdb, 0x54, 0x3a, 0xfc, 0x20, 0xf8, 0xf6, 0x2a, + 0x5f, 0xe7, 0x17, 0x82, 0x18, 0x05, 0x3e, 0x07, 0xef, 0x5c, 0xc3, 0x32, 0x1c, 0x69, 0xf1, 0xfd, + 0x81, 0xd8, 0x24, 0x25, 0x46, 0x15, 0x1f, 0x01, 0xec, 0xf2, 0x20, 0x82, 0x9b, 0xd3, 0x2b, 0xd6, + 0xde, 0x50, 0x9f, 0x71, 0x01, 0xd3, 0x94, 0x56, 0xb4, 0xaa, 0x0b, 0xd6, 0xde, 0xb2, 0xfb, 0x8f, + 0xff, 0x8c, 0x60, 0x3e, 0xcc, 0x8c, 0x8f, 0xc0, 0x97, 0x55, 0x99, 0xe5, 0xdf, 0xcf, 0x28, 0x6f, + 0xf2, 0xac, 0x2c, 0xb2, 0x83, 0x14, 0xbf, 0xcd, 0xf2, 0xea, 0xf5, 0x2b, 0xc5, 0xab, 0x7c, 0xae, + 0xe2, 0x3b, 0x08, 0x1f, 0xc2, 0xb4, 0xa3, 0x9d, 0xc8, 0x4e, 0x9c, 0x95, 0x45, 0x3a, 0x04, 0x17, + 0xe0, 0x6d, 0x84, 0xe0, 0x8a, 0x74, 0x23, 0x3b, 0x99, 0xae, 0x2c, 0x62, 0x00, 0xcd, 0x71, 0xb1, + 0x51, 0xdc, 0x38, 0xb2, 0x93, 0x99, 0xe6, 0x1a, 0x00, 0x8f, 0x60, 0xde, 0x94, 0x78, 0x5f, 0x96, + 0xb4, 0x56, 0x92, 0x49, 0x64, 0x27, 0xc1, 0x12, 0x5b, 0x7b, 0xbe, 0xed, 0xc8, 0x95, 0x45, 0x6e, + 0x69, 0x55, 0x74, 0xd3, 0x40, 0x17, 0xed, 0x0d, 0xa2, 0xbf, 0xee, 0x48, 0x15, 0x3d, 0xd4, 0x1e, + 0x07, 0xe0, 0x37, 0x6e, 0x9f, 0x51, 0x1e, 0xdf, 0xc0, 0x7c, 0x9d, 0xcb, 0x82, 0x9d, 0x57, 0xff, + 0x1f, 0x8b, 0xa7, 0xfd, 0x87, 0x54, 0xb5, 0x0e, 0xda, 0x5a, 0xeb, 0x54, 0x7e, 0x29, 0x3f, 0xb1, + 0x5a, 0x76, 0x6f, 0x88, 0x31, 0xcc, 0x2e, 0x32, 0x5e, 0xb1, 0xf2, 0x63, 0xc6, 0x78, 0x2a, 0x43, + 0x27, 0x72, 0x12, 0x9f, 0x0c, 0xb0, 0xf8, 0x31, 0x04, 0xbd, 0x1e, 0xd5, 0x43, 0xdf, 0x50, 0xde, + 0x4c, 0x94, 0x4b, 0xf4, 0x59, 0x49, 0x7a, 0x8d, 0x0c, 0x24, 0x7e, 0x2b, 0x61, 0xe0, 0x77, 0xf5, + 0xf1, 0x09, 0x38, 0x59, 0x2a, 0xf5, 0xb5, 0xf7, 0x19, 0xa9, 0x04, 0x98, 0x80, 0x7b, 0xc9, 0x6a, + 0xd3, 0xc7, 0xbf, 0x3d, 0xd3, 0x8a, 0xe3, 0x09, 0xb8, 0x6a, 0xac, 0x96, 0xbf, 0x61, 0x72, 0xa2, + 0x77, 0x0a, 0xdf, 0xc1, 0xd4, 0xac, 0x11, 0x3e, 0x68, 0x23, 0x6f, 0xed, 0xd5, 0xc2, 0x64, 0xec, + 0x2d, 0x4d, 0x6c, 0xbd, 0xb0, 0xf1, 0x0d, 0x78, 0xad, 0xd5, 0x68, 0xb6, 0x60, 0x68, 0xfd, 0xbe, + 0xc8, 0xcd, 0x44, 0xc3, 0x2f, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0xe2, 0xfa, 0x94, 0xa4, 0xda, + 0x03, 0x00, 0x00, +} diff --git a/proto/pilosa.proto b/proto/pilosa.proto new file mode 100644 index 000000000..ae6e40860 --- /dev/null +++ b/proto/pilosa.proto @@ -0,0 +1,56 @@ +syntax = "proto3"; +package proto; + +message QueryPQLRequest { + string index = 1; + string pql = 2; +} + + +message RowResponse{ + repeated ColumnInfo headers = 1; + repeated ColumnResponse columns = 2; +} + +message ColumnInfo { + string name = 1; + string datatype = 2; +} + +message ColumnResponse{ + oneof columnVal { + string stringVal = 1; + uint64 uint64Val = 2; + int64 int64Val = 3; + bool boolVal = 4; + bytes blobVal = 5; + Uint64Array uint64ArrayVal = 6; + StringArray stringArrayVal = 7; + } +} + +message InspectRequest { + string index = 1; + IdsOrKeys columns = 2; + repeated string filterFields = 3; +} + +message Uint64Array { + repeated uint64 vals = 1; +} + +message StringArray { + repeated string vals = 1; +} + +message IdsOrKeys { + oneof type { + Uint64Array ids = 1; + StringArray keys = 2; + } +} + +service Pilosa { + rpc QueryPQL(QueryPQLRequest) returns (stream RowResponse) {}; + rpc Inspect(InspectRequest) returns (stream RowResponse) {}; +} diff --git a/row.go b/row.go index fd2104967..90e19d4b1 100644 --- a/row.go +++ b/row.go @@ -347,6 +347,17 @@ func (s *rowSegment) Freeze() { s.data.Freeze() } +/* +// Raw returns the row segment as a byte slice. +// It may be used by the gRPC server to deliver results +// as a roaring bitmap instead of a stream of RowResults. +func (s *rowSegment) Raw() (uint64, []byte) { + var buf bytes.Buffer + s.data.WriteTo(&buf) + return s.shard, buf.Bytes() +} +*/ + // Merge adds chunks from other to s. // Chunks in s are overwritten if they exist in other. func (s *rowSegment) Merge(other *rowSegment) { diff --git a/server/config.go b/server/config.go index d33af0596..379fe6816 100644 --- a/server/config.go +++ b/server/config.go @@ -53,6 +53,9 @@ type Config struct { // Bind is the host:port on which Pilosa will listen. Bind string `toml:"bind"` + // BindGRPC is the host:port on which Pilosa will bind for gRPC. + BindGRPC string `toml:"bind-grpc"` + // Advertise is the address advertised by the server to other nodes // in the cluster. It should be reachable by all other nodes and should // route to an interface that Bind is listening on. @@ -161,6 +164,7 @@ func NewConfig() *Config { c := &Config{ DataDir: "~/.pilosa", Bind: ":10101", + BindGRPC: ":20101", MaxWritesPerRequest: 5000, // We default these Max File/Map counts very high. This is basically a @@ -233,6 +237,13 @@ func (cfg *Config) validateAddrs(ctx context.Context) error { } cfg.Bind = schemeHostPortString(listenScheme, listenHost, listenPort) + // Validate the gRPC listen address. + grpcListenScheme, grpcListenHost, grpcListenPort, err := validateListenAddr(ctx, cfg.BindGRPC) + if err != nil { + return errors.Wrap(err, "validating grpc listen address") + } + cfg.BindGRPC = schemeHostPortString(grpcListenScheme, grpcListenHost, grpcListenPort) + return nil } diff --git a/server/grpc.go b/server/grpc.go new file mode 100644 index 000000000..f35113001 --- /dev/null +++ b/server/grpc.go @@ -0,0 +1,584 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "context" + "fmt" + "log" + "net" + + "github.com/pilosa/pilosa/v2" + pb "github.com/pilosa/pilosa/v2/proto" + "github.com/pkg/errors" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/reflection" + "google.golang.org/grpc/status" +) + +// grpcHandler contains methods which handle the various gRPC requests. +type grpcHandler struct { + api *pilosa.API +} + +// QueryPQL handles the PQL request and sends RowResponses to the stream. +func (h grpcHandler) QueryPQL(req *pb.QueryPQLRequest, stream pb.Pilosa_QueryPQLServer) error { + query := pilosa.QueryRequest{ + Index: req.Index, + Query: req.Pql, + } + resp, err := h.api.Query(context.Background(), &query) + if err != nil { + return status.Error(codes.Unknown, err.Error()) + } + for row := range makeRows(resp) { + err = stream.Send(row) + if err != nil { + return status.Error(codes.Unknown, err.Error()) + } + } + + return nil +} + +// Inspect handles the inpspect request and sends an InspectResponse to the stream. +func (h grpcHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectServer) error { + index, err := h.api.Index(context.Background(), req.Index) + if err != nil { + return errors.Wrap(err, "getting index") + } + + var fields []*pilosa.Field + for _, field := range index.Fields() { + if len(req.FilterFields) > 0 { + for _, filter := range req.FilterFields { + if filter == field.Name() { + fields = append(fields, field) + break + } + + } + } else { + fields = append(fields, field) + } + } + + // If there are no matching fields, then don't return any records. + if len(fields) == 0 { + return nil + } + + if ints, ok := req.Columns.Type.(*pb.IdsOrKeys_Ids); ok { + ci := []*pb.ColumnInfo{ + {Name: "_id", Datatype: "uint64"}, + } + for _, field := range fields { + ci = append(ci, &pb.ColumnInfo{Name: field.Name(), Datatype: field.Type()}) // TODO: field.Type likely doesn't align with supported datatypes + } + + for _, col := range ints.Ids.Vals { + rowResp := &pb.RowResponse{ + Headers: ci, + Columns: []*pb.ColumnResponse{ + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{col}}, + }, + } + ci = nil // only include headers with the first row + + for _, field := range fields { + // TODO: handle `time` fields + switch field.Type() { + case "set": + pql := fmt.Sprintf("Rows(%s, column=%d)", field.Name(), col) + query := pilosa.QueryRequest{ + Index: req.Index, + Query: pql, + } + resp, err := h.api.Query(context.Background(), &query) + if err != nil { + return errors.Wrap(err, "querying rows") + } + + ids, ok := resp.Results[0].(pilosa.RowIdentifiers) + if !ok { + return errors.Wrap(err, "getting row identifiers") + } + + if len(ids.Keys) > 0 { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringArrayVal{&pb.StringArray{Vals: ids.Keys}}}) + } else { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64ArrayVal{&pb.Uint64Array{Vals: ids.Rows}}}) + } + + case "mutex": + pql := fmt.Sprintf("Rows(%s, column=%d)", field.Name(), col) + query := pilosa.QueryRequest{ + Index: req.Index, + Query: pql, + } + resp, err := h.api.Query(context.Background(), &query) + if err != nil { + return errors.Wrap(err, "querying rows") + } + + ids, ok := resp.Results[0].(pilosa.RowIdentifiers) + if !ok { + return errors.Wrap(err, "getting row identifiers") + } + + if len(ids.Keys) == 1 { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{ids.Keys[0]}}) + } else if len(ids.Rows) == 1 { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{ids.Rows[0]}}) + } else { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: nil}) + } + + case "int": + value, exists, err := field.Value(col) + if err != nil { + return errors.Wrap(err, "getting int field value for column") + } else if exists { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{value}}) + } else { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: nil}) + } + + case "bool": + pql := fmt.Sprintf("Rows(%s, column=%d)", field.Name(), col) + query := pilosa.QueryRequest{ + Index: req.Index, + Query: pql, + } + resp, err := h.api.Query(context.Background(), &query) + if err != nil { + return errors.Wrap(err, "querying rows") + } + + ids, ok := resp.Results[0].(pilosa.RowIdentifiers) + if !ok { + return errors.Wrap(err, "getting row identifiers") + } + + if len(ids.Rows) == 1 { + var bval bool + if ids.Rows[0] == 1 { + bval = true + } + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_BoolVal{bval}}) + } else { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: nil}) + } + } + } + + if err := stream.Send(rowResp); err != nil { + return errors.Wrap(err, "sending response to stream") + } + } + + } else if keys, ok := req.Columns.Type.(*pb.IdsOrKeys_Keys); ok { + ci := []*pb.ColumnInfo{ + {Name: "_id", Datatype: "string"}, + } + for _, field := range fields { + ci = append(ci, &pb.ColumnInfo{Name: field.Name(), Datatype: field.Type()}) // TODO: field.Type likely doesn't align with supported datatypes + } + + for _, col := range keys.Keys.Vals { + rowResp := &pb.RowResponse{ + Headers: ci, + Columns: []*pb.ColumnResponse{ + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{col}}, + }, + } + ci = nil // only include headers with the first row + + for _, field := range fields { + // TODO: handle `time` fields + switch field.Type() { + case "set": + pql := fmt.Sprintf("Rows(%s, column=\"%s\")", field.Name(), col) + query := pilosa.QueryRequest{ + Index: req.Index, + Query: pql, + } + resp, err := h.api.Query(context.Background(), &query) + if err != nil { + return errors.Wrap(err, "querying rows") + } + + ids, ok := resp.Results[0].(pilosa.RowIdentifiers) + if !ok { + return errors.Wrap(err, "getting row identifiers") + } + + if len(ids.Keys) > 0 { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringArrayVal{&pb.StringArray{Vals: ids.Keys}}}) + } else { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64ArrayVal{&pb.Uint64Array{Vals: ids.Rows}}}) + } + + case "mutex": + pql := fmt.Sprintf("Rows(%s, column=\"%s\")", field.Name(), col) + query := pilosa.QueryRequest{ + Index: req.Index, + Query: pql, + } + resp, err := h.api.Query(context.Background(), &query) + if err != nil { + return errors.Wrap(err, "querying rows") + } + + ids, ok := resp.Results[0].(pilosa.RowIdentifiers) + if !ok { + return errors.Wrap(err, "getting row identifiers") + } + + if len(ids.Keys) == 1 { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{ids.Keys[0]}}) + } else if len(ids.Rows) == 1 { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{ids.Rows[0]}}) + } else { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: nil}) + } + + case "int": + // Translate column key. + id, err := index.TranslateStore().TranslateKey(col) + if err != nil { + return errors.Wrap(err, "translating column key") + } + + value, exists, err := field.Value(id) + if err != nil { + return errors.Wrap(err, "getting int field value for column") + } else if exists { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Int64Val{value}}) + } else { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: nil}) + } + + case "bool": + pql := fmt.Sprintf("Rows(%s, column=\"%s\")", field.Name(), col) + query := pilosa.QueryRequest{ + Index: req.Index, + Query: pql, + } + resp, err := h.api.Query(context.Background(), &query) + if err != nil { + return errors.Wrap(err, "querying rows") + } + + ids, ok := resp.Results[0].(pilosa.RowIdentifiers) + if !ok { + return errors.Wrap(err, "getting row identifiers") + } + + if len(ids.Rows) == 1 { + var bval bool + if ids.Rows[0] == 1 { + bval = true + } + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_BoolVal{bval}}) + } else { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: nil}) + } + } + } + + if err := stream.Send(rowResp); err != nil { + return errors.Wrap(err, "sending response to stream") + } + } + + } + return nil +} + +// I think ideally this would be plugged in the executor somewhere +// in order to get some concurrency benefit but we can +// start with the combined response +func makeRows(resp pilosa.QueryResponse) chan *pb.RowResponse { + results := make(chan *pb.RowResponse) + go func() { + for _, result := range resp.Results { + switch r := result.(type) { + case *pilosa.Row: + if len(r.Keys) > 0 { + // Column keys + ci := []*pb.ColumnInfo{ + {Name: "_id", Datatype: "string"}, + } + for _, x := range r.Keys { + results <- &pb.RowResponse{ + Headers: ci, + Columns: []*pb.ColumnResponse{ + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{x}}, + }} + ci = nil //only send on the first + } + } else { + // Column IDs + ci := []*pb.ColumnInfo{ + {Name: "_id", Datatype: "uint64"}, + } + for _, x := range r.Columns() { + results <- &pb.RowResponse{ + Headers: ci, + Columns: []*pb.ColumnResponse{ + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{x}}, + }} + ci = nil //only send on the first + } + + // The following will return roaring segments. + // This is commented out for now until we decide how we want to use this. + /* + // Roaring segments + ci := []*pb.ColumnInfo{ + // TODO: + {Name: "shard", Datatype: "uint64"}, + {Name: "segment", Datatype: "roaring"}, + } + for _, x := range r.Segments() { + shard, b := x.Raw() + results <- &pb.RowResponse{ + Headers: ci, + Columns: []*pb.ColumnResponse{ + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_IntVal{int64(shard)}}, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_BlobVal{b}}, + }} + ci = nil //only send on the first + } + */ + } + case pilosa.Pair: + if r.Key != "" { + results <- &pb.RowResponse{ + Headers: []*pb.ColumnInfo{ + {Name: "_id", Datatype: "string"}, + {Name: "count", Datatype: "uint64"}, + }, + Columns: []*pb.ColumnResponse{ + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{r.Key}}, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{r.Count}}, + }, + } + } else { + results <- &pb.RowResponse{ + Headers: []*pb.ColumnInfo{ + {Name: "_id", Datatype: "uint64"}, + {Name: "count", Datatype: "uint64"}, + }, + Columns: []*pb.ColumnResponse{ + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{r.ID}}, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{r.Count}}, + }, + } + } + case []pilosa.Pair: + // Determine if the ID has string keys. + var stringKeys bool + if len(r) > 0 { + if r[0].Key != "" { + stringKeys = true + } + } + + dtype := "uint64" + if stringKeys { + dtype = "string" + } + ci := []*pb.ColumnInfo{ + {Name: "_id", Datatype: dtype}, + {Name: "count", Datatype: "uint64"}, + } + for _, pair := range r { + if stringKeys { + results <- &pb.RowResponse{ + Headers: ci, + Columns: []*pb.ColumnResponse{ + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{pair.Key}}, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{uint64(pair.Count)}}, + }, + } + } else { + results <- &pb.RowResponse{ + Headers: ci, + Columns: []*pb.ColumnResponse{ + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{uint64(pair.ID)}}, + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{uint64(pair.Count)}}, + }, + } + } + ci = nil //only send on the first + } + case []pilosa.GroupCount: + for i, gc := range r { + var ci []*pb.ColumnInfo + if i == 0 { + for _, fieldRow := range gc.Group { + if fieldRow.RowKey != "" { + ci = append(ci, &pb.ColumnInfo{Name: fieldRow.Field, Datatype: "string"}) + } else { + ci = append(ci, &pb.ColumnInfo{Name: fieldRow.Field, Datatype: "uint64"}) + } + } + ci = append(ci, &pb.ColumnInfo{Name: "count", Datatype: "uint64"}) + } + rowResp := &pb.RowResponse{ + Headers: ci, + Columns: []*pb.ColumnResponse{}, + } + + for _, fieldRow := range gc.Group { + if fieldRow.RowKey != "" { + rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{fieldRow.RowKey}}) + } else { + rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{uint64(fieldRow.RowID)}}) + } + } + rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{uint64(gc.Count)}}) + results <- rowResp + } + case pilosa.RowIdentifiers: + if len(r.Keys) > 0 { + ci := []*pb.ColumnInfo{{Name: "_id", Datatype: "string"}} + for _, key := range r.Keys { + results <- &pb.RowResponse{ + Headers: ci, + Columns: []*pb.ColumnResponse{ + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_StringVal{key}}, + }} + ci = nil + } + } else { + ci := []*pb.ColumnInfo{{Name: "_id", Datatype: "uint64"}} + for _, id := range r.Rows { + results <- &pb.RowResponse{ + Headers: ci, + Columns: []*pb.ColumnResponse{ + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{uint64(id)}}, + }} + ci = nil + } + } + case uint64: + ci := []*pb.ColumnInfo{{Name: "count", Datatype: "uint64"}} + results <- &pb.RowResponse{ + Headers: ci, + Columns: []*pb.ColumnResponse{ + &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64Val{uint64(r)}}, + }} + default: + log.Printf("unhandled %T\n", r) + break + } + } + close(results) + }() + return results +} + +func makeItems(p pilosa.RowIdentifiers) *pb.IdsOrKeys { + if len(p.Keys) == 0 { + //use Rows + results := make([]uint64, len(p.Rows)) + for i, id := range p.Rows { + results[i] = id + } + return &pb.IdsOrKeys{Type: &pb.IdsOrKeys_Ids{Ids: &pb.Uint64Array{Vals: results}}} + } + results := make([]string, len(p.Keys)) + for i, key := range p.Keys { + results[i] = key + } + return &pb.IdsOrKeys{Type: &pb.IdsOrKeys_Keys{Keys: &pb.StringArray{Vals: results}}} +} + +type grpcServer struct { + api *pilosa.API + hostPort string +} + +type grpcServerOption func(s *grpcServer) error + +func OptGRPCServerAPI(api *pilosa.API) grpcServerOption { + return func(s *grpcServer) error { + s.api = api + return nil + } +} + +func OptGRPCServerURI(uri *pilosa.URI) grpcServerOption { + hostport := fmt.Sprintf("%s:%d", uri.Host, uri.Port) + return func(h *grpcServer) error { + h.hostPort = hostport + return nil + } +} + +func (s *grpcServer) Serve() error { + // create listener + lis, err := net.Listen("tcp", s.hostPort) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + log.Printf("enabled grpc listening on %s", s.hostPort) + + // create grpc server + srv := grpc.NewServer() + pb.RegisterPilosaServer(srv, grpcHandler{api: s.api}) + + // register the server so its services are available to grpc_cli and others + reflection.Register(srv) + + // and start... + if err := srv.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } + return nil +} + +func NewGRPCServer(opts ...grpcServerOption) (*grpcServer, error) { + server := &grpcServer{} + for _, opt := range opts { + err := opt(server) + if err != nil { + return nil, errors.Wrap(err, "applying option") + } + } + return server, nil +} diff --git a/server/grpc_internal_test.go b/server/grpc_internal_test.go new file mode 100644 index 000000000..f43fa7cb3 --- /dev/null +++ b/server/grpc_internal_test.go @@ -0,0 +1,254 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "testing" + + "github.com/pilosa/pilosa/v2" +) + +func TestGRPC(t *testing.T) { + t.Run("makeRows", func(t *testing.T) { + type expHeader struct { + name string + dataType string + } + + type expColumn interface{} + + tests := []struct { + result interface{} + expHeaders []expHeader + expColumns [][]expColumn + }{ + // Row (uint64) + { + pilosa.NewRow(10, 11, 12), + []expHeader{ + {"_id", "uint64"}, + }, + [][]expColumn{ + {uint64(10)}, + {uint64(11)}, + {uint64(12)}, + }, + }, + // Row (string) + { + &pilosa.Row{Keys: []string{"ten", "eleven", "twelve"}}, + []expHeader{ + {"_id", "string"}, + }, + [][]expColumn{ + {"ten"}, + {"eleven"}, + {"twelve"}, + }, + }, + // Pair (uint64) + { + pilosa.Pair{ID: 10, Count: 123}, + []expHeader{ + {"_id", "uint64"}, + {"count", "uint64"}, + }, + [][]expColumn{ + {uint64(10), uint64(123)}, + }, + }, + // Pair (string) + { + pilosa.Pair{Key: "ten", Count: 123}, + []expHeader{ + {"_id", "string"}, + {"count", "uint64"}, + }, + [][]expColumn{ + {string("ten"), uint64(123)}, + }, + }, + // []Pair (uint64) + { + []pilosa.Pair{ + {ID: 10, Count: 123}, + {ID: 11, Count: 456}, + }, + []expHeader{ + {"_id", "uint64"}, + {"count", "uint64"}, + }, + [][]expColumn{ + {uint64(10), uint64(123)}, + {uint64(11), uint64(456)}, + }, + }, + // []Pair (string) + { + []pilosa.Pair{ + {Key: "ten", Count: 123}, + {Key: "eleven", Count: 456}, + }, + []expHeader{ + {"_id", "string"}, + {"count", "uint64"}, + }, + [][]expColumn{ + {"ten", uint64(123)}, + {"eleven", uint64(456)}, + }, + }, + // []GroupCount (uint64) + { + []pilosa.GroupCount{ + { + []pilosa.FieldRow{ + {Field: "a", RowID: 10}, + {Field: "b", RowID: 11}, + }, + 123, + }, + { + []pilosa.FieldRow{ + {Field: "a", RowID: 10}, + {Field: "b", RowID: 12}, + }, + 456, + }, + }, + []expHeader{ + {"a", "uint64"}, + {"b", "uint64"}, + {"count", "uint64"}, + }, + [][]expColumn{ + {uint64(10), uint64(11), uint64(123)}, + {uint64(10), uint64(12), uint64(456)}, + }, + }, + // []GroupCount (string) + { + []pilosa.GroupCount{ + { + []pilosa.FieldRow{ + {Field: "a", RowKey: "ten"}, + {Field: "b", RowKey: "eleven"}, + }, + 123, + }, + { + []pilosa.FieldRow{ + {Field: "a", RowKey: "ten"}, + {Field: "b", RowKey: "twelve"}, + }, + 456, + }, + }, + []expHeader{ + {"a", "string"}, + {"b", "string"}, + {"count", "uint64"}, + }, + [][]expColumn{ + {"ten", "eleven", uint64(123)}, + {"ten", "twelve", uint64(456)}, + }, + }, + // RowIdentifiers (uint64) + { + pilosa.RowIdentifiers{ + Rows: []uint64{10, 11, 12}, + }, + []expHeader{ + {"_id", "uint64"}, + }, + [][]expColumn{ + {uint64(10)}, + {uint64(11)}, + {uint64(12)}, + }, + }, + // RowIdentifiers (string) + { + pilosa.RowIdentifiers{ + Keys: []string{"ten", "eleven", "twelve"}, + }, + []expHeader{ + {"_id", "string"}, + }, + [][]expColumn{ + {"ten"}, + {"eleven"}, + {"twelve"}, + }, + }, + // uint64 + { + uint64(123), + []expHeader{ + {"count", "uint64"}, + }, + [][]expColumn{ + {uint64(123)}, + }, + }, + } + + for ti, test := range tests { + results := make([]interface{}, 0) + results = append(results, test.result) + + qr := pilosa.QueryResponse{} + qr.Results = results + + ch := makeRows(qr) + + cnt := 0 + for row := range ch { + // Ensure headers match (on the first row). + if cnt == 0 { + for i, header := range row.GetHeaders() { + if header.Name != test.expHeaders[i].name { + t.Fatalf("test %d expected header name: %s, but got: %s", ti, test.expHeaders[i].name, header.Name) + } + if header.Datatype != test.expHeaders[i].dataType { + t.Fatalf("test %d expected header data type: %s, but got: %s", ti, test.expHeaders[i].dataType, header.Datatype) + } + } + } + + // Ensure column data matches. + for i, column := range row.GetColumns() { + switch v := test.expColumns[cnt][i].(type) { + case string: + val := column.GetStringVal() + if val != v { + t.Fatalf("test %d expected column val: %v, but got: %v", ti, v, val) + } + case uint64: + val := column.GetUint64Val() + if val != v { + t.Fatalf("test %d expected column val: %v, but got: %v", ti, v, val) + } + default: + t.Fatalf("test %d has unhandled data type: %T", ti, v) + } + } + + cnt++ + } + } + }) +} diff --git a/server/server.go b/server/server.go index 758c18310..64d20ce04 100644 --- a/server/server.go +++ b/server/server.go @@ -80,6 +80,7 @@ type Command struct { logger loggerLogger Handler pilosa.Handler + grpcServer *grpcServer API *pilosa.API ln net.Listener listenURI *pilosa.URI @@ -163,6 +164,11 @@ func (m *Command) Start() (err error) { } m.logger.Printf("listening as %s\n", m.listenURI) + go func() { + if err := m.grpcServer.Serve(); err != nil { + m.logger.Printf("grpc server error: %v", err) + } + }() close(m.Started) return nil @@ -251,6 +257,11 @@ func (m *Command) SetupServer() error { return errors.Wrap(err, "processing bind address") } + grpcURI, err := pilosa.NewURIFromAddress(m.Config.BindGRPC) + if err != nil { + return errors.Wrap(err, "processing bind grpc address") + } + // Setup TLS var TLSConfig *tls.Config if uri.Scheme == "https" { @@ -351,7 +362,12 @@ func (m *Command) SetupServer() error { http.OptHandlerListener(m.ln), http.OptHandlerCloseTimeout(m.closeTimeout), ) - return errors.Wrap(err, "new handler") + if err != nil { + return errors.Wrap(err, "new handler") + } + + m.grpcServer, err = NewGRPCServer(OptGRPCServerAPI(m.API), OptGRPCServerURI(grpcURI)) + return errors.Wrap(err, "new grpc server") } // setupNetworking sets up internode communication based on the configuration. diff --git a/test/pilosa.go b/test/pilosa.go index fab3c65be..73935290c 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -73,6 +73,9 @@ func newCommand(opts ...server.CommandOption) *Command { if m.Config.Bind == defaultConf.Bind { m.Config.Bind = "http://localhost:0" } + if m.Config.BindGRPC == defaultConf.BindGRPC { + m.Config.BindGRPC = "http://localhost:0" + } m.Config.Translation.MapSize = 140000 m.Config.WorkerPoolSize = 2