From 1b210080c61879c575546fbf56bf9917fd02e6e7 Mon Sep 17 00:00:00 2001 From: Nia Weiss Date: Tue, 29 Sep 2020 13:54:16 -0400 Subject: [PATCH 01/53] fix race condition when resetting translation Previously, we never waited for translation sync goroutines to stop. That issue should be mostly harmless in the normal path. Additionally, this waits for the translation sync to shut down when stopping the server. --- boltdb/translate_test.go | 25 +++++++++++++++++++++++++ holder.go | 16 +++++++++++++--- server.go | 5 +++++ 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/boltdb/translate_test.go b/boltdb/translate_test.go index 8c7a23c18..8c3915f00 100644 --- a/boltdb/translate_test.go +++ b/boltdb/translate_test.go @@ -20,6 +20,7 @@ import ( "io/ioutil" "os" "reflect" + "strconv" "testing" "time" @@ -216,6 +217,30 @@ func TestTranslateStore_TranslateIDs(t *testing.T) { } } +func TestTranslateStore_MaxID(t *testing.T) { + s := MustOpenNewTranslateStore() + defer MustCloseTranslateStore(s) + + // Generate a bunch of keys. + var lastk uint64 + for i := 0; i < 1026; i++ { + k, err := s.TranslateKey(strconv.Itoa(i), true) + if err != nil { + t.Fatalf("translating %d: %v", i, err) + } + lastk = k + } + + // Verify the max ID. + max, err := s.MaxID() + if err != nil { + t.Fatalf("checking max ID: %v", err) + } + if max != lastk { + t.Fatalf("last key is %d but max is %d", lastk, max) + } +} + func TestTranslateStore_EntryReader(t *testing.T) { t.Run("OK", func(t *testing.T) { s := MustOpenNewTranslateStore() diff --git a/holder.go b/holder.go index 1b8700020..5d8691a92 100644 --- a/holder.go +++ b/holder.go @@ -1275,6 +1275,8 @@ type holderSyncer struct { // Translation sync handling. readers []TranslateEntryReader + syncers errgroup.Group + // Stats Stats stats.StatsClient @@ -1569,6 +1571,7 @@ func (s *holderSyncer) stopTranslationSync() error { return rd.Close() }) } + g.Go(s.syncers.Wait) return g.Wait() } @@ -1658,7 +1661,11 @@ func (s *holderSyncer) initializeIndexTranslateReplication() error { } s.readers = append(s.readers, rd) - go func() { defer rd.Close(); s.readIndexTranslateReader(rd) }() + s.syncers.Go(func() error { + defer rd.Close() + s.readIndexTranslateReader(rd) + return nil + }) } return nil @@ -1697,8 +1704,11 @@ func (s *holderSyncer) initializeFieldTranslateReplication() error { } s.readers = append(s.readers, rd) - go func() { defer rd.Close(); s.readFieldTranslateReader(rd) }() - + s.syncers.Go(func() error { + defer rd.Close() + s.readFieldTranslateReader(rd) + return nil + }) return nil } diff --git a/server.go b/server.go index a2ceadc01..a93df180a 100644 --- a/server.go +++ b/server.go @@ -534,10 +534,12 @@ func (s *Server) Close() error { s.wg.Wait() var errh error + var errhs error var errc error if s.cluster != nil { errc = s.cluster.close() } + errhs = s.syncer.stopTranslationSync() if s.holder != nil { errh = s.holder.Close() } @@ -553,6 +555,9 @@ func (s *Server) Close() error { if errh != nil { return errors.Wrap(errh, "closing holder") } + if errhs != nil { + return errors.Wrap(errhs, "terminating holder translation sync") + } if errc != nil { return errors.Wrap(errc, "closing cluster") } From c1e50ed909aee7198826e174d93245c8427e1385 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 17 Sep 2020 11:20:58 -0500 Subject: [PATCH 02/53] Remove VDS references from pilosa proto file, begin porting VDSM service --- Makefile | 1 + proto/pilosa.pb.go | 626 ++------------ proto/pilosa.proto | 43 +- proto/vdsm/vdsm.pb.go | 1902 +++++++++++++++++++++++++++++++++++++++++ proto/vdsm/vdsm.proto | 146 ++++ server/grpc.go | 132 ++- 6 files changed, 2200 insertions(+), 650 deletions(-) create mode 100644 proto/vdsm/vdsm.pb.go create mode 100644 proto/vdsm/vdsm.proto diff --git a/Makefile b/Makefile index ba3d994a8..f6ca2fe1f 100644 --- a/Makefile +++ b/Makefile @@ -169,6 +169,7 @@ generate-pql: require-peg # dunno if protoc-gen-gofast is actually needed here generate-proto-grpc: require-protoc require-protoc-gen-gofast protoc -I proto proto/pilosa.proto --go_out=plugins=grpc:proto + protoc -I proto proto/vdsm/vdsm.proto --go_out=plugins=grpc:proto # `go generate` all needed packages generate: generate-protoc generate-statik generate-stringer generate-pql diff --git a/proto/pilosa.pb.go b/proto/pilosa.pb.go index a48ae519b..45ac65612 100644 --- a/proto/pilosa.pb.go +++ b/proto/pilosa.pb.go @@ -24,349 +24,6 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package -type VDS struct { - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *VDS) Reset() { *m = VDS{} } -func (m *VDS) String() string { return proto.CompactTextString(m) } -func (*VDS) ProtoMessage() {} -func (*VDS) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{0} -} - -func (m *VDS) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_VDS.Unmarshal(m, b) -} -func (m *VDS) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_VDS.Marshal(b, m, deterministic) -} -func (m *VDS) XXX_Merge(src proto.Message) { - xxx_messageInfo_VDS.Merge(m, src) -} -func (m *VDS) XXX_Size() int { - return xxx_messageInfo_VDS.Size(m) -} -func (m *VDS) XXX_DiscardUnknown() { - xxx_messageInfo_VDS.DiscardUnknown(m) -} - -var xxx_messageInfo_VDS proto.InternalMessageInfo - -func (m *VDS) GetName() string { - if m != nil { - return m.Name - } - return "" -} - -type GetVDSsRequest struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *GetVDSsRequest) Reset() { *m = GetVDSsRequest{} } -func (m *GetVDSsRequest) String() string { return proto.CompactTextString(m) } -func (*GetVDSsRequest) ProtoMessage() {} -func (*GetVDSsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{1} -} - -func (m *GetVDSsRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetVDSsRequest.Unmarshal(m, b) -} -func (m *GetVDSsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetVDSsRequest.Marshal(b, m, deterministic) -} -func (m *GetVDSsRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetVDSsRequest.Merge(m, src) -} -func (m *GetVDSsRequest) XXX_Size() int { - return xxx_messageInfo_GetVDSsRequest.Size(m) -} -func (m *GetVDSsRequest) XXX_DiscardUnknown() { - xxx_messageInfo_GetVDSsRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_GetVDSsRequest proto.InternalMessageInfo - -type GetVDSsResponse struct { - Vdss []*VDS `protobuf:"bytes,1,rep,name=vdss,proto3" json:"vdss,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *GetVDSsResponse) Reset() { *m = GetVDSsResponse{} } -func (m *GetVDSsResponse) String() string { return proto.CompactTextString(m) } -func (*GetVDSsResponse) ProtoMessage() {} -func (*GetVDSsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{2} -} - -func (m *GetVDSsResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetVDSsResponse.Unmarshal(m, b) -} -func (m *GetVDSsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetVDSsResponse.Marshal(b, m, deterministic) -} -func (m *GetVDSsResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetVDSsResponse.Merge(m, src) -} -func (m *GetVDSsResponse) XXX_Size() int { - return xxx_messageInfo_GetVDSsResponse.Size(m) -} -func (m *GetVDSsResponse) XXX_DiscardUnknown() { - xxx_messageInfo_GetVDSsResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_GetVDSsResponse proto.InternalMessageInfo - -func (m *GetVDSsResponse) GetVdss() []*VDS { - if m != nil { - return m.Vdss - } - return nil -} - -type GetVDSRequest struct { - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *GetVDSRequest) Reset() { *m = GetVDSRequest{} } -func (m *GetVDSRequest) String() string { return proto.CompactTextString(m) } -func (*GetVDSRequest) ProtoMessage() {} -func (*GetVDSRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{3} -} - -func (m *GetVDSRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetVDSRequest.Unmarshal(m, b) -} -func (m *GetVDSRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetVDSRequest.Marshal(b, m, deterministic) -} -func (m *GetVDSRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetVDSRequest.Merge(m, src) -} -func (m *GetVDSRequest) XXX_Size() int { - return xxx_messageInfo_GetVDSRequest.Size(m) -} -func (m *GetVDSRequest) XXX_DiscardUnknown() { - xxx_messageInfo_GetVDSRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_GetVDSRequest proto.InternalMessageInfo - -func (m *GetVDSRequest) GetName() string { - if m != nil { - return m.Name - } - return "" -} - -type GetVDSResponse struct { - Vds *VDS `protobuf:"bytes,1,opt,name=vds,proto3" json:"vds,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *GetVDSResponse) Reset() { *m = GetVDSResponse{} } -func (m *GetVDSResponse) String() string { return proto.CompactTextString(m) } -func (*GetVDSResponse) ProtoMessage() {} -func (*GetVDSResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{4} -} - -func (m *GetVDSResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetVDSResponse.Unmarshal(m, b) -} -func (m *GetVDSResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetVDSResponse.Marshal(b, m, deterministic) -} -func (m *GetVDSResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetVDSResponse.Merge(m, src) -} -func (m *GetVDSResponse) XXX_Size() int { - return xxx_messageInfo_GetVDSResponse.Size(m) -} -func (m *GetVDSResponse) XXX_DiscardUnknown() { - xxx_messageInfo_GetVDSResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_GetVDSResponse proto.InternalMessageInfo - -func (m *GetVDSResponse) GetVds() *VDS { - if m != nil { - return m.Vds - } - return nil -} - -type PostVDSRequest struct { - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - Keys bool `protobuf:"varint,2,opt,name=keys,proto3" json:"keys,omitempty"` - TrackExistence bool `protobuf:"varint,3,opt,name=trackExistence,proto3" json:"trackExistence,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *PostVDSRequest) Reset() { *m = PostVDSRequest{} } -func (m *PostVDSRequest) String() string { return proto.CompactTextString(m) } -func (*PostVDSRequest) ProtoMessage() {} -func (*PostVDSRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{5} -} - -func (m *PostVDSRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_PostVDSRequest.Unmarshal(m, b) -} -func (m *PostVDSRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_PostVDSRequest.Marshal(b, m, deterministic) -} -func (m *PostVDSRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_PostVDSRequest.Merge(m, src) -} -func (m *PostVDSRequest) XXX_Size() int { - return xxx_messageInfo_PostVDSRequest.Size(m) -} -func (m *PostVDSRequest) XXX_DiscardUnknown() { - xxx_messageInfo_PostVDSRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_PostVDSRequest proto.InternalMessageInfo - -func (m *PostVDSRequest) GetName() string { - if m != nil { - return m.Name - } - return "" -} - -func (m *PostVDSRequest) GetKeys() bool { - if m != nil { - return m.Keys - } - return false -} - -func (m *PostVDSRequest) GetTrackExistence() bool { - if m != nil { - return m.TrackExistence - } - return false -} - -type PostVDSResponse struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *PostVDSResponse) Reset() { *m = PostVDSResponse{} } -func (m *PostVDSResponse) String() string { return proto.CompactTextString(m) } -func (*PostVDSResponse) ProtoMessage() {} -func (*PostVDSResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{6} -} - -func (m *PostVDSResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_PostVDSResponse.Unmarshal(m, b) -} -func (m *PostVDSResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_PostVDSResponse.Marshal(b, m, deterministic) -} -func (m *PostVDSResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_PostVDSResponse.Merge(m, src) -} -func (m *PostVDSResponse) XXX_Size() int { - return xxx_messageInfo_PostVDSResponse.Size(m) -} -func (m *PostVDSResponse) XXX_DiscardUnknown() { - xxx_messageInfo_PostVDSResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_PostVDSResponse proto.InternalMessageInfo - -type DeleteVDSRequest struct { - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *DeleteVDSRequest) Reset() { *m = DeleteVDSRequest{} } -func (m *DeleteVDSRequest) String() string { return proto.CompactTextString(m) } -func (*DeleteVDSRequest) ProtoMessage() {} -func (*DeleteVDSRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{7} -} - -func (m *DeleteVDSRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_DeleteVDSRequest.Unmarshal(m, b) -} -func (m *DeleteVDSRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_DeleteVDSRequest.Marshal(b, m, deterministic) -} -func (m *DeleteVDSRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeleteVDSRequest.Merge(m, src) -} -func (m *DeleteVDSRequest) XXX_Size() int { - return xxx_messageInfo_DeleteVDSRequest.Size(m) -} -func (m *DeleteVDSRequest) XXX_DiscardUnknown() { - xxx_messageInfo_DeleteVDSRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_DeleteVDSRequest proto.InternalMessageInfo - -func (m *DeleteVDSRequest) GetName() string { - if m != nil { - return m.Name - } - return "" -} - -type DeleteVDSResponse struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *DeleteVDSResponse) Reset() { *m = DeleteVDSResponse{} } -func (m *DeleteVDSResponse) String() string { return proto.CompactTextString(m) } -func (*DeleteVDSResponse) ProtoMessage() {} -func (*DeleteVDSResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{8} -} - -func (m *DeleteVDSResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_DeleteVDSResponse.Unmarshal(m, b) -} -func (m *DeleteVDSResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_DeleteVDSResponse.Marshal(b, m, deterministic) -} -func (m *DeleteVDSResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeleteVDSResponse.Merge(m, src) -} -func (m *DeleteVDSResponse) XXX_Size() int { - return xxx_messageInfo_DeleteVDSResponse.Size(m) -} -func (m *DeleteVDSResponse) XXX_DiscardUnknown() { - xxx_messageInfo_DeleteVDSResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_DeleteVDSResponse proto.InternalMessageInfo - type QueryPQLRequest struct { Index string `protobuf:"bytes,1,opt,name=index,proto3" json:"index,omitempty"` Pql string `protobuf:"bytes,2,opt,name=pql,proto3" json:"pql,omitempty"` @@ -379,7 +36,7 @@ func (m *QueryPQLRequest) Reset() { *m = QueryPQLRequest{} } func (m *QueryPQLRequest) String() string { return proto.CompactTextString(m) } func (*QueryPQLRequest) ProtoMessage() {} func (*QueryPQLRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{9} + return fileDescriptor_ef0691a44d1e275c, []int{0} } func (m *QueryPQLRequest) XXX_Unmarshal(b []byte) error { @@ -425,7 +82,7 @@ func (m *QuerySQLRequest) Reset() { *m = QuerySQLRequest{} } func (m *QuerySQLRequest) String() string { return proto.CompactTextString(m) } func (*QuerySQLRequest) ProtoMessage() {} func (*QuerySQLRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{10} + return fileDescriptor_ef0691a44d1e275c, []int{1} } func (m *QuerySQLRequest) XXX_Unmarshal(b []byte) error { @@ -465,7 +122,7 @@ func (m *StatusError) Reset() { *m = StatusError{} } func (m *StatusError) String() string { return proto.CompactTextString(m) } func (*StatusError) ProtoMessage() {} func (*StatusError) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{11} + return fileDescriptor_ef0691a44d1e275c, []int{2} } func (m *StatusError) XXX_Unmarshal(b []byte) error { @@ -513,7 +170,7 @@ func (m *RowResponse) Reset() { *m = RowResponse{} } func (m *RowResponse) String() string { return proto.CompactTextString(m) } func (*RowResponse) ProtoMessage() {} func (*RowResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{12} + return fileDescriptor_ef0691a44d1e275c, []int{3} } func (m *RowResponse) XXX_Unmarshal(b []byte) error { @@ -566,7 +223,7 @@ func (m *Row) Reset() { *m = Row{} } func (m *Row) String() string { return proto.CompactTextString(m) } func (*Row) ProtoMessage() {} func (*Row) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{13} + return fileDescriptor_ef0691a44d1e275c, []int{4} } func (m *Row) XXX_Unmarshal(b []byte) error { @@ -607,7 +264,7 @@ func (m *TableResponse) Reset() { *m = TableResponse{} } func (m *TableResponse) String() string { return proto.CompactTextString(m) } func (*TableResponse) ProtoMessage() {} func (*TableResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{14} + return fileDescriptor_ef0691a44d1e275c, []int{5} } func (m *TableResponse) XXX_Unmarshal(b []byte) error { @@ -661,7 +318,7 @@ func (m *ColumnInfo) Reset() { *m = ColumnInfo{} } func (m *ColumnInfo) String() string { return proto.CompactTextString(m) } func (*ColumnInfo) ProtoMessage() {} func (*ColumnInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{15} + return fileDescriptor_ef0691a44d1e275c, []int{6} } func (m *ColumnInfo) XXX_Unmarshal(b []byte) error { @@ -717,7 +374,7 @@ func (m *ColumnResponse) Reset() { *m = ColumnResponse{} } func (m *ColumnResponse) String() string { return proto.CompactTextString(m) } func (*ColumnResponse) ProtoMessage() {} func (*ColumnResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{16} + return fileDescriptor_ef0691a44d1e275c, []int{7} } func (m *ColumnResponse) XXX_Unmarshal(b []byte) error { @@ -893,7 +550,7 @@ func (m *Decimal) Reset() { *m = Decimal{} } func (m *Decimal) String() string { return proto.CompactTextString(m) } func (*Decimal) ProtoMessage() {} func (*Decimal) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{17} + return fileDescriptor_ef0691a44d1e275c, []int{8} } func (m *Decimal) XXX_Unmarshal(b []byte) error { @@ -944,7 +601,7 @@ func (m *InspectRequest) Reset() { *m = InspectRequest{} } func (m *InspectRequest) String() string { return proto.CompactTextString(m) } func (*InspectRequest) ProtoMessage() {} func (*InspectRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{18} + return fileDescriptor_ef0691a44d1e275c, []int{9} } func (m *InspectRequest) XXX_Unmarshal(b []byte) error { @@ -1018,7 +675,7 @@ func (m *Uint64Array) Reset() { *m = Uint64Array{} } func (m *Uint64Array) String() string { return proto.CompactTextString(m) } func (*Uint64Array) ProtoMessage() {} func (*Uint64Array) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{19} + return fileDescriptor_ef0691a44d1e275c, []int{10} } func (m *Uint64Array) XXX_Unmarshal(b []byte) error { @@ -1057,7 +714,7 @@ func (m *StringArray) Reset() { *m = StringArray{} } func (m *StringArray) String() string { return proto.CompactTextString(m) } func (*StringArray) ProtoMessage() {} func (*StringArray) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{20} + return fileDescriptor_ef0691a44d1e275c, []int{11} } func (m *StringArray) XXX_Unmarshal(b []byte) error { @@ -1099,7 +756,7 @@ func (m *IdsOrKeys) Reset() { *m = IdsOrKeys{} } func (m *IdsOrKeys) String() string { return proto.CompactTextString(m) } func (*IdsOrKeys) ProtoMessage() {} func (*IdsOrKeys) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{21} + return fileDescriptor_ef0691a44d1e275c, []int{12} } func (m *IdsOrKeys) XXX_Unmarshal(b []byte) error { @@ -1166,15 +823,6 @@ func (*IdsOrKeys) XXX_OneofWrappers() []interface{} { } func init() { - proto.RegisterType((*VDS)(nil), "pilosa.VDS") - proto.RegisterType((*GetVDSsRequest)(nil), "pilosa.GetVDSsRequest") - proto.RegisterType((*GetVDSsResponse)(nil), "pilosa.GetVDSsResponse") - proto.RegisterType((*GetVDSRequest)(nil), "pilosa.GetVDSRequest") - proto.RegisterType((*GetVDSResponse)(nil), "pilosa.GetVDSResponse") - proto.RegisterType((*PostVDSRequest)(nil), "pilosa.PostVDSRequest") - proto.RegisterType((*PostVDSResponse)(nil), "pilosa.PostVDSResponse") - proto.RegisterType((*DeleteVDSRequest)(nil), "pilosa.DeleteVDSRequest") - proto.RegisterType((*DeleteVDSResponse)(nil), "pilosa.DeleteVDSResponse") proto.RegisterType((*QueryPQLRequest)(nil), "pilosa.QueryPQLRequest") proto.RegisterType((*QuerySQLRequest)(nil), "pilosa.QuerySQLRequest") proto.RegisterType((*StatusError)(nil), "pilosa.StatusError") @@ -1193,64 +841,52 @@ func init() { func init() { proto.RegisterFile("pilosa.proto", fileDescriptor_ef0691a44d1e275c) } var fileDescriptor_ef0691a44d1e275c = []byte{ - // 897 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0xdd, 0x72, 0xdb, 0x44, - 0x14, 0xb6, 0x90, 0x6a, 0x59, 0xc7, 0x89, 0x93, 0x6c, 0x20, 0xb8, 0x1e, 0xa0, 0x66, 0x3b, 0x53, - 0xcc, 0xc0, 0x94, 0x12, 0x28, 0x4c, 0x21, 0x5c, 0x34, 0x75, 0xc1, 0x19, 0x60, 0x70, 0xd7, 0xd4, - 0xd7, 0x6c, 0xac, 0x75, 0xd0, 0x54, 0xd1, 0x3a, 0x5a, 0x39, 0xa9, 0x5f, 0x80, 0x37, 0xe0, 0x0d, - 0x78, 0x0b, 0xae, 0x78, 0x33, 0x66, 0x7f, 0x25, 0xb9, 0x31, 0x2d, 0xb9, 0xd3, 0x39, 0xdf, 0x77, - 0xfe, 0xf6, 0xec, 0x9e, 0x23, 0xd8, 0x5a, 0x24, 0x29, 0x17, 0xf4, 0xfe, 0x22, 0xe7, 0x05, 0x47, - 0x4d, 0x2d, 0xe1, 0xdb, 0xe0, 0x4f, 0x87, 0x13, 0x84, 0x20, 0xc8, 0xe8, 0x39, 0xeb, 0x7a, 0x7d, - 0x6f, 0x10, 0x11, 0xf5, 0x8d, 0x77, 0xa1, 0xf3, 0x03, 0x2b, 0xa6, 0xc3, 0x89, 0x20, 0xec, 0x62, - 0xc9, 0x44, 0x81, 0x0f, 0x61, 0xc7, 0x69, 0xc4, 0x82, 0x67, 0x82, 0xa1, 0x3b, 0x10, 0x5c, 0xc6, - 0x42, 0x74, 0xbd, 0xbe, 0x3f, 0x68, 0x1f, 0xb6, 0xef, 0x9b, 0x20, 0xd3, 0xe1, 0x84, 0x28, 0x00, - 0xdf, 0x85, 0x6d, 0x6d, 0x63, 0x9c, 0x5c, 0x1b, 0xea, 0x33, 0x1b, 0xca, 0xf9, 0x7d, 0x1f, 0xfc, - 0xcb, 0x58, 0x28, 0xd2, 0x9a, 0x5b, 0xa9, 0xc7, 0xbf, 0x41, 0x67, 0xcc, 0xc5, 0x6b, 0xdc, 0x4a, - 0xdd, 0x0b, 0xb6, 0x12, 0xdd, 0xb7, 0xfa, 0xde, 0xa0, 0x45, 0xd4, 0x37, 0xba, 0x07, 0x9d, 0x22, - 0xa7, 0xb3, 0x17, 0x4f, 0x5f, 0x26, 0xa2, 0x60, 0xd9, 0x8c, 0x75, 0x7d, 0x85, 0xae, 0x69, 0xf1, - 0x1e, 0xec, 0xb8, 0x08, 0x3a, 0x27, 0x7c, 0x0f, 0x76, 0x87, 0x2c, 0x65, 0x05, 0x7b, 0x4d, 0x35, - 0xfb, 0xb0, 0x57, 0xe1, 0x19, 0xe3, 0x47, 0xb0, 0xf3, 0x6c, 0xc9, 0xf2, 0xd5, 0xf8, 0xd9, 0x4f, - 0xd6, 0xf6, 0x6d, 0xb8, 0x95, 0x64, 0x31, 0x7b, 0x69, 0x8c, 0xb5, 0x80, 0x76, 0xc1, 0x5f, 0x5c, - 0xa4, 0x2a, 0xe7, 0x88, 0xc8, 0x4f, 0x7c, 0xd7, 0x98, 0x4e, 0x4a, 0xd3, 0x5d, 0xf0, 0xc5, 0x45, - 0x6a, 0x0c, 0xe5, 0x27, 0xfe, 0x16, 0xda, 0x93, 0x82, 0x16, 0x4b, 0xf1, 0x34, 0xcf, 0x79, 0x2e, - 0xf3, 0x7a, 0xc2, 0x63, 0x9d, 0xd7, 0x36, 0x51, 0xdf, 0xa8, 0x0b, 0xe1, 0xcf, 0x4c, 0x08, 0x7a, - 0xc6, 0x8c, 0x77, 0x2b, 0xe2, 0xbf, 0x3c, 0x68, 0x13, 0x7e, 0xe5, 0x4e, 0xff, 0x53, 0x08, 0x7f, - 0x67, 0x34, 0x66, 0xb9, 0x6d, 0x2c, 0xb2, 0x1d, 0x78, 0xc2, 0xd3, 0xe5, 0x79, 0x76, 0x92, 0xcd, - 0x39, 0xb1, 0x14, 0xf4, 0x00, 0xc2, 0x99, 0x52, 0xcb, 0x93, 0x96, 0xec, 0x83, 0x3a, 0xdb, 0xba, - 0x25, 0x96, 0x86, 0x1e, 0xd6, 0x92, 0x55, 0x1d, 0x68, 0x1f, 0xee, 0x5b, 0xab, 0x0a, 0x44, 0xaa, - 0x3c, 0xfc, 0x35, 0xf8, 0x84, 0x5f, 0x55, 0xe3, 0x79, 0x6f, 0x14, 0x0f, 0xff, 0xe9, 0xc1, 0xf6, - 0xaf, 0xf4, 0x34, 0x65, 0x37, 0xac, 0xf0, 0x0e, 0x04, 0x39, 0xbf, 0xb2, 0xe5, 0xb9, 0xeb, 0x28, - 0x8f, 0x4c, 0x01, 0x37, 0x2d, 0xe8, 0x08, 0xa0, 0x0c, 0x77, 0xed, 0x15, 0xee, 0x41, 0x2b, 0xa6, - 0x05, 0x2d, 0x56, 0x0b, 0xdb, 0x34, 0x27, 0xe3, 0x3f, 0x7c, 0xe8, 0xd4, 0x2b, 0x46, 0x1f, 0x40, - 0x24, 0x8a, 0x3c, 0xc9, 0xce, 0xa6, 0xd4, 0xdc, 0x8e, 0x51, 0x83, 0x94, 0x2a, 0x89, 0x2f, 0x93, - 0xac, 0xf8, 0xea, 0x4b, 0x89, 0x4b, 0x7f, 0x81, 0xc4, 0x9d, 0x0a, 0xbd, 0x07, 0x2d, 0x07, 0xcb, - 0x22, 0xfc, 0x51, 0x83, 0x38, 0x0d, 0xea, 0x41, 0x78, 0xca, 0x79, 0x2a, 0xc1, 0x40, 0x3e, 0x9a, - 0x51, 0x83, 0x58, 0x85, 0xc2, 0x52, 0x7e, 0x2a, 0xb1, 0x5b, 0x7d, 0x6f, 0xb0, 0xa5, 0x30, 0xad, - 0x40, 0xdf, 0x41, 0x47, 0x87, 0x78, 0x9c, 0xe7, 0x74, 0x25, 0x29, 0xcd, 0xfa, 0x01, 0x3d, 0x2f, - 0xd1, 0x51, 0x83, 0xac, 0x91, 0xa5, 0xb9, 0xae, 0xc0, 0x99, 0x87, 0xeb, 0xe7, 0xeb, 0x50, 0x69, - 0x5e, 0x27, 0xa3, 0x3e, 0xc0, 0x3c, 0xe5, 0xd4, 0x54, 0xd5, 0xea, 0x7b, 0x03, 0x6f, 0xd4, 0x20, - 0x15, 0x1d, 0xfa, 0x1c, 0x20, 0x66, 0xb3, 0xe4, 0x9c, 0xaa, 0xd2, 0x22, 0xe5, 0x7c, 0xc7, 0x3a, - 0x1f, 0x6a, 0x44, 0x9a, 0x94, 0xa4, 0xe3, 0x36, 0x44, 0xfa, 0x72, 0x4d, 0x69, 0x8a, 0x1f, 0x42, - 0x68, 0x58, 0xf2, 0x4d, 0x5f, 0xd2, 0x74, 0xa9, 0x9b, 0xe8, 0x13, 0x2d, 0x48, 0xad, 0x98, 0xd1, - 0x54, 0xb7, 0xd0, 0x27, 0x5a, 0xc0, 0x7f, 0x7b, 0xd0, 0x39, 0xc9, 0xc4, 0x82, 0xcd, 0x8a, 0xff, - 0x1e, 0x09, 0x9f, 0x54, 0x1f, 0x98, 0x4c, 0x6e, 0xcf, 0x26, 0x77, 0x12, 0x8b, 0x5f, 0xf2, 0x1f, - 0xd9, 0x4a, 0x94, 0x6f, 0x0b, 0xc3, 0xd6, 0x3c, 0x49, 0x0b, 0x96, 0x7f, 0x9f, 0xb0, 0x34, 0x16, - 0x5d, 0xbf, 0xef, 0x0f, 0x22, 0x52, 0xd3, 0xc9, 0x30, 0x69, 0x72, 0x9e, 0x14, 0xaa, 0x8d, 0x01, - 0xd1, 0x02, 0x3a, 0x80, 0x26, 0x9f, 0xcf, 0x05, 0x2b, 0x54, 0x07, 0x03, 0x62, 0x24, 0xc9, 0xbe, - 0x90, 0xf3, 0x47, 0x75, 0x2d, 0x22, 0x5a, 0xc0, 0x1f, 0x42, 0xbb, 0xd2, 0x36, 0x79, 0x79, 0x2f, - 0x69, 0xaa, 0x5f, 0x53, 0x40, 0xd4, 0xb7, 0xa4, 0x54, 0x5a, 0x53, 0xa3, 0x44, 0x86, 0x72, 0x06, - 0x91, 0xab, 0x01, 0x7d, 0x04, 0x7e, 0xe2, 0x86, 0xfe, 0x86, 0xcb, 0x21, 0x19, 0xe8, 0xe3, 0xca, - 0x60, 0xdf, 0x78, 0x0f, 0x14, 0xe5, 0xb8, 0x09, 0x81, 0x7c, 0x2c, 0x87, 0xff, 0x04, 0xd0, 0x1c, - 0x2b, 0x1a, 0x3a, 0x82, 0xd0, 0xac, 0x31, 0xe4, 0x26, 0x47, 0x7d, 0xd3, 0xf5, 0xde, 0x7d, 0x45, - 0x6f, 0xc6, 0x78, 0x03, 0x3d, 0x82, 0xa6, 0x56, 0xa2, 0x77, 0xea, 0x24, 0x6b, 0x7b, 0xb0, 0xae, - 0x76, 0xa6, 0x47, 0x10, 0x9a, 0x9d, 0x52, 0x06, 0xae, 0xaf, 0xb1, 0x32, 0xf0, 0xfa, 0xf2, 0x69, - 0xa0, 0x63, 0x88, 0xdc, 0x5a, 0x41, 0xdd, 0xf2, 0x7a, 0xd6, 0x37, 0x52, 0xef, 0xf6, 0x35, 0x48, - 0x25, 0x83, 0x96, 0x5d, 0x25, 0xc8, 0x85, 0x5a, 0x5b, 0x2e, 0xbd, 0xfd, 0xea, 0x7c, 0x73, 0xb6, - 0x0f, 0x3c, 0xf4, 0x18, 0xb6, 0x2d, 0xf7, 0x79, 0x46, 0xf3, 0xd5, 0x66, 0x17, 0xee, 0x68, 0x6a, - 0x53, 0xb7, 0x92, 0xc0, 0xf8, 0x95, 0x04, 0xc6, 0xff, 0x23, 0x81, 0xf1, 0xf5, 0x09, 0x8c, 0xdf, - 0x20, 0x81, 0x6f, 0x20, 0x34, 0x6f, 0xae, 0xec, 0x41, 0xfd, 0x11, 0x6e, 0x0c, 0x7f, 0xda, 0x54, - 0xff, 0x4e, 0x5f, 0xfc, 0x1b, 0x00, 0x00, 0xff, 0xff, 0x1e, 0xff, 0x07, 0x47, 0x4b, 0x09, 0x00, - 0x00, + // 718 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x55, 0xdd, 0x52, 0xd3, 0x4e, + 0x14, 0x6f, 0x48, 0x68, 0x9b, 0x53, 0xbe, 0xfe, 0xcb, 0x5f, 0xcc, 0x30, 0x8e, 0xc6, 0x78, 0x61, + 0x1c, 0x1d, 0x06, 0x51, 0x74, 0x54, 0xbc, 0x00, 0xd4, 0x29, 0xa3, 0x8e, 0x65, 0x11, 0xee, 0xb7, + 0xcd, 0x16, 0x33, 0x6e, 0xb3, 0x6d, 0x36, 0x05, 0xfb, 0x02, 0xbe, 0x81, 0x6f, 0xe0, 0x5b, 0x78, + 0xef, 0x73, 0x39, 0xbb, 0x9b, 0x4d, 0x13, 0xb4, 0x0e, 0x72, 0xb7, 0xe7, 0xfc, 0x7e, 0xe7, 0x2b, + 0xe7, 0x23, 0xb0, 0x30, 0x8c, 0x19, 0x17, 0x64, 0x63, 0x98, 0xf2, 0x8c, 0xa3, 0xba, 0x96, 0x82, + 0x67, 0xb0, 0x7c, 0x38, 0xa6, 0xe9, 0xa4, 0x73, 0xf8, 0x0e, 0xd3, 0xd1, 0x98, 0x8a, 0x0c, 0xfd, + 0x0f, 0xf3, 0x71, 0x12, 0xd1, 0x2f, 0x9e, 0xe5, 0x5b, 0xa1, 0x8b, 0xb5, 0x80, 0x56, 0xc0, 0x1e, + 0x8e, 0x98, 0x37, 0xa7, 0x74, 0xf2, 0x19, 0xdc, 0xc9, 0x4d, 0x8f, 0xa6, 0xa6, 0x2b, 0x60, 0x8b, + 0x11, 0xcb, 0x0d, 0xe5, 0x33, 0x78, 0x01, 0xad, 0xa3, 0x8c, 0x64, 0x63, 0xf1, 0x3a, 0x4d, 0x79, + 0x8a, 0x10, 0x38, 0xfb, 0x3c, 0xa2, 0x8a, 0xb1, 0x88, 0xd5, 0x1b, 0x79, 0xd0, 0x78, 0x4f, 0x85, + 0x20, 0xa7, 0x34, 0xf7, 0x6e, 0xc4, 0xe0, 0xbb, 0x05, 0x2d, 0xcc, 0xcf, 0x31, 0x15, 0x43, 0x9e, + 0x08, 0x8a, 0x1e, 0x40, 0xe3, 0x13, 0x25, 0x11, 0x4d, 0x85, 0x67, 0xf9, 0x76, 0xd8, 0xda, 0x42, + 0x1b, 0x79, 0x51, 0xfb, 0x9c, 0x8d, 0x07, 0xc9, 0x41, 0xd2, 0xe7, 0xd8, 0x50, 0xd0, 0x26, 0x34, + 0x7a, 0x4a, 0x2d, 0xbc, 0x39, 0xc5, 0x5e, 0xab, 0xb2, 0x8d, 0x5b, 0x6c, 0x68, 0x68, 0xbb, 0x92, + 0xac, 0x67, 0xfb, 0x56, 0xd8, 0xda, 0x5a, 0x35, 0x56, 0x25, 0x08, 0x97, 0x79, 0xc1, 0x53, 0xb0, + 0x31, 0x3f, 0x2f, 0xc7, 0xb3, 0x2e, 0x15, 0x2f, 0xf8, 0x66, 0xc1, 0xe2, 0x47, 0xd2, 0x65, 0xf4, + 0x8a, 0x15, 0xde, 0x02, 0x27, 0xe5, 0xe7, 0xa6, 0xbc, 0x96, 0xa1, 0xca, 0x4f, 0xa6, 0x80, 0xab, + 0x16, 0xb4, 0x03, 0x30, 0x0d, 0x27, 0x7b, 0x96, 0x90, 0x01, 0xcd, 0xbb, 0xaa, 0xde, 0x68, 0x1d, + 0x9a, 0x11, 0xc9, 0x48, 0x36, 0x19, 0x9a, 0xa6, 0x15, 0x72, 0xf0, 0xd5, 0x86, 0xa5, 0x6a, 0xc5, + 0xe8, 0x26, 0xb8, 0x22, 0x4b, 0xe3, 0xe4, 0xf4, 0x84, 0xe4, 0xd3, 0xd1, 0xae, 0xe1, 0xa9, 0x4a, + 0xe2, 0xe3, 0x38, 0xc9, 0x9e, 0x3c, 0x96, 0xb8, 0xf4, 0xe7, 0x48, 0xbc, 0x50, 0xa1, 0x1b, 0xd0, + 0x2c, 0x60, 0x59, 0x84, 0xdd, 0xae, 0xe1, 0x42, 0x83, 0xd6, 0xa1, 0xd1, 0xe5, 0x9c, 0x49, 0xd0, + 0xf1, 0xad, 0xb0, 0xd9, 0xae, 0x61, 0xa3, 0x50, 0x18, 0xe3, 0x5d, 0x89, 0xcd, 0xfb, 0x56, 0xb8, + 0xa0, 0x30, 0xad, 0x40, 0x2f, 0x61, 0x49, 0x87, 0xd8, 0x4d, 0x53, 0x32, 0x91, 0x94, 0x7a, 0xf5, + 0x03, 0x1d, 0x4f, 0xd1, 0x76, 0x0d, 0x5f, 0x20, 0x4b, 0x73, 0x5d, 0x41, 0x61, 0xde, 0xb8, 0xf8, + 0x7d, 0x0b, 0x54, 0x9a, 0x57, 0xc9, 0xc8, 0x07, 0xe8, 0x33, 0x4e, 0xf2, 0xaa, 0x9a, 0xbe, 0x15, + 0x5a, 0xed, 0x1a, 0x2e, 0xe9, 0xd0, 0x43, 0x80, 0x88, 0xf6, 0xe2, 0x01, 0x51, 0xa5, 0xb9, 0xca, + 0xf9, 0xb2, 0x71, 0xfe, 0x4a, 0x23, 0xd2, 0x64, 0x4a, 0xda, 0x6b, 0x81, 0xab, 0x87, 0xeb, 0x84, + 0xb0, 0x60, 0x1b, 0x1a, 0x39, 0x4b, 0xee, 0xf4, 0x19, 0x61, 0x63, 0xdd, 0x44, 0x1b, 0x6b, 0x41, + 0x6a, 0x45, 0x8f, 0x30, 0xdd, 0x42, 0x1b, 0x6b, 0x21, 0xf8, 0x61, 0xc1, 0xd2, 0x41, 0x22, 0x86, + 0xb4, 0x97, 0xfd, 0xfd, 0x24, 0xdc, 0x2f, 0x2f, 0x98, 0x4c, 0xee, 0x3f, 0x93, 0xdc, 0x41, 0x24, + 0x3e, 0xa4, 0x6f, 0xe9, 0x44, 0x4c, 0x77, 0x2b, 0x80, 0x85, 0x7e, 0xcc, 0x32, 0x9a, 0xbe, 0x89, + 0x29, 0x8b, 0x84, 0x67, 0xfb, 0x76, 0xe8, 0xe2, 0x8a, 0x4e, 0x86, 0x61, 0xf1, 0x20, 0xce, 0x54, + 0x1b, 0x1d, 0xac, 0x05, 0xb4, 0x06, 0x75, 0xde, 0xef, 0x0b, 0x9a, 0xa9, 0x0e, 0x3a, 0x38, 0x97, + 0x24, 0x7b, 0x24, 0xef, 0x8f, 0xea, 0x9a, 0x8b, 0xb5, 0x10, 0xdc, 0x86, 0x56, 0xa9, 0x6d, 0x72, + 0x78, 0xcf, 0x08, 0xd3, 0xdb, 0xe4, 0x60, 0xf5, 0x96, 0x94, 0x52, 0x6b, 0x2a, 0x14, 0x37, 0xa7, + 0x9c, 0x82, 0x5b, 0xd4, 0x80, 0xee, 0x82, 0x1d, 0x47, 0x42, 0xd5, 0x3e, 0x73, 0x38, 0x24, 0x03, + 0xdd, 0x03, 0xe7, 0x33, 0x9d, 0x98, 0xaf, 0x31, 0x63, 0x0e, 0x14, 0x65, 0xaf, 0x0e, 0x8e, 0x5c, + 0x96, 0xad, 0x9f, 0x73, 0x50, 0xef, 0x28, 0x1a, 0xda, 0x81, 0xa6, 0xb9, 0xa7, 0xe8, 0xba, 0xb1, + 0xbd, 0x70, 0x61, 0xd7, 0x57, 0xcb, 0x4b, 0x9e, 0xaf, 0x57, 0x50, 0xdb, 0xb4, 0xd0, 0x2e, 0x2c, + 0x1a, 0xee, 0x71, 0x42, 0xd2, 0xc9, 0x6c, 0x17, 0xd7, 0x0c, 0x50, 0x39, 0x3d, 0x41, 0xad, 0x48, + 0xa0, 0xf3, 0x5b, 0x02, 0x9d, 0x7f, 0x48, 0xa0, 0xf3, 0xe7, 0x04, 0x3a, 0x97, 0x48, 0xe0, 0x39, + 0x34, 0xf2, 0xc1, 0x43, 0xc5, 0xed, 0xac, 0x4e, 0xe2, 0xcc, 0xf0, 0xdd, 0xba, 0xfa, 0xaf, 0x3d, + 0xfa, 0x15, 0x00, 0x00, 0xff, 0xff, 0x7e, 0x8d, 0x53, 0x0b, 0xe7, 0x06, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -1265,10 +901,6 @@ const _ = grpc.SupportPackageIsVersion4 // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. type PilosaClient interface { - GetVDSs(ctx context.Context, in *GetVDSsRequest, opts ...grpc.CallOption) (*GetVDSsResponse, error) - GetVDS(ctx context.Context, in *GetVDSRequest, opts ...grpc.CallOption) (*GetVDSResponse, error) - PostVDS(ctx context.Context, in *PostVDSRequest, opts ...grpc.CallOption) (*PostVDSResponse, error) - DeleteVDS(ctx context.Context, in *DeleteVDSRequest, opts ...grpc.CallOption) (*DeleteVDSResponse, error) QuerySQL(ctx context.Context, in *QuerySQLRequest, opts ...grpc.CallOption) (Pilosa_QuerySQLClient, error) QuerySQLUnary(ctx context.Context, in *QuerySQLRequest, opts ...grpc.CallOption) (*TableResponse, error) QueryPQL(ctx context.Context, in *QueryPQLRequest, opts ...grpc.CallOption) (Pilosa_QueryPQLClient, error) @@ -1284,42 +916,6 @@ func NewPilosaClient(cc *grpc.ClientConn) PilosaClient { return &pilosaClient{cc} } -func (c *pilosaClient) GetVDSs(ctx context.Context, in *GetVDSsRequest, opts ...grpc.CallOption) (*GetVDSsResponse, error) { - out := new(GetVDSsResponse) - err := c.cc.Invoke(ctx, "/pilosa.Pilosa/GetVDSs", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *pilosaClient) GetVDS(ctx context.Context, in *GetVDSRequest, opts ...grpc.CallOption) (*GetVDSResponse, error) { - out := new(GetVDSResponse) - err := c.cc.Invoke(ctx, "/pilosa.Pilosa/GetVDS", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *pilosaClient) PostVDS(ctx context.Context, in *PostVDSRequest, opts ...grpc.CallOption) (*PostVDSResponse, error) { - out := new(PostVDSResponse) - err := c.cc.Invoke(ctx, "/pilosa.Pilosa/PostVDS", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *pilosaClient) DeleteVDS(ctx context.Context, in *DeleteVDSRequest, opts ...grpc.CallOption) (*DeleteVDSResponse, error) { - out := new(DeleteVDSResponse) - err := c.cc.Invoke(ctx, "/pilosa.Pilosa/DeleteVDS", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - func (c *pilosaClient) QuerySQL(ctx context.Context, in *QuerySQLRequest, opts ...grpc.CallOption) (Pilosa_QuerySQLClient, error) { stream, err := c.cc.NewStream(ctx, &_Pilosa_serviceDesc.Streams[0], "/pilosa.Pilosa/QuerySQL", opts...) if err != nil { @@ -1436,10 +1032,6 @@ func (x *pilosaInspectClient) Recv() (*RowResponse, error) { // PilosaServer is the server API for Pilosa service. type PilosaServer interface { - GetVDSs(context.Context, *GetVDSsRequest) (*GetVDSsResponse, error) - GetVDS(context.Context, *GetVDSRequest) (*GetVDSResponse, error) - PostVDS(context.Context, *PostVDSRequest) (*PostVDSResponse, error) - DeleteVDS(context.Context, *DeleteVDSRequest) (*DeleteVDSResponse, error) QuerySQL(*QuerySQLRequest, Pilosa_QuerySQLServer) error QuerySQLUnary(context.Context, *QuerySQLRequest) (*TableResponse, error) QueryPQL(*QueryPQLRequest, Pilosa_QueryPQLServer) error @@ -1451,18 +1043,6 @@ type PilosaServer interface { type UnimplementedPilosaServer struct { } -func (*UnimplementedPilosaServer) GetVDSs(ctx context.Context, req *GetVDSsRequest) (*GetVDSsResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetVDSs not implemented") -} -func (*UnimplementedPilosaServer) GetVDS(ctx context.Context, req *GetVDSRequest) (*GetVDSResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetVDS not implemented") -} -func (*UnimplementedPilosaServer) PostVDS(ctx context.Context, req *PostVDSRequest) (*PostVDSResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method PostVDS not implemented") -} -func (*UnimplementedPilosaServer) DeleteVDS(ctx context.Context, req *DeleteVDSRequest) (*DeleteVDSResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method DeleteVDS not implemented") -} func (*UnimplementedPilosaServer) QuerySQL(req *QuerySQLRequest, srv Pilosa_QuerySQLServer) error { return status.Errorf(codes.Unimplemented, "method QuerySQL not implemented") } @@ -1483,78 +1063,6 @@ func RegisterPilosaServer(s *grpc.Server, srv PilosaServer) { s.RegisterService(&_Pilosa_serviceDesc, srv) } -func _Pilosa_GetVDSs_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(GetVDSsRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(PilosaServer).GetVDSs(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/pilosa.Pilosa/GetVDSs", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(PilosaServer).GetVDSs(ctx, req.(*GetVDSsRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Pilosa_GetVDS_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(GetVDSRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(PilosaServer).GetVDS(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/pilosa.Pilosa/GetVDS", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(PilosaServer).GetVDS(ctx, req.(*GetVDSRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Pilosa_PostVDS_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(PostVDSRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(PilosaServer).PostVDS(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/pilosa.Pilosa/PostVDS", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(PilosaServer).PostVDS(ctx, req.(*PostVDSRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Pilosa_DeleteVDS_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(DeleteVDSRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(PilosaServer).DeleteVDS(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/pilosa.Pilosa/DeleteVDS", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(PilosaServer).DeleteVDS(ctx, req.(*DeleteVDSRequest)) - } - return interceptor(ctx, in, info, handler) -} - func _Pilosa_QuerySQL_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(QuerySQLRequest) if err := stream.RecvMsg(m); err != nil { @@ -1658,22 +1166,6 @@ var _Pilosa_serviceDesc = grpc.ServiceDesc{ ServiceName: "pilosa.Pilosa", HandlerType: (*PilosaServer)(nil), Methods: []grpc.MethodDesc{ - { - MethodName: "GetVDSs", - Handler: _Pilosa_GetVDSs_Handler, - }, - { - MethodName: "GetVDS", - Handler: _Pilosa_GetVDS_Handler, - }, - { - MethodName: "PostVDS", - Handler: _Pilosa_PostVDS_Handler, - }, - { - MethodName: "DeleteVDS", - Handler: _Pilosa_DeleteVDS_Handler, - }, { MethodName: "QuerySQLUnary", Handler: _Pilosa_QuerySQLUnary_Handler, diff --git a/proto/pilosa.proto b/proto/pilosa.proto index f51a45502..1fc966083 100644 --- a/proto/pilosa.proto +++ b/proto/pilosa.proto @@ -1,42 +1,7 @@ syntax = "proto3"; package pilosa; -import "public.proto"; - -message VDS { - string name = 1; -} - -message GetVDSsRequest { -} - -message GetVDSsResponse { - repeated VDS vdss = 1; -} - -message GetVDSRequest { - string name = 1; -} - -message GetVDSResponse { - VDS vds = 1; -} - -message PostVDSRequest { - string name = 1; - bool keys = 2; - bool trackExistence = 3; -} - -message PostVDSResponse { -} - -message DeleteVDSRequest { - string name = 1; -} - -message DeleteVDSResponse { -} +//import "public.proto"; message QueryPQLRequest { string index = 1; @@ -117,14 +82,10 @@ message IdsOrKeys { } service Pilosa { - rpc GetVDSs(GetVDSsRequest) returns (GetVDSsResponse) {}; - rpc GetVDS(GetVDSRequest) returns (GetVDSResponse) {}; - rpc PostVDS(PostVDSRequest) returns (PostVDSResponse) {}; - rpc DeleteVDS(DeleteVDSRequest) returns (DeleteVDSResponse) {}; rpc QuerySQL(QuerySQLRequest) returns (stream RowResponse) {}; rpc QuerySQLUnary(QuerySQLRequest) returns (TableResponse) {}; rpc QueryPQL(QueryPQLRequest) returns (stream RowResponse) {}; rpc QueryPQLUnary(QueryPQLRequest) returns (TableResponse) {}; rpc Inspect(InspectRequest) returns (stream RowResponse) {}; - rpc ImportAtomicRecord(stream AtomicRecord) returns (AtomicImportResponse) {}; + //rpc ImportAtomicRecord(stream AtomicRecord) returns (AtomicImportResponse) {}; } diff --git a/proto/vdsm/vdsm.pb.go b/proto/vdsm/vdsm.pb.go new file mode 100644 index 000000000..2d4deaf42 --- /dev/null +++ b/proto/vdsm/vdsm.pb.go @@ -0,0 +1,1902 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: vdsm/vdsm.proto + +package vdsm + +import ( + context "context" + fmt "fmt" + proto "github.com/golang/protobuf/proto" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + math "math" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package + +// deprecated +type DataSource struct { + Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Description string `protobuf:"bytes,3,opt,name=description,proto3" json:"description,omitempty"` + Type string `protobuf:"bytes,4,opt,name=type,proto3" json:"type,omitempty"` + Config map[string]string `protobuf:"bytes,5,rep,name=config,proto3" json:"config,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Status string `protobuf:"bytes,6,opt,name=status,proto3" json:"status,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DataSource) Reset() { *m = DataSource{} } +func (m *DataSource) String() string { return proto.CompactTextString(m) } +func (*DataSource) ProtoMessage() {} +func (*DataSource) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{0} +} + +func (m *DataSource) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DataSource.Unmarshal(m, b) +} +func (m *DataSource) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DataSource.Marshal(b, m, deterministic) +} +func (m *DataSource) XXX_Merge(src proto.Message) { + xxx_messageInfo_DataSource.Merge(m, src) +} +func (m *DataSource) XXX_Size() int { + return xxx_messageInfo_DataSource.Size(m) +} +func (m *DataSource) XXX_DiscardUnknown() { + xxx_messageInfo_DataSource.DiscardUnknown(m) +} + +var xxx_messageInfo_DataSource proto.InternalMessageInfo + +func (m *DataSource) GetId() int64 { + if m != nil { + return m.Id + } + return 0 +} + +func (m *DataSource) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *DataSource) GetDescription() string { + if m != nil { + return m.Description + } + return "" +} + +func (m *DataSource) GetType() string { + if m != nil { + return m.Type + } + return "" +} + +func (m *DataSource) GetConfig() map[string]string { + if m != nil { + return m.Config + } + return nil +} + +func (m *DataSource) GetStatus() string { + if m != nil { + return m.Status + } + return "" +} + +type VDS struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Description string `protobuf:"bytes,3,opt,name=description,proto3" json:"description,omitempty"` + PilosaIndex string `protobuf:"bytes,4,opt,name=pilosa_index,json=pilosaIndex,proto3" json:"pilosa_index,omitempty"` + Datasources []*DataSource `protobuf:"bytes,5,rep,name=datasources,proto3" json:"datasources,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *VDS) Reset() { *m = VDS{} } +func (m *VDS) String() string { return proto.CompactTextString(m) } +func (*VDS) ProtoMessage() {} +func (*VDS) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{1} +} + +func (m *VDS) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_VDS.Unmarshal(m, b) +} +func (m *VDS) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_VDS.Marshal(b, m, deterministic) +} +func (m *VDS) XXX_Merge(src proto.Message) { + xxx_messageInfo_VDS.Merge(m, src) +} +func (m *VDS) XXX_Size() int { + return xxx_messageInfo_VDS.Size(m) +} +func (m *VDS) XXX_DiscardUnknown() { + xxx_messageInfo_VDS.DiscardUnknown(m) +} + +var xxx_messageInfo_VDS proto.InternalMessageInfo + +func (m *VDS) GetId() string { + if m != nil { + return m.Id + } + return "" +} + +func (m *VDS) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *VDS) GetDescription() string { + if m != nil { + return m.Description + } + return "" +} + +func (m *VDS) GetPilosaIndex() string { + if m != nil { + return m.PilosaIndex + } + return "" +} + +func (m *VDS) GetDatasources() []*DataSource { + if m != nil { + return m.Datasources + } + return nil +} + +type GetVDSsRequest struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetVDSsRequest) Reset() { *m = GetVDSsRequest{} } +func (m *GetVDSsRequest) String() string { return proto.CompactTextString(m) } +func (*GetVDSsRequest) ProtoMessage() {} +func (*GetVDSsRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{2} +} + +func (m *GetVDSsRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetVDSsRequest.Unmarshal(m, b) +} +func (m *GetVDSsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetVDSsRequest.Marshal(b, m, deterministic) +} +func (m *GetVDSsRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetVDSsRequest.Merge(m, src) +} +func (m *GetVDSsRequest) XXX_Size() int { + return xxx_messageInfo_GetVDSsRequest.Size(m) +} +func (m *GetVDSsRequest) XXX_DiscardUnknown() { + xxx_messageInfo_GetVDSsRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_GetVDSsRequest proto.InternalMessageInfo + +type GetVDSsResponse struct { + Vdss []*VDS `protobuf:"bytes,1,rep,name=vdss,proto3" json:"vdss,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetVDSsResponse) Reset() { *m = GetVDSsResponse{} } +func (m *GetVDSsResponse) String() string { return proto.CompactTextString(m) } +func (*GetVDSsResponse) ProtoMessage() {} +func (*GetVDSsResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{3} +} + +func (m *GetVDSsResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetVDSsResponse.Unmarshal(m, b) +} +func (m *GetVDSsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetVDSsResponse.Marshal(b, m, deterministic) +} +func (m *GetVDSsResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetVDSsResponse.Merge(m, src) +} +func (m *GetVDSsResponse) XXX_Size() int { + return xxx_messageInfo_GetVDSsResponse.Size(m) +} +func (m *GetVDSsResponse) XXX_DiscardUnknown() { + xxx_messageInfo_GetVDSsResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_GetVDSsResponse proto.InternalMessageInfo + +func (m *GetVDSsResponse) GetVdss() []*VDS { + if m != nil { + return m.Vdss + } + return nil +} + +type GetVDSRequest struct { + // Types that are valid to be assigned to IdOrName: + // *GetVDSRequest_Id + // *GetVDSRequest_Name + IdOrName isGetVDSRequest_IdOrName `protobuf_oneof:"idOrName"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetVDSRequest) Reset() { *m = GetVDSRequest{} } +func (m *GetVDSRequest) String() string { return proto.CompactTextString(m) } +func (*GetVDSRequest) ProtoMessage() {} +func (*GetVDSRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{4} +} + +func (m *GetVDSRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetVDSRequest.Unmarshal(m, b) +} +func (m *GetVDSRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetVDSRequest.Marshal(b, m, deterministic) +} +func (m *GetVDSRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetVDSRequest.Merge(m, src) +} +func (m *GetVDSRequest) XXX_Size() int { + return xxx_messageInfo_GetVDSRequest.Size(m) +} +func (m *GetVDSRequest) XXX_DiscardUnknown() { + xxx_messageInfo_GetVDSRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_GetVDSRequest proto.InternalMessageInfo + +type isGetVDSRequest_IdOrName interface { + isGetVDSRequest_IdOrName() +} + +type GetVDSRequest_Id struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3,oneof"` +} + +type GetVDSRequest_Name struct { + Name string `protobuf:"bytes,2,opt,name=name,proto3,oneof"` +} + +func (*GetVDSRequest_Id) isGetVDSRequest_IdOrName() {} + +func (*GetVDSRequest_Name) isGetVDSRequest_IdOrName() {} + +func (m *GetVDSRequest) GetIdOrName() isGetVDSRequest_IdOrName { + if m != nil { + return m.IdOrName + } + return nil +} + +func (m *GetVDSRequest) GetId() string { + if x, ok := m.GetIdOrName().(*GetVDSRequest_Id); ok { + return x.Id + } + return "" +} + +func (m *GetVDSRequest) GetName() string { + if x, ok := m.GetIdOrName().(*GetVDSRequest_Name); ok { + return x.Name + } + return "" +} + +// XXX_OneofWrappers is for the internal use of the proto package. +func (*GetVDSRequest) XXX_OneofWrappers() []interface{} { + return []interface{}{ + (*GetVDSRequest_Id)(nil), + (*GetVDSRequest_Name)(nil), + } +} + +type GetVDSResponse struct { + Vds *VDS `protobuf:"bytes,1,opt,name=vds,proto3" json:"vds,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetVDSResponse) Reset() { *m = GetVDSResponse{} } +func (m *GetVDSResponse) String() string { return proto.CompactTextString(m) } +func (*GetVDSResponse) ProtoMessage() {} +func (*GetVDSResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{5} +} + +func (m *GetVDSResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetVDSResponse.Unmarshal(m, b) +} +func (m *GetVDSResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetVDSResponse.Marshal(b, m, deterministic) +} +func (m *GetVDSResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetVDSResponse.Merge(m, src) +} +func (m *GetVDSResponse) XXX_Size() int { + return xxx_messageInfo_GetVDSResponse.Size(m) +} +func (m *GetVDSResponse) XXX_DiscardUnknown() { + xxx_messageInfo_GetVDSResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_GetVDSResponse proto.InternalMessageInfo + +func (m *GetVDSResponse) GetVds() *VDS { + if m != nil { + return m.Vds + } + return nil +} + +type PostVDSRequest struct { + Definition string `protobuf:"bytes,1,opt,name=definition,proto3" json:"definition,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *PostVDSRequest) Reset() { *m = PostVDSRequest{} } +func (m *PostVDSRequest) String() string { return proto.CompactTextString(m) } +func (*PostVDSRequest) ProtoMessage() {} +func (*PostVDSRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{6} +} + +func (m *PostVDSRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_PostVDSRequest.Unmarshal(m, b) +} +func (m *PostVDSRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_PostVDSRequest.Marshal(b, m, deterministic) +} +func (m *PostVDSRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_PostVDSRequest.Merge(m, src) +} +func (m *PostVDSRequest) XXX_Size() int { + return xxx_messageInfo_PostVDSRequest.Size(m) +} +func (m *PostVDSRequest) XXX_DiscardUnknown() { + xxx_messageInfo_PostVDSRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_PostVDSRequest proto.InternalMessageInfo + +func (m *PostVDSRequest) GetDefinition() string { + if m != nil { + return m.Definition + } + return "" +} + +type PostVDSResponse struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Uri string `protobuf:"bytes,2,opt,name=uri,proto3" json:"uri,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *PostVDSResponse) Reset() { *m = PostVDSResponse{} } +func (m *PostVDSResponse) String() string { return proto.CompactTextString(m) } +func (*PostVDSResponse) ProtoMessage() {} +func (*PostVDSResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{7} +} + +func (m *PostVDSResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_PostVDSResponse.Unmarshal(m, b) +} +func (m *PostVDSResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_PostVDSResponse.Marshal(b, m, deterministic) +} +func (m *PostVDSResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_PostVDSResponse.Merge(m, src) +} +func (m *PostVDSResponse) XXX_Size() int { + return xxx_messageInfo_PostVDSResponse.Size(m) +} +func (m *PostVDSResponse) XXX_DiscardUnknown() { + xxx_messageInfo_PostVDSResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_PostVDSResponse proto.InternalMessageInfo + +func (m *PostVDSResponse) GetId() string { + if m != nil { + return m.Id + } + return "" +} + +func (m *PostVDSResponse) GetUri() string { + if m != nil { + return m.Uri + } + return "" +} + +type DeleteVDSRequest struct { + // Types that are valid to be assigned to IdOrName: + // *DeleteVDSRequest_Id + // *DeleteVDSRequest_Name + IdOrName isDeleteVDSRequest_IdOrName `protobuf_oneof:"idOrName"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DeleteVDSRequest) Reset() { *m = DeleteVDSRequest{} } +func (m *DeleteVDSRequest) String() string { return proto.CompactTextString(m) } +func (*DeleteVDSRequest) ProtoMessage() {} +func (*DeleteVDSRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{8} +} + +func (m *DeleteVDSRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DeleteVDSRequest.Unmarshal(m, b) +} +func (m *DeleteVDSRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DeleteVDSRequest.Marshal(b, m, deterministic) +} +func (m *DeleteVDSRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteVDSRequest.Merge(m, src) +} +func (m *DeleteVDSRequest) XXX_Size() int { + return xxx_messageInfo_DeleteVDSRequest.Size(m) +} +func (m *DeleteVDSRequest) XXX_DiscardUnknown() { + xxx_messageInfo_DeleteVDSRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_DeleteVDSRequest proto.InternalMessageInfo + +type isDeleteVDSRequest_IdOrName interface { + isDeleteVDSRequest_IdOrName() +} + +type DeleteVDSRequest_Id struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3,oneof"` +} + +type DeleteVDSRequest_Name struct { + Name string `protobuf:"bytes,2,opt,name=name,proto3,oneof"` +} + +func (*DeleteVDSRequest_Id) isDeleteVDSRequest_IdOrName() {} + +func (*DeleteVDSRequest_Name) isDeleteVDSRequest_IdOrName() {} + +func (m *DeleteVDSRequest) GetIdOrName() isDeleteVDSRequest_IdOrName { + if m != nil { + return m.IdOrName + } + return nil +} + +func (m *DeleteVDSRequest) GetId() string { + if x, ok := m.GetIdOrName().(*DeleteVDSRequest_Id); ok { + return x.Id + } + return "" +} + +func (m *DeleteVDSRequest) GetName() string { + if x, ok := m.GetIdOrName().(*DeleteVDSRequest_Name); ok { + return x.Name + } + return "" +} + +// XXX_OneofWrappers is for the internal use of the proto package. +func (*DeleteVDSRequest) XXX_OneofWrappers() []interface{} { + return []interface{}{ + (*DeleteVDSRequest_Id)(nil), + (*DeleteVDSRequest_Name)(nil), + } +} + +type DeleteVDSResponse struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DeleteVDSResponse) Reset() { *m = DeleteVDSResponse{} } +func (m *DeleteVDSResponse) String() string { return proto.CompactTextString(m) } +func (*DeleteVDSResponse) ProtoMessage() {} +func (*DeleteVDSResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{9} +} + +func (m *DeleteVDSResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DeleteVDSResponse.Unmarshal(m, b) +} +func (m *DeleteVDSResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DeleteVDSResponse.Marshal(b, m, deterministic) +} +func (m *DeleteVDSResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteVDSResponse.Merge(m, src) +} +func (m *DeleteVDSResponse) XXX_Size() int { + return xxx_messageInfo_DeleteVDSResponse.Size(m) +} +func (m *DeleteVDSResponse) XXX_DiscardUnknown() { + xxx_messageInfo_DeleteVDSResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_DeleteVDSResponse proto.InternalMessageInfo + +type QueryPQLRequest struct { + Vds string `protobuf:"bytes,1,opt,name=vds,proto3" json:"vds,omitempty"` + Pql string `protobuf:"bytes,2,opt,name=pql,proto3" json:"pql,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *QueryPQLRequest) Reset() { *m = QueryPQLRequest{} } +func (m *QueryPQLRequest) String() string { return proto.CompactTextString(m) } +func (*QueryPQLRequest) ProtoMessage() {} +func (*QueryPQLRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{10} +} + +func (m *QueryPQLRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_QueryPQLRequest.Unmarshal(m, b) +} +func (m *QueryPQLRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_QueryPQLRequest.Marshal(b, m, deterministic) +} +func (m *QueryPQLRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_QueryPQLRequest.Merge(m, src) +} +func (m *QueryPQLRequest) XXX_Size() int { + return xxx_messageInfo_QueryPQLRequest.Size(m) +} +func (m *QueryPQLRequest) XXX_DiscardUnknown() { + xxx_messageInfo_QueryPQLRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_QueryPQLRequest proto.InternalMessageInfo + +func (m *QueryPQLRequest) GetVds() string { + if m != nil { + return m.Vds + } + return "" +} + +func (m *QueryPQLRequest) GetPql() string { + if m != nil { + return m.Pql + } + return "" +} + +type QuerySQLRequest struct { + Sql string `protobuf:"bytes,1,opt,name=sql,proto3" json:"sql,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *QuerySQLRequest) Reset() { *m = QuerySQLRequest{} } +func (m *QuerySQLRequest) String() string { return proto.CompactTextString(m) } +func (*QuerySQLRequest) ProtoMessage() {} +func (*QuerySQLRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{11} +} + +func (m *QuerySQLRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_QuerySQLRequest.Unmarshal(m, b) +} +func (m *QuerySQLRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_QuerySQLRequest.Marshal(b, m, deterministic) +} +func (m *QuerySQLRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_QuerySQLRequest.Merge(m, src) +} +func (m *QuerySQLRequest) XXX_Size() int { + return xxx_messageInfo_QuerySQLRequest.Size(m) +} +func (m *QuerySQLRequest) XXX_DiscardUnknown() { + xxx_messageInfo_QuerySQLRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_QuerySQLRequest proto.InternalMessageInfo + +func (m *QuerySQLRequest) GetSql() string { + if m != nil { + return m.Sql + } + return "" +} + +type StatusError struct { + Code uint32 `protobuf:"varint,1,opt,name=Code,proto3" json:"Code,omitempty"` + Message string `protobuf:"bytes,2,opt,name=Message,proto3" json:"Message,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *StatusError) Reset() { *m = StatusError{} } +func (m *StatusError) String() string { return proto.CompactTextString(m) } +func (*StatusError) ProtoMessage() {} +func (*StatusError) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{12} +} + +func (m *StatusError) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_StatusError.Unmarshal(m, b) +} +func (m *StatusError) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_StatusError.Marshal(b, m, deterministic) +} +func (m *StatusError) XXX_Merge(src proto.Message) { + xxx_messageInfo_StatusError.Merge(m, src) +} +func (m *StatusError) XXX_Size() int { + return xxx_messageInfo_StatusError.Size(m) +} +func (m *StatusError) XXX_DiscardUnknown() { + xxx_messageInfo_StatusError.DiscardUnknown(m) +} + +var xxx_messageInfo_StatusError proto.InternalMessageInfo + +func (m *StatusError) GetCode() uint32 { + if m != nil { + return m.Code + } + return 0 +} + +func (m *StatusError) GetMessage() string { + if m != nil { + return m.Message + } + return "" +} + +type RowResponse struct { + Headers []*ColumnInfo `protobuf:"bytes,1,rep,name=headers,proto3" json:"headers,omitempty"` + Columns []*ColumnResponse `protobuf:"bytes,2,rep,name=columns,proto3" json:"columns,omitempty"` + StatusError *StatusError `protobuf:"bytes,3,opt,name=StatusError,proto3" json:"StatusError,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *RowResponse) Reset() { *m = RowResponse{} } +func (m *RowResponse) String() string { return proto.CompactTextString(m) } +func (*RowResponse) ProtoMessage() {} +func (*RowResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{13} +} + +func (m *RowResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_RowResponse.Unmarshal(m, b) +} +func (m *RowResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_RowResponse.Marshal(b, m, deterministic) +} +func (m *RowResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_RowResponse.Merge(m, src) +} +func (m *RowResponse) XXX_Size() int { + return xxx_messageInfo_RowResponse.Size(m) +} +func (m *RowResponse) XXX_DiscardUnknown() { + xxx_messageInfo_RowResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_RowResponse proto.InternalMessageInfo + +func (m *RowResponse) GetHeaders() []*ColumnInfo { + if m != nil { + return m.Headers + } + return nil +} + +func (m *RowResponse) GetColumns() []*ColumnResponse { + if m != nil { + return m.Columns + } + return nil +} + +func (m *RowResponse) GetStatusError() *StatusError { + if m != nil { + return m.StatusError + } + return nil +} + +type Row struct { + Columns []*ColumnResponse `protobuf:"bytes,1,rep,name=columns,proto3" json:"columns,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Row) Reset() { *m = Row{} } +func (m *Row) String() string { return proto.CompactTextString(m) } +func (*Row) ProtoMessage() {} +func (*Row) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{14} +} + +func (m *Row) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Row.Unmarshal(m, b) +} +func (m *Row) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Row.Marshal(b, m, deterministic) +} +func (m *Row) XXX_Merge(src proto.Message) { + xxx_messageInfo_Row.Merge(m, src) +} +func (m *Row) XXX_Size() int { + return xxx_messageInfo_Row.Size(m) +} +func (m *Row) XXX_DiscardUnknown() { + xxx_messageInfo_Row.DiscardUnknown(m) +} + +var xxx_messageInfo_Row proto.InternalMessageInfo + +func (m *Row) GetColumns() []*ColumnResponse { + if m != nil { + return m.Columns + } + return nil +} + +type TableResponse struct { + Headers []*ColumnInfo `protobuf:"bytes,1,rep,name=headers,proto3" json:"headers,omitempty"` + Rows []*Row `protobuf:"bytes,2,rep,name=rows,proto3" json:"rows,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *TableResponse) Reset() { *m = TableResponse{} } +func (m *TableResponse) String() string { return proto.CompactTextString(m) } +func (*TableResponse) ProtoMessage() {} +func (*TableResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{15} +} + +func (m *TableResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_TableResponse.Unmarshal(m, b) +} +func (m *TableResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_TableResponse.Marshal(b, m, deterministic) +} +func (m *TableResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_TableResponse.Merge(m, src) +} +func (m *TableResponse) XXX_Size() int { + return xxx_messageInfo_TableResponse.Size(m) +} +func (m *TableResponse) XXX_DiscardUnknown() { + xxx_messageInfo_TableResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_TableResponse proto.InternalMessageInfo + +func (m *TableResponse) GetHeaders() []*ColumnInfo { + if m != nil { + return m.Headers + } + return nil +} + +func (m *TableResponse) GetRows() []*Row { + if m != nil { + return m.Rows + } + return nil +} + +type ColumnInfo struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Datatype string `protobuf:"bytes,2,opt,name=datatype,proto3" json:"datatype,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ColumnInfo) Reset() { *m = ColumnInfo{} } +func (m *ColumnInfo) String() string { return proto.CompactTextString(m) } +func (*ColumnInfo) ProtoMessage() {} +func (*ColumnInfo) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{16} +} + +func (m *ColumnInfo) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ColumnInfo.Unmarshal(m, b) +} +func (m *ColumnInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ColumnInfo.Marshal(b, m, deterministic) +} +func (m *ColumnInfo) XXX_Merge(src proto.Message) { + xxx_messageInfo_ColumnInfo.Merge(m, src) +} +func (m *ColumnInfo) XXX_Size() int { + return xxx_messageInfo_ColumnInfo.Size(m) +} +func (m *ColumnInfo) XXX_DiscardUnknown() { + xxx_messageInfo_ColumnInfo.DiscardUnknown(m) +} + +var xxx_messageInfo_ColumnInfo proto.InternalMessageInfo + +func (m *ColumnInfo) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *ColumnInfo) GetDatatype() string { + if m != nil { + return m.Datatype + } + return "" +} + +type ColumnResponse struct { + // Types that are valid to be assigned to ColumnVal: + // *ColumnResponse_StringVal + // *ColumnResponse_Uint64Val + // *ColumnResponse_Int64Val + // *ColumnResponse_BoolVal + // *ColumnResponse_BlobVal + // *ColumnResponse_Uint64ArrayVal + // *ColumnResponse_StringArrayVal + // *ColumnResponse_Float64Val + // *ColumnResponse_DecimalVal + ColumnVal isColumnResponse_ColumnVal `protobuf_oneof:"columnVal"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ColumnResponse) Reset() { *m = ColumnResponse{} } +func (m *ColumnResponse) String() string { return proto.CompactTextString(m) } +func (*ColumnResponse) ProtoMessage() {} +func (*ColumnResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{17} +} + +func (m *ColumnResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ColumnResponse.Unmarshal(m, b) +} +func (m *ColumnResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ColumnResponse.Marshal(b, m, deterministic) +} +func (m *ColumnResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ColumnResponse.Merge(m, src) +} +func (m *ColumnResponse) XXX_Size() int { + return xxx_messageInfo_ColumnResponse.Size(m) +} +func (m *ColumnResponse) XXX_DiscardUnknown() { + xxx_messageInfo_ColumnResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_ColumnResponse proto.InternalMessageInfo + +type isColumnResponse_ColumnVal interface { + isColumnResponse_ColumnVal() +} + +type ColumnResponse_StringVal struct { + StringVal string `protobuf:"bytes,1,opt,name=stringVal,proto3,oneof"` +} + +type ColumnResponse_Uint64Val struct { + Uint64Val uint64 `protobuf:"varint,2,opt,name=uint64Val,proto3,oneof"` +} + +type ColumnResponse_Int64Val struct { + Int64Val int64 `protobuf:"varint,3,opt,name=int64Val,proto3,oneof"` +} + +type ColumnResponse_BoolVal struct { + BoolVal bool `protobuf:"varint,4,opt,name=boolVal,proto3,oneof"` +} + +type ColumnResponse_BlobVal struct { + BlobVal []byte `protobuf:"bytes,5,opt,name=blobVal,proto3,oneof"` +} + +type ColumnResponse_Uint64ArrayVal struct { + Uint64ArrayVal *Uint64Array `protobuf:"bytes,6,opt,name=uint64ArrayVal,proto3,oneof"` +} + +type ColumnResponse_StringArrayVal struct { + StringArrayVal *StringArray `protobuf:"bytes,7,opt,name=stringArrayVal,proto3,oneof"` +} + +type ColumnResponse_Float64Val struct { + Float64Val float64 `protobuf:"fixed64,8,opt,name=float64Val,proto3,oneof"` +} + +type ColumnResponse_DecimalVal struct { + DecimalVal *Decimal `protobuf:"bytes,9,opt,name=decimalVal,proto3,oneof"` +} + +func (*ColumnResponse_StringVal) isColumnResponse_ColumnVal() {} + +func (*ColumnResponse_Uint64Val) isColumnResponse_ColumnVal() {} + +func (*ColumnResponse_Int64Val) isColumnResponse_ColumnVal() {} + +func (*ColumnResponse_BoolVal) isColumnResponse_ColumnVal() {} + +func (*ColumnResponse_BlobVal) isColumnResponse_ColumnVal() {} + +func (*ColumnResponse_Uint64ArrayVal) isColumnResponse_ColumnVal() {} + +func (*ColumnResponse_StringArrayVal) isColumnResponse_ColumnVal() {} + +func (*ColumnResponse_Float64Val) isColumnResponse_ColumnVal() {} + +func (*ColumnResponse_DecimalVal) isColumnResponse_ColumnVal() {} + +func (m *ColumnResponse) GetColumnVal() isColumnResponse_ColumnVal { + if m != nil { + return m.ColumnVal + } + return nil +} + +func (m *ColumnResponse) GetStringVal() string { + if x, ok := m.GetColumnVal().(*ColumnResponse_StringVal); ok { + return x.StringVal + } + return "" +} + +func (m *ColumnResponse) GetUint64Val() uint64 { + if x, ok := m.GetColumnVal().(*ColumnResponse_Uint64Val); ok { + return x.Uint64Val + } + return 0 +} + +func (m *ColumnResponse) GetInt64Val() int64 { + if x, ok := m.GetColumnVal().(*ColumnResponse_Int64Val); ok { + return x.Int64Val + } + return 0 +} + +func (m *ColumnResponse) GetBoolVal() bool { + if x, ok := m.GetColumnVal().(*ColumnResponse_BoolVal); ok { + return x.BoolVal + } + return false +} + +func (m *ColumnResponse) GetBlobVal() []byte { + if x, ok := m.GetColumnVal().(*ColumnResponse_BlobVal); ok { + return x.BlobVal + } + return nil +} + +func (m *ColumnResponse) GetUint64ArrayVal() *Uint64Array { + if x, ok := m.GetColumnVal().(*ColumnResponse_Uint64ArrayVal); ok { + return x.Uint64ArrayVal + } + return nil +} + +func (m *ColumnResponse) GetStringArrayVal() *StringArray { + if x, ok := m.GetColumnVal().(*ColumnResponse_StringArrayVal); ok { + return x.StringArrayVal + } + return nil +} + +func (m *ColumnResponse) GetFloat64Val() float64 { + if x, ok := m.GetColumnVal().(*ColumnResponse_Float64Val); ok { + return x.Float64Val + } + return 0 +} + +func (m *ColumnResponse) GetDecimalVal() *Decimal { + if x, ok := m.GetColumnVal().(*ColumnResponse_DecimalVal); ok { + return x.DecimalVal + } + return nil +} + +// XXX_OneofWrappers is for the internal use of the proto package. +func (*ColumnResponse) XXX_OneofWrappers() []interface{} { + return []interface{}{ + (*ColumnResponse_StringVal)(nil), + (*ColumnResponse_Uint64Val)(nil), + (*ColumnResponse_Int64Val)(nil), + (*ColumnResponse_BoolVal)(nil), + (*ColumnResponse_BlobVal)(nil), + (*ColumnResponse_Uint64ArrayVal)(nil), + (*ColumnResponse_StringArrayVal)(nil), + (*ColumnResponse_Float64Val)(nil), + (*ColumnResponse_DecimalVal)(nil), + } +} + +type Decimal struct { + Value int64 `protobuf:"varint,1,opt,name=value,proto3" json:"value,omitempty"` + Scale int64 `protobuf:"varint,2,opt,name=scale,proto3" json:"scale,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Decimal) Reset() { *m = Decimal{} } +func (m *Decimal) String() string { return proto.CompactTextString(m) } +func (*Decimal) ProtoMessage() {} +func (*Decimal) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{18} +} + +func (m *Decimal) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Decimal.Unmarshal(m, b) +} +func (m *Decimal) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Decimal.Marshal(b, m, deterministic) +} +func (m *Decimal) XXX_Merge(src proto.Message) { + xxx_messageInfo_Decimal.Merge(m, src) +} +func (m *Decimal) XXX_Size() int { + return xxx_messageInfo_Decimal.Size(m) +} +func (m *Decimal) XXX_DiscardUnknown() { + xxx_messageInfo_Decimal.DiscardUnknown(m) +} + +var xxx_messageInfo_Decimal proto.InternalMessageInfo + +func (m *Decimal) GetValue() int64 { + if m != nil { + return m.Value + } + return 0 +} + +func (m *Decimal) GetScale() int64 { + if m != nil { + return m.Scale + } + return 0 +} + +type InspectRequest struct { + Vds string `protobuf:"bytes,1,opt,name=vds,proto3" json:"vds,omitempty"` + Records *IdsOrKeys `protobuf:"bytes,2,opt,name=records,proto3" json:"records,omitempty"` + FilterFields []string `protobuf:"bytes,3,rep,name=filterFields,proto3" json:"filterFields,omitempty"` + Limit uint64 `protobuf:"varint,4,opt,name=limit,proto3" json:"limit,omitempty"` + Offset uint64 `protobuf:"varint,5,opt,name=offset,proto3" json:"offset,omitempty"` + Query string `protobuf:"bytes,6,opt,name=query,proto3" json:"query,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *InspectRequest) Reset() { *m = InspectRequest{} } +func (m *InspectRequest) String() string { return proto.CompactTextString(m) } +func (*InspectRequest) ProtoMessage() {} +func (*InspectRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{19} +} + +func (m *InspectRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_InspectRequest.Unmarshal(m, b) +} +func (m *InspectRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_InspectRequest.Marshal(b, m, deterministic) +} +func (m *InspectRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_InspectRequest.Merge(m, src) +} +func (m *InspectRequest) XXX_Size() int { + return xxx_messageInfo_InspectRequest.Size(m) +} +func (m *InspectRequest) XXX_DiscardUnknown() { + xxx_messageInfo_InspectRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_InspectRequest proto.InternalMessageInfo + +func (m *InspectRequest) GetVds() string { + if m != nil { + return m.Vds + } + return "" +} + +func (m *InspectRequest) GetRecords() *IdsOrKeys { + if m != nil { + return m.Records + } + return nil +} + +func (m *InspectRequest) GetFilterFields() []string { + if m != nil { + return m.FilterFields + } + return nil +} + +func (m *InspectRequest) GetLimit() uint64 { + if m != nil { + return m.Limit + } + return 0 +} + +func (m *InspectRequest) GetOffset() uint64 { + if m != nil { + return m.Offset + } + return 0 +} + +func (m *InspectRequest) GetQuery() string { + if m != nil { + return m.Query + } + return "" +} + +type Uint64Array struct { + Vals []uint64 `protobuf:"varint,1,rep,packed,name=vals,proto3" json:"vals,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Uint64Array) Reset() { *m = Uint64Array{} } +func (m *Uint64Array) String() string { return proto.CompactTextString(m) } +func (*Uint64Array) ProtoMessage() {} +func (*Uint64Array) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{20} +} + +func (m *Uint64Array) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Uint64Array.Unmarshal(m, b) +} +func (m *Uint64Array) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Uint64Array.Marshal(b, m, deterministic) +} +func (m *Uint64Array) XXX_Merge(src proto.Message) { + xxx_messageInfo_Uint64Array.Merge(m, src) +} +func (m *Uint64Array) XXX_Size() int { + return xxx_messageInfo_Uint64Array.Size(m) +} +func (m *Uint64Array) XXX_DiscardUnknown() { + xxx_messageInfo_Uint64Array.DiscardUnknown(m) +} + +var xxx_messageInfo_Uint64Array proto.InternalMessageInfo + +func (m *Uint64Array) GetVals() []uint64 { + if m != nil { + return m.Vals + } + return nil +} + +type StringArray struct { + Vals []string `protobuf:"bytes,1,rep,name=vals,proto3" json:"vals,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *StringArray) Reset() { *m = StringArray{} } +func (m *StringArray) String() string { return proto.CompactTextString(m) } +func (*StringArray) ProtoMessage() {} +func (*StringArray) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{21} +} + +func (m *StringArray) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_StringArray.Unmarshal(m, b) +} +func (m *StringArray) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_StringArray.Marshal(b, m, deterministic) +} +func (m *StringArray) XXX_Merge(src proto.Message) { + xxx_messageInfo_StringArray.Merge(m, src) +} +func (m *StringArray) XXX_Size() int { + return xxx_messageInfo_StringArray.Size(m) +} +func (m *StringArray) XXX_DiscardUnknown() { + xxx_messageInfo_StringArray.DiscardUnknown(m) +} + +var xxx_messageInfo_StringArray proto.InternalMessageInfo + +func (m *StringArray) GetVals() []string { + if m != nil { + return m.Vals + } + return nil +} + +type IdsOrKeys struct { + // Types that are valid to be assigned to Type: + // *IdsOrKeys_Ids + // *IdsOrKeys_Keys + Type isIdsOrKeys_Type `protobuf_oneof:"type"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *IdsOrKeys) Reset() { *m = IdsOrKeys{} } +func (m *IdsOrKeys) String() string { return proto.CompactTextString(m) } +func (*IdsOrKeys) ProtoMessage() {} +func (*IdsOrKeys) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{22} +} + +func (m *IdsOrKeys) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_IdsOrKeys.Unmarshal(m, b) +} +func (m *IdsOrKeys) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_IdsOrKeys.Marshal(b, m, deterministic) +} +func (m *IdsOrKeys) XXX_Merge(src proto.Message) { + xxx_messageInfo_IdsOrKeys.Merge(m, src) +} +func (m *IdsOrKeys) XXX_Size() int { + return xxx_messageInfo_IdsOrKeys.Size(m) +} +func (m *IdsOrKeys) XXX_DiscardUnknown() { + xxx_messageInfo_IdsOrKeys.DiscardUnknown(m) +} + +var xxx_messageInfo_IdsOrKeys proto.InternalMessageInfo + +type isIdsOrKeys_Type interface { + isIdsOrKeys_Type() +} + +type IdsOrKeys_Ids struct { + Ids *Uint64Array `protobuf:"bytes,1,opt,name=ids,proto3,oneof"` +} + +type IdsOrKeys_Keys struct { + Keys *StringArray `protobuf:"bytes,2,opt,name=keys,proto3,oneof"` +} + +func (*IdsOrKeys_Ids) isIdsOrKeys_Type() {} + +func (*IdsOrKeys_Keys) isIdsOrKeys_Type() {} + +func (m *IdsOrKeys) GetType() isIdsOrKeys_Type { + if m != nil { + return m.Type + } + return nil +} + +func (m *IdsOrKeys) GetIds() *Uint64Array { + if x, ok := m.GetType().(*IdsOrKeys_Ids); ok { + return x.Ids + } + return nil +} + +func (m *IdsOrKeys) GetKeys() *StringArray { + if x, ok := m.GetType().(*IdsOrKeys_Keys); ok { + return x.Keys + } + return nil +} + +// XXX_OneofWrappers is for the internal use of the proto package. +func (*IdsOrKeys) XXX_OneofWrappers() []interface{} { + return []interface{}{ + (*IdsOrKeys_Ids)(nil), + (*IdsOrKeys_Keys)(nil), + } +} + +func init() { + proto.RegisterType((*DataSource)(nil), "vdsm.DataSource") + proto.RegisterMapType((map[string]string)(nil), "vdsm.DataSource.ConfigEntry") + proto.RegisterType((*VDS)(nil), "vdsm.VDS") + proto.RegisterType((*GetVDSsRequest)(nil), "vdsm.GetVDSsRequest") + proto.RegisterType((*GetVDSsResponse)(nil), "vdsm.GetVDSsResponse") + proto.RegisterType((*GetVDSRequest)(nil), "vdsm.GetVDSRequest") + proto.RegisterType((*GetVDSResponse)(nil), "vdsm.GetVDSResponse") + proto.RegisterType((*PostVDSRequest)(nil), "vdsm.PostVDSRequest") + proto.RegisterType((*PostVDSResponse)(nil), "vdsm.PostVDSResponse") + proto.RegisterType((*DeleteVDSRequest)(nil), "vdsm.DeleteVDSRequest") + proto.RegisterType((*DeleteVDSResponse)(nil), "vdsm.DeleteVDSResponse") + proto.RegisterType((*QueryPQLRequest)(nil), "vdsm.QueryPQLRequest") + proto.RegisterType((*QuerySQLRequest)(nil), "vdsm.QuerySQLRequest") + proto.RegisterType((*StatusError)(nil), "vdsm.StatusError") + proto.RegisterType((*RowResponse)(nil), "vdsm.RowResponse") + proto.RegisterType((*Row)(nil), "vdsm.Row") + proto.RegisterType((*TableResponse)(nil), "vdsm.TableResponse") + proto.RegisterType((*ColumnInfo)(nil), "vdsm.ColumnInfo") + proto.RegisterType((*ColumnResponse)(nil), "vdsm.ColumnResponse") + proto.RegisterType((*Decimal)(nil), "vdsm.Decimal") + proto.RegisterType((*InspectRequest)(nil), "vdsm.InspectRequest") + proto.RegisterType((*Uint64Array)(nil), "vdsm.Uint64Array") + proto.RegisterType((*StringArray)(nil), "vdsm.StringArray") + proto.RegisterType((*IdsOrKeys)(nil), "vdsm.IdsOrKeys") +} + +func init() { proto.RegisterFile("vdsm/vdsm.proto", fileDescriptor_3849c3e549495f50) } + +var fileDescriptor_3849c3e549495f50 = []byte{ + // 1065 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0xdd, 0x6e, 0x1b, 0x45, + 0x14, 0xf6, 0x66, 0x1d, 0xdb, 0x7b, 0x36, 0x71, 0x9c, 0x69, 0x28, 0x96, 0x69, 0x2b, 0x77, 0x10, + 0xc2, 0x20, 0x91, 0x46, 0x49, 0x53, 0x05, 0x0a, 0x48, 0x24, 0x69, 0x71, 0x04, 0xa5, 0xee, 0x98, + 0xe6, 0x82, 0x1b, 0x34, 0xf1, 0x8e, 0xc3, 0xaa, 0xe3, 0x1d, 0x7b, 0x67, 0x9d, 0xe0, 0x3b, 0x9e, + 0x04, 0x24, 0xde, 0x81, 0xe7, 0xe2, 0x15, 0xd0, 0xfc, 0xad, 0x77, 0x4d, 0x22, 0x2a, 0x7a, 0x63, + 0xed, 0x7c, 0xe7, 0x3b, 0x7f, 0x73, 0xce, 0x9c, 0x63, 0xd8, 0xba, 0x8a, 0xe4, 0xe4, 0x91, 0xfa, + 0xd9, 0x9d, 0xa6, 0x22, 0x13, 0xa8, 0xaa, 0xbe, 0xf1, 0xdf, 0x1e, 0xc0, 0x29, 0xcd, 0xe8, 0x50, + 0xcc, 0xd3, 0x11, 0x43, 0x4d, 0x58, 0x8b, 0xa3, 0xb6, 0xd7, 0xf5, 0x7a, 0x3e, 0x59, 0x8b, 0x23, + 0x84, 0xa0, 0x9a, 0xd0, 0x09, 0x6b, 0xaf, 0x75, 0xbd, 0x5e, 0x40, 0xf4, 0x37, 0xea, 0x42, 0x18, + 0x31, 0x39, 0x4a, 0xe3, 0x69, 0x16, 0x8b, 0xa4, 0xed, 0x6b, 0x51, 0x11, 0x52, 0x5a, 0xd9, 0x62, + 0xca, 0xda, 0x55, 0xa3, 0xa5, 0xbe, 0xd1, 0x63, 0xa8, 0x8d, 0x44, 0x32, 0x8e, 0x2f, 0xdb, 0xeb, + 0x5d, 0xbf, 0x17, 0xee, 0xdf, 0xdb, 0xd5, 0xb1, 0x2c, 0x7d, 0xef, 0x9e, 0x68, 0xf1, 0xb3, 0x24, + 0x4b, 0x17, 0xc4, 0x72, 0xd1, 0x5d, 0xa8, 0xc9, 0x8c, 0x66, 0x73, 0xd9, 0xae, 0x69, 0x5b, 0xf6, + 0xd4, 0xf9, 0x1c, 0xc2, 0x02, 0x1d, 0xb5, 0xc0, 0x7f, 0xc3, 0x16, 0x3a, 0xee, 0x80, 0xa8, 0x4f, + 0xb4, 0x03, 0xeb, 0x57, 0x94, 0xcf, 0x5d, 0xe4, 0xe6, 0xf0, 0xc5, 0xda, 0x91, 0x87, 0xff, 0xf4, + 0xc0, 0x3f, 0x3f, 0x1d, 0x16, 0x52, 0x0d, 0xde, 0x21, 0xd5, 0x87, 0xb0, 0x31, 0x8d, 0xb9, 0x90, + 0xf4, 0xe7, 0x38, 0x89, 0xd8, 0xaf, 0x36, 0xe5, 0xd0, 0x60, 0x67, 0x0a, 0x42, 0xfb, 0x10, 0x46, + 0x34, 0xa3, 0x52, 0x67, 0x29, 0x6d, 0xfa, 0xad, 0xd5, 0xf4, 0x49, 0x91, 0x84, 0x5b, 0xd0, 0xfc, + 0x96, 0x65, 0xe7, 0xa7, 0x43, 0x49, 0xd8, 0x6c, 0xce, 0x64, 0x86, 0xf7, 0x60, 0x2b, 0x47, 0xe4, + 0x54, 0x24, 0x92, 0xa1, 0xfb, 0xa0, 0x6a, 0x28, 0xdb, 0x9e, 0xb6, 0x18, 0x18, 0x8b, 0xe7, 0xa7, + 0x43, 0xa2, 0x61, 0x7c, 0x02, 0x9b, 0x46, 0xc3, 0x9a, 0x40, 0xad, 0x65, 0xc6, 0xfd, 0x8a, 0xce, + 0x79, 0xa7, 0x98, 0x73, 0xbf, 0x62, 0xb2, 0x3e, 0x06, 0x68, 0xc4, 0xd1, 0xcb, 0xf4, 0x07, 0x3a, + 0x61, 0xf8, 0x33, 0x17, 0x48, 0xee, 0xf5, 0x03, 0xf0, 0xaf, 0x22, 0xa9, 0xcd, 0x94, 0x9c, 0x2a, + 0x14, 0xef, 0x41, 0x73, 0x20, 0x64, 0xd1, 0xe9, 0x03, 0x80, 0x88, 0x8d, 0xe3, 0x24, 0xd6, 0x37, + 0x68, 0xae, 0xbb, 0x80, 0xe0, 0x03, 0xd8, 0xca, 0x35, 0xac, 0x87, 0xd5, 0xca, 0xb4, 0xc0, 0x9f, + 0xa7, 0xb1, 0x2d, 0x8c, 0xfa, 0xc4, 0xcf, 0xa1, 0x75, 0xca, 0x38, 0xcb, 0xd8, 0x3b, 0x66, 0x77, + 0x07, 0xb6, 0x0b, 0x76, 0x8c, 0x7b, 0x7c, 0x08, 0x5b, 0xaf, 0xe6, 0x2c, 0x5d, 0x0c, 0x5e, 0x7d, + 0xbf, 0xb4, 0x9d, 0xe7, 0x1c, 0xe8, 0x44, 0x15, 0x32, 0x9d, 0x71, 0x17, 0xd3, 0x74, 0xc6, 0xf1, + 0x87, 0x56, 0x6d, 0x58, 0x52, 0x93, 0x33, 0xee, 0xd4, 0xe4, 0x8c, 0xe3, 0xa7, 0x10, 0x0e, 0x75, + 0x07, 0x3f, 0x4b, 0x53, 0x91, 0xaa, 0x9e, 0x3b, 0x11, 0x11, 0xd3, 0x8c, 0x4d, 0xa2, 0xbf, 0x51, + 0x1b, 0xea, 0x2f, 0x98, 0x94, 0xf4, 0xd2, 0xb5, 0xa2, 0x3b, 0xe2, 0xdf, 0x3d, 0x08, 0x89, 0xb8, + 0xce, 0xef, 0xe9, 0x53, 0xa8, 0xff, 0xc2, 0x68, 0xc4, 0x52, 0xd7, 0x02, 0xb6, 0xa9, 0x4e, 0x04, + 0x9f, 0x4f, 0x92, 0xb3, 0x64, 0x2c, 0x88, 0x23, 0xa0, 0x5d, 0xa8, 0x8f, 0x34, 0x2c, 0xdb, 0x6b, + 0x9a, 0xbb, 0x53, 0xe4, 0x3a, 0x93, 0xc4, 0x91, 0xd0, 0x41, 0x29, 0x50, 0xdd, 0xf9, 0xe1, 0xfe, + 0xb6, 0xd1, 0x29, 0x08, 0x48, 0x91, 0x85, 0x0f, 0xc1, 0x27, 0xe2, 0xba, 0xe8, 0xcb, 0x7b, 0x0b, + 0x5f, 0xf8, 0x27, 0xd8, 0xfc, 0x91, 0x5e, 0x70, 0xf6, 0xbf, 0x12, 0xbb, 0x0f, 0xd5, 0x54, 0x5c, + 0xbb, 0xac, 0x6c, 0x3f, 0xaa, 0x5b, 0xd2, 0x30, 0xfe, 0x12, 0x60, 0xa9, 0x95, 0xbf, 0x71, 0xaf, + 0xf0, 0xc6, 0x3b, 0xd0, 0x50, 0x2f, 0x4f, 0x0f, 0x2c, 0x73, 0xe1, 0xf9, 0x19, 0xff, 0xe6, 0x43, + 0xb3, 0x1c, 0x35, 0x7a, 0x00, 0x81, 0xcc, 0xd2, 0x38, 0xb9, 0x3c, 0xa7, 0x3c, 0xef, 0xb6, 0x25, + 0xa4, 0xe4, 0xf3, 0x38, 0xc9, 0x9e, 0x3c, 0x56, 0x72, 0x65, 0xaf, 0xaa, 0xe4, 0x39, 0x84, 0xee, + 0x41, 0x23, 0x17, 0xab, 0x5b, 0xf5, 0xfb, 0x15, 0x92, 0x23, 0xa8, 0x03, 0xf5, 0x0b, 0x21, 0xb8, + 0x12, 0xaa, 0x49, 0xd2, 0xe8, 0x57, 0x88, 0x03, 0xb4, 0x8c, 0x8b, 0x0b, 0x25, 0x5b, 0xef, 0x7a, + 0xbd, 0x0d, 0x2d, 0x33, 0x00, 0x7a, 0x0a, 0x4d, 0xe3, 0xe2, 0x9b, 0x34, 0xa5, 0x0b, 0x45, 0xa9, + 0x15, 0x2b, 0xf6, 0x7a, 0x29, 0xeb, 0x57, 0xc8, 0x0a, 0x55, 0x29, 0x9b, 0xf8, 0x73, 0xe5, 0x7a, + 0xb9, 0xdc, 0xb9, 0x4c, 0x29, 0x97, 0xa9, 0xa8, 0x0b, 0x30, 0xe6, 0x82, 0xda, 0x8c, 0x1a, 0x5d, + 0xaf, 0xe7, 0xf5, 0x2b, 0xa4, 0x80, 0xa1, 0x47, 0x6a, 0x02, 0x8c, 0xe2, 0x09, 0xd5, 0x69, 0x05, + 0xda, 0xf4, 0xa6, 0x1d, 0x7f, 0x06, 0x57, 0x0a, 0x4b, 0xca, 0x71, 0x08, 0x81, 0x69, 0x8d, 0x73, + 0xca, 0xf1, 0x21, 0xd4, 0x2d, 0x6b, 0x39, 0xd3, 0xcd, 0x7e, 0x32, 0x07, 0x85, 0xca, 0x11, 0xe5, + 0xa6, 0x78, 0x3e, 0x31, 0x07, 0xfc, 0x97, 0x07, 0xcd, 0xb3, 0x44, 0x4e, 0xd9, 0x28, 0xbb, 0xfd, + 0x11, 0x7f, 0x02, 0xf5, 0x94, 0x8d, 0x44, 0x1a, 0x49, 0xad, 0x1c, 0xee, 0x6f, 0x99, 0xb0, 0xce, + 0x22, 0xf9, 0x32, 0xfd, 0x8e, 0x2d, 0x24, 0x71, 0x72, 0x84, 0x61, 0x63, 0x1c, 0xf3, 0x8c, 0xa5, + 0xcf, 0x63, 0xc6, 0x23, 0xd9, 0xf6, 0xbb, 0x7e, 0x2f, 0x20, 0x25, 0x4c, 0x45, 0xc2, 0xe3, 0x49, + 0x9c, 0xe9, 0xd2, 0x55, 0x89, 0x39, 0xa8, 0x15, 0x26, 0xc6, 0x63, 0xc9, 0x32, 0x5d, 0xb5, 0x2a, + 0xb1, 0x27, 0xc5, 0x9e, 0xa9, 0x79, 0x61, 0x37, 0x9b, 0x39, 0xe0, 0x87, 0x10, 0x16, 0x8a, 0xa5, + 0x1a, 0xf6, 0x8a, 0x72, 0xf3, 0x0c, 0xaa, 0x44, 0x7f, 0x2b, 0x4a, 0xa1, 0x24, 0x25, 0x4a, 0x60, + 0x29, 0x23, 0x08, 0xf2, 0x1c, 0xd0, 0x47, 0xe0, 0xc7, 0xf9, 0xc0, 0xbe, 0xb1, 0x21, 0x94, 0x1c, + 0x7d, 0x0c, 0xd5, 0x37, 0x6c, 0xe1, 0x6e, 0xe2, 0xc6, 0xda, 0x6b, 0xc2, 0x71, 0xcd, 0x6c, 0xf7, + 0xfd, 0x3f, 0xaa, 0xd0, 0x78, 0x21, 0x38, 0x1b, 0xcd, 0x39, 0x45, 0x47, 0x50, 0xb7, 0xeb, 0x09, + 0xd9, 0xd7, 0x5e, 0xde, 0x5f, 0x9d, 0xf7, 0x56, 0x50, 0x3b, 0x6c, 0x2b, 0xe8, 0x10, 0x6a, 0x06, + 0x44, 0x77, 0x8a, 0x14, 0xa7, 0xb7, 0x53, 0x06, 0x73, 0xb5, 0x23, 0xa8, 0xdb, 0xbd, 0xe1, 0x1c, + 0x96, 0x17, 0x8f, 0x73, 0xb8, 0xb2, 0x5c, 0x70, 0x05, 0x7d, 0x0d, 0x41, 0x3e, 0xf4, 0xd1, 0x5d, + 0xd7, 0x88, 0xe5, 0x6d, 0xd2, 0x79, 0xff, 0x5f, 0x78, 0xc1, 0x73, 0xc3, 0x0d, 0x7a, 0x64, 0x9d, + 0xac, 0x0c, 0xfe, 0xce, 0xf6, 0x72, 0x0c, 0xe5, 0x7a, 0x7b, 0x1e, 0xfa, 0x0a, 0x36, 0x1d, 0xf3, + 0x75, 0x42, 0xd3, 0xc5, 0x6d, 0xea, 0xf6, 0x22, 0x4a, 0x43, 0xb1, 0xe0, 0x78, 0xb0, 0xe2, 0x78, + 0xf0, 0xd6, 0x8e, 0x07, 0x37, 0x39, 0x1e, 0xfc, 0xa7, 0xe3, 0x27, 0x50, 0xb7, 0x6f, 0xc9, 0xdd, + 0x75, 0xf9, 0x69, 0xdd, 0xe2, 0xf6, 0xa2, 0xa6, 0xff, 0x69, 0x1e, 0xfc, 0x13, 0x00, 0x00, 0xff, + 0xff, 0xa0, 0xf1, 0x1f, 0xe4, 0x7c, 0x0a, 0x00, 0x00, +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// MoleculaClient is the client API for Molecula service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type MoleculaClient interface { + GetVDSs(ctx context.Context, in *GetVDSsRequest, opts ...grpc.CallOption) (*GetVDSsResponse, error) + GetVDS(ctx context.Context, in *GetVDSRequest, opts ...grpc.CallOption) (*GetVDSResponse, error) + PostVDS(ctx context.Context, in *PostVDSRequest, opts ...grpc.CallOption) (*PostVDSResponse, error) + DeleteVDS(ctx context.Context, in *DeleteVDSRequest, opts ...grpc.CallOption) (*DeleteVDSResponse, error) + QuerySQL(ctx context.Context, in *QuerySQLRequest, opts ...grpc.CallOption) (Molecula_QuerySQLClient, error) + QuerySQLUnary(ctx context.Context, in *QuerySQLRequest, opts ...grpc.CallOption) (*TableResponse, error) + QueryPQL(ctx context.Context, in *QueryPQLRequest, opts ...grpc.CallOption) (Molecula_QueryPQLClient, error) + QueryPQLUnary(ctx context.Context, in *QueryPQLRequest, opts ...grpc.CallOption) (*TableResponse, error) + Inspect(ctx context.Context, in *InspectRequest, opts ...grpc.CallOption) (Molecula_InspectClient, error) +} + +type moleculaClient struct { + cc *grpc.ClientConn +} + +func NewMoleculaClient(cc *grpc.ClientConn) MoleculaClient { + return &moleculaClient{cc} +} + +func (c *moleculaClient) GetVDSs(ctx context.Context, in *GetVDSsRequest, opts ...grpc.CallOption) (*GetVDSsResponse, error) { + out := new(GetVDSsResponse) + err := c.cc.Invoke(ctx, "/vdsm.Molecula/GetVDSs", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *moleculaClient) GetVDS(ctx context.Context, in *GetVDSRequest, opts ...grpc.CallOption) (*GetVDSResponse, error) { + out := new(GetVDSResponse) + err := c.cc.Invoke(ctx, "/vdsm.Molecula/GetVDS", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *moleculaClient) PostVDS(ctx context.Context, in *PostVDSRequest, opts ...grpc.CallOption) (*PostVDSResponse, error) { + out := new(PostVDSResponse) + err := c.cc.Invoke(ctx, "/vdsm.Molecula/PostVDS", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *moleculaClient) DeleteVDS(ctx context.Context, in *DeleteVDSRequest, opts ...grpc.CallOption) (*DeleteVDSResponse, error) { + out := new(DeleteVDSResponse) + err := c.cc.Invoke(ctx, "/vdsm.Molecula/DeleteVDS", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *moleculaClient) QuerySQL(ctx context.Context, in *QuerySQLRequest, opts ...grpc.CallOption) (Molecula_QuerySQLClient, error) { + stream, err := c.cc.NewStream(ctx, &_Molecula_serviceDesc.Streams[0], "/vdsm.Molecula/QuerySQL", opts...) + if err != nil { + return nil, err + } + x := &moleculaQuerySQLClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type Molecula_QuerySQLClient interface { + Recv() (*RowResponse, error) + grpc.ClientStream +} + +type moleculaQuerySQLClient struct { + grpc.ClientStream +} + +func (x *moleculaQuerySQLClient) Recv() (*RowResponse, error) { + m := new(RowResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *moleculaClient) QuerySQLUnary(ctx context.Context, in *QuerySQLRequest, opts ...grpc.CallOption) (*TableResponse, error) { + out := new(TableResponse) + err := c.cc.Invoke(ctx, "/vdsm.Molecula/QuerySQLUnary", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *moleculaClient) QueryPQL(ctx context.Context, in *QueryPQLRequest, opts ...grpc.CallOption) (Molecula_QueryPQLClient, error) { + stream, err := c.cc.NewStream(ctx, &_Molecula_serviceDesc.Streams[1], "/vdsm.Molecula/QueryPQL", opts...) + if err != nil { + return nil, err + } + x := &moleculaQueryPQLClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type Molecula_QueryPQLClient interface { + Recv() (*RowResponse, error) + grpc.ClientStream +} + +type moleculaQueryPQLClient struct { + grpc.ClientStream +} + +func (x *moleculaQueryPQLClient) Recv() (*RowResponse, error) { + m := new(RowResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *moleculaClient) QueryPQLUnary(ctx context.Context, in *QueryPQLRequest, opts ...grpc.CallOption) (*TableResponse, error) { + out := new(TableResponse) + err := c.cc.Invoke(ctx, "/vdsm.Molecula/QueryPQLUnary", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *moleculaClient) Inspect(ctx context.Context, in *InspectRequest, opts ...grpc.CallOption) (Molecula_InspectClient, error) { + stream, err := c.cc.NewStream(ctx, &_Molecula_serviceDesc.Streams[2], "/vdsm.Molecula/Inspect", opts...) + if err != nil { + return nil, err + } + x := &moleculaInspectClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type Molecula_InspectClient interface { + Recv() (*RowResponse, error) + grpc.ClientStream +} + +type moleculaInspectClient struct { + grpc.ClientStream +} + +func (x *moleculaInspectClient) Recv() (*RowResponse, error) { + m := new(RowResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// MoleculaServer is the server API for Molecula service. +type MoleculaServer interface { + GetVDSs(context.Context, *GetVDSsRequest) (*GetVDSsResponse, error) + GetVDS(context.Context, *GetVDSRequest) (*GetVDSResponse, error) + PostVDS(context.Context, *PostVDSRequest) (*PostVDSResponse, error) + DeleteVDS(context.Context, *DeleteVDSRequest) (*DeleteVDSResponse, error) + QuerySQL(*QuerySQLRequest, Molecula_QuerySQLServer) error + QuerySQLUnary(context.Context, *QuerySQLRequest) (*TableResponse, error) + QueryPQL(*QueryPQLRequest, Molecula_QueryPQLServer) error + QueryPQLUnary(context.Context, *QueryPQLRequest) (*TableResponse, error) + Inspect(*InspectRequest, Molecula_InspectServer) error +} + +// UnimplementedMoleculaServer can be embedded to have forward compatible implementations. +type UnimplementedMoleculaServer struct { +} + +func (*UnimplementedMoleculaServer) GetVDSs(ctx context.Context, req *GetVDSsRequest) (*GetVDSsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetVDSs not implemented") +} +func (*UnimplementedMoleculaServer) GetVDS(ctx context.Context, req *GetVDSRequest) (*GetVDSResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetVDS not implemented") +} +func (*UnimplementedMoleculaServer) PostVDS(ctx context.Context, req *PostVDSRequest) (*PostVDSResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method PostVDS not implemented") +} +func (*UnimplementedMoleculaServer) DeleteVDS(ctx context.Context, req *DeleteVDSRequest) (*DeleteVDSResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteVDS not implemented") +} +func (*UnimplementedMoleculaServer) QuerySQL(req *QuerySQLRequest, srv Molecula_QuerySQLServer) error { + return status.Errorf(codes.Unimplemented, "method QuerySQL not implemented") +} +func (*UnimplementedMoleculaServer) QuerySQLUnary(ctx context.Context, req *QuerySQLRequest) (*TableResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method QuerySQLUnary not implemented") +} +func (*UnimplementedMoleculaServer) QueryPQL(req *QueryPQLRequest, srv Molecula_QueryPQLServer) error { + return status.Errorf(codes.Unimplemented, "method QueryPQL not implemented") +} +func (*UnimplementedMoleculaServer) QueryPQLUnary(ctx context.Context, req *QueryPQLRequest) (*TableResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method QueryPQLUnary not implemented") +} +func (*UnimplementedMoleculaServer) Inspect(req *InspectRequest, srv Molecula_InspectServer) error { + return status.Errorf(codes.Unimplemented, "method Inspect not implemented") +} + +func RegisterMoleculaServer(s *grpc.Server, srv MoleculaServer) { + s.RegisterService(&_Molecula_serviceDesc, srv) +} + +func _Molecula_GetVDSs_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetVDSsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MoleculaServer).GetVDSs(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/vdsm.Molecula/GetVDSs", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MoleculaServer).GetVDSs(ctx, req.(*GetVDSsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Molecula_GetVDS_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetVDSRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MoleculaServer).GetVDS(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/vdsm.Molecula/GetVDS", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MoleculaServer).GetVDS(ctx, req.(*GetVDSRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Molecula_PostVDS_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PostVDSRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MoleculaServer).PostVDS(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/vdsm.Molecula/PostVDS", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MoleculaServer).PostVDS(ctx, req.(*PostVDSRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Molecula_DeleteVDS_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteVDSRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MoleculaServer).DeleteVDS(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/vdsm.Molecula/DeleteVDS", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MoleculaServer).DeleteVDS(ctx, req.(*DeleteVDSRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Molecula_QuerySQL_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(QuerySQLRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(MoleculaServer).QuerySQL(m, &moleculaQuerySQLServer{stream}) +} + +type Molecula_QuerySQLServer interface { + Send(*RowResponse) error + grpc.ServerStream +} + +type moleculaQuerySQLServer struct { + grpc.ServerStream +} + +func (x *moleculaQuerySQLServer) Send(m *RowResponse) error { + return x.ServerStream.SendMsg(m) +} + +func _Molecula_QuerySQLUnary_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(QuerySQLRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MoleculaServer).QuerySQLUnary(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/vdsm.Molecula/QuerySQLUnary", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MoleculaServer).QuerySQLUnary(ctx, req.(*QuerySQLRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Molecula_QueryPQL_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(QueryPQLRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(MoleculaServer).QueryPQL(m, &moleculaQueryPQLServer{stream}) +} + +type Molecula_QueryPQLServer interface { + Send(*RowResponse) error + grpc.ServerStream +} + +type moleculaQueryPQLServer struct { + grpc.ServerStream +} + +func (x *moleculaQueryPQLServer) Send(m *RowResponse) error { + return x.ServerStream.SendMsg(m) +} + +func _Molecula_QueryPQLUnary_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(QueryPQLRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MoleculaServer).QueryPQLUnary(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/vdsm.Molecula/QueryPQLUnary", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MoleculaServer).QueryPQLUnary(ctx, req.(*QueryPQLRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Molecula_Inspect_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(InspectRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(MoleculaServer).Inspect(m, &moleculaInspectServer{stream}) +} + +type Molecula_InspectServer interface { + Send(*RowResponse) error + grpc.ServerStream +} + +type moleculaInspectServer struct { + grpc.ServerStream +} + +func (x *moleculaInspectServer) Send(m *RowResponse) error { + return x.ServerStream.SendMsg(m) +} + +var _Molecula_serviceDesc = grpc.ServiceDesc{ + ServiceName: "vdsm.Molecula", + HandlerType: (*MoleculaServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetVDSs", + Handler: _Molecula_GetVDSs_Handler, + }, + { + MethodName: "GetVDS", + Handler: _Molecula_GetVDS_Handler, + }, + { + MethodName: "PostVDS", + Handler: _Molecula_PostVDS_Handler, + }, + { + MethodName: "DeleteVDS", + Handler: _Molecula_DeleteVDS_Handler, + }, + { + MethodName: "QuerySQLUnary", + Handler: _Molecula_QuerySQLUnary_Handler, + }, + { + MethodName: "QueryPQLUnary", + Handler: _Molecula_QueryPQLUnary_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "QuerySQL", + Handler: _Molecula_QuerySQL_Handler, + ServerStreams: true, + }, + { + StreamName: "QueryPQL", + Handler: _Molecula_QueryPQL_Handler, + ServerStreams: true, + }, + { + StreamName: "Inspect", + Handler: _Molecula_Inspect_Handler, + ServerStreams: true, + }, + }, + Metadata: "vdsm/vdsm.proto", +} diff --git a/proto/vdsm/vdsm.proto b/proto/vdsm/vdsm.proto new file mode 100644 index 000000000..2798b2cc5 --- /dev/null +++ b/proto/vdsm/vdsm.proto @@ -0,0 +1,146 @@ +syntax = "proto3"; +package vdsm; + +// deprecated +message DataSource { + int64 id = 1; + string name = 2; + string description = 3; + string type = 4; + map config = 5; + string status = 6; +} + +message VDS { + string id = 1; + string name = 2; + string description = 3; + string pilosa_index = 4; + repeated DataSource datasources = 5; +} + +message GetVDSsRequest { +} + +message GetVDSsResponse { + repeated VDS vdss = 1; +} + +message GetVDSRequest { + oneof idOrName { + string id = 1; + string name = 2; + } +} + +message GetVDSResponse { + VDS vds = 1; +} + +message PostVDSRequest { + string definition = 1; +} + +message PostVDSResponse { + string id = 1; + string uri = 2; +} + +message DeleteVDSRequest { + oneof idOrName { + string id = 1; + string name = 2; + } +} + +message DeleteVDSResponse { +} + +message QueryPQLRequest { + string vds = 1; + string pql = 2; +} + +message QuerySQLRequest { + string sql = 1; +} + +message StatusError{ + uint32 Code = 1; + string Message = 2; +} + +message RowResponse{ + repeated ColumnInfo headers = 1; + repeated ColumnResponse columns = 2; + StatusError StatusError = 3; +} + +message Row { + repeated ColumnResponse columns = 1; +} + +message TableResponse{ + repeated ColumnInfo headers = 1; + repeated Row rows = 2; +} + +message ColumnInfo { + string name = 1; + string datatype = 2; +} + +message ColumnResponse{ + oneof columnVal { + string stringVal = 1; + uint64 uint64Val = 2; + int64 int64Val = 3; + bool boolVal = 4; + bytes blobVal = 5; + Uint64Array uint64ArrayVal = 6; + StringArray stringArrayVal = 7; + double float64Val = 8; + Decimal decimalVal = 9; + } +} + +message Decimal { + int64 value = 1; + int64 scale = 2; +} + +message InspectRequest { + string vds = 1; + IdsOrKeys records = 2; + repeated string filterFields = 3; + uint64 limit = 4; + uint64 offset = 5; + string query = 6; +} + +message Uint64Array { + repeated uint64 vals = 1; +} + +message StringArray { + repeated string vals = 1; +} + +message IdsOrKeys { + oneof type { + Uint64Array ids = 1; + StringArray keys = 2; + } +} + +service Molecula { + rpc GetVDSs(GetVDSsRequest) returns (GetVDSsResponse) {}; + rpc GetVDS(GetVDSRequest) returns (GetVDSResponse) {}; + rpc PostVDS(PostVDSRequest) returns (PostVDSResponse) {}; + rpc DeleteVDS(DeleteVDSRequest) returns (DeleteVDSResponse) {}; + rpc QuerySQL(QuerySQLRequest) returns (stream RowResponse) {}; + rpc QuerySQLUnary(QuerySQLRequest) returns (TableResponse) {}; + rpc QueryPQL(QueryPQLRequest) returns (stream RowResponse) {}; + rpc QueryPQLUnary(QueryPQLRequest) returns (TableResponse) {}; + rpc Inspect(InspectRequest) returns (stream RowResponse) {}; +} diff --git a/server/grpc.go b/server/grpc.go index ec99f0685..a84e059f9 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -26,6 +26,7 @@ import ( "github.com/pilosa/pilosa/v2" "github.com/pilosa/pilosa/v2/logger" pb "github.com/pilosa/pilosa/v2/proto" + vdsm_pb "github.com/pilosa/pilosa/v2/proto/vdsm" "github.com/pilosa/pilosa/v2/stats" "github.com/pkg/errors" "google.golang.org/grpc" @@ -125,48 +126,6 @@ func errToStatusError(err error) error { return status.Error(codes.Unknown, err.Error()) } -// GetVDSs returns a single VDS given a name -func (h *GRPCHandler) GetVDS(ctx context.Context, req *pb.GetVDSRequest) (*pb.GetVDSResponse, error) { - // TODO: Return all schema information associated with the VDS. - // It's obviously not very useful to return the same data as given. - schema := h.api.Schema(ctx) - for _, index := range schema { - if req.Name == index.Name { - return &pb.GetVDSResponse{Vds: &pb.VDS{Name: index.Name}}, nil - } - } - return nil, status.Error(codes.NotFound, fmt.Sprintf("VDS with name %s not found", req.Name)) -} - -// GetVDSs returns a list of all VDSs -func (h *GRPCHandler) GetVDSs(ctx context.Context, req *pb.GetVDSsRequest) (*pb.GetVDSsResponse, error) { - schema := h.api.Schema(ctx) - vdss := make([]*pb.VDS, len(schema)) - for i, index := range schema { - vdss[i] = &pb.VDS{Name: index.Name} - } - return &pb.GetVDSsResponse{Vdss: vdss}, nil -} - -// PostVDS creates a new VDS -func (h *GRPCHandler) PostVDS(ctx context.Context, req *pb.PostVDSRequest) (*pb.PostVDSResponse, error) { - opts := pilosa.IndexOptions{Keys: req.Keys, TrackExistence: req.TrackExistence} - _, err := h.api.CreateIndex(ctx, req.Name, opts) - if err != nil { - return nil, errToStatusError(err) - } - return &pb.PostVDSResponse{}, nil -} - -// DeleteVDS deletes a VDS -func (h *GRPCHandler) DeleteVDS(ctx context.Context, req *pb.DeleteVDSRequest) (*pb.DeleteVDSResponse, error) { - err := h.api.DeleteIndex(ctx, req.Name) - if err != nil { - return nil, errToStatusError(err) - } - return &pb.DeleteVDSResponse{}, nil -} - func (h *GRPCHandler) execSQL(ctx context.Context, queryStr string) (pb.ToRowser, error) { h.stats.Count(pilosa.MetricSqlQueries, 1, 1) return execSQL(ctx, h.api, h.logger, queryStr) @@ -292,6 +251,94 @@ func (h *GRPCHandler) QueryPQLUnary(ctx context.Context, req *pb.QueryPQLRequest return table, errToStatusError(nil) } +// VDSMGRPCHandler contains methods which handle the various gRPC requests, ported from VDSM. +type VDSMGRPCHandler struct { + api *pilosa.API + logger logger.Logger + stats stats.StatsClient +} + +func NewVDSMGRPCHandler(api *pilosa.API) *VDSMGRPCHandler { + return &VDSMGRPCHandler{api: api, logger: logger.NopLogger, stats: stats.NopStatsClient} +} + +func (h *VDSMGRPCHandler) WithLogger(logger logger.Logger) *VDSMGRPCHandler { + h.logger = logger + return h +} + +func (h *VDSMGRPCHandler) WithStats(stats stats.StatsClient) *VDSMGRPCHandler { + h.stats = stats + return h +} + +// GetVDSs returns a single VDS given a name +func (h *VDSMGRPCHandler) GetVDS(ctx context.Context, req *vdsm_pb.GetVDSRequest) (*vdsm_pb.GetVDSResponse, error) { + typedIdOrName := req.GetIdOrName() + switch idOrName := typedIdOrName.(type) { + case *vdsm_pb.GetVDSRequest_Id: + return nil, status.Error(codes.InvalidArgument, "VDS IDs are no longer supported") + case *vdsm_pb.GetVDSRequest_Name: + schema := h.api.Schema(ctx) + for _, index := range schema { + if idOrName.Name == index.Name { + return &vdsm_pb.GetVDSResponse{Vds: &vdsm_pb.VDS{Name: index.Name}}, nil + } + } + return nil, status.Error(codes.NotFound, fmt.Sprintf("VDS with name %s not found", idOrName.Name)) + default: + return nil, status.Error(codes.NotFound, "VDS not found") + } +} + +// GetVDSs returns a list of all VDSs +func (h *VDSMGRPCHandler) GetVDSs(ctx context.Context, req *vdsm_pb.GetVDSsRequest) (*vdsm_pb.GetVDSsResponse, error) { + schema := h.api.Schema(ctx) + vdss := make([]*vdsm_pb.VDS, len(schema)) + for i, index := range schema { + vdss[i] = &vdsm_pb.VDS{Name: index.Name} + } + return &vdsm_pb.GetVDSsResponse{Vdss: vdss}, nil +} + +// PostVDS creates a new VDS +func (*VDSMGRPCHandler) PostVDS(ctx context.Context, req *vdsm_pb.PostVDSRequest) (*vdsm_pb.PostVDSResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method PostVDS not implemented") +} + +// DeleteVDS deletes a VDS +func (h *VDSMGRPCHandler) DeleteVDS(ctx context.Context, req *vdsm_pb.DeleteVDSRequest) (*vdsm_pb.DeleteVDSResponse, error) { + typedIdOrName := req.GetIdOrName() + switch idOrName := typedIdOrName.(type) { + case *vdsm_pb.DeleteVDSRequest_Id: + return nil, status.Error(codes.InvalidArgument, "VDS IDs are no longer supported") + case *vdsm_pb.DeleteVDSRequest_Name: + err := h.api.DeleteIndex(ctx, idOrName.Name) + if err != nil { + return nil, errToStatusError(err) + } + return &vdsm_pb.DeleteVDSResponse{}, nil + default: + return nil, status.Error(codes.NotFound, "") + } +} + +func (*VDSMGRPCHandler) QuerySQL(req *vdsm_pb.QuerySQLRequest, srv vdsm_pb.Molecula_QuerySQLServer) error { + return status.Errorf(codes.Unimplemented, "method QuerySQL not implemented") +} +func (*VDSMGRPCHandler) QuerySQLUnary(ctx context.Context, req *vdsm_pb.QuerySQLRequest) (*vdsm_pb.TableResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method QuerySQLUnary not implemented") +} +func (*VDSMGRPCHandler) QueryPQL(req *vdsm_pb.QueryPQLRequest, srv vdsm_pb.Molecula_QueryPQLServer) error { + return status.Errorf(codes.Unimplemented, "method QueryPQL not implemented") +} +func (*VDSMGRPCHandler) QueryPQLUnary(ctx context.Context, req *vdsm_pb.QueryPQLRequest) (*vdsm_pb.TableResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method QueryPQLUnary not implemented") +} +func (*VDSMGRPCHandler) Inspect(req *vdsm_pb.InspectRequest, srv vdsm_pb.Molecula_InspectServer) error { + return status.Errorf(codes.Unimplemented, "method Inspect not implemented") +} + // ResultUint64 is a wrapper around a uint64 result type // so that we can implement the ToTabler and ToRowser // interfaces. @@ -1145,6 +1192,7 @@ func (s *grpcServer) Serve(tlsConfig *tls.Config) error { s.mu.Lock() s.grpcServer = grpc.NewServer(opts...) pb.RegisterPilosaServer(s.grpcServer, NewGRPCHandler(s.api).WithLogger(s.logger).WithStats(s.stats)) + vdsm_pb.RegisterMoleculaServer(s.grpcServer, NewVDSMGRPCHandler(s.api).WithLogger(s.logger).WithStats(s.stats)) // register the server so its services are available to grpc_cli and others reflection.Register(s.grpcServer) From 83704e7973f358a4ad942352f0b53e5c785a8e8c Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 17 Sep 2020 14:21:56 -0500 Subject: [PATCH 03/53] Proxy unary methods to Pilosa gRPC handler --- proto/vdsm/vdsm.pb.go | 151 ++++++++++++++++++++++-------------------- proto/vdsm/vdsm.proto | 3 +- server/grpc.go | 89 ++++++++++++++++++++++--- 3 files changed, 160 insertions(+), 83 deletions(-) diff --git a/proto/vdsm/vdsm.pb.go b/proto/vdsm/vdsm.pb.go index 2d4deaf42..edb45edde 100644 --- a/proto/vdsm/vdsm.pb.go +++ b/proto/vdsm/vdsm.pb.go @@ -562,7 +562,7 @@ func (m *DeleteVDSResponse) XXX_DiscardUnknown() { var xxx_messageInfo_DeleteVDSResponse proto.InternalMessageInfo type QueryPQLRequest struct { - Vds string `protobuf:"bytes,1,opt,name=vds,proto3" json:"vds,omitempty"` + Index string `protobuf:"bytes,1,opt,name=index,proto3" json:"index,omitempty"` Pql string `protobuf:"bytes,2,opt,name=pql,proto3" json:"pql,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -594,9 +594,9 @@ func (m *QueryPQLRequest) XXX_DiscardUnknown() { var xxx_messageInfo_QueryPQLRequest proto.InternalMessageInfo -func (m *QueryPQLRequest) GetVds() string { +func (m *QueryPQLRequest) GetIndex() string { if m != nil { - return m.Vds + return m.Index } return "" } @@ -791,6 +791,7 @@ func (m *Row) GetColumns() []*ColumnResponse { type TableResponse struct { Headers []*ColumnInfo `protobuf:"bytes,1,rep,name=headers,proto3" json:"headers,omitempty"` Rows []*Row `protobuf:"bytes,2,rep,name=rows,proto3" json:"rows,omitempty"` + StatusError *StatusError `protobuf:"bytes,3,opt,name=StatusError,proto3" json:"StatusError,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -835,6 +836,13 @@ func (m *TableResponse) GetRows() []*Row { return nil } +func (m *TableResponse) GetStatusError() *StatusError { + if m != nil { + return m.StatusError + } + return nil +} + type ColumnInfo struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` Datatype string `protobuf:"bytes,2,opt,name=datatype,proto3" json:"datatype,omitempty"` @@ -1381,74 +1389,75 @@ func init() { func init() { proto.RegisterFile("vdsm/vdsm.proto", fileDescriptor_3849c3e549495f50) } var fileDescriptor_3849c3e549495f50 = []byte{ - // 1065 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0xdd, 0x6e, 0x1b, 0x45, - 0x14, 0xf6, 0x66, 0x1d, 0xdb, 0x7b, 0x36, 0x71, 0x9c, 0x69, 0x28, 0x96, 0x69, 0x2b, 0x77, 0x10, - 0xc2, 0x20, 0x91, 0x46, 0x49, 0x53, 0x05, 0x0a, 0x48, 0x24, 0x69, 0x71, 0x04, 0xa5, 0xee, 0x98, - 0xe6, 0x82, 0x1b, 0x34, 0xf1, 0x8e, 0xc3, 0xaa, 0xe3, 0x1d, 0x7b, 0x67, 0x9d, 0xe0, 0x3b, 0x9e, - 0x04, 0x24, 0xde, 0x81, 0xe7, 0xe2, 0x15, 0xd0, 0xfc, 0xad, 0x77, 0x4d, 0x22, 0x2a, 0x7a, 0x63, - 0xed, 0x7c, 0xe7, 0x3b, 0x7f, 0x73, 0xce, 0x9c, 0x63, 0xd8, 0xba, 0x8a, 0xe4, 0xe4, 0x91, 0xfa, - 0xd9, 0x9d, 0xa6, 0x22, 0x13, 0xa8, 0xaa, 0xbe, 0xf1, 0xdf, 0x1e, 0xc0, 0x29, 0xcd, 0xe8, 0x50, - 0xcc, 0xd3, 0x11, 0x43, 0x4d, 0x58, 0x8b, 0xa3, 0xb6, 0xd7, 0xf5, 0x7a, 0x3e, 0x59, 0x8b, 0x23, - 0x84, 0xa0, 0x9a, 0xd0, 0x09, 0x6b, 0xaf, 0x75, 0xbd, 0x5e, 0x40, 0xf4, 0x37, 0xea, 0x42, 0x18, - 0x31, 0x39, 0x4a, 0xe3, 0x69, 0x16, 0x8b, 0xa4, 0xed, 0x6b, 0x51, 0x11, 0x52, 0x5a, 0xd9, 0x62, - 0xca, 0xda, 0x55, 0xa3, 0xa5, 0xbe, 0xd1, 0x63, 0xa8, 0x8d, 0x44, 0x32, 0x8e, 0x2f, 0xdb, 0xeb, - 0x5d, 0xbf, 0x17, 0xee, 0xdf, 0xdb, 0xd5, 0xb1, 0x2c, 0x7d, 0xef, 0x9e, 0x68, 0xf1, 0xb3, 0x24, - 0x4b, 0x17, 0xc4, 0x72, 0xd1, 0x5d, 0xa8, 0xc9, 0x8c, 0x66, 0x73, 0xd9, 0xae, 0x69, 0x5b, 0xf6, - 0xd4, 0xf9, 0x1c, 0xc2, 0x02, 0x1d, 0xb5, 0xc0, 0x7f, 0xc3, 0x16, 0x3a, 0xee, 0x80, 0xa8, 0x4f, - 0xb4, 0x03, 0xeb, 0x57, 0x94, 0xcf, 0x5d, 0xe4, 0xe6, 0xf0, 0xc5, 0xda, 0x91, 0x87, 0xff, 0xf4, - 0xc0, 0x3f, 0x3f, 0x1d, 0x16, 0x52, 0x0d, 0xde, 0x21, 0xd5, 0x87, 0xb0, 0x31, 0x8d, 0xb9, 0x90, - 0xf4, 0xe7, 0x38, 0x89, 0xd8, 0xaf, 0x36, 0xe5, 0xd0, 0x60, 0x67, 0x0a, 0x42, 0xfb, 0x10, 0x46, - 0x34, 0xa3, 0x52, 0x67, 0x29, 0x6d, 0xfa, 0xad, 0xd5, 0xf4, 0x49, 0x91, 0x84, 0x5b, 0xd0, 0xfc, - 0x96, 0x65, 0xe7, 0xa7, 0x43, 0x49, 0xd8, 0x6c, 0xce, 0x64, 0x86, 0xf7, 0x60, 0x2b, 0x47, 0xe4, - 0x54, 0x24, 0x92, 0xa1, 0xfb, 0xa0, 0x6a, 0x28, 0xdb, 0x9e, 0xb6, 0x18, 0x18, 0x8b, 0xe7, 0xa7, - 0x43, 0xa2, 0x61, 0x7c, 0x02, 0x9b, 0x46, 0xc3, 0x9a, 0x40, 0xad, 0x65, 0xc6, 0xfd, 0x8a, 0xce, - 0x79, 0xa7, 0x98, 0x73, 0xbf, 0x62, 0xb2, 0x3e, 0x06, 0x68, 0xc4, 0xd1, 0xcb, 0xf4, 0x07, 0x3a, - 0x61, 0xf8, 0x33, 0x17, 0x48, 0xee, 0xf5, 0x03, 0xf0, 0xaf, 0x22, 0xa9, 0xcd, 0x94, 0x9c, 0x2a, - 0x14, 0xef, 0x41, 0x73, 0x20, 0x64, 0xd1, 0xe9, 0x03, 0x80, 0x88, 0x8d, 0xe3, 0x24, 0xd6, 0x37, - 0x68, 0xae, 0xbb, 0x80, 0xe0, 0x03, 0xd8, 0xca, 0x35, 0xac, 0x87, 0xd5, 0xca, 0xb4, 0xc0, 0x9f, - 0xa7, 0xb1, 0x2d, 0x8c, 0xfa, 0xc4, 0xcf, 0xa1, 0x75, 0xca, 0x38, 0xcb, 0xd8, 0x3b, 0x66, 0x77, - 0x07, 0xb6, 0x0b, 0x76, 0x8c, 0x7b, 0x7c, 0x08, 0x5b, 0xaf, 0xe6, 0x2c, 0x5d, 0x0c, 0x5e, 0x7d, - 0xbf, 0xb4, 0x9d, 0xe7, 0x1c, 0xe8, 0x44, 0x15, 0x32, 0x9d, 0x71, 0x17, 0xd3, 0x74, 0xc6, 0xf1, - 0x87, 0x56, 0x6d, 0x58, 0x52, 0x93, 0x33, 0xee, 0xd4, 0xe4, 0x8c, 0xe3, 0xa7, 0x10, 0x0e, 0x75, - 0x07, 0x3f, 0x4b, 0x53, 0x91, 0xaa, 0x9e, 0x3b, 0x11, 0x11, 0xd3, 0x8c, 0x4d, 0xa2, 0xbf, 0x51, - 0x1b, 0xea, 0x2f, 0x98, 0x94, 0xf4, 0xd2, 0xb5, 0xa2, 0x3b, 0xe2, 0xdf, 0x3d, 0x08, 0x89, 0xb8, - 0xce, 0xef, 0xe9, 0x53, 0xa8, 0xff, 0xc2, 0x68, 0xc4, 0x52, 0xd7, 0x02, 0xb6, 0xa9, 0x4e, 0x04, - 0x9f, 0x4f, 0x92, 0xb3, 0x64, 0x2c, 0x88, 0x23, 0xa0, 0x5d, 0xa8, 0x8f, 0x34, 0x2c, 0xdb, 0x6b, - 0x9a, 0xbb, 0x53, 0xe4, 0x3a, 0x93, 0xc4, 0x91, 0xd0, 0x41, 0x29, 0x50, 0xdd, 0xf9, 0xe1, 0xfe, - 0xb6, 0xd1, 0x29, 0x08, 0x48, 0x91, 0x85, 0x0f, 0xc1, 0x27, 0xe2, 0xba, 0xe8, 0xcb, 0x7b, 0x0b, - 0x5f, 0xf8, 0x27, 0xd8, 0xfc, 0x91, 0x5e, 0x70, 0xf6, 0xbf, 0x12, 0xbb, 0x0f, 0xd5, 0x54, 0x5c, - 0xbb, 0xac, 0x6c, 0x3f, 0xaa, 0x5b, 0xd2, 0x30, 0xfe, 0x12, 0x60, 0xa9, 0x95, 0xbf, 0x71, 0xaf, - 0xf0, 0xc6, 0x3b, 0xd0, 0x50, 0x2f, 0x4f, 0x0f, 0x2c, 0x73, 0xe1, 0xf9, 0x19, 0xff, 0xe6, 0x43, - 0xb3, 0x1c, 0x35, 0x7a, 0x00, 0x81, 0xcc, 0xd2, 0x38, 0xb9, 0x3c, 0xa7, 0x3c, 0xef, 0xb6, 0x25, - 0xa4, 0xe4, 0xf3, 0x38, 0xc9, 0x9e, 0x3c, 0x56, 0x72, 0x65, 0xaf, 0xaa, 0xe4, 0x39, 0x84, 0xee, - 0x41, 0x23, 0x17, 0xab, 0x5b, 0xf5, 0xfb, 0x15, 0x92, 0x23, 0xa8, 0x03, 0xf5, 0x0b, 0x21, 0xb8, - 0x12, 0xaa, 0x49, 0xd2, 0xe8, 0x57, 0x88, 0x03, 0xb4, 0x8c, 0x8b, 0x0b, 0x25, 0x5b, 0xef, 0x7a, - 0xbd, 0x0d, 0x2d, 0x33, 0x00, 0x7a, 0x0a, 0x4d, 0xe3, 0xe2, 0x9b, 0x34, 0xa5, 0x0b, 0x45, 0xa9, - 0x15, 0x2b, 0xf6, 0x7a, 0x29, 0xeb, 0x57, 0xc8, 0x0a, 0x55, 0x29, 0x9b, 0xf8, 0x73, 0xe5, 0x7a, - 0xb9, 0xdc, 0xb9, 0x4c, 0x29, 0x97, 0xa9, 0xa8, 0x0b, 0x30, 0xe6, 0x82, 0xda, 0x8c, 0x1a, 0x5d, - 0xaf, 0xe7, 0xf5, 0x2b, 0xa4, 0x80, 0xa1, 0x47, 0x6a, 0x02, 0x8c, 0xe2, 0x09, 0xd5, 0x69, 0x05, - 0xda, 0xf4, 0xa6, 0x1d, 0x7f, 0x06, 0x57, 0x0a, 0x4b, 0xca, 0x71, 0x08, 0x81, 0x69, 0x8d, 0x73, - 0xca, 0xf1, 0x21, 0xd4, 0x2d, 0x6b, 0x39, 0xd3, 0xcd, 0x7e, 0x32, 0x07, 0x85, 0xca, 0x11, 0xe5, - 0xa6, 0x78, 0x3e, 0x31, 0x07, 0xfc, 0x97, 0x07, 0xcd, 0xb3, 0x44, 0x4e, 0xd9, 0x28, 0xbb, 0xfd, - 0x11, 0x7f, 0x02, 0xf5, 0x94, 0x8d, 0x44, 0x1a, 0x49, 0xad, 0x1c, 0xee, 0x6f, 0x99, 0xb0, 0xce, - 0x22, 0xf9, 0x32, 0xfd, 0x8e, 0x2d, 0x24, 0x71, 0x72, 0x84, 0x61, 0x63, 0x1c, 0xf3, 0x8c, 0xa5, - 0xcf, 0x63, 0xc6, 0x23, 0xd9, 0xf6, 0xbb, 0x7e, 0x2f, 0x20, 0x25, 0x4c, 0x45, 0xc2, 0xe3, 0x49, - 0x9c, 0xe9, 0xd2, 0x55, 0x89, 0x39, 0xa8, 0x15, 0x26, 0xc6, 0x63, 0xc9, 0x32, 0x5d, 0xb5, 0x2a, - 0xb1, 0x27, 0xc5, 0x9e, 0xa9, 0x79, 0x61, 0x37, 0x9b, 0x39, 0xe0, 0x87, 0x10, 0x16, 0x8a, 0xa5, - 0x1a, 0xf6, 0x8a, 0x72, 0xf3, 0x0c, 0xaa, 0x44, 0x7f, 0x2b, 0x4a, 0xa1, 0x24, 0x25, 0x4a, 0x60, - 0x29, 0x23, 0x08, 0xf2, 0x1c, 0xd0, 0x47, 0xe0, 0xc7, 0xf9, 0xc0, 0xbe, 0xb1, 0x21, 0x94, 0x1c, - 0x7d, 0x0c, 0xd5, 0x37, 0x6c, 0xe1, 0x6e, 0xe2, 0xc6, 0xda, 0x6b, 0xc2, 0x71, 0xcd, 0x6c, 0xf7, - 0xfd, 0x3f, 0xaa, 0xd0, 0x78, 0x21, 0x38, 0x1b, 0xcd, 0x39, 0x45, 0x47, 0x50, 0xb7, 0xeb, 0x09, - 0xd9, 0xd7, 0x5e, 0xde, 0x5f, 0x9d, 0xf7, 0x56, 0x50, 0x3b, 0x6c, 0x2b, 0xe8, 0x10, 0x6a, 0x06, - 0x44, 0x77, 0x8a, 0x14, 0xa7, 0xb7, 0x53, 0x06, 0x73, 0xb5, 0x23, 0xa8, 0xdb, 0xbd, 0xe1, 0x1c, - 0x96, 0x17, 0x8f, 0x73, 0xb8, 0xb2, 0x5c, 0x70, 0x05, 0x7d, 0x0d, 0x41, 0x3e, 0xf4, 0xd1, 0x5d, - 0xd7, 0x88, 0xe5, 0x6d, 0xd2, 0x79, 0xff, 0x5f, 0x78, 0xc1, 0x73, 0xc3, 0x0d, 0x7a, 0x64, 0x9d, - 0xac, 0x0c, 0xfe, 0xce, 0xf6, 0x72, 0x0c, 0xe5, 0x7a, 0x7b, 0x1e, 0xfa, 0x0a, 0x36, 0x1d, 0xf3, - 0x75, 0x42, 0xd3, 0xc5, 0x6d, 0xea, 0xf6, 0x22, 0x4a, 0x43, 0xb1, 0xe0, 0x78, 0xb0, 0xe2, 0x78, - 0xf0, 0xd6, 0x8e, 0x07, 0x37, 0x39, 0x1e, 0xfc, 0xa7, 0xe3, 0x27, 0x50, 0xb7, 0x6f, 0xc9, 0xdd, - 0x75, 0xf9, 0x69, 0xdd, 0xe2, 0xf6, 0xa2, 0xa6, 0xff, 0x69, 0x1e, 0xfc, 0x13, 0x00, 0x00, 0xff, - 0xff, 0xa0, 0xf1, 0x1f, 0xe4, 0x7c, 0x0a, 0x00, 0x00, + // 1081 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0xef, 0x6e, 0x1b, 0x45, + 0x10, 0xf7, 0xe5, 0x1c, 0xdb, 0x37, 0x97, 0x38, 0xce, 0x36, 0x14, 0xcb, 0xb4, 0x95, 0x7b, 0x08, + 0x61, 0x90, 0x48, 0xa3, 0xa4, 0xa9, 0x52, 0x0a, 0x48, 0x24, 0x69, 0x71, 0x04, 0xa5, 0xee, 0x9a, + 0xe6, 0x2b, 0xda, 0xf8, 0xd6, 0xe1, 0xd4, 0xf5, 0xad, 0x7d, 0x7b, 0x4e, 0xf0, 0x37, 0xde, 0x80, + 0x37, 0x00, 0x89, 0x77, 0xe0, 0xb9, 0x78, 0x05, 0xb4, 0xff, 0xce, 0x7b, 0x26, 0x15, 0x55, 0xfb, + 0xc5, 0xda, 0xfd, 0xcd, 0x6f, 0xfe, 0xed, 0xcc, 0xcd, 0x18, 0xb6, 0xae, 0x62, 0x31, 0x79, 0x20, + 0x7f, 0x76, 0xa7, 0x19, 0xcf, 0x39, 0xaa, 0xca, 0x73, 0xf4, 0x8f, 0x07, 0x70, 0x4a, 0x72, 0x32, + 0xe4, 0xf3, 0x6c, 0x44, 0x51, 0x13, 0xd6, 0x92, 0xb8, 0xed, 0x75, 0xbd, 0x9e, 0x8f, 0xd7, 0x92, + 0x18, 0x21, 0xa8, 0xa6, 0x64, 0x42, 0xdb, 0x6b, 0x5d, 0xaf, 0x17, 0x60, 0x75, 0x46, 0x5d, 0x08, + 0x63, 0x2a, 0x46, 0x59, 0x32, 0xcd, 0x13, 0x9e, 0xb6, 0x7d, 0x25, 0x72, 0x21, 0xa9, 0x95, 0x2f, + 0xa6, 0xb4, 0x5d, 0xd5, 0x5a, 0xf2, 0x8c, 0x1e, 0x42, 0x6d, 0xc4, 0xd3, 0x71, 0x72, 0xd9, 0x5e, + 0xef, 0xfa, 0xbd, 0x70, 0xff, 0xce, 0xae, 0x8a, 0x65, 0xe9, 0x7b, 0xf7, 0x44, 0x89, 0x9f, 0xa6, + 0x79, 0xb6, 0xc0, 0x86, 0x8b, 0x6e, 0x43, 0x4d, 0xe4, 0x24, 0x9f, 0x8b, 0x76, 0x4d, 0xd9, 0x32, + 0xb7, 0xce, 0x63, 0x08, 0x1d, 0x3a, 0x6a, 0x81, 0xff, 0x9a, 0x2e, 0x54, 0xdc, 0x01, 0x96, 0x47, + 0xb4, 0x03, 0xeb, 0x57, 0x84, 0xcd, 0x6d, 0xe4, 0xfa, 0xf2, 0xe5, 0xda, 0x91, 0x17, 0xfd, 0xe5, + 0x81, 0x7f, 0x7e, 0x3a, 0x74, 0x52, 0x0d, 0xde, 0x23, 0xd5, 0xfb, 0xb0, 0x31, 0x4d, 0x18, 0x17, + 0xe4, 0xe7, 0x24, 0x8d, 0xe9, 0xaf, 0x26, 0xe5, 0x50, 0x63, 0x67, 0x12, 0x42, 0xfb, 0x10, 0xc6, + 0x24, 0x27, 0x42, 0x65, 0x29, 0x4c, 0xfa, 0xad, 0xd5, 0xf4, 0xb1, 0x4b, 0x8a, 0x5a, 0xd0, 0xfc, + 0x8e, 0xe6, 0xe7, 0xa7, 0x43, 0x81, 0xe9, 0x6c, 0x4e, 0x45, 0x1e, 0xed, 0xc1, 0x56, 0x81, 0x88, + 0x29, 0x4f, 0x05, 0x45, 0x77, 0x41, 0xd6, 0x50, 0xb4, 0x3d, 0x65, 0x31, 0xd0, 0x16, 0xcf, 0x4f, + 0x87, 0x58, 0xc1, 0xd1, 0x09, 0x6c, 0x6a, 0x0d, 0x63, 0x02, 0xb5, 0x96, 0x19, 0xf7, 0x2b, 0x2a, + 0xe7, 0x1d, 0x37, 0xe7, 0x7e, 0x45, 0x67, 0x7d, 0x0c, 0xd0, 0x48, 0xe2, 0x17, 0xd9, 0x8f, 0x64, + 0x42, 0xa3, 0x2f, 0x6c, 0x20, 0x85, 0xd7, 0x8f, 0xc0, 0xbf, 0x8a, 0x85, 0x32, 0x53, 0x72, 0x2a, + 0xd1, 0x68, 0x0f, 0x9a, 0x03, 0x2e, 0x5c, 0xa7, 0xf7, 0x00, 0x62, 0x3a, 0x4e, 0xd2, 0x44, 0xbd, + 0xa0, 0x7e, 0x6e, 0x07, 0x89, 0x0e, 0x60, 0xab, 0xd0, 0x30, 0x1e, 0x56, 0x2b, 0xd3, 0x02, 0x7f, + 0x9e, 0x25, 0xa6, 0x30, 0xf2, 0x18, 0x3d, 0x83, 0xd6, 0x29, 0x65, 0x34, 0xa7, 0xef, 0x99, 0xdd, + 0x2d, 0xd8, 0x76, 0xec, 0x68, 0xf7, 0xd1, 0x63, 0xd8, 0x7a, 0x39, 0xa7, 0xd9, 0x62, 0xf0, 0xf2, + 0x07, 0x6b, 0x7b, 0x07, 0xd6, 0x75, 0x79, 0x75, 0x50, 0xfa, 0x22, 0xe3, 0x9a, 0xce, 0x98, 0x8d, + 0x6b, 0x3a, 0x63, 0xd1, 0xc7, 0x46, 0x75, 0xb8, 0x54, 0x6d, 0x81, 0x2f, 0x66, 0xcc, 0xb6, 0xa6, + 0x98, 0xb1, 0xe8, 0x09, 0x84, 0x43, 0xd5, 0xc5, 0x4f, 0xb3, 0x8c, 0x67, 0xb2, 0xef, 0x4e, 0x78, + 0x4c, 0x15, 0x63, 0x13, 0xab, 0x33, 0x6a, 0x43, 0xfd, 0x39, 0x15, 0x82, 0x5c, 0xda, 0x76, 0xb4, + 0xd7, 0xe8, 0x0f, 0x0f, 0x42, 0xcc, 0xaf, 0x8b, 0xb7, 0xfa, 0x1c, 0xea, 0xbf, 0x50, 0x12, 0xd3, + 0xcc, 0xb6, 0x81, 0x69, 0xac, 0x13, 0xce, 0xe6, 0x93, 0xf4, 0x2c, 0x1d, 0x73, 0x6c, 0x09, 0x68, + 0x17, 0xea, 0x23, 0x05, 0x8b, 0xf6, 0x9a, 0xe2, 0xee, 0xb8, 0x5c, 0x6b, 0x12, 0x5b, 0x12, 0x3a, + 0x28, 0x05, 0xaa, 0xba, 0x3f, 0xdc, 0xdf, 0xd6, 0x3a, 0x8e, 0x00, 0xbb, 0xac, 0xe8, 0x10, 0x7c, + 0xcc, 0xaf, 0x5d, 0x5f, 0xde, 0x5b, 0xf8, 0x8a, 0x7e, 0xf7, 0x60, 0xf3, 0x27, 0x72, 0xc1, 0xe8, + 0x3b, 0x65, 0x76, 0x17, 0xaa, 0x19, 0xbf, 0xb6, 0x69, 0x99, 0xa6, 0x94, 0xcf, 0xa4, 0xe0, 0x77, + 0x4b, 0xe4, 0x2b, 0x80, 0xa5, 0xab, 0x62, 0x3a, 0x78, 0xce, 0x74, 0xe8, 0x40, 0x43, 0x7e, 0xb3, + 0x6a, 0xd4, 0xe9, 0x32, 0x15, 0xf7, 0xe8, 0x37, 0x1f, 0x9a, 0xe5, 0x5c, 0xd1, 0x3d, 0x08, 0x44, + 0x9e, 0x25, 0xe9, 0xe5, 0x39, 0x61, 0x45, 0x9f, 0x2e, 0x21, 0x29, 0x9f, 0x27, 0x69, 0xfe, 0xe8, + 0xa1, 0x94, 0x4b, 0x7b, 0x55, 0x29, 0x2f, 0x20, 0x74, 0x07, 0x1a, 0x85, 0x58, 0xa6, 0xe0, 0xf7, + 0x2b, 0xb8, 0x40, 0x50, 0x07, 0xea, 0x17, 0x9c, 0x33, 0x29, 0x94, 0x33, 0xa8, 0xd1, 0xaf, 0x60, + 0x0b, 0x28, 0x19, 0xe3, 0x17, 0x52, 0xb6, 0xde, 0xf5, 0x7a, 0x1b, 0x4a, 0xa6, 0x01, 0xf4, 0x04, + 0x9a, 0xda, 0xc5, 0xb7, 0x59, 0x46, 0x16, 0x92, 0x52, 0x73, 0x9f, 0xe7, 0xd5, 0x52, 0xd6, 0xaf, + 0xe0, 0x15, 0xaa, 0x54, 0xd6, 0xf1, 0x17, 0xca, 0xf5, 0xf2, 0xdb, 0x16, 0x32, 0xa9, 0x5c, 0xa6, + 0xa2, 0x2e, 0xc0, 0x98, 0x71, 0x62, 0x32, 0x6a, 0x74, 0xbd, 0x9e, 0xd7, 0xaf, 0x60, 0x07, 0x43, + 0x0f, 0xe4, 0xec, 0x18, 0x25, 0x13, 0xa2, 0xd2, 0x0a, 0x94, 0xe9, 0x4d, 0x33, 0x38, 0x35, 0x2e, + 0x15, 0x96, 0x94, 0xe3, 0x10, 0x02, 0xdd, 0x50, 0xe7, 0x84, 0x45, 0x87, 0x50, 0x37, 0xac, 0xe5, + 0x36, 0xd0, 0x9b, 0x4d, 0x5f, 0x24, 0x2a, 0x46, 0x84, 0xe9, 0xe2, 0xf9, 0x58, 0x5f, 0xa2, 0xbf, + 0x3d, 0x68, 0x9e, 0xa5, 0x62, 0x4a, 0x47, 0xb9, 0xf3, 0x0d, 0xdb, 0x91, 0x17, 0xa8, 0x39, 0x87, + 0x3e, 0x83, 0x7a, 0x46, 0x47, 0x3c, 0x8b, 0x85, 0x52, 0x0e, 0xf7, 0xb7, 0x74, 0x58, 0x67, 0xb1, + 0x78, 0x91, 0x7d, 0x4f, 0x17, 0x02, 0x5b, 0x39, 0x8a, 0x60, 0x63, 0x9c, 0xb0, 0x9c, 0x66, 0xcf, + 0x12, 0xca, 0x62, 0xd1, 0xf6, 0xbb, 0x7e, 0x2f, 0xc0, 0x25, 0x4c, 0x46, 0xc2, 0x92, 0x49, 0x92, + 0xab, 0xd2, 0x55, 0xb1, 0xbe, 0xc8, 0xe5, 0xc7, 0xc7, 0x63, 0x41, 0x73, 0x55, 0xb5, 0x2a, 0x36, + 0x37, 0xc9, 0x9e, 0xc9, 0x29, 0x63, 0x76, 0xa2, 0xbe, 0x44, 0xf7, 0x21, 0x74, 0x8a, 0x25, 0x1b, + 0xf6, 0x8a, 0x30, 0xfd, 0xed, 0x54, 0xb1, 0x3a, 0x4b, 0x8a, 0x53, 0x92, 0x12, 0x25, 0x30, 0x94, + 0x11, 0x04, 0x45, 0x0e, 0xe8, 0x13, 0xf0, 0x93, 0x62, 0xd4, 0xdf, 0xd8, 0x10, 0x52, 0x8e, 0x3e, + 0x85, 0xea, 0x6b, 0xba, 0xb0, 0x2f, 0x71, 0x63, 0xed, 0x15, 0xe1, 0xb8, 0xa6, 0xff, 0x17, 0xec, + 0xff, 0x59, 0x85, 0xc6, 0x73, 0xce, 0xe8, 0x68, 0xce, 0x08, 0x3a, 0x82, 0xba, 0x59, 0x6c, 0xc8, + 0xcc, 0x88, 0xf2, 0xe6, 0xeb, 0x7c, 0xb0, 0x82, 0x9a, 0x31, 0x5d, 0x41, 0x87, 0x50, 0xd3, 0x20, + 0xba, 0xe5, 0x52, 0xac, 0xde, 0x4e, 0x19, 0x2c, 0xd4, 0x8e, 0xa0, 0x6e, 0x36, 0x8e, 0x75, 0x58, + 0x5e, 0x59, 0xd6, 0xe1, 0xca, 0x5a, 0x8a, 0x2a, 0xe8, 0x1b, 0x08, 0x8a, 0x75, 0x81, 0x6e, 0xdb, + 0x46, 0x2c, 0xef, 0xa1, 0xce, 0x87, 0xff, 0xc1, 0x1d, 0xcf, 0x0d, 0xbb, 0x1e, 0x90, 0x71, 0xb2, + 0xb2, 0x2e, 0x3a, 0xdb, 0xcb, 0xd9, 0x55, 0xe8, 0xed, 0x79, 0xe8, 0x6b, 0xd8, 0xb4, 0xcc, 0x57, + 0x29, 0xc9, 0x16, 0x6f, 0x52, 0x37, 0x0f, 0x51, 0x9a, 0xa4, 0x8e, 0xe3, 0xc1, 0x8a, 0xe3, 0xc1, + 0x5b, 0x3b, 0x1e, 0xdc, 0xe4, 0x78, 0xf0, 0xbf, 0x8e, 0x1f, 0x41, 0xdd, 0x7c, 0x4b, 0xf6, 0xad, + 0xcb, 0x9f, 0xd6, 0x1b, 0xdc, 0x5e, 0xd4, 0xd4, 0x7f, 0xd4, 0x83, 0x7f, 0x03, 0x00, 0x00, 0xff, + 0xff, 0x5a, 0xe9, 0x98, 0xbf, 0xb6, 0x0a, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/proto/vdsm/vdsm.proto b/proto/vdsm/vdsm.proto index 2798b2cc5..34b13de70 100644 --- a/proto/vdsm/vdsm.proto +++ b/proto/vdsm/vdsm.proto @@ -57,7 +57,7 @@ message DeleteVDSResponse { } message QueryPQLRequest { - string vds = 1; + string index = 1; string pql = 2; } @@ -83,6 +83,7 @@ message Row { message TableResponse{ repeated ColumnInfo headers = 1; repeated Row rows = 2; + StatusError StatusError = 3; } message ColumnInfo { diff --git a/server/grpc.go b/server/grpc.go index a84e059f9..7dba6558e 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -253,13 +253,14 @@ func (h *GRPCHandler) QueryPQLUnary(ctx context.Context, req *pb.QueryPQLRequest // VDSMGRPCHandler contains methods which handle the various gRPC requests, ported from VDSM. type VDSMGRPCHandler struct { - api *pilosa.API - logger logger.Logger - stats stats.StatsClient + grpcHandler *GRPCHandler + api *pilosa.API + logger logger.Logger + stats stats.StatsClient } -func NewVDSMGRPCHandler(api *pilosa.API) *VDSMGRPCHandler { - return &VDSMGRPCHandler{api: api, logger: logger.NopLogger, stats: stats.NopStatsClient} +func NewVDSMGRPCHandler(grpcHandler *GRPCHandler, api *pilosa.API) *VDSMGRPCHandler { + return &VDSMGRPCHandler{grpcHandler: grpcHandler, api: api, logger: logger.NopLogger, stats: stats.NopStatsClient} } func (h *VDSMGRPCHandler) WithLogger(logger logger.Logger) *VDSMGRPCHandler { @@ -303,6 +304,7 @@ func (h *VDSMGRPCHandler) GetVDSs(ctx context.Context, req *vdsm_pb.GetVDSsReque // PostVDS creates a new VDS func (*VDSMGRPCHandler) PostVDS(ctx context.Context, req *vdsm_pb.PostVDSRequest) (*vdsm_pb.PostVDSResponse, error) { + // Pilosa doesn't implement VDSD files, so this is unimplemented return nil, status.Errorf(codes.Unimplemented, "method PostVDS not implemented") } @@ -326,15 +328,31 @@ func (h *VDSMGRPCHandler) DeleteVDS(ctx context.Context, req *vdsm_pb.DeleteVDSR func (*VDSMGRPCHandler) QuerySQL(req *vdsm_pb.QuerySQLRequest, srv vdsm_pb.Molecula_QuerySQLServer) error { return status.Errorf(codes.Unimplemented, "method QuerySQL not implemented") } -func (*VDSMGRPCHandler) QuerySQLUnary(ctx context.Context, req *vdsm_pb.QuerySQLRequest) (*vdsm_pb.TableResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method QuerySQLUnary not implemented") + +func (h *VDSMGRPCHandler) QuerySQLUnary(ctx context.Context, req *vdsm_pb.QuerySQLRequest) (*vdsm_pb.TableResponse, error) { + pReq := pb.QuerySQLRequest(*req) + resp, err := h.grpcHandler.QuerySQLUnary(ctx, &pReq) + if err != nil { + return nil, err + } + pResp, err := tableToTable(resp) + return pResp, err } + func (*VDSMGRPCHandler) QueryPQL(req *vdsm_pb.QueryPQLRequest, srv vdsm_pb.Molecula_QueryPQLServer) error { return status.Errorf(codes.Unimplemented, "method QueryPQL not implemented") } -func (*VDSMGRPCHandler) QueryPQLUnary(ctx context.Context, req *vdsm_pb.QueryPQLRequest) (*vdsm_pb.TableResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method QueryPQLUnary not implemented") + +func (h *VDSMGRPCHandler) QueryPQLUnary(ctx context.Context, req *vdsm_pb.QueryPQLRequest) (*vdsm_pb.TableResponse, error) { + pReq := pb.QueryPQLRequest(*req) + resp, err := h.grpcHandler.QueryPQLUnary(ctx, &pReq) + if err != nil { + return nil, err + } + pResp, err := tableToTable(resp) + return pResp, err } + func (*VDSMGRPCHandler) Inspect(req *vdsm_pb.InspectRequest, srv vdsm_pb.Molecula_InspectServer) error { return status.Errorf(codes.Unimplemented, "method Inspect not implemented") } @@ -1191,8 +1209,9 @@ func (s *grpcServer) Serve(tlsConfig *tls.Config) error { // create grpc server s.mu.Lock() s.grpcServer = grpc.NewServer(opts...) - pb.RegisterPilosaServer(s.grpcServer, NewGRPCHandler(s.api).WithLogger(s.logger).WithStats(s.stats)) - vdsm_pb.RegisterMoleculaServer(s.grpcServer, NewVDSMGRPCHandler(s.api).WithLogger(s.logger).WithStats(s.stats)) + grpcHandler := NewGRPCHandler(s.api).WithLogger(s.logger).WithStats(s.stats) + pb.RegisterPilosaServer(s.grpcServer, grpcHandler) + vdsm_pb.RegisterMoleculaServer(s.grpcServer, NewVDSMGRPCHandler(grpcHandler, s.api).WithLogger(s.logger).WithStats(s.stats)) // register the server so its services are available to grpc_cli and others reflection.Register(s.grpcServer) @@ -1227,3 +1246,51 @@ func NewGRPCServer(opts ...grpcServerOption) (*grpcServer, error) { } return server, nil } + +// tableToTable is a helper function used by QueryPQLUnary +// to convert a pb.TableResponse into a vdsm_pb.TableResponse. +func tableToTable(ptbl *pb.TableResponse) (*vdsm_pb.TableResponse, error) { + vtbl := &vdsm_pb.TableResponse{ + Headers: make([]*vdsm_pb.ColumnInfo, len(ptbl.Headers)), + Rows: make([]*vdsm_pb.Row, len(ptbl.Rows)), + } + + // Headers + for i, info := range ptbl.Headers { + vtbl.Headers[i] = &vdsm_pb.ColumnInfo{Name: info.Name, Datatype: info.Datatype} + } + + // Rows + for r := range ptbl.Rows { + columns := make([]*vdsm_pb.ColumnResponse, len(ptbl.Rows[r].Columns)) + for i, col := range ptbl.Rows[r].Columns { + switch v := col.GetColumnVal().(type) { + case *pb.ColumnResponse_StringVal: + columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_StringVal{StringVal: v.StringVal}} + case *pb.ColumnResponse_Uint64Val: + columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_Uint64Val{Uint64Val: v.Uint64Val}} + case *pb.ColumnResponse_Int64Val: + columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_Int64Val{Int64Val: v.Int64Val}} + case *pb.ColumnResponse_BoolVal: + columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_BoolVal{BoolVal: v.BoolVal}} + case *pb.ColumnResponse_BlobVal: + columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_BlobVal{BlobVal: v.BlobVal}} + case *pb.ColumnResponse_Uint64ArrayVal: + columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_Uint64ArrayVal{Uint64ArrayVal: &vdsm_pb.Uint64Array{Vals: v.Uint64ArrayVal.Vals}}} + case *pb.ColumnResponse_StringArrayVal: + columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_StringArrayVal{StringArrayVal: &vdsm_pb.StringArray{Vals: v.StringArrayVal.Vals}}} + case *pb.ColumnResponse_Float64Val: + columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_Float64Val{Float64Val: v.Float64Val}} + case *pb.ColumnResponse_DecimalVal: + columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_DecimalVal{DecimalVal: &vdsm_pb.Decimal{Value: v.DecimalVal.Value, Scale: v.DecimalVal.Scale}}} + case nil: + columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: nil} + default: + return nil, errors.Errorf("unhandled columnval type: %T", col.GetColumnVal()) + } + } + vtbl.Rows[r] = &vdsm_pb.Row{Columns: columns} + } + + return vtbl, nil +} From fdce92eb404cee2bd2283a93e066f9cd380c2551 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 17 Sep 2020 16:27:48 -0500 Subject: [PATCH 04/53] Use shared types from pilosa proto file and enable generic proxying of query calls --- Makefile | 11 +- proto/interface.go | 2 +- proto/pilosa.pb.go | 96 ++-- proto/pilosa.proto | 2 + proto/vdsm/vdsm.pb.go | 1008 ++++------------------------------------- proto/vdsm/vdsm.proto | 92 +--- server/grpc.go | 126 +++--- 7 files changed, 211 insertions(+), 1126 deletions(-) diff --git a/Makefile b/Makefile index f6ca2fe1f..52d246c64 100644 --- a/Makefile +++ b/Makefile @@ -166,10 +166,14 @@ generate-stringer: generate-pql: require-peg cd pql && peg -inline pql.peg && cd .. -# dunno if protoc-gen-gofast is actually needed here -generate-proto-grpc: require-protoc require-protoc-gen-gofast +generate-proto-grpc: require-protoc require-protoc-gen-go protoc -I proto proto/pilosa.proto --go_out=plugins=grpc:proto protoc -I proto proto/vdsm/vdsm.proto --go_out=plugins=grpc:proto + # TODO: Modify above commands and remove the below mv if possible. + # See https://go-review.googlesource.com/c/protobuf/+/219298/ for info on --go-opt + # I couldn't get it to work during development - Cody + cp -r proto/github.com/pilosa/pilosa/v2/proto/ proto/ + rm -rf proto/github.com # `go generate` all needed packages generate: generate-protoc generate-statik generate-stringer generate-pql @@ -387,6 +391,9 @@ install-stringer: install-protoc-gen-gofast: GO111MODULE=off go get -u github.com/gogo/protobuf/protoc-gen-gofast +install-protoc-gen-go: + GO111MODULE=off go get -u github.com/golang/protobuf/protoc-gen-go + install-protoc: @echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html diff --git a/proto/interface.go b/proto/interface.go index af2599707..77ba3fd8d 100644 --- a/proto/interface.go +++ b/proto/interface.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa +package proto import ( "fmt" diff --git a/proto/pilosa.pb.go b/proto/pilosa.pb.go index 45ac65612..973b6e173 100644 --- a/proto/pilosa.pb.go +++ b/proto/pilosa.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: pilosa.proto -package pilosa +package proto import ( context "context" @@ -841,52 +841,54 @@ func init() { func init() { proto.RegisterFile("pilosa.proto", fileDescriptor_ef0691a44d1e275c) } var fileDescriptor_ef0691a44d1e275c = []byte{ - // 718 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x55, 0xdd, 0x52, 0xd3, 0x4e, - 0x14, 0x6f, 0x48, 0x68, 0x9b, 0x53, 0xbe, 0xfe, 0xcb, 0x5f, 0xcc, 0x30, 0x8e, 0xc6, 0x78, 0x61, - 0x1c, 0x1d, 0x06, 0x51, 0x74, 0x54, 0xbc, 0x00, 0xd4, 0x29, 0xa3, 0x8e, 0x65, 0x11, 0xee, 0xb7, - 0xcd, 0x16, 0x33, 0x6e, 0xb3, 0x6d, 0x36, 0x05, 0xfb, 0x02, 0xbe, 0x81, 0x6f, 0xe0, 0x5b, 0x78, - 0xef, 0x73, 0x39, 0xbb, 0x9b, 0x4d, 0x13, 0xb4, 0x0e, 0x72, 0xb7, 0xe7, 0xfc, 0x7e, 0xe7, 0x2b, - 0xe7, 0x23, 0xb0, 0x30, 0x8c, 0x19, 0x17, 0x64, 0x63, 0x98, 0xf2, 0x8c, 0xa3, 0xba, 0x96, 0x82, - 0x67, 0xb0, 0x7c, 0x38, 0xa6, 0xe9, 0xa4, 0x73, 0xf8, 0x0e, 0xd3, 0xd1, 0x98, 0x8a, 0x0c, 0xfd, - 0x0f, 0xf3, 0x71, 0x12, 0xd1, 0x2f, 0x9e, 0xe5, 0x5b, 0xa1, 0x8b, 0xb5, 0x80, 0x56, 0xc0, 0x1e, - 0x8e, 0x98, 0x37, 0xa7, 0x74, 0xf2, 0x19, 0xdc, 0xc9, 0x4d, 0x8f, 0xa6, 0xa6, 0x2b, 0x60, 0x8b, - 0x11, 0xcb, 0x0d, 0xe5, 0x33, 0x78, 0x01, 0xad, 0xa3, 0x8c, 0x64, 0x63, 0xf1, 0x3a, 0x4d, 0x79, - 0x8a, 0x10, 0x38, 0xfb, 0x3c, 0xa2, 0x8a, 0xb1, 0x88, 0xd5, 0x1b, 0x79, 0xd0, 0x78, 0x4f, 0x85, - 0x20, 0xa7, 0x34, 0xf7, 0x6e, 0xc4, 0xe0, 0xbb, 0x05, 0x2d, 0xcc, 0xcf, 0x31, 0x15, 0x43, 0x9e, - 0x08, 0x8a, 0x1e, 0x40, 0xe3, 0x13, 0x25, 0x11, 0x4d, 0x85, 0x67, 0xf9, 0x76, 0xd8, 0xda, 0x42, - 0x1b, 0x79, 0x51, 0xfb, 0x9c, 0x8d, 0x07, 0xc9, 0x41, 0xd2, 0xe7, 0xd8, 0x50, 0xd0, 0x26, 0x34, - 0x7a, 0x4a, 0x2d, 0xbc, 0x39, 0xc5, 0x5e, 0xab, 0xb2, 0x8d, 0x5b, 0x6c, 0x68, 0x68, 0xbb, 0x92, - 0xac, 0x67, 0xfb, 0x56, 0xd8, 0xda, 0x5a, 0x35, 0x56, 0x25, 0x08, 0x97, 0x79, 0xc1, 0x53, 0xb0, - 0x31, 0x3f, 0x2f, 0xc7, 0xb3, 0x2e, 0x15, 0x2f, 0xf8, 0x66, 0xc1, 0xe2, 0x47, 0xd2, 0x65, 0xf4, - 0x8a, 0x15, 0xde, 0x02, 0x27, 0xe5, 0xe7, 0xa6, 0xbc, 0x96, 0xa1, 0xca, 0x4f, 0xa6, 0x80, 0xab, - 0x16, 0xb4, 0x03, 0x30, 0x0d, 0x27, 0x7b, 0x96, 0x90, 0x01, 0xcd, 0xbb, 0xaa, 0xde, 0x68, 0x1d, - 0x9a, 0x11, 0xc9, 0x48, 0x36, 0x19, 0x9a, 0xa6, 0x15, 0x72, 0xf0, 0xd5, 0x86, 0xa5, 0x6a, 0xc5, - 0xe8, 0x26, 0xb8, 0x22, 0x4b, 0xe3, 0xe4, 0xf4, 0x84, 0xe4, 0xd3, 0xd1, 0xae, 0xe1, 0xa9, 0x4a, - 0xe2, 0xe3, 0x38, 0xc9, 0x9e, 0x3c, 0x96, 0xb8, 0xf4, 0xe7, 0x48, 0xbc, 0x50, 0xa1, 0x1b, 0xd0, - 0x2c, 0x60, 0x59, 0x84, 0xdd, 0xae, 0xe1, 0x42, 0x83, 0xd6, 0xa1, 0xd1, 0xe5, 0x9c, 0x49, 0xd0, - 0xf1, 0xad, 0xb0, 0xd9, 0xae, 0x61, 0xa3, 0x50, 0x18, 0xe3, 0x5d, 0x89, 0xcd, 0xfb, 0x56, 0xb8, - 0xa0, 0x30, 0xad, 0x40, 0x2f, 0x61, 0x49, 0x87, 0xd8, 0x4d, 0x53, 0x32, 0x91, 0x94, 0x7a, 0xf5, - 0x03, 0x1d, 0x4f, 0xd1, 0x76, 0x0d, 0x5f, 0x20, 0x4b, 0x73, 0x5d, 0x41, 0x61, 0xde, 0xb8, 0xf8, - 0x7d, 0x0b, 0x54, 0x9a, 0x57, 0xc9, 0xc8, 0x07, 0xe8, 0x33, 0x4e, 0xf2, 0xaa, 0x9a, 0xbe, 0x15, - 0x5a, 0xed, 0x1a, 0x2e, 0xe9, 0xd0, 0x43, 0x80, 0x88, 0xf6, 0xe2, 0x01, 0x51, 0xa5, 0xb9, 0xca, - 0xf9, 0xb2, 0x71, 0xfe, 0x4a, 0x23, 0xd2, 0x64, 0x4a, 0xda, 0x6b, 0x81, 0xab, 0x87, 0xeb, 0x84, - 0xb0, 0x60, 0x1b, 0x1a, 0x39, 0x4b, 0xee, 0xf4, 0x19, 0x61, 0x63, 0xdd, 0x44, 0x1b, 0x6b, 0x41, - 0x6a, 0x45, 0x8f, 0x30, 0xdd, 0x42, 0x1b, 0x6b, 0x21, 0xf8, 0x61, 0xc1, 0xd2, 0x41, 0x22, 0x86, - 0xb4, 0x97, 0xfd, 0xfd, 0x24, 0xdc, 0x2f, 0x2f, 0x98, 0x4c, 0xee, 0x3f, 0x93, 0xdc, 0x41, 0x24, - 0x3e, 0xa4, 0x6f, 0xe9, 0x44, 0x4c, 0x77, 0x2b, 0x80, 0x85, 0x7e, 0xcc, 0x32, 0x9a, 0xbe, 0x89, - 0x29, 0x8b, 0x84, 0x67, 0xfb, 0x76, 0xe8, 0xe2, 0x8a, 0x4e, 0x86, 0x61, 0xf1, 0x20, 0xce, 0x54, - 0x1b, 0x1d, 0xac, 0x05, 0xb4, 0x06, 0x75, 0xde, 0xef, 0x0b, 0x9a, 0xa9, 0x0e, 0x3a, 0x38, 0x97, - 0x24, 0x7b, 0x24, 0xef, 0x8f, 0xea, 0x9a, 0x8b, 0xb5, 0x10, 0xdc, 0x86, 0x56, 0xa9, 0x6d, 0x72, - 0x78, 0xcf, 0x08, 0xd3, 0xdb, 0xe4, 0x60, 0xf5, 0x96, 0x94, 0x52, 0x6b, 0x2a, 0x14, 0x37, 0xa7, - 0x9c, 0x82, 0x5b, 0xd4, 0x80, 0xee, 0x82, 0x1d, 0x47, 0x42, 0xd5, 0x3e, 0x73, 0x38, 0x24, 0x03, - 0xdd, 0x03, 0xe7, 0x33, 0x9d, 0x98, 0xaf, 0x31, 0x63, 0x0e, 0x14, 0x65, 0xaf, 0x0e, 0x8e, 0x5c, - 0x96, 0xad, 0x9f, 0x73, 0x50, 0xef, 0x28, 0x1a, 0xda, 0x81, 0xa6, 0xb9, 0xa7, 0xe8, 0xba, 0xb1, - 0xbd, 0x70, 0x61, 0xd7, 0x57, 0xcb, 0x4b, 0x9e, 0xaf, 0x57, 0x50, 0xdb, 0xb4, 0xd0, 0x2e, 0x2c, - 0x1a, 0xee, 0x71, 0x42, 0xd2, 0xc9, 0x6c, 0x17, 0xd7, 0x0c, 0x50, 0x39, 0x3d, 0x41, 0xad, 0x48, - 0xa0, 0xf3, 0x5b, 0x02, 0x9d, 0x7f, 0x48, 0xa0, 0xf3, 0xe7, 0x04, 0x3a, 0x97, 0x48, 0xe0, 0x39, - 0x34, 0xf2, 0xc1, 0x43, 0xc5, 0xed, 0xac, 0x4e, 0xe2, 0xcc, 0xf0, 0xdd, 0xba, 0xfa, 0xaf, 0x3d, - 0xfa, 0x15, 0x00, 0x00, 0xff, 0xff, 0x7e, 0x8d, 0x53, 0x0b, 0xe7, 0x06, 0x00, 0x00, + // 745 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x55, 0xdd, 0x72, 0xd3, 0x3a, + 0x10, 0x8e, 0x6b, 0x37, 0x89, 0x37, 0xfd, 0x3b, 0xea, 0x39, 0x3d, 0x99, 0xce, 0x99, 0x83, 0xeb, + 0x5e, 0x10, 0x06, 0xa6, 0x2d, 0x81, 0xc2, 0x00, 0xe5, 0xa2, 0x2d, 0x30, 0xe9, 0x00, 0x43, 0xaa, + 0xd2, 0x5e, 0x70, 0xa7, 0xc4, 0x4a, 0xea, 0x41, 0xb1, 0x12, 0xcb, 0x69, 0xc9, 0x0b, 0xf0, 0x06, + 0xbc, 0x01, 0x6f, 0xc1, 0x3d, 0xcf, 0xc5, 0x48, 0xb2, 0x1c, 0xbb, 0x10, 0xa6, 0xf4, 0xca, 0xda, + 0xfd, 0xbe, 0xd5, 0xee, 0x6a, 0x7f, 0x0c, 0x0b, 0xc3, 0x90, 0x71, 0x41, 0xb6, 0x86, 0x31, 0x4f, + 0x38, 0x2a, 0x6b, 0xc9, 0x7f, 0x02, 0xcb, 0xc7, 0x63, 0x1a, 0x4f, 0xda, 0xc7, 0x6f, 0x30, 0x1d, + 0x8d, 0xa9, 0x48, 0xd0, 0xdf, 0x30, 0x1f, 0x46, 0x01, 0xfd, 0x54, 0xb7, 0x3c, 0xab, 0xe1, 0x62, + 0x2d, 0xa0, 0x15, 0xb0, 0x87, 0x23, 0x56, 0x9f, 0x53, 0x3a, 0x79, 0xf4, 0x37, 0x53, 0xd3, 0x93, + 0xa9, 0xe9, 0x0a, 0xd8, 0x62, 0xc4, 0x52, 0x43, 0x79, 0xf4, 0x9f, 0x41, 0xed, 0x24, 0x21, 0xc9, + 0x58, 0xbc, 0x8c, 0x63, 0x1e, 0x23, 0x04, 0xce, 0x21, 0x0f, 0xa8, 0x62, 0x2c, 0x62, 0x75, 0x46, + 0x75, 0xa8, 0xbc, 0xa5, 0x42, 0x90, 0x3e, 0x4d, 0x6f, 0x37, 0xa2, 0xff, 0xd5, 0x82, 0x1a, 0xe6, + 0x97, 0x98, 0x8a, 0x21, 0x8f, 0x04, 0x45, 0xf7, 0xa0, 0x72, 0x4e, 0x49, 0x40, 0x63, 0x51, 0xb7, + 0x3c, 0xbb, 0x51, 0x6b, 0xa2, 0xad, 0x34, 0xa9, 0x43, 0xce, 0xc6, 0x83, 0xe8, 0x28, 0xea, 0x71, + 0x6c, 0x28, 0x68, 0x07, 0x2a, 0x5d, 0xa5, 0x16, 0xf5, 0x39, 0xc5, 0x5e, 0x2b, 0xb2, 0xcd, 0xb5, + 0xd8, 0xd0, 0xd0, 0x6e, 0x21, 0xd8, 0xba, 0xed, 0x59, 0x8d, 0x5a, 0x73, 0xd5, 0x58, 0xe5, 0x20, + 0x9c, 0xe7, 0xf9, 0x8f, 0xc1, 0xc6, 0xfc, 0x32, 0xef, 0xcf, 0xba, 0x96, 0x3f, 0xff, 0x8b, 0x05, + 0x8b, 0xef, 0x49, 0x87, 0xd1, 0x1b, 0x66, 0x78, 0x0b, 0x9c, 0x98, 0x5f, 0x9a, 0xf4, 0x6a, 0x86, + 0x2a, 0x9f, 0x4c, 0x01, 0x37, 0x4d, 0x68, 0x0f, 0x60, 0xea, 0x4e, 0xd6, 0x2c, 0x22, 0x03, 0x9a, + 0x56, 0x55, 0x9d, 0xd1, 0x3a, 0x54, 0x03, 0x92, 0x90, 0x64, 0x32, 0x34, 0x45, 0xcb, 0x64, 0xff, + 0xb3, 0x0d, 0x4b, 0xc5, 0x8c, 0xd1, 0xff, 0xe0, 0x8a, 0x24, 0x0e, 0xa3, 0xfe, 0x19, 0x49, 0xbb, + 0xa3, 0x55, 0xc2, 0x53, 0x95, 0xc4, 0xc7, 0x61, 0x94, 0x3c, 0x7a, 0x28, 0x71, 0x79, 0x9f, 0x23, + 0xf1, 0x4c, 0x85, 0xfe, 0x83, 0x6a, 0x06, 0xcb, 0x24, 0xec, 0x56, 0x09, 0x67, 0x1a, 0xb4, 0x0e, + 0x95, 0x0e, 0xe7, 0x4c, 0x82, 0x8e, 0x67, 0x35, 0xaa, 0xad, 0x12, 0x36, 0x0a, 0x85, 0x31, 0xde, + 0x91, 0xd8, 0xbc, 0x67, 0x35, 0x16, 0x14, 0xa6, 0x15, 0xe8, 0x39, 0x2c, 0x69, 0x17, 0xfb, 0x71, + 0x4c, 0x26, 0x92, 0x52, 0x2e, 0x3e, 0xd0, 0xe9, 0x14, 0x6d, 0x95, 0xf0, 0x15, 0xb2, 0x34, 0xd7, + 0x19, 0x64, 0xe6, 0x95, 0xab, 0xef, 0x9b, 0xa1, 0xd2, 0xbc, 0x48, 0x46, 0x1e, 0x40, 0x8f, 0x71, + 0x92, 0x66, 0x55, 0xf5, 0xac, 0x86, 0xd5, 0x2a, 0xe1, 0x9c, 0x0e, 0xdd, 0x07, 0x08, 0x68, 0x37, + 0x1c, 0x10, 0x95, 0x9a, 0xab, 0x2e, 0x5f, 0x36, 0x97, 0xbf, 0xd0, 0x88, 0x34, 0x99, 0x92, 0x0e, + 0x6a, 0xe0, 0xea, 0xe6, 0x3a, 0x23, 0xcc, 0xdf, 0x85, 0x4a, 0xca, 0x92, 0x33, 0x7d, 0x41, 0xd8, + 0x58, 0x17, 0xd1, 0xc6, 0x5a, 0x90, 0x5a, 0xd1, 0x25, 0x4c, 0x97, 0xd0, 0xc6, 0x5a, 0xf0, 0xbf, + 0x59, 0xb0, 0x74, 0x14, 0x89, 0x21, 0xed, 0x26, 0xbf, 0x5f, 0x09, 0x77, 0xf3, 0x03, 0x26, 0x83, + 0xfb, 0xcb, 0x04, 0x77, 0x14, 0x88, 0x77, 0xf1, 0x6b, 0x3a, 0x11, 0xd3, 0xd9, 0xf2, 0x61, 0xa1, + 0x17, 0xb2, 0x84, 0xc6, 0xaf, 0x42, 0xca, 0x02, 0x51, 0xb7, 0x3d, 0xbb, 0xe1, 0xe2, 0x82, 0x4e, + 0xba, 0x61, 0xe1, 0x20, 0x4c, 0x54, 0x19, 0x1d, 0xac, 0x05, 0xb4, 0x06, 0x65, 0xde, 0xeb, 0x09, + 0x9a, 0xa8, 0x0a, 0x3a, 0x38, 0x95, 0x24, 0x7b, 0x24, 0xf7, 0x8f, 0xaa, 0x9a, 0x8b, 0xb5, 0xe0, + 0x6f, 0x40, 0x2d, 0x57, 0x36, 0xd9, 0xbc, 0x17, 0x84, 0xe9, 0x69, 0x72, 0xb0, 0x3a, 0x4b, 0x4a, + 0xae, 0x34, 0x05, 0x8a, 0x9b, 0x52, 0xfa, 0xe0, 0x66, 0x39, 0xa0, 0xdb, 0x60, 0x87, 0x81, 0x50, + 0xb9, 0xcf, 0x6c, 0x0e, 0xc9, 0x40, 0x77, 0xc0, 0xf9, 0x48, 0x27, 0xe6, 0x35, 0x66, 0xf4, 0x81, + 0xa2, 0x1c, 0x94, 0xc1, 0x91, 0xc3, 0xd2, 0xfc, 0x3e, 0x07, 0xe5, 0xb6, 0xa2, 0xa1, 0x3d, 0xa8, + 0x9a, 0x7d, 0x8a, 0xfe, 0x35, 0xb6, 0x57, 0x36, 0xec, 0xfa, 0x6a, 0x7e, 0xc8, 0xd3, 0xf1, 0xf2, + 0x4b, 0x3b, 0x16, 0xda, 0x87, 0x45, 0xc3, 0x3d, 0x8d, 0x48, 0x3c, 0x99, 0x7d, 0xc5, 0x3f, 0x06, + 0x28, 0xac, 0x1e, 0xbf, 0x94, 0x05, 0xd0, 0xfe, 0x29, 0x80, 0xf6, 0x1f, 0x04, 0xd0, 0xfe, 0x75, + 0x00, 0xed, 0x6b, 0x04, 0xf0, 0x14, 0x2a, 0x69, 0xe3, 0xa1, 0x6c, 0x77, 0x16, 0x3b, 0x71, 0xa6, + 0xfb, 0x83, 0xcd, 0x0f, 0x1b, 0xfd, 0x30, 0x39, 0x1f, 0x77, 0xb6, 0xba, 0x7c, 0xb0, 0xad, 0x49, + 0xe6, 0x73, 0xd1, 0xdc, 0x56, 0x7f, 0xbd, 0x4e, 0x59, 0x7d, 0x1e, 0xfc, 0x08, 0x00, 0x00, 0xff, + 0xff, 0x60, 0xce, 0x2e, 0x49, 0x0c, 0x07, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/proto/pilosa.proto b/proto/pilosa.proto index 1fc966083..05bfbd9e4 100644 --- a/proto/pilosa.proto +++ b/proto/pilosa.proto @@ -3,6 +3,8 @@ package pilosa; //import "public.proto"; +option go_package = "github.com/pilosa/pilosa/v2/proto"; + message QueryPQLRequest { string index = 1; string pql = 2; diff --git a/proto/vdsm/vdsm.pb.go b/proto/vdsm/vdsm.pb.go index edb45edde..99f04cda2 100644 --- a/proto/vdsm/vdsm.pb.go +++ b/proto/vdsm/vdsm.pb.go @@ -7,6 +7,7 @@ import ( context "context" fmt "fmt" proto "github.com/golang/protobuf/proto" + proto1 "github.com/pilosa/pilosa/v2/proto" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" @@ -561,804 +562,6 @@ func (m *DeleteVDSResponse) XXX_DiscardUnknown() { var xxx_messageInfo_DeleteVDSResponse proto.InternalMessageInfo -type QueryPQLRequest struct { - Index string `protobuf:"bytes,1,opt,name=index,proto3" json:"index,omitempty"` - Pql string `protobuf:"bytes,2,opt,name=pql,proto3" json:"pql,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *QueryPQLRequest) Reset() { *m = QueryPQLRequest{} } -func (m *QueryPQLRequest) String() string { return proto.CompactTextString(m) } -func (*QueryPQLRequest) ProtoMessage() {} -func (*QueryPQLRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_3849c3e549495f50, []int{10} -} - -func (m *QueryPQLRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_QueryPQLRequest.Unmarshal(m, b) -} -func (m *QueryPQLRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_QueryPQLRequest.Marshal(b, m, deterministic) -} -func (m *QueryPQLRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_QueryPQLRequest.Merge(m, src) -} -func (m *QueryPQLRequest) XXX_Size() int { - return xxx_messageInfo_QueryPQLRequest.Size(m) -} -func (m *QueryPQLRequest) XXX_DiscardUnknown() { - xxx_messageInfo_QueryPQLRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_QueryPQLRequest proto.InternalMessageInfo - -func (m *QueryPQLRequest) GetIndex() string { - if m != nil { - return m.Index - } - return "" -} - -func (m *QueryPQLRequest) GetPql() string { - if m != nil { - return m.Pql - } - return "" -} - -type QuerySQLRequest struct { - Sql string `protobuf:"bytes,1,opt,name=sql,proto3" json:"sql,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *QuerySQLRequest) Reset() { *m = QuerySQLRequest{} } -func (m *QuerySQLRequest) String() string { return proto.CompactTextString(m) } -func (*QuerySQLRequest) ProtoMessage() {} -func (*QuerySQLRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_3849c3e549495f50, []int{11} -} - -func (m *QuerySQLRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_QuerySQLRequest.Unmarshal(m, b) -} -func (m *QuerySQLRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_QuerySQLRequest.Marshal(b, m, deterministic) -} -func (m *QuerySQLRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_QuerySQLRequest.Merge(m, src) -} -func (m *QuerySQLRequest) XXX_Size() int { - return xxx_messageInfo_QuerySQLRequest.Size(m) -} -func (m *QuerySQLRequest) XXX_DiscardUnknown() { - xxx_messageInfo_QuerySQLRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_QuerySQLRequest proto.InternalMessageInfo - -func (m *QuerySQLRequest) GetSql() string { - if m != nil { - return m.Sql - } - return "" -} - -type StatusError struct { - Code uint32 `protobuf:"varint,1,opt,name=Code,proto3" json:"Code,omitempty"` - Message string `protobuf:"bytes,2,opt,name=Message,proto3" json:"Message,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *StatusError) Reset() { *m = StatusError{} } -func (m *StatusError) String() string { return proto.CompactTextString(m) } -func (*StatusError) ProtoMessage() {} -func (*StatusError) Descriptor() ([]byte, []int) { - return fileDescriptor_3849c3e549495f50, []int{12} -} - -func (m *StatusError) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_StatusError.Unmarshal(m, b) -} -func (m *StatusError) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_StatusError.Marshal(b, m, deterministic) -} -func (m *StatusError) XXX_Merge(src proto.Message) { - xxx_messageInfo_StatusError.Merge(m, src) -} -func (m *StatusError) XXX_Size() int { - return xxx_messageInfo_StatusError.Size(m) -} -func (m *StatusError) XXX_DiscardUnknown() { - xxx_messageInfo_StatusError.DiscardUnknown(m) -} - -var xxx_messageInfo_StatusError proto.InternalMessageInfo - -func (m *StatusError) GetCode() uint32 { - if m != nil { - return m.Code - } - return 0 -} - -func (m *StatusError) GetMessage() string { - if m != nil { - return m.Message - } - return "" -} - -type RowResponse struct { - Headers []*ColumnInfo `protobuf:"bytes,1,rep,name=headers,proto3" json:"headers,omitempty"` - Columns []*ColumnResponse `protobuf:"bytes,2,rep,name=columns,proto3" json:"columns,omitempty"` - StatusError *StatusError `protobuf:"bytes,3,opt,name=StatusError,proto3" json:"StatusError,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *RowResponse) Reset() { *m = RowResponse{} } -func (m *RowResponse) String() string { return proto.CompactTextString(m) } -func (*RowResponse) ProtoMessage() {} -func (*RowResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_3849c3e549495f50, []int{13} -} - -func (m *RowResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_RowResponse.Unmarshal(m, b) -} -func (m *RowResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_RowResponse.Marshal(b, m, deterministic) -} -func (m *RowResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_RowResponse.Merge(m, src) -} -func (m *RowResponse) XXX_Size() int { - return xxx_messageInfo_RowResponse.Size(m) -} -func (m *RowResponse) XXX_DiscardUnknown() { - xxx_messageInfo_RowResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_RowResponse proto.InternalMessageInfo - -func (m *RowResponse) GetHeaders() []*ColumnInfo { - if m != nil { - return m.Headers - } - return nil -} - -func (m *RowResponse) GetColumns() []*ColumnResponse { - if m != nil { - return m.Columns - } - return nil -} - -func (m *RowResponse) GetStatusError() *StatusError { - if m != nil { - return m.StatusError - } - return nil -} - -type Row struct { - Columns []*ColumnResponse `protobuf:"bytes,1,rep,name=columns,proto3" json:"columns,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Row) Reset() { *m = Row{} } -func (m *Row) String() string { return proto.CompactTextString(m) } -func (*Row) ProtoMessage() {} -func (*Row) Descriptor() ([]byte, []int) { - return fileDescriptor_3849c3e549495f50, []int{14} -} - -func (m *Row) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Row.Unmarshal(m, b) -} -func (m *Row) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Row.Marshal(b, m, deterministic) -} -func (m *Row) XXX_Merge(src proto.Message) { - xxx_messageInfo_Row.Merge(m, src) -} -func (m *Row) XXX_Size() int { - return xxx_messageInfo_Row.Size(m) -} -func (m *Row) XXX_DiscardUnknown() { - xxx_messageInfo_Row.DiscardUnknown(m) -} - -var xxx_messageInfo_Row proto.InternalMessageInfo - -func (m *Row) GetColumns() []*ColumnResponse { - if m != nil { - return m.Columns - } - return nil -} - -type TableResponse struct { - Headers []*ColumnInfo `protobuf:"bytes,1,rep,name=headers,proto3" json:"headers,omitempty"` - Rows []*Row `protobuf:"bytes,2,rep,name=rows,proto3" json:"rows,omitempty"` - StatusError *StatusError `protobuf:"bytes,3,opt,name=StatusError,proto3" json:"StatusError,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *TableResponse) Reset() { *m = TableResponse{} } -func (m *TableResponse) String() string { return proto.CompactTextString(m) } -func (*TableResponse) ProtoMessage() {} -func (*TableResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_3849c3e549495f50, []int{15} -} - -func (m *TableResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_TableResponse.Unmarshal(m, b) -} -func (m *TableResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_TableResponse.Marshal(b, m, deterministic) -} -func (m *TableResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_TableResponse.Merge(m, src) -} -func (m *TableResponse) XXX_Size() int { - return xxx_messageInfo_TableResponse.Size(m) -} -func (m *TableResponse) XXX_DiscardUnknown() { - xxx_messageInfo_TableResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_TableResponse proto.InternalMessageInfo - -func (m *TableResponse) GetHeaders() []*ColumnInfo { - if m != nil { - return m.Headers - } - return nil -} - -func (m *TableResponse) GetRows() []*Row { - if m != nil { - return m.Rows - } - return nil -} - -func (m *TableResponse) GetStatusError() *StatusError { - if m != nil { - return m.StatusError - } - return nil -} - -type ColumnInfo struct { - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - Datatype string `protobuf:"bytes,2,opt,name=datatype,proto3" json:"datatype,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ColumnInfo) Reset() { *m = ColumnInfo{} } -func (m *ColumnInfo) String() string { return proto.CompactTextString(m) } -func (*ColumnInfo) ProtoMessage() {} -func (*ColumnInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_3849c3e549495f50, []int{16} -} - -func (m *ColumnInfo) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ColumnInfo.Unmarshal(m, b) -} -func (m *ColumnInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ColumnInfo.Marshal(b, m, deterministic) -} -func (m *ColumnInfo) XXX_Merge(src proto.Message) { - xxx_messageInfo_ColumnInfo.Merge(m, src) -} -func (m *ColumnInfo) XXX_Size() int { - return xxx_messageInfo_ColumnInfo.Size(m) -} -func (m *ColumnInfo) XXX_DiscardUnknown() { - xxx_messageInfo_ColumnInfo.DiscardUnknown(m) -} - -var xxx_messageInfo_ColumnInfo proto.InternalMessageInfo - -func (m *ColumnInfo) GetName() string { - if m != nil { - return m.Name - } - return "" -} - -func (m *ColumnInfo) GetDatatype() string { - if m != nil { - return m.Datatype - } - return "" -} - -type ColumnResponse struct { - // Types that are valid to be assigned to ColumnVal: - // *ColumnResponse_StringVal - // *ColumnResponse_Uint64Val - // *ColumnResponse_Int64Val - // *ColumnResponse_BoolVal - // *ColumnResponse_BlobVal - // *ColumnResponse_Uint64ArrayVal - // *ColumnResponse_StringArrayVal - // *ColumnResponse_Float64Val - // *ColumnResponse_DecimalVal - ColumnVal isColumnResponse_ColumnVal `protobuf_oneof:"columnVal"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ColumnResponse) Reset() { *m = ColumnResponse{} } -func (m *ColumnResponse) String() string { return proto.CompactTextString(m) } -func (*ColumnResponse) ProtoMessage() {} -func (*ColumnResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_3849c3e549495f50, []int{17} -} - -func (m *ColumnResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ColumnResponse.Unmarshal(m, b) -} -func (m *ColumnResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ColumnResponse.Marshal(b, m, deterministic) -} -func (m *ColumnResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_ColumnResponse.Merge(m, src) -} -func (m *ColumnResponse) XXX_Size() int { - return xxx_messageInfo_ColumnResponse.Size(m) -} -func (m *ColumnResponse) XXX_DiscardUnknown() { - xxx_messageInfo_ColumnResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_ColumnResponse proto.InternalMessageInfo - -type isColumnResponse_ColumnVal interface { - isColumnResponse_ColumnVal() -} - -type ColumnResponse_StringVal struct { - StringVal string `protobuf:"bytes,1,opt,name=stringVal,proto3,oneof"` -} - -type ColumnResponse_Uint64Val struct { - Uint64Val uint64 `protobuf:"varint,2,opt,name=uint64Val,proto3,oneof"` -} - -type ColumnResponse_Int64Val struct { - Int64Val int64 `protobuf:"varint,3,opt,name=int64Val,proto3,oneof"` -} - -type ColumnResponse_BoolVal struct { - BoolVal bool `protobuf:"varint,4,opt,name=boolVal,proto3,oneof"` -} - -type ColumnResponse_BlobVal struct { - BlobVal []byte `protobuf:"bytes,5,opt,name=blobVal,proto3,oneof"` -} - -type ColumnResponse_Uint64ArrayVal struct { - Uint64ArrayVal *Uint64Array `protobuf:"bytes,6,opt,name=uint64ArrayVal,proto3,oneof"` -} - -type ColumnResponse_StringArrayVal struct { - StringArrayVal *StringArray `protobuf:"bytes,7,opt,name=stringArrayVal,proto3,oneof"` -} - -type ColumnResponse_Float64Val struct { - Float64Val float64 `protobuf:"fixed64,8,opt,name=float64Val,proto3,oneof"` -} - -type ColumnResponse_DecimalVal struct { - DecimalVal *Decimal `protobuf:"bytes,9,opt,name=decimalVal,proto3,oneof"` -} - -func (*ColumnResponse_StringVal) isColumnResponse_ColumnVal() {} - -func (*ColumnResponse_Uint64Val) isColumnResponse_ColumnVal() {} - -func (*ColumnResponse_Int64Val) isColumnResponse_ColumnVal() {} - -func (*ColumnResponse_BoolVal) isColumnResponse_ColumnVal() {} - -func (*ColumnResponse_BlobVal) isColumnResponse_ColumnVal() {} - -func (*ColumnResponse_Uint64ArrayVal) isColumnResponse_ColumnVal() {} - -func (*ColumnResponse_StringArrayVal) isColumnResponse_ColumnVal() {} - -func (*ColumnResponse_Float64Val) isColumnResponse_ColumnVal() {} - -func (*ColumnResponse_DecimalVal) isColumnResponse_ColumnVal() {} - -func (m *ColumnResponse) GetColumnVal() isColumnResponse_ColumnVal { - if m != nil { - return m.ColumnVal - } - return nil -} - -func (m *ColumnResponse) GetStringVal() string { - if x, ok := m.GetColumnVal().(*ColumnResponse_StringVal); ok { - return x.StringVal - } - return "" -} - -func (m *ColumnResponse) GetUint64Val() uint64 { - if x, ok := m.GetColumnVal().(*ColumnResponse_Uint64Val); ok { - return x.Uint64Val - } - return 0 -} - -func (m *ColumnResponse) GetInt64Val() int64 { - if x, ok := m.GetColumnVal().(*ColumnResponse_Int64Val); ok { - return x.Int64Val - } - return 0 -} - -func (m *ColumnResponse) GetBoolVal() bool { - if x, ok := m.GetColumnVal().(*ColumnResponse_BoolVal); ok { - return x.BoolVal - } - return false -} - -func (m *ColumnResponse) GetBlobVal() []byte { - if x, ok := m.GetColumnVal().(*ColumnResponse_BlobVal); ok { - return x.BlobVal - } - return nil -} - -func (m *ColumnResponse) GetUint64ArrayVal() *Uint64Array { - if x, ok := m.GetColumnVal().(*ColumnResponse_Uint64ArrayVal); ok { - return x.Uint64ArrayVal - } - return nil -} - -func (m *ColumnResponse) GetStringArrayVal() *StringArray { - if x, ok := m.GetColumnVal().(*ColumnResponse_StringArrayVal); ok { - return x.StringArrayVal - } - return nil -} - -func (m *ColumnResponse) GetFloat64Val() float64 { - if x, ok := m.GetColumnVal().(*ColumnResponse_Float64Val); ok { - return x.Float64Val - } - return 0 -} - -func (m *ColumnResponse) GetDecimalVal() *Decimal { - if x, ok := m.GetColumnVal().(*ColumnResponse_DecimalVal); ok { - return x.DecimalVal - } - return nil -} - -// XXX_OneofWrappers is for the internal use of the proto package. -func (*ColumnResponse) XXX_OneofWrappers() []interface{} { - return []interface{}{ - (*ColumnResponse_StringVal)(nil), - (*ColumnResponse_Uint64Val)(nil), - (*ColumnResponse_Int64Val)(nil), - (*ColumnResponse_BoolVal)(nil), - (*ColumnResponse_BlobVal)(nil), - (*ColumnResponse_Uint64ArrayVal)(nil), - (*ColumnResponse_StringArrayVal)(nil), - (*ColumnResponse_Float64Val)(nil), - (*ColumnResponse_DecimalVal)(nil), - } -} - -type Decimal struct { - Value int64 `protobuf:"varint,1,opt,name=value,proto3" json:"value,omitempty"` - Scale int64 `protobuf:"varint,2,opt,name=scale,proto3" json:"scale,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Decimal) Reset() { *m = Decimal{} } -func (m *Decimal) String() string { return proto.CompactTextString(m) } -func (*Decimal) ProtoMessage() {} -func (*Decimal) Descriptor() ([]byte, []int) { - return fileDescriptor_3849c3e549495f50, []int{18} -} - -func (m *Decimal) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Decimal.Unmarshal(m, b) -} -func (m *Decimal) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Decimal.Marshal(b, m, deterministic) -} -func (m *Decimal) XXX_Merge(src proto.Message) { - xxx_messageInfo_Decimal.Merge(m, src) -} -func (m *Decimal) XXX_Size() int { - return xxx_messageInfo_Decimal.Size(m) -} -func (m *Decimal) XXX_DiscardUnknown() { - xxx_messageInfo_Decimal.DiscardUnknown(m) -} - -var xxx_messageInfo_Decimal proto.InternalMessageInfo - -func (m *Decimal) GetValue() int64 { - if m != nil { - return m.Value - } - return 0 -} - -func (m *Decimal) GetScale() int64 { - if m != nil { - return m.Scale - } - return 0 -} - -type InspectRequest struct { - Vds string `protobuf:"bytes,1,opt,name=vds,proto3" json:"vds,omitempty"` - Records *IdsOrKeys `protobuf:"bytes,2,opt,name=records,proto3" json:"records,omitempty"` - FilterFields []string `protobuf:"bytes,3,rep,name=filterFields,proto3" json:"filterFields,omitempty"` - Limit uint64 `protobuf:"varint,4,opt,name=limit,proto3" json:"limit,omitempty"` - Offset uint64 `protobuf:"varint,5,opt,name=offset,proto3" json:"offset,omitempty"` - Query string `protobuf:"bytes,6,opt,name=query,proto3" json:"query,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *InspectRequest) Reset() { *m = InspectRequest{} } -func (m *InspectRequest) String() string { return proto.CompactTextString(m) } -func (*InspectRequest) ProtoMessage() {} -func (*InspectRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_3849c3e549495f50, []int{19} -} - -func (m *InspectRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_InspectRequest.Unmarshal(m, b) -} -func (m *InspectRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_InspectRequest.Marshal(b, m, deterministic) -} -func (m *InspectRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_InspectRequest.Merge(m, src) -} -func (m *InspectRequest) XXX_Size() int { - return xxx_messageInfo_InspectRequest.Size(m) -} -func (m *InspectRequest) XXX_DiscardUnknown() { - xxx_messageInfo_InspectRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_InspectRequest proto.InternalMessageInfo - -func (m *InspectRequest) GetVds() string { - if m != nil { - return m.Vds - } - return "" -} - -func (m *InspectRequest) GetRecords() *IdsOrKeys { - if m != nil { - return m.Records - } - return nil -} - -func (m *InspectRequest) GetFilterFields() []string { - if m != nil { - return m.FilterFields - } - return nil -} - -func (m *InspectRequest) GetLimit() uint64 { - if m != nil { - return m.Limit - } - return 0 -} - -func (m *InspectRequest) GetOffset() uint64 { - if m != nil { - return m.Offset - } - return 0 -} - -func (m *InspectRequest) GetQuery() string { - if m != nil { - return m.Query - } - return "" -} - -type Uint64Array struct { - Vals []uint64 `protobuf:"varint,1,rep,packed,name=vals,proto3" json:"vals,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Uint64Array) Reset() { *m = Uint64Array{} } -func (m *Uint64Array) String() string { return proto.CompactTextString(m) } -func (*Uint64Array) ProtoMessage() {} -func (*Uint64Array) Descriptor() ([]byte, []int) { - return fileDescriptor_3849c3e549495f50, []int{20} -} - -func (m *Uint64Array) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Uint64Array.Unmarshal(m, b) -} -func (m *Uint64Array) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Uint64Array.Marshal(b, m, deterministic) -} -func (m *Uint64Array) XXX_Merge(src proto.Message) { - xxx_messageInfo_Uint64Array.Merge(m, src) -} -func (m *Uint64Array) XXX_Size() int { - return xxx_messageInfo_Uint64Array.Size(m) -} -func (m *Uint64Array) XXX_DiscardUnknown() { - xxx_messageInfo_Uint64Array.DiscardUnknown(m) -} - -var xxx_messageInfo_Uint64Array proto.InternalMessageInfo - -func (m *Uint64Array) GetVals() []uint64 { - if m != nil { - return m.Vals - } - return nil -} - -type StringArray struct { - Vals []string `protobuf:"bytes,1,rep,name=vals,proto3" json:"vals,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *StringArray) Reset() { *m = StringArray{} } -func (m *StringArray) String() string { return proto.CompactTextString(m) } -func (*StringArray) ProtoMessage() {} -func (*StringArray) Descriptor() ([]byte, []int) { - return fileDescriptor_3849c3e549495f50, []int{21} -} - -func (m *StringArray) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_StringArray.Unmarshal(m, b) -} -func (m *StringArray) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_StringArray.Marshal(b, m, deterministic) -} -func (m *StringArray) XXX_Merge(src proto.Message) { - xxx_messageInfo_StringArray.Merge(m, src) -} -func (m *StringArray) XXX_Size() int { - return xxx_messageInfo_StringArray.Size(m) -} -func (m *StringArray) XXX_DiscardUnknown() { - xxx_messageInfo_StringArray.DiscardUnknown(m) -} - -var xxx_messageInfo_StringArray proto.InternalMessageInfo - -func (m *StringArray) GetVals() []string { - if m != nil { - return m.Vals - } - return nil -} - -type IdsOrKeys struct { - // Types that are valid to be assigned to Type: - // *IdsOrKeys_Ids - // *IdsOrKeys_Keys - Type isIdsOrKeys_Type `protobuf_oneof:"type"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *IdsOrKeys) Reset() { *m = IdsOrKeys{} } -func (m *IdsOrKeys) String() string { return proto.CompactTextString(m) } -func (*IdsOrKeys) ProtoMessage() {} -func (*IdsOrKeys) Descriptor() ([]byte, []int) { - return fileDescriptor_3849c3e549495f50, []int{22} -} - -func (m *IdsOrKeys) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_IdsOrKeys.Unmarshal(m, b) -} -func (m *IdsOrKeys) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_IdsOrKeys.Marshal(b, m, deterministic) -} -func (m *IdsOrKeys) XXX_Merge(src proto.Message) { - xxx_messageInfo_IdsOrKeys.Merge(m, src) -} -func (m *IdsOrKeys) XXX_Size() int { - return xxx_messageInfo_IdsOrKeys.Size(m) -} -func (m *IdsOrKeys) XXX_DiscardUnknown() { - xxx_messageInfo_IdsOrKeys.DiscardUnknown(m) -} - -var xxx_messageInfo_IdsOrKeys proto.InternalMessageInfo - -type isIdsOrKeys_Type interface { - isIdsOrKeys_Type() -} - -type IdsOrKeys_Ids struct { - Ids *Uint64Array `protobuf:"bytes,1,opt,name=ids,proto3,oneof"` -} - -type IdsOrKeys_Keys struct { - Keys *StringArray `protobuf:"bytes,2,opt,name=keys,proto3,oneof"` -} - -func (*IdsOrKeys_Ids) isIdsOrKeys_Type() {} - -func (*IdsOrKeys_Keys) isIdsOrKeys_Type() {} - -func (m *IdsOrKeys) GetType() isIdsOrKeys_Type { - if m != nil { - return m.Type - } - return nil -} - -func (m *IdsOrKeys) GetIds() *Uint64Array { - if x, ok := m.GetType().(*IdsOrKeys_Ids); ok { - return x.Ids - } - return nil -} - -func (m *IdsOrKeys) GetKeys() *StringArray { - if x, ok := m.GetType().(*IdsOrKeys_Keys); ok { - return x.Keys - } - return nil -} - -// XXX_OneofWrappers is for the internal use of the proto package. -func (*IdsOrKeys) XXX_OneofWrappers() []interface{} { - return []interface{}{ - (*IdsOrKeys_Ids)(nil), - (*IdsOrKeys_Keys)(nil), - } -} - func init() { proto.RegisterType((*DataSource)(nil), "vdsm.DataSource") proto.RegisterMapType((map[string]string)(nil), "vdsm.DataSource.ConfigEntry") @@ -1371,93 +574,50 @@ func init() { proto.RegisterType((*PostVDSResponse)(nil), "vdsm.PostVDSResponse") proto.RegisterType((*DeleteVDSRequest)(nil), "vdsm.DeleteVDSRequest") proto.RegisterType((*DeleteVDSResponse)(nil), "vdsm.DeleteVDSResponse") - proto.RegisterType((*QueryPQLRequest)(nil), "vdsm.QueryPQLRequest") - proto.RegisterType((*QuerySQLRequest)(nil), "vdsm.QuerySQLRequest") - proto.RegisterType((*StatusError)(nil), "vdsm.StatusError") - proto.RegisterType((*RowResponse)(nil), "vdsm.RowResponse") - proto.RegisterType((*Row)(nil), "vdsm.Row") - proto.RegisterType((*TableResponse)(nil), "vdsm.TableResponse") - proto.RegisterType((*ColumnInfo)(nil), "vdsm.ColumnInfo") - proto.RegisterType((*ColumnResponse)(nil), "vdsm.ColumnResponse") - proto.RegisterType((*Decimal)(nil), "vdsm.Decimal") - proto.RegisterType((*InspectRequest)(nil), "vdsm.InspectRequest") - proto.RegisterType((*Uint64Array)(nil), "vdsm.Uint64Array") - proto.RegisterType((*StringArray)(nil), "vdsm.StringArray") - proto.RegisterType((*IdsOrKeys)(nil), "vdsm.IdsOrKeys") } func init() { proto.RegisterFile("vdsm/vdsm.proto", fileDescriptor_3849c3e549495f50) } var fileDescriptor_3849c3e549495f50 = []byte{ - // 1081 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0xef, 0x6e, 0x1b, 0x45, - 0x10, 0xf7, 0xe5, 0x1c, 0xdb, 0x37, 0x97, 0x38, 0xce, 0x36, 0x14, 0xcb, 0xb4, 0x95, 0x7b, 0x08, - 0x61, 0x90, 0x48, 0xa3, 0xa4, 0xa9, 0x52, 0x0a, 0x48, 0x24, 0x69, 0x71, 0x04, 0xa5, 0xee, 0x9a, - 0xe6, 0x2b, 0xda, 0xf8, 0xd6, 0xe1, 0xd4, 0xf5, 0xad, 0x7d, 0x7b, 0x4e, 0xf0, 0x37, 0xde, 0x80, - 0x37, 0x00, 0x89, 0x77, 0xe0, 0xb9, 0x78, 0x05, 0xb4, 0xff, 0xce, 0x7b, 0x26, 0x15, 0x55, 0xfb, - 0xc5, 0xda, 0xfd, 0xcd, 0x6f, 0xfe, 0xed, 0xcc, 0xcd, 0x18, 0xb6, 0xae, 0x62, 0x31, 0x79, 0x20, - 0x7f, 0x76, 0xa7, 0x19, 0xcf, 0x39, 0xaa, 0xca, 0x73, 0xf4, 0x8f, 0x07, 0x70, 0x4a, 0x72, 0x32, - 0xe4, 0xf3, 0x6c, 0x44, 0x51, 0x13, 0xd6, 0x92, 0xb8, 0xed, 0x75, 0xbd, 0x9e, 0x8f, 0xd7, 0x92, - 0x18, 0x21, 0xa8, 0xa6, 0x64, 0x42, 0xdb, 0x6b, 0x5d, 0xaf, 0x17, 0x60, 0x75, 0x46, 0x5d, 0x08, - 0x63, 0x2a, 0x46, 0x59, 0x32, 0xcd, 0x13, 0x9e, 0xb6, 0x7d, 0x25, 0x72, 0x21, 0xa9, 0x95, 0x2f, - 0xa6, 0xb4, 0x5d, 0xd5, 0x5a, 0xf2, 0x8c, 0x1e, 0x42, 0x6d, 0xc4, 0xd3, 0x71, 0x72, 0xd9, 0x5e, - 0xef, 0xfa, 0xbd, 0x70, 0xff, 0xce, 0xae, 0x8a, 0x65, 0xe9, 0x7b, 0xf7, 0x44, 0x89, 0x9f, 0xa6, - 0x79, 0xb6, 0xc0, 0x86, 0x8b, 0x6e, 0x43, 0x4d, 0xe4, 0x24, 0x9f, 0x8b, 0x76, 0x4d, 0xd9, 0x32, - 0xb7, 0xce, 0x63, 0x08, 0x1d, 0x3a, 0x6a, 0x81, 0xff, 0x9a, 0x2e, 0x54, 0xdc, 0x01, 0x96, 0x47, - 0xb4, 0x03, 0xeb, 0x57, 0x84, 0xcd, 0x6d, 0xe4, 0xfa, 0xf2, 0xe5, 0xda, 0x91, 0x17, 0xfd, 0xe5, - 0x81, 0x7f, 0x7e, 0x3a, 0x74, 0x52, 0x0d, 0xde, 0x23, 0xd5, 0xfb, 0xb0, 0x31, 0x4d, 0x18, 0x17, - 0xe4, 0xe7, 0x24, 0x8d, 0xe9, 0xaf, 0x26, 0xe5, 0x50, 0x63, 0x67, 0x12, 0x42, 0xfb, 0x10, 0xc6, - 0x24, 0x27, 0x42, 0x65, 0x29, 0x4c, 0xfa, 0xad, 0xd5, 0xf4, 0xb1, 0x4b, 0x8a, 0x5a, 0xd0, 0xfc, - 0x8e, 0xe6, 0xe7, 0xa7, 0x43, 0x81, 0xe9, 0x6c, 0x4e, 0x45, 0x1e, 0xed, 0xc1, 0x56, 0x81, 0x88, - 0x29, 0x4f, 0x05, 0x45, 0x77, 0x41, 0xd6, 0x50, 0xb4, 0x3d, 0x65, 0x31, 0xd0, 0x16, 0xcf, 0x4f, - 0x87, 0x58, 0xc1, 0xd1, 0x09, 0x6c, 0x6a, 0x0d, 0x63, 0x02, 0xb5, 0x96, 0x19, 0xf7, 0x2b, 0x2a, - 0xe7, 0x1d, 0x37, 0xe7, 0x7e, 0x45, 0x67, 0x7d, 0x0c, 0xd0, 0x48, 0xe2, 0x17, 0xd9, 0x8f, 0x64, - 0x42, 0xa3, 0x2f, 0x6c, 0x20, 0x85, 0xd7, 0x8f, 0xc0, 0xbf, 0x8a, 0x85, 0x32, 0x53, 0x72, 0x2a, - 0xd1, 0x68, 0x0f, 0x9a, 0x03, 0x2e, 0x5c, 0xa7, 0xf7, 0x00, 0x62, 0x3a, 0x4e, 0xd2, 0x44, 0xbd, - 0xa0, 0x7e, 0x6e, 0x07, 0x89, 0x0e, 0x60, 0xab, 0xd0, 0x30, 0x1e, 0x56, 0x2b, 0xd3, 0x02, 0x7f, - 0x9e, 0x25, 0xa6, 0x30, 0xf2, 0x18, 0x3d, 0x83, 0xd6, 0x29, 0x65, 0x34, 0xa7, 0xef, 0x99, 0xdd, - 0x2d, 0xd8, 0x76, 0xec, 0x68, 0xf7, 0xd1, 0x63, 0xd8, 0x7a, 0x39, 0xa7, 0xd9, 0x62, 0xf0, 0xf2, - 0x07, 0x6b, 0x7b, 0x07, 0xd6, 0x75, 0x79, 0x75, 0x50, 0xfa, 0x22, 0xe3, 0x9a, 0xce, 0x98, 0x8d, - 0x6b, 0x3a, 0x63, 0xd1, 0xc7, 0x46, 0x75, 0xb8, 0x54, 0x6d, 0x81, 0x2f, 0x66, 0xcc, 0xb6, 0xa6, - 0x98, 0xb1, 0xe8, 0x09, 0x84, 0x43, 0xd5, 0xc5, 0x4f, 0xb3, 0x8c, 0x67, 0xb2, 0xef, 0x4e, 0x78, - 0x4c, 0x15, 0x63, 0x13, 0xab, 0x33, 0x6a, 0x43, 0xfd, 0x39, 0x15, 0x82, 0x5c, 0xda, 0x76, 0xb4, - 0xd7, 0xe8, 0x0f, 0x0f, 0x42, 0xcc, 0xaf, 0x8b, 0xb7, 0xfa, 0x1c, 0xea, 0xbf, 0x50, 0x12, 0xd3, - 0xcc, 0xb6, 0x81, 0x69, 0xac, 0x13, 0xce, 0xe6, 0x93, 0xf4, 0x2c, 0x1d, 0x73, 0x6c, 0x09, 0x68, - 0x17, 0xea, 0x23, 0x05, 0x8b, 0xf6, 0x9a, 0xe2, 0xee, 0xb8, 0x5c, 0x6b, 0x12, 0x5b, 0x12, 0x3a, - 0x28, 0x05, 0xaa, 0xba, 0x3f, 0xdc, 0xdf, 0xd6, 0x3a, 0x8e, 0x00, 0xbb, 0xac, 0xe8, 0x10, 0x7c, - 0xcc, 0xaf, 0x5d, 0x5f, 0xde, 0x5b, 0xf8, 0x8a, 0x7e, 0xf7, 0x60, 0xf3, 0x27, 0x72, 0xc1, 0xe8, - 0x3b, 0x65, 0x76, 0x17, 0xaa, 0x19, 0xbf, 0xb6, 0x69, 0x99, 0xa6, 0x94, 0xcf, 0xa4, 0xe0, 0x77, - 0x4b, 0xe4, 0x2b, 0x80, 0xa5, 0xab, 0x62, 0x3a, 0x78, 0xce, 0x74, 0xe8, 0x40, 0x43, 0x7e, 0xb3, - 0x6a, 0xd4, 0xe9, 0x32, 0x15, 0xf7, 0xe8, 0x37, 0x1f, 0x9a, 0xe5, 0x5c, 0xd1, 0x3d, 0x08, 0x44, - 0x9e, 0x25, 0xe9, 0xe5, 0x39, 0x61, 0x45, 0x9f, 0x2e, 0x21, 0x29, 0x9f, 0x27, 0x69, 0xfe, 0xe8, - 0xa1, 0x94, 0x4b, 0x7b, 0x55, 0x29, 0x2f, 0x20, 0x74, 0x07, 0x1a, 0x85, 0x58, 0xa6, 0xe0, 0xf7, - 0x2b, 0xb8, 0x40, 0x50, 0x07, 0xea, 0x17, 0x9c, 0x33, 0x29, 0x94, 0x33, 0xa8, 0xd1, 0xaf, 0x60, - 0x0b, 0x28, 0x19, 0xe3, 0x17, 0x52, 0xb6, 0xde, 0xf5, 0x7a, 0x1b, 0x4a, 0xa6, 0x01, 0xf4, 0x04, - 0x9a, 0xda, 0xc5, 0xb7, 0x59, 0x46, 0x16, 0x92, 0x52, 0x73, 0x9f, 0xe7, 0xd5, 0x52, 0xd6, 0xaf, - 0xe0, 0x15, 0xaa, 0x54, 0xd6, 0xf1, 0x17, 0xca, 0xf5, 0xf2, 0xdb, 0x16, 0x32, 0xa9, 0x5c, 0xa6, - 0xa2, 0x2e, 0xc0, 0x98, 0x71, 0x62, 0x32, 0x6a, 0x74, 0xbd, 0x9e, 0xd7, 0xaf, 0x60, 0x07, 0x43, - 0x0f, 0xe4, 0xec, 0x18, 0x25, 0x13, 0xa2, 0xd2, 0x0a, 0x94, 0xe9, 0x4d, 0x33, 0x38, 0x35, 0x2e, - 0x15, 0x96, 0x94, 0xe3, 0x10, 0x02, 0xdd, 0x50, 0xe7, 0x84, 0x45, 0x87, 0x50, 0x37, 0xac, 0xe5, - 0x36, 0xd0, 0x9b, 0x4d, 0x5f, 0x24, 0x2a, 0x46, 0x84, 0xe9, 0xe2, 0xf9, 0x58, 0x5f, 0xa2, 0xbf, - 0x3d, 0x68, 0x9e, 0xa5, 0x62, 0x4a, 0x47, 0xb9, 0xf3, 0x0d, 0xdb, 0x91, 0x17, 0xa8, 0x39, 0x87, - 0x3e, 0x83, 0x7a, 0x46, 0x47, 0x3c, 0x8b, 0x85, 0x52, 0x0e, 0xf7, 0xb7, 0x74, 0x58, 0x67, 0xb1, - 0x78, 0x91, 0x7d, 0x4f, 0x17, 0x02, 0x5b, 0x39, 0x8a, 0x60, 0x63, 0x9c, 0xb0, 0x9c, 0x66, 0xcf, - 0x12, 0xca, 0x62, 0xd1, 0xf6, 0xbb, 0x7e, 0x2f, 0xc0, 0x25, 0x4c, 0x46, 0xc2, 0x92, 0x49, 0x92, - 0xab, 0xd2, 0x55, 0xb1, 0xbe, 0xc8, 0xe5, 0xc7, 0xc7, 0x63, 0x41, 0x73, 0x55, 0xb5, 0x2a, 0x36, - 0x37, 0xc9, 0x9e, 0xc9, 0x29, 0x63, 0x76, 0xa2, 0xbe, 0x44, 0xf7, 0x21, 0x74, 0x8a, 0x25, 0x1b, - 0xf6, 0x8a, 0x30, 0xfd, 0xed, 0x54, 0xb1, 0x3a, 0x4b, 0x8a, 0x53, 0x92, 0x12, 0x25, 0x30, 0x94, - 0x11, 0x04, 0x45, 0x0e, 0xe8, 0x13, 0xf0, 0x93, 0x62, 0xd4, 0xdf, 0xd8, 0x10, 0x52, 0x8e, 0x3e, - 0x85, 0xea, 0x6b, 0xba, 0xb0, 0x2f, 0x71, 0x63, 0xed, 0x15, 0xe1, 0xb8, 0xa6, 0xff, 0x17, 0xec, - 0xff, 0x59, 0x85, 0xc6, 0x73, 0xce, 0xe8, 0x68, 0xce, 0x08, 0x3a, 0x82, 0xba, 0x59, 0x6c, 0xc8, - 0xcc, 0x88, 0xf2, 0xe6, 0xeb, 0x7c, 0xb0, 0x82, 0x9a, 0x31, 0x5d, 0x41, 0x87, 0x50, 0xd3, 0x20, - 0xba, 0xe5, 0x52, 0xac, 0xde, 0x4e, 0x19, 0x2c, 0xd4, 0x8e, 0xa0, 0x6e, 0x36, 0x8e, 0x75, 0x58, - 0x5e, 0x59, 0xd6, 0xe1, 0xca, 0x5a, 0x8a, 0x2a, 0xe8, 0x1b, 0x08, 0x8a, 0x75, 0x81, 0x6e, 0xdb, - 0x46, 0x2c, 0xef, 0xa1, 0xce, 0x87, 0xff, 0xc1, 0x1d, 0xcf, 0x0d, 0xbb, 0x1e, 0x90, 0x71, 0xb2, - 0xb2, 0x2e, 0x3a, 0xdb, 0xcb, 0xd9, 0x55, 0xe8, 0xed, 0x79, 0xe8, 0x6b, 0xd8, 0xb4, 0xcc, 0x57, - 0x29, 0xc9, 0x16, 0x6f, 0x52, 0x37, 0x0f, 0x51, 0x9a, 0xa4, 0x8e, 0xe3, 0xc1, 0x8a, 0xe3, 0xc1, - 0x5b, 0x3b, 0x1e, 0xdc, 0xe4, 0x78, 0xf0, 0xbf, 0x8e, 0x1f, 0x41, 0xdd, 0x7c, 0x4b, 0xf6, 0xad, - 0xcb, 0x9f, 0xd6, 0x1b, 0xdc, 0x5e, 0xd4, 0xd4, 0x7f, 0xd4, 0x83, 0x7f, 0x03, 0x00, 0x00, 0xff, - 0xff, 0x5a, 0xe9, 0x98, 0xbf, 0xb6, 0x0a, 0x00, 0x00, + // 604 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x54, 0xcd, 0x6e, 0xd3, 0x4c, + 0x14, 0x8d, 0xe3, 0x34, 0x6d, 0x6e, 0xbe, 0xb6, 0xfe, 0xa6, 0x69, 0x6b, 0x85, 0x1f, 0x05, 0x2f, + 0x50, 0x36, 0xa4, 0x51, 0x0a, 0x52, 0xa8, 0x10, 0x12, 0x6d, 0x80, 0x56, 0xe2, 0x27, 0x75, 0x20, + 0x0b, 0x36, 0x68, 0x62, 0x4f, 0xcb, 0x08, 0xc7, 0x36, 0x9e, 0x71, 0x20, 0xaf, 0xc3, 0x2b, 0xf0, + 0x60, 0xbc, 0x02, 0xf2, 0xfc, 0x38, 0x4e, 0x5a, 0x04, 0xa8, 0x9b, 0xf6, 0xfa, 0xcc, 0x3d, 0xf7, + 0x9c, 0x73, 0x27, 0x1a, 0xd8, 0x9e, 0xf9, 0x6c, 0x7a, 0x90, 0xfd, 0xe9, 0xc4, 0x49, 0xc4, 0x23, + 0x54, 0xc9, 0xea, 0xe6, 0x7f, 0x31, 0x0d, 0x22, 0x86, 0x25, 0xe6, 0xfc, 0x34, 0x00, 0x06, 0x98, + 0xe3, 0x51, 0x94, 0x26, 0x1e, 0x41, 0x5b, 0x50, 0xa6, 0xbe, 0x6d, 0xb4, 0x8c, 0xb6, 0xe9, 0x96, + 0xa9, 0x8f, 0x10, 0x54, 0x42, 0x3c, 0x25, 0x76, 0xb9, 0x65, 0xb4, 0x6b, 0xae, 0xa8, 0x51, 0x0b, + 0xea, 0x3e, 0x61, 0x5e, 0x42, 0x63, 0x4e, 0xa3, 0xd0, 0x36, 0xc5, 0x51, 0x11, 0xca, 0x58, 0x7c, + 0x1e, 0x13, 0xbb, 0x22, 0x59, 0x59, 0x8d, 0x1e, 0x42, 0xd5, 0x8b, 0xc2, 0x0b, 0x7a, 0x69, 0xaf, + 0xb5, 0xcc, 0x76, 0xbd, 0x77, 0xbb, 0x23, 0x9c, 0x2d, 0xb4, 0x3b, 0x27, 0xe2, 0xf8, 0x79, 0xc8, + 0x93, 0xb9, 0xab, 0x7a, 0xd1, 0x1e, 0x54, 0x19, 0xc7, 0x3c, 0x65, 0x76, 0x55, 0xcc, 0x52, 0x5f, + 0xcd, 0xc7, 0x50, 0x2f, 0xb4, 0x23, 0x0b, 0xcc, 0xcf, 0x64, 0x2e, 0x7c, 0xd7, 0xdc, 0xac, 0x44, + 0x0d, 0x58, 0x9b, 0xe1, 0x20, 0xd5, 0xce, 0xe5, 0xc7, 0x51, 0xb9, 0x6f, 0x38, 0xdf, 0x0d, 0x30, + 0xc7, 0x83, 0x51, 0x21, 0x6a, 0xed, 0x06, 0x51, 0xef, 0x81, 0xda, 0xe7, 0x47, 0x1a, 0xfa, 0xe4, + 0x9b, 0x8a, 0x5c, 0x97, 0xd8, 0x59, 0x06, 0xa1, 0x1e, 0xd4, 0x7d, 0xcc, 0x31, 0x13, 0x29, 0x99, + 0x8a, 0x6f, 0xad, 0xc6, 0x77, 0x8b, 0x4d, 0x8e, 0x05, 0x5b, 0x2f, 0x09, 0x1f, 0x0f, 0x46, 0xcc, + 0x25, 0x5f, 0x52, 0xc2, 0xb8, 0xd3, 0x85, 0xed, 0x1c, 0x61, 0x71, 0x14, 0x32, 0x82, 0xee, 0x40, + 0x76, 0xa3, 0xcc, 0x36, 0xc4, 0xc4, 0x9a, 0x9c, 0x38, 0x1e, 0x8c, 0x5c, 0x01, 0x3b, 0x27, 0xb0, + 0x29, 0x19, 0x6a, 0x04, 0xb2, 0x16, 0x89, 0x4f, 0x4b, 0x22, 0x73, 0xa3, 0x98, 0xf9, 0xb4, 0x24, + 0x53, 0x1f, 0x03, 0x6c, 0x50, 0xff, 0x6d, 0xf2, 0x06, 0x4f, 0x89, 0xf3, 0x40, 0x1b, 0xc9, 0x55, + 0x6f, 0x81, 0x39, 0xf3, 0x99, 0x18, 0xb3, 0x24, 0x9a, 0xa1, 0x4e, 0x17, 0xb6, 0x86, 0x11, 0x2b, + 0x8a, 0xde, 0x05, 0xf0, 0xc9, 0x05, 0x0d, 0xa9, 0xd8, 0xa0, 0x5c, 0x77, 0x01, 0x71, 0x0e, 0x61, + 0x3b, 0x67, 0x28, 0x85, 0xd5, 0x9b, 0xb1, 0xc0, 0x4c, 0x13, 0xaa, 0x2e, 0x26, 0x2b, 0x9d, 0x17, + 0x60, 0x0d, 0x48, 0x40, 0x38, 0xb9, 0x61, 0xba, 0x1d, 0xf8, 0xbf, 0x30, 0x47, 0xca, 0xf7, 0x7e, + 0x54, 0x60, 0xe3, 0x75, 0x14, 0x10, 0x2f, 0x0d, 0x30, 0xea, 0xc3, 0xba, 0x5a, 0x3b, 0x6a, 0xc8, + 0xac, 0xcb, 0xf7, 0xd2, 0xdc, 0x5d, 0x41, 0xe5, 0x10, 0xa7, 0x84, 0x1e, 0x41, 0x55, 0x82, 0x68, + 0xa7, 0xd8, 0xa2, 0x79, 0x8d, 0x65, 0x30, 0xa7, 0xf5, 0x61, 0x5d, 0xed, 0x43, 0x0b, 0x2e, 0x2f, + 0x54, 0x0b, 0xae, 0x2c, 0xcd, 0x29, 0xa1, 0xa7, 0x50, 0xcb, 0xc3, 0xa0, 0x3d, 0xf5, 0xfb, 0x5a, + 0xd9, 0x52, 0x73, 0xff, 0x0a, 0x9e, 0xf3, 0x9f, 0xc0, 0xc6, 0x79, 0x4a, 0x92, 0xf9, 0xe8, 0xfc, + 0x15, 0xda, 0xef, 0xa8, 0x57, 0x42, 0x23, 0x9a, 0xbf, 0xa3, 0x0f, 0xdc, 0xe8, 0xeb, 0x82, 0xdb, + 0x35, 0xd0, 0x33, 0xd8, 0xd4, 0xbd, 0xef, 0x43, 0x9c, 0xcc, 0x7f, 0x3f, 0x62, 0x57, 0x1f, 0xbc, + 0xc3, 0x93, 0x80, 0x5c, 0x63, 0x60, 0x78, 0xc5, 0xc0, 0xf0, 0x1f, 0x0c, 0x0c, 0xaf, 0x37, 0x30, + 0xfc, 0x0b, 0x03, 0x47, 0xb0, 0x7e, 0x16, 0xb2, 0x98, 0x78, 0x1c, 0xed, 0xe9, 0x1e, 0x05, 0xfc, + 0x49, 0xfe, 0xb8, 0xfd, 0xe1, 0xfe, 0x25, 0xe5, 0x9f, 0xd2, 0x49, 0xc7, 0x8b, 0xa6, 0x07, 0xb2, + 0x49, 0xff, 0x9b, 0xf5, 0x0e, 0xc4, 0x6b, 0x2b, 0x1e, 0xe3, 0x49, 0x55, 0xd4, 0x87, 0xbf, 0x02, + 0x00, 0x00, 0xff, 0xff, 0x39, 0x92, 0xf5, 0x8e, 0xa0, 0x05, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -1476,11 +636,11 @@ type MoleculaClient interface { GetVDS(ctx context.Context, in *GetVDSRequest, opts ...grpc.CallOption) (*GetVDSResponse, error) PostVDS(ctx context.Context, in *PostVDSRequest, opts ...grpc.CallOption) (*PostVDSResponse, error) DeleteVDS(ctx context.Context, in *DeleteVDSRequest, opts ...grpc.CallOption) (*DeleteVDSResponse, error) - QuerySQL(ctx context.Context, in *QuerySQLRequest, opts ...grpc.CallOption) (Molecula_QuerySQLClient, error) - QuerySQLUnary(ctx context.Context, in *QuerySQLRequest, opts ...grpc.CallOption) (*TableResponse, error) - QueryPQL(ctx context.Context, in *QueryPQLRequest, opts ...grpc.CallOption) (Molecula_QueryPQLClient, error) - QueryPQLUnary(ctx context.Context, in *QueryPQLRequest, opts ...grpc.CallOption) (*TableResponse, error) - Inspect(ctx context.Context, in *InspectRequest, opts ...grpc.CallOption) (Molecula_InspectClient, error) + QuerySQL(ctx context.Context, in *proto1.QuerySQLRequest, opts ...grpc.CallOption) (Molecula_QuerySQLClient, error) + QuerySQLUnary(ctx context.Context, in *proto1.QuerySQLRequest, opts ...grpc.CallOption) (*proto1.TableResponse, error) + QueryPQL(ctx context.Context, in *proto1.QueryPQLRequest, opts ...grpc.CallOption) (Molecula_QueryPQLClient, error) + QueryPQLUnary(ctx context.Context, in *proto1.QueryPQLRequest, opts ...grpc.CallOption) (*proto1.TableResponse, error) + Inspect(ctx context.Context, in *proto1.InspectRequest, opts ...grpc.CallOption) (Molecula_InspectClient, error) } type moleculaClient struct { @@ -1527,7 +687,7 @@ func (c *moleculaClient) DeleteVDS(ctx context.Context, in *DeleteVDSRequest, op return out, nil } -func (c *moleculaClient) QuerySQL(ctx context.Context, in *QuerySQLRequest, opts ...grpc.CallOption) (Molecula_QuerySQLClient, error) { +func (c *moleculaClient) QuerySQL(ctx context.Context, in *proto1.QuerySQLRequest, opts ...grpc.CallOption) (Molecula_QuerySQLClient, error) { stream, err := c.cc.NewStream(ctx, &_Molecula_serviceDesc.Streams[0], "/vdsm.Molecula/QuerySQL", opts...) if err != nil { return nil, err @@ -1543,7 +703,7 @@ func (c *moleculaClient) QuerySQL(ctx context.Context, in *QuerySQLRequest, opts } type Molecula_QuerySQLClient interface { - Recv() (*RowResponse, error) + Recv() (*proto1.RowResponse, error) grpc.ClientStream } @@ -1551,16 +711,16 @@ type moleculaQuerySQLClient struct { grpc.ClientStream } -func (x *moleculaQuerySQLClient) Recv() (*RowResponse, error) { - m := new(RowResponse) +func (x *moleculaQuerySQLClient) Recv() (*proto1.RowResponse, error) { + m := new(proto1.RowResponse) if err := x.ClientStream.RecvMsg(m); err != nil { return nil, err } return m, nil } -func (c *moleculaClient) QuerySQLUnary(ctx context.Context, in *QuerySQLRequest, opts ...grpc.CallOption) (*TableResponse, error) { - out := new(TableResponse) +func (c *moleculaClient) QuerySQLUnary(ctx context.Context, in *proto1.QuerySQLRequest, opts ...grpc.CallOption) (*proto1.TableResponse, error) { + out := new(proto1.TableResponse) err := c.cc.Invoke(ctx, "/vdsm.Molecula/QuerySQLUnary", in, out, opts...) if err != nil { return nil, err @@ -1568,7 +728,7 @@ func (c *moleculaClient) QuerySQLUnary(ctx context.Context, in *QuerySQLRequest, return out, nil } -func (c *moleculaClient) QueryPQL(ctx context.Context, in *QueryPQLRequest, opts ...grpc.CallOption) (Molecula_QueryPQLClient, error) { +func (c *moleculaClient) QueryPQL(ctx context.Context, in *proto1.QueryPQLRequest, opts ...grpc.CallOption) (Molecula_QueryPQLClient, error) { stream, err := c.cc.NewStream(ctx, &_Molecula_serviceDesc.Streams[1], "/vdsm.Molecula/QueryPQL", opts...) if err != nil { return nil, err @@ -1584,7 +744,7 @@ func (c *moleculaClient) QueryPQL(ctx context.Context, in *QueryPQLRequest, opts } type Molecula_QueryPQLClient interface { - Recv() (*RowResponse, error) + Recv() (*proto1.RowResponse, error) grpc.ClientStream } @@ -1592,16 +752,16 @@ type moleculaQueryPQLClient struct { grpc.ClientStream } -func (x *moleculaQueryPQLClient) Recv() (*RowResponse, error) { - m := new(RowResponse) +func (x *moleculaQueryPQLClient) Recv() (*proto1.RowResponse, error) { + m := new(proto1.RowResponse) if err := x.ClientStream.RecvMsg(m); err != nil { return nil, err } return m, nil } -func (c *moleculaClient) QueryPQLUnary(ctx context.Context, in *QueryPQLRequest, opts ...grpc.CallOption) (*TableResponse, error) { - out := new(TableResponse) +func (c *moleculaClient) QueryPQLUnary(ctx context.Context, in *proto1.QueryPQLRequest, opts ...grpc.CallOption) (*proto1.TableResponse, error) { + out := new(proto1.TableResponse) err := c.cc.Invoke(ctx, "/vdsm.Molecula/QueryPQLUnary", in, out, opts...) if err != nil { return nil, err @@ -1609,7 +769,7 @@ func (c *moleculaClient) QueryPQLUnary(ctx context.Context, in *QueryPQLRequest, return out, nil } -func (c *moleculaClient) Inspect(ctx context.Context, in *InspectRequest, opts ...grpc.CallOption) (Molecula_InspectClient, error) { +func (c *moleculaClient) Inspect(ctx context.Context, in *proto1.InspectRequest, opts ...grpc.CallOption) (Molecula_InspectClient, error) { stream, err := c.cc.NewStream(ctx, &_Molecula_serviceDesc.Streams[2], "/vdsm.Molecula/Inspect", opts...) if err != nil { return nil, err @@ -1625,7 +785,7 @@ func (c *moleculaClient) Inspect(ctx context.Context, in *InspectRequest, opts . } type Molecula_InspectClient interface { - Recv() (*RowResponse, error) + Recv() (*proto1.RowResponse, error) grpc.ClientStream } @@ -1633,8 +793,8 @@ type moleculaInspectClient struct { grpc.ClientStream } -func (x *moleculaInspectClient) Recv() (*RowResponse, error) { - m := new(RowResponse) +func (x *moleculaInspectClient) Recv() (*proto1.RowResponse, error) { + m := new(proto1.RowResponse) if err := x.ClientStream.RecvMsg(m); err != nil { return nil, err } @@ -1647,11 +807,11 @@ type MoleculaServer interface { GetVDS(context.Context, *GetVDSRequest) (*GetVDSResponse, error) PostVDS(context.Context, *PostVDSRequest) (*PostVDSResponse, error) DeleteVDS(context.Context, *DeleteVDSRequest) (*DeleteVDSResponse, error) - QuerySQL(*QuerySQLRequest, Molecula_QuerySQLServer) error - QuerySQLUnary(context.Context, *QuerySQLRequest) (*TableResponse, error) - QueryPQL(*QueryPQLRequest, Molecula_QueryPQLServer) error - QueryPQLUnary(context.Context, *QueryPQLRequest) (*TableResponse, error) - Inspect(*InspectRequest, Molecula_InspectServer) error + QuerySQL(*proto1.QuerySQLRequest, Molecula_QuerySQLServer) error + QuerySQLUnary(context.Context, *proto1.QuerySQLRequest) (*proto1.TableResponse, error) + QueryPQL(*proto1.QueryPQLRequest, Molecula_QueryPQLServer) error + QueryPQLUnary(context.Context, *proto1.QueryPQLRequest) (*proto1.TableResponse, error) + Inspect(*proto1.InspectRequest, Molecula_InspectServer) error } // UnimplementedMoleculaServer can be embedded to have forward compatible implementations. @@ -1670,19 +830,19 @@ func (*UnimplementedMoleculaServer) PostVDS(ctx context.Context, req *PostVDSReq func (*UnimplementedMoleculaServer) DeleteVDS(ctx context.Context, req *DeleteVDSRequest) (*DeleteVDSResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method DeleteVDS not implemented") } -func (*UnimplementedMoleculaServer) QuerySQL(req *QuerySQLRequest, srv Molecula_QuerySQLServer) error { +func (*UnimplementedMoleculaServer) QuerySQL(req *proto1.QuerySQLRequest, srv Molecula_QuerySQLServer) error { return status.Errorf(codes.Unimplemented, "method QuerySQL not implemented") } -func (*UnimplementedMoleculaServer) QuerySQLUnary(ctx context.Context, req *QuerySQLRequest) (*TableResponse, error) { +func (*UnimplementedMoleculaServer) QuerySQLUnary(ctx context.Context, req *proto1.QuerySQLRequest) (*proto1.TableResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method QuerySQLUnary not implemented") } -func (*UnimplementedMoleculaServer) QueryPQL(req *QueryPQLRequest, srv Molecula_QueryPQLServer) error { +func (*UnimplementedMoleculaServer) QueryPQL(req *proto1.QueryPQLRequest, srv Molecula_QueryPQLServer) error { return status.Errorf(codes.Unimplemented, "method QueryPQL not implemented") } -func (*UnimplementedMoleculaServer) QueryPQLUnary(ctx context.Context, req *QueryPQLRequest) (*TableResponse, error) { +func (*UnimplementedMoleculaServer) QueryPQLUnary(ctx context.Context, req *proto1.QueryPQLRequest) (*proto1.TableResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method QueryPQLUnary not implemented") } -func (*UnimplementedMoleculaServer) Inspect(req *InspectRequest, srv Molecula_InspectServer) error { +func (*UnimplementedMoleculaServer) Inspect(req *proto1.InspectRequest, srv Molecula_InspectServer) error { return status.Errorf(codes.Unimplemented, "method Inspect not implemented") } @@ -1763,7 +923,7 @@ func _Molecula_DeleteVDS_Handler(srv interface{}, ctx context.Context, dec func( } func _Molecula_QuerySQL_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(QuerySQLRequest) + m := new(proto1.QuerySQLRequest) if err := stream.RecvMsg(m); err != nil { return err } @@ -1771,7 +931,7 @@ func _Molecula_QuerySQL_Handler(srv interface{}, stream grpc.ServerStream) error } type Molecula_QuerySQLServer interface { - Send(*RowResponse) error + Send(*proto1.RowResponse) error grpc.ServerStream } @@ -1779,12 +939,12 @@ type moleculaQuerySQLServer struct { grpc.ServerStream } -func (x *moleculaQuerySQLServer) Send(m *RowResponse) error { +func (x *moleculaQuerySQLServer) Send(m *proto1.RowResponse) error { return x.ServerStream.SendMsg(m) } func _Molecula_QuerySQLUnary_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(QuerySQLRequest) + in := new(proto1.QuerySQLRequest) if err := dec(in); err != nil { return nil, err } @@ -1796,13 +956,13 @@ func _Molecula_QuerySQLUnary_Handler(srv interface{}, ctx context.Context, dec f FullMethod: "/vdsm.Molecula/QuerySQLUnary", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(MoleculaServer).QuerySQLUnary(ctx, req.(*QuerySQLRequest)) + return srv.(MoleculaServer).QuerySQLUnary(ctx, req.(*proto1.QuerySQLRequest)) } return interceptor(ctx, in, info, handler) } func _Molecula_QueryPQL_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(QueryPQLRequest) + m := new(proto1.QueryPQLRequest) if err := stream.RecvMsg(m); err != nil { return err } @@ -1810,7 +970,7 @@ func _Molecula_QueryPQL_Handler(srv interface{}, stream grpc.ServerStream) error } type Molecula_QueryPQLServer interface { - Send(*RowResponse) error + Send(*proto1.RowResponse) error grpc.ServerStream } @@ -1818,12 +978,12 @@ type moleculaQueryPQLServer struct { grpc.ServerStream } -func (x *moleculaQueryPQLServer) Send(m *RowResponse) error { +func (x *moleculaQueryPQLServer) Send(m *proto1.RowResponse) error { return x.ServerStream.SendMsg(m) } func _Molecula_QueryPQLUnary_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(QueryPQLRequest) + in := new(proto1.QueryPQLRequest) if err := dec(in); err != nil { return nil, err } @@ -1835,13 +995,13 @@ func _Molecula_QueryPQLUnary_Handler(srv interface{}, ctx context.Context, dec f FullMethod: "/vdsm.Molecula/QueryPQLUnary", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(MoleculaServer).QueryPQLUnary(ctx, req.(*QueryPQLRequest)) + return srv.(MoleculaServer).QueryPQLUnary(ctx, req.(*proto1.QueryPQLRequest)) } return interceptor(ctx, in, info, handler) } func _Molecula_Inspect_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(InspectRequest) + m := new(proto1.InspectRequest) if err := stream.RecvMsg(m); err != nil { return err } @@ -1849,7 +1009,7 @@ func _Molecula_Inspect_Handler(srv interface{}, stream grpc.ServerStream) error } type Molecula_InspectServer interface { - Send(*RowResponse) error + Send(*proto1.RowResponse) error grpc.ServerStream } @@ -1857,7 +1017,7 @@ type moleculaInspectServer struct { grpc.ServerStream } -func (x *moleculaInspectServer) Send(m *RowResponse) error { +func (x *moleculaInspectServer) Send(m *proto1.RowResponse) error { return x.ServerStream.SendMsg(m) } diff --git a/proto/vdsm/vdsm.proto b/proto/vdsm/vdsm.proto index 34b13de70..b72622ac9 100644 --- a/proto/vdsm/vdsm.proto +++ b/proto/vdsm/vdsm.proto @@ -1,6 +1,10 @@ syntax = "proto3"; package vdsm; +option go_package = "github.com/pilosa/pilosa/v2/proto/vdsm"; + +import "pilosa.proto"; + // deprecated message DataSource { int64 id = 1; @@ -56,92 +60,14 @@ message DeleteVDSRequest { message DeleteVDSResponse { } -message QueryPQLRequest { - string index = 1; - string pql = 2; -} - -message QuerySQLRequest { - string sql = 1; -} - -message StatusError{ - uint32 Code = 1; - string Message = 2; -} - -message RowResponse{ - repeated ColumnInfo headers = 1; - repeated ColumnResponse columns = 2; - StatusError StatusError = 3; -} - -message Row { - repeated ColumnResponse columns = 1; -} - -message TableResponse{ - repeated ColumnInfo headers = 1; - repeated Row rows = 2; - StatusError StatusError = 3; -} - -message ColumnInfo { - string name = 1; - string datatype = 2; -} - -message ColumnResponse{ - oneof columnVal { - string stringVal = 1; - uint64 uint64Val = 2; - int64 int64Val = 3; - bool boolVal = 4; - bytes blobVal = 5; - Uint64Array uint64ArrayVal = 6; - StringArray stringArrayVal = 7; - double float64Val = 8; - Decimal decimalVal = 9; - } -} - -message Decimal { - int64 value = 1; - int64 scale = 2; -} - -message InspectRequest { - string vds = 1; - IdsOrKeys records = 2; - repeated string filterFields = 3; - uint64 limit = 4; - uint64 offset = 5; - string query = 6; -} - -message Uint64Array { - repeated uint64 vals = 1; -} - -message StringArray { - repeated string vals = 1; -} - -message IdsOrKeys { - oneof type { - Uint64Array ids = 1; - StringArray keys = 2; - } -} - service Molecula { rpc GetVDSs(GetVDSsRequest) returns (GetVDSsResponse) {}; rpc GetVDS(GetVDSRequest) returns (GetVDSResponse) {}; rpc PostVDS(PostVDSRequest) returns (PostVDSResponse) {}; rpc DeleteVDS(DeleteVDSRequest) returns (DeleteVDSResponse) {}; - rpc QuerySQL(QuerySQLRequest) returns (stream RowResponse) {}; - rpc QuerySQLUnary(QuerySQLRequest) returns (TableResponse) {}; - rpc QueryPQL(QueryPQLRequest) returns (stream RowResponse) {}; - rpc QueryPQLUnary(QueryPQLRequest) returns (TableResponse) {}; - rpc Inspect(InspectRequest) returns (stream RowResponse) {}; + rpc QuerySQL(pilosa.QuerySQLRequest) returns (stream pilosa.RowResponse) {}; + rpc QuerySQLUnary(pilosa.QuerySQLRequest) returns (pilosa.TableResponse) {}; + rpc QueryPQL(pilosa.QueryPQLRequest) returns (stream pilosa.RowResponse) {}; + rpc QueryPQLUnary(pilosa.QueryPQLRequest) returns (pilosa.TableResponse) {}; + rpc Inspect(pilosa.InspectRequest) returns (stream pilosa.RowResponse) {}; } diff --git a/server/grpc.go b/server/grpc.go index 7dba6558e..fb3c92251 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -325,36 +325,24 @@ func (h *VDSMGRPCHandler) DeleteVDS(ctx context.Context, req *vdsm_pb.DeleteVDSR } } -func (*VDSMGRPCHandler) QuerySQL(req *vdsm_pb.QuerySQLRequest, srv vdsm_pb.Molecula_QuerySQLServer) error { - return status.Errorf(codes.Unimplemented, "method QuerySQL not implemented") +func (h *VDSMGRPCHandler) QuerySQL(req *pb.QuerySQLRequest, srv vdsm_pb.Molecula_QuerySQLServer) error { + return h.grpcHandler.QuerySQL(req, srv) } -func (h *VDSMGRPCHandler) QuerySQLUnary(ctx context.Context, req *vdsm_pb.QuerySQLRequest) (*vdsm_pb.TableResponse, error) { - pReq := pb.QuerySQLRequest(*req) - resp, err := h.grpcHandler.QuerySQLUnary(ctx, &pReq) - if err != nil { - return nil, err - } - pResp, err := tableToTable(resp) - return pResp, err +func (h *VDSMGRPCHandler) QuerySQLUnary(ctx context.Context, req *pb.QuerySQLRequest) (*pb.TableResponse, error) { + return h.grpcHandler.QuerySQLUnary(ctx, req) } -func (*VDSMGRPCHandler) QueryPQL(req *vdsm_pb.QueryPQLRequest, srv vdsm_pb.Molecula_QueryPQLServer) error { - return status.Errorf(codes.Unimplemented, "method QueryPQL not implemented") +func (h *VDSMGRPCHandler) QueryPQL(req *pb.QueryPQLRequest, srv vdsm_pb.Molecula_QueryPQLServer) error { + return h.grpcHandler.QueryPQL(req, srv) } -func (h *VDSMGRPCHandler) QueryPQLUnary(ctx context.Context, req *vdsm_pb.QueryPQLRequest) (*vdsm_pb.TableResponse, error) { - pReq := pb.QueryPQLRequest(*req) - resp, err := h.grpcHandler.QueryPQLUnary(ctx, &pReq) - if err != nil { - return nil, err - } - pResp, err := tableToTable(resp) - return pResp, err +func (h *VDSMGRPCHandler) QueryPQLUnary(ctx context.Context, req *pb.QueryPQLRequest) (*pb.TableResponse, error) { + return h.grpcHandler.QueryPQLUnary(ctx, req) } -func (*VDSMGRPCHandler) Inspect(req *vdsm_pb.InspectRequest, srv vdsm_pb.Molecula_InspectServer) error { - return status.Errorf(codes.Unimplemented, "method Inspect not implemented") +func (h *VDSMGRPCHandler) Inspect(req *pb.InspectRequest, srv vdsm_pb.Molecula_InspectServer) error { + return h.grpcHandler.Inspect(req, srv) } // ResultUint64 is a wrapper around a uint64 result type @@ -1247,50 +1235,50 @@ func NewGRPCServer(opts ...grpcServerOption) (*grpcServer, error) { return server, nil } -// tableToTable is a helper function used by QueryPQLUnary -// to convert a pb.TableResponse into a vdsm_pb.TableResponse. -func tableToTable(ptbl *pb.TableResponse) (*vdsm_pb.TableResponse, error) { - vtbl := &vdsm_pb.TableResponse{ - Headers: make([]*vdsm_pb.ColumnInfo, len(ptbl.Headers)), - Rows: make([]*vdsm_pb.Row, len(ptbl.Rows)), - } - - // Headers - for i, info := range ptbl.Headers { - vtbl.Headers[i] = &vdsm_pb.ColumnInfo{Name: info.Name, Datatype: info.Datatype} - } - - // Rows - for r := range ptbl.Rows { - columns := make([]*vdsm_pb.ColumnResponse, len(ptbl.Rows[r].Columns)) - for i, col := range ptbl.Rows[r].Columns { - switch v := col.GetColumnVal().(type) { - case *pb.ColumnResponse_StringVal: - columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_StringVal{StringVal: v.StringVal}} - case *pb.ColumnResponse_Uint64Val: - columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_Uint64Val{Uint64Val: v.Uint64Val}} - case *pb.ColumnResponse_Int64Val: - columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_Int64Val{Int64Val: v.Int64Val}} - case *pb.ColumnResponse_BoolVal: - columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_BoolVal{BoolVal: v.BoolVal}} - case *pb.ColumnResponse_BlobVal: - columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_BlobVal{BlobVal: v.BlobVal}} - case *pb.ColumnResponse_Uint64ArrayVal: - columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_Uint64ArrayVal{Uint64ArrayVal: &vdsm_pb.Uint64Array{Vals: v.Uint64ArrayVal.Vals}}} - case *pb.ColumnResponse_StringArrayVal: - columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_StringArrayVal{StringArrayVal: &vdsm_pb.StringArray{Vals: v.StringArrayVal.Vals}}} - case *pb.ColumnResponse_Float64Val: - columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_Float64Val{Float64Val: v.Float64Val}} - case *pb.ColumnResponse_DecimalVal: - columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_DecimalVal{DecimalVal: &vdsm_pb.Decimal{Value: v.DecimalVal.Value, Scale: v.DecimalVal.Scale}}} - case nil: - columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: nil} - default: - return nil, errors.Errorf("unhandled columnval type: %T", col.GetColumnVal()) - } - } - vtbl.Rows[r] = &vdsm_pb.Row{Columns: columns} - } - - return vtbl, nil -} +//// tableToTable is a helper function used by QueryPQLUnary +//// to convert a pb.TableResponse into a vdsm_pb.TableResponse. +//func tableToTable(ptbl *pb.TableResponse) (*vdsm_pb.TableResponse, error) { +// vtbl := &vdsm_pb.TableResponse{ +// Headers: make([]*vdsm_pb.ColumnInfo, len(ptbl.Headers)), +// Rows: make([]*vdsm_pb.Row, len(ptbl.Rows)), +// } +// +// // Headers +// for i, info := range ptbl.Headers { +// vtbl.Headers[i] = &vdsm_pb.ColumnInfo{Name: info.Name, Datatype: info.Datatype} +// } +// +// // Rows +// for r := range ptbl.Rows { +// columns := make([]*vdsm_pb.ColumnResponse, len(ptbl.Rows[r].Columns)) +// for i, col := range ptbl.Rows[r].Columns { +// switch v := col.GetColumnVal().(type) { +// case *pb.ColumnResponse_StringVal: +// columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_StringVal{StringVal: v.StringVal}} +// case *pb.ColumnResponse_Uint64Val: +// columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_Uint64Val{Uint64Val: v.Uint64Val}} +// case *pb.ColumnResponse_Int64Val: +// columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_Int64Val{Int64Val: v.Int64Val}} +// case *pb.ColumnResponse_BoolVal: +// columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_BoolVal{BoolVal: v.BoolVal}} +// case *pb.ColumnResponse_BlobVal: +// columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_BlobVal{BlobVal: v.BlobVal}} +// case *pb.ColumnResponse_Uint64ArrayVal: +// columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_Uint64ArrayVal{Uint64ArrayVal: &vdsm_pb.Uint64Array{Vals: v.Uint64ArrayVal.Vals}}} +// case *pb.ColumnResponse_StringArrayVal: +// columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_StringArrayVal{StringArrayVal: &vdsm_pb.StringArray{Vals: v.StringArrayVal.Vals}}} +// case *pb.ColumnResponse_Float64Val: +// columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_Float64Val{Float64Val: v.Float64Val}} +// case *pb.ColumnResponse_DecimalVal: +// columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_DecimalVal{DecimalVal: &vdsm_pb.Decimal{Value: v.DecimalVal.Value, Scale: v.DecimalVal.Scale}}} +// case nil: +// columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: nil} +// default: +// return nil, errors.Errorf("unhandled columnval type: %T", col.GetColumnVal()) +// } +// } +// vtbl.Rows[r] = &vdsm_pb.Row{Columns: columns} +// } +// +// return vtbl, nil +//} From 949ca882fc0f94e96361068a82a168b0b1997d8b Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 17 Sep 2020 16:34:13 -0500 Subject: [PATCH 05/53] Remove unused helper func --- server/grpc.go | 48 ------------------------------------------------ 1 file changed, 48 deletions(-) diff --git a/server/grpc.go b/server/grpc.go index fb3c92251..f378fd403 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -1234,51 +1234,3 @@ func NewGRPCServer(opts ...grpcServerOption) (*grpcServer, error) { } return server, nil } - -//// tableToTable is a helper function used by QueryPQLUnary -//// to convert a pb.TableResponse into a vdsm_pb.TableResponse. -//func tableToTable(ptbl *pb.TableResponse) (*vdsm_pb.TableResponse, error) { -// vtbl := &vdsm_pb.TableResponse{ -// Headers: make([]*vdsm_pb.ColumnInfo, len(ptbl.Headers)), -// Rows: make([]*vdsm_pb.Row, len(ptbl.Rows)), -// } -// -// // Headers -// for i, info := range ptbl.Headers { -// vtbl.Headers[i] = &vdsm_pb.ColumnInfo{Name: info.Name, Datatype: info.Datatype} -// } -// -// // Rows -// for r := range ptbl.Rows { -// columns := make([]*vdsm_pb.ColumnResponse, len(ptbl.Rows[r].Columns)) -// for i, col := range ptbl.Rows[r].Columns { -// switch v := col.GetColumnVal().(type) { -// case *pb.ColumnResponse_StringVal: -// columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_StringVal{StringVal: v.StringVal}} -// case *pb.ColumnResponse_Uint64Val: -// columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_Uint64Val{Uint64Val: v.Uint64Val}} -// case *pb.ColumnResponse_Int64Val: -// columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_Int64Val{Int64Val: v.Int64Val}} -// case *pb.ColumnResponse_BoolVal: -// columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_BoolVal{BoolVal: v.BoolVal}} -// case *pb.ColumnResponse_BlobVal: -// columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_BlobVal{BlobVal: v.BlobVal}} -// case *pb.ColumnResponse_Uint64ArrayVal: -// columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_Uint64ArrayVal{Uint64ArrayVal: &vdsm_pb.Uint64Array{Vals: v.Uint64ArrayVal.Vals}}} -// case *pb.ColumnResponse_StringArrayVal: -// columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_StringArrayVal{StringArrayVal: &vdsm_pb.StringArray{Vals: v.StringArrayVal.Vals}}} -// case *pb.ColumnResponse_Float64Val: -// columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_Float64Val{Float64Val: v.Float64Val}} -// case *pb.ColumnResponse_DecimalVal: -// columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: &vdsm_pb.ColumnResponse_DecimalVal{DecimalVal: &vdsm_pb.Decimal{Value: v.DecimalVal.Value, Scale: v.DecimalVal.Scale}}} -// case nil: -// columns[i] = &vdsm_pb.ColumnResponse{ColumnVal: nil} -// default: -// return nil, errors.Errorf("unhandled columnval type: %T", col.GetColumnVal()) -// } -// } -// vtbl.Rows[r] = &vdsm_pb.Row{Columns: columns} -// } -// -// return vtbl, nil -//} From 0e8564e2bebd67a3892e8b0754ad1ad83cd1476e Mon Sep 17 00:00:00 2001 From: Jason Aten Date: Tue, 22 Sep 2020 12:07:31 -0500 Subject: [PATCH 06/53] fix license exceptions for vdsm/proto --- license.exceptions | 2 ++ 1 file changed, 2 insertions(+) diff --git a/license.exceptions b/license.exceptions index ffb08a26e..5fb7846e8 100644 --- a/license.exceptions +++ b/license.exceptions @@ -19,3 +19,5 @@ ./cmd/pilosa-keydump/vprint.go ./cmd/pilosa-keydump/keydump.go ./synthload/vprint.go +./proto/vdsm/vdsm.proto +./proto/vdsm/vdsm.pb.go From e0f98703788b323e564eb5d61ba3e58edb19e809 Mon Sep 17 00:00:00 2001 From: Nia Weiss Date: Thu, 1 Oct 2020 09:35:05 -0400 Subject: [PATCH 07/53] fix misaligned bitmaps in Distinct This resolves an issue in which a filter would eliminate all bits in all shards other than shard 0. --- executor.go | 6 +++--- executor_test.go | 24 ++++++++++++------------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/executor.go b/executor.go index 774f137fe..aa94cb3ce 100644 --- a/executor.go +++ b/executor.go @@ -1425,7 +1425,7 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str tx, finisher := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard}) defer finisher(&err) - existsBitmap, err := tx.OffsetRange(index, fieldName, view, shard, 0, ShardWidth*0, ShardWidth*1) + existsBitmap, err := tx.OffsetRange(index, fieldName, view, shard, ShardWidth*shard, ShardWidth*0, ShardWidth*1) if err != nil { return result, err } @@ -1436,7 +1436,7 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str return result, nil } - signBitmap, err := tx.OffsetRange(index, fieldName, view, shard, 0, ShardWidth*1, ShardWidth*2) + signBitmap, err := tx.OffsetRange(index, fieldName, view, shard, ShardWidth*shard, ShardWidth*1, ShardWidth*2) if err != nil { return result, nil } @@ -1444,7 +1444,7 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str dataBitmaps := make([]*roaring.Bitmap, depth) for i := uint64(0); i < depth; i++ { - dataBitmaps[i], err = tx.OffsetRange(index, fieldName, view, shard, 0, ShardWidth*(i+2), ShardWidth*(i+3)) + dataBitmaps[i], err = tx.OffsetRange(index, fieldName, view, shard, ShardWidth*shard, ShardWidth*(i+2), ShardWidth*(i+3)) if err != nil { return result, err } diff --git a/executor_test.go b/executor_test.go index a34041844..d173ecd1b 100644 --- a/executor_test.go +++ b/executor_test.go @@ -5293,7 +5293,7 @@ func TestExecutor_ForeignIndex(t *testing.T) { ) // Populate parent data. - c.Query(t, "parent", ` + c.Query(t, "parent", fmt.Sprintf(` Set("one", general=1) Set("two", general=1) Set("three", general=1) @@ -5302,25 +5302,25 @@ func TestExecutor_ForeignIndex(t *testing.T) { Set("twenty-two", general=2) Set("twenty-three", general=2) - Set("one", general=3) - Set("twenty-one", general=3) - `) + Set("one", general=%d) + Set("twenty-one", general=%d) + `, ShardWidth, ShardWidth)) // Populate child data. - c.Query(t, "child", ` + c.Query(t, "child", fmt.Sprintf(` Set(1, parent_id="one") Set(2, parent_id="two") - Set(3, parent_id="one") + Set(%d, parent_id="one") Set(4, parent_id="twenty-one") - `) + `, ShardWidth)) // Populate color data. - c.Query(t, "child", ` + c.Query(t, "child", fmt.Sprintf(` Set(1, color="red") Set(2, color="blue") - Set(3, color="blue") + Set(%d, color="blue") Set(4, color="red") - `) + `, ShardWidth)) distinct := c.Query(t, "child", `Distinct(index="child", field="parent_id")`).Results[0].(pilosa.SignedRow) if !sameStringSlice(distinct.Pos.Keys, []string{"one", "two", "twenty-one"}) { @@ -5328,7 +5328,7 @@ func TestExecutor_ForeignIndex(t *testing.T) { } eq := c.Query(t, "child", `Row(parent_id=="one")`).Results[0].(*pilosa.Row) - if !reflect.DeepEqual(eq.Columns(), []uint64{1, 3}) { + if !reflect.DeepEqual(eq.Columns(), []uint64{1, ShardWidth}) { t.Fatalf("unexpected columns: %v", eq.Columns()) } @@ -5337,7 +5337,7 @@ func TestExecutor_ForeignIndex(t *testing.T) { t.Fatalf("unexpected columns: %v", neq.Columns()) } - join := c.Query(t, "parent", `Intersect(Row(general=3), Distinct(Row(color="blue"), index="child", field="parent_id"))`).Results[0].(*pilosa.Row) + join := c.Query(t, "parent", fmt.Sprintf(`Intersect(Row(general=%d), Distinct(Row(color="blue"), index="child", field="parent_id"))`, ShardWidth)).Results[0].(*pilosa.Row) if !reflect.DeepEqual(join.Keys, []string{"one"}) { t.Fatalf("unexpected keys: %v", join.Keys) } From 474261f12f1a513b4aa765006c470b9445b9e450 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 30 Sep 2020 07:14:35 -0500 Subject: [PATCH 08/53] Add single-node 'bytesOnDisk' object to /status response --- api.go | 66 ++++++++++++++++++++++++++++++++++++++++++ http/handler.go | 27 +++++++++++++---- server/handler_test.go | 5 ++++ 3 files changed, 92 insertions(+), 6 deletions(-) diff --git a/api.go b/api.go index b6e8187c5..a307e323d 100644 --- a/api.go +++ b/api.go @@ -25,6 +25,8 @@ import ( "io/ioutil" "math" "net/url" + "os" + "path" "sort" "strconv" "strings" @@ -780,6 +782,70 @@ func (api *API) Node() *Node { return &node } +// Usage gets the disk usage per index +func (api *API) Usage() (map[string]int64, int64, error) { + indexSizes := make(map[string]int64, 0) + var totalSize int64 + + dirName, err := expandDirName(api.server.dataDir) + if err != nil { + return indexSizes, totalSize, errors.Wrap(err, "expanding data directory") + } + dir, err := os.Open(dirName) + if err != nil { + return indexSizes, totalSize, errors.Wrap(err, "opening data directory") + } + defer dir.Close() + + files, err := dir.Readdir(-1) + if err != nil { + return indexSizes, totalSize, errors.Wrap(err, "reading data directory") + } + + for _, file := range files { + if !file.IsDir() { + continue + } + fullName := path.Join(dirName, file.Name()) + indexSizes[file.Name()], err = diskUsage(fullName) + if err != nil { + break + } + totalSize += indexSizes[file.Name()] + } + + return indexSizes, totalSize, nil +} + +func diskUsage(fname string) (int64, error) { + var size int64 + + dir, err := os.Open(fname) + if err != nil { + return 0, errors.Wrap(err, "opening data subdirectory") + } + defer dir.Close() + + files, err := dir.Readdir(-1) + if err != nil { + return 0, errors.Wrap(err, "reading data subdirectory") + } + + for _, file := range files { + if file.IsDir() { + sz, err := diskUsage(path.Join(fname, file.Name())) + if err != nil { + return 0, err + } + size += sz + } else { + size += file.Size() + } + } + + return size, nil +} + // RecalculateCaches forces all TopN caches to be updated. // This is done internally within a TopN query, but a user may want to do it ahead of time? func (api *API) RecalculateCaches(ctx context.Context) error { diff --git a/http/handler.go b/http/handler.go index c1020c941..606aaffa4 100644 --- a/http/handler.go +++ b/http/handler.go @@ -663,10 +663,19 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) return } + usageIndexes, usageTotal, err := h.api.Usage() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + usage := diskUsage{ + Total: usageTotal, + Indexes: usageIndexes, + } status := getStatusResponse{ - State: h.api.State(), - Nodes: h.api.Hosts(r.Context()), - LocalID: h.api.Node().ID, + State: h.api.State(), + Nodes: h.api.Hosts(r.Context()), + LocalID: h.api.Node().ID, + BytesOnDisk: usage, } w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(status); err != nil { @@ -722,9 +731,15 @@ type getSchemaResponse struct { } type getStatusResponse struct { - State string `json:"state"` - Nodes []*pilosa.Node `json:"nodes"` - LocalID string `json:"localID"` + State string `json:"state"` + Nodes []*pilosa.Node `json:"nodes"` + LocalID string `json:"localID"` + BytesOnDisk diskUsage `json:"bytesOnDisk"` +} + +type diskUsage struct { + Total int64 `json:"total"` + Indexes map[string]int64 `json:"indexes"` } func hash(s string) string { diff --git a/server/handler_test.go b/server/handler_test.go index 4a1323452..c093d873d 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -379,6 +379,11 @@ func TestHandler_Endpoints(t *testing.T) { if len(ret["nodes"].([]interface{})) != 1 { t.Fatalf("wrong length nodes list: %#v", ret) } + usage := ret["bytesOnDisk"].(map[string]interface{}) + indexes := usage["indexes"].(map[string]interface{}) + if len(indexes) != 2 { + t.Fatalf("wrong length index size list: %#v", indexes) + } }) t.Run("Metrics", func(t *testing.T) { From 6f2fe3afe14861cb4a11734330a6cdf4a93e5e16 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 30 Sep 2020 07:43:48 -0500 Subject: [PATCH 09/53] Address linter issue --- api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api.go b/api.go index a307e323d..19a433315 100644 --- a/api.go +++ b/api.go @@ -784,7 +784,7 @@ func (api *API) Node() *Node { // Usage gets the disk usage per index func (api *API) Usage() (map[string]int64, int64, error) { - indexSizes := make(map[string]int64, 0) + indexSizes := make(map[string]int64) var totalSize int64 dirName, err := expandDirName(api.server.dataDir) From b6af0e521957b208548bdf5d0a0e25dadf34afdb Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Thu, 1 Oct 2020 10:25:28 -0500 Subject: [PATCH 10/53] Move usage data to new ui-specific endpoint --- http/handler.go | 53 +++++++++++++++++++++++++++++------------- server/handler_test.go | 9 +++++++ 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/http/handler.go b/http/handler.go index 606aaffa4..f000a365c 100644 --- a/http/handler.go +++ b/http/handler.go @@ -392,6 +392,8 @@ func newRouter(handler *Handler) http.Handler { router.HandleFunc("/queries", handler.handleGetActiveQueries).Methods("GET").Name("GetActiveQueries") router.HandleFunc("/version", handler.handleGetVersion).Methods("GET").Name("GetVersion") + router.HandleFunc("/ui/usage", handler.handleGetUsage).Methods("GET").Name("GetUsage") + // /internal endpoints are for internal use only; they may change at any time. // DO NOT rely on these for external applications! router.HandleFunc("/internal/cluster/message", handler.handlePostClusterMessage).Methods("POST").Name("PostClusterMessage") @@ -657,8 +659,8 @@ func (h *Handler) handlePostSchema(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) } -// handleGetStatus handles GET /status requests. -func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { +// handleGetUsage handles GET /ui/usage requests. +func (h *Handler) handleGetUsage(w http.ResponseWriter, r *http.Request) { if !validHeaderAcceptJSON(r.Header) { http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) return @@ -667,15 +669,40 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } - usage := diskUsage{ + + disk := diskUsage{ Total: usageTotal, Indexes: usageIndexes, } + + usage := getUsageResponse{ + Disk: disk, + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(usage); err != nil { + h.logger.Printf("write status response error: %s", err) + } +} + +type getUsageResponse struct { + Disk diskUsage `json:"bytesOnDisk"` +} +type diskUsage struct { + Total int64 `json:"total"` + Indexes map[string]int64 `json:"indexes"` +} + +// handleGetStatus handles GET /status requests. +func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { + if !validHeaderAcceptJSON(r.Header) { + http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) + return + } status := getStatusResponse{ - State: h.api.State(), - Nodes: h.api.Hosts(r.Context()), - LocalID: h.api.Node().ID, - BytesOnDisk: usage, + State: h.api.State(), + Nodes: h.api.Hosts(r.Context()), + LocalID: h.api.Node().ID, } w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(status); err != nil { @@ -731,15 +758,9 @@ type getSchemaResponse struct { } type getStatusResponse struct { - State string `json:"state"` - Nodes []*pilosa.Node `json:"nodes"` - LocalID string `json:"localID"` - BytesOnDisk diskUsage `json:"bytesOnDisk"` -} - -type diskUsage struct { - Total int64 `json:"total"` - Indexes map[string]int64 `json:"indexes"` + State string `json:"state"` + Nodes []*pilosa.Node `json:"nodes"` + LocalID string `json:"localID"` } func hash(s string) string { diff --git a/server/handler_test.go b/server/handler_test.go index c093d873d..47e4698b0 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -379,6 +379,15 @@ func TestHandler_Endpoints(t *testing.T) { if len(ret["nodes"].([]interface{})) != 1 { t.Fatalf("wrong length nodes list: %#v", ret) } + }) + + t.Run("UI/usage", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/ui/usage", nil)) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + ret := mustJSONDecode(t, w.Body) usage := ret["bytesOnDisk"].(map[string]interface{}) indexes := usage["indexes"].(map[string]interface{}) if len(indexes) != 2 { From 664c911791cd1532097fac6140ad5b1172776432 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 7 Aug 2020 10:31:09 -0500 Subject: [PATCH 11/53] Include replicaN in /info response --- api.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api.go b/api.go index 19a433315..501187512 100644 --- a/api.go +++ b/api.go @@ -1758,6 +1758,7 @@ func (api *API) Info() serverInfo { CPUType: si.CPUModel(), Memory: mem, TxSrc: api.holder.txf.TxType(), + ReplicaN: api.cluster.ReplicaN, } } @@ -2004,6 +2005,7 @@ func (api *API) TranslateFieldDB(ctx context.Context, indexName, fieldName strin type serverInfo struct { ShardWidth uint64 `json:"shardWidth"` + ReplicaN int `json:"replicaN"` Memory uint64 `json:"memory"` CPUType string `json:"cpuType"` CPUPhysicalCores int `json:"cpuPhysicalCores"` From 8fad2b92c2048ba2a336aaaa16e86901623ef93d Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 7 Aug 2020 11:14:24 -0500 Subject: [PATCH 12/53] Add all node states to /status response --- api.go | 6 ++++++ cluster.go | 6 ++++++ http/handler.go | 8 +++++--- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/api.go b/api.go index 501187512..4e11cc838 100644 --- a/api.go +++ b/api.go @@ -776,6 +776,12 @@ func (api *API) Hosts(ctx context.Context) []*Node { return api.cluster.Nodes() } +func (api *API) HostStates(ctx context.Context) map[string]string { + span, _ := tracing.StartSpanFromContext(ctx, "API.HostStates") + defer span.Finish() + return api.cluster.AllNodeStates() +} + // Node gets the ID, URI and coordinator status for this particular node. func (api *API) Node() *Node { node := api.server.node() diff --git a/cluster.go b/cluster.go index 6e5450cf4..1762be18d 100644 --- a/cluster.go +++ b/cluster.go @@ -700,6 +700,12 @@ func (c *cluster) Nodes() []*Node { return ret } +func (c *cluster) AllNodeStates() map[string]string { + c.mu.RLock() + defer c.mu.RUnlock() + return c.Topology.nodeStates +} + // removeNodeBasicSorted removes a node from the cluster, maintaining the sort // order. Returns true if the node was removed. unprotected. func (c *cluster) removeNodeBasicSorted(nodeID string) bool { diff --git a/http/handler.go b/http/handler.go index f000a365c..186e857ea 100644 --- a/http/handler.go +++ b/http/handler.go @@ -702,6 +702,7 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { status := getStatusResponse{ State: h.api.State(), Nodes: h.api.Hosts(r.Context()), + NodeStates: h.api.HostStates(r.Context()), LocalID: h.api.Node().ID, } w.Header().Set("Content-Type", "application/json") @@ -758,9 +759,10 @@ type getSchemaResponse struct { } type getStatusResponse struct { - State string `json:"state"` - Nodes []*pilosa.Node `json:"nodes"` - LocalID string `json:"localID"` + State string `json:"state"` + Nodes []*pilosa.Node `json:"nodes"` + NodeStates map[string]string `json:"nodeStates"` + LocalID string `json:"localID"` } func hash(s string) string { From 5705b45864ba52256b1917f231a302a6545de58a Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 11 Aug 2020 10:57:59 -0500 Subject: [PATCH 13/53] Add a CLI flag for cluster name and include it in /status response --- api.go | 10 ++++++++++ cluster.go | 3 +++ ctl/server.go | 1 + http/handler.go | 20 ++++++++++++-------- server.go | 8 ++++++++ server/config.go | 1 + server/server.go | 1 + 7 files changed, 36 insertions(+), 8 deletions(-) diff --git a/api.go b/api.go index 4e11cc838..232a510e8 100644 --- a/api.go +++ b/api.go @@ -1739,6 +1739,16 @@ func (api *API) State() string { return api.cluster.State() } +// ClusterID returns the cluster ID. +func (api *API) ClusterID() string { + return api.cluster.id +} + +// ClusterName returns the cluster name. +func (api *API) ClusterName() string { + return api.cluster.Name +} + // Version returns the Pilosa version. func (api *API) Version() string { return strings.TrimPrefix(Version, "v") diff --git a/cluster.go b/cluster.go index 1762be18d..f31a819e5 100644 --- a/cluster.go +++ b/cluster.go @@ -208,6 +208,9 @@ type cluster struct { // nolint: maligned // The number of replicas a partition has. ReplicaN int + // Human-readable name of the cluster. + Name string + // Threshold for logging long-running queries // TODO(2.0) move this out of cluster. (why is it here??) longQueryTime time.Duration diff --git a/ctl/server.go b/ctl/server.go index 968b2bd6d..221f3918f 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -47,6 +47,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.") flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster. Only used for testing.") flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Duration that will trigger log and stat messages for slow queries.") + flags.StringVar(&srv.Config.Cluster.Name, "cluster.name", srv.Config.Cluster.Name, "Human-readable name for the cluster.") // Translation flags.StringVarP(&srv.Config.Translation.PrimaryURL, "translation.primary-url", "", srv.Config.Translation.PrimaryURL, "DEPRECATED: URL for primary translation node for replication.") diff --git a/http/handler.go b/http/handler.go index 186e857ea..6f08664a4 100644 --- a/http/handler.go +++ b/http/handler.go @@ -700,10 +700,12 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { return } status := getStatusResponse{ - State: h.api.State(), - Nodes: h.api.Hosts(r.Context()), - NodeStates: h.api.HostStates(r.Context()), - LocalID: h.api.Node().ID, + State: h.api.State(), + Nodes: h.api.Hosts(r.Context()), + NodeStates: h.api.HostStates(r.Context()), + LocalID: h.api.Node().ID, + ClusterID: h.api.ClusterID(), + ClusterName: h.api.ClusterName(), } w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(status); err != nil { @@ -759,10 +761,12 @@ type getSchemaResponse struct { } type getStatusResponse struct { - State string `json:"state"` - Nodes []*pilosa.Node `json:"nodes"` - NodeStates map[string]string `json:"nodeStates"` - LocalID string `json:"localID"` + State string `json:"state"` + Nodes []*pilosa.Node `json:"nodes"` + NodeStates map[string]string `json:"nodeStates"` + LocalID string `json:"localID"` + ClusterID string `json:"clusterID"` + ClusterName string `json:"clusterName"` } func hash(s string) string { diff --git a/server.go b/server.go index a93df180a..a011ab0ba 100644 --- a/server.go +++ b/server.go @@ -269,6 +269,14 @@ func OptServerClusterDisabled(disabled bool, hosts []string) ServerOption { } } +// OptServerClusterName sets the human-readable cluster name. +func OptServerClusterName(name string) ServerOption { + return func(s *Server) error { + s.cluster.Name = name + return nil + } +} + // OptServerSerializer is a functional option on Server // used to set the serializer. func OptServerSerializer(ser Serializer) ServerOption { diff --git a/server/config.go b/server/config.go index c51a7f95f..3e8575c81 100644 --- a/server/config.go +++ b/server/config.go @@ -122,6 +122,7 @@ type Config struct { Coordinator bool `toml:"coordinator"` ReplicaN int `toml:"replicas"` Hosts []string `toml:"hosts"` + Name string `toml:"name"` // TODO(2.0) move this out of cluster. (why is it here??) LongQueryTime toml.Duration `toml:"long-query-time"` } `toml:"cluster"` diff --git a/server/server.go b/server/server.go index 1b7dea316..077c8a55c 100644 --- a/server/server.go +++ b/server/server.go @@ -406,6 +406,7 @@ func (m *Command) SetupServer() error { pilosa.OptServerGRPCURI(advertiseGRPCURI), pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)), pilosa.OptServerClusterDisabled(m.Config.Cluster.Disabled, m.Config.Cluster.Hosts), + pilosa.OptServerClusterName(m.Config.Cluster.Name), pilosa.OptServerSerializer(proto.Serializer{}), pilosa.OptServerTxsrc(m.Config.Txsrc), coordinatorOpt, From 1a04e94fc179d480a0578b19c1318c0170739fcb Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Thu, 20 Aug 2020 17:52:25 -0500 Subject: [PATCH 14/53] Remove clusterID and default name to ID --- api.go | 8 +++----- http/handler.go | 2 -- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/api.go b/api.go index 232a510e8..018eb8091 100644 --- a/api.go +++ b/api.go @@ -1739,13 +1739,11 @@ func (api *API) State() string { return api.cluster.State() } -// ClusterID returns the cluster ID. -func (api *API) ClusterID() string { - return api.cluster.id -} - // ClusterName returns the cluster name. func (api *API) ClusterName() string { + if api.cluster.Name == "" { + return api.cluster.id + } return api.cluster.Name } diff --git a/http/handler.go b/http/handler.go index 6f08664a4..63e51ef6d 100644 --- a/http/handler.go +++ b/http/handler.go @@ -704,7 +704,6 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { Nodes: h.api.Hosts(r.Context()), NodeStates: h.api.HostStates(r.Context()), LocalID: h.api.Node().ID, - ClusterID: h.api.ClusterID(), ClusterName: h.api.ClusterName(), } w.Header().Set("Content-Type", "application/json") @@ -765,7 +764,6 @@ type getStatusResponse struct { Nodes []*pilosa.Node `json:"nodes"` NodeStates map[string]string `json:"nodeStates"` LocalID string `json:"localID"` - ClusterID string `json:"clusterID"` ClusterName string `json:"clusterName"` } From 46d010fe188b8ed42643c6af464b2bdada2e247d Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 30 Sep 2020 07:23:39 -0500 Subject: [PATCH 15/53] Remove 'NodeStates' from /status until it is reliable --- http/handler.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/http/handler.go b/http/handler.go index 63e51ef6d..c91adef68 100644 --- a/http/handler.go +++ b/http/handler.go @@ -702,7 +702,6 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { status := getStatusResponse{ State: h.api.State(), Nodes: h.api.Hosts(r.Context()), - NodeStates: h.api.HostStates(r.Context()), LocalID: h.api.Node().ID, ClusterName: h.api.ClusterName(), } @@ -760,11 +759,10 @@ type getSchemaResponse struct { } type getStatusResponse struct { - State string `json:"state"` - Nodes []*pilosa.Node `json:"nodes"` - NodeStates map[string]string `json:"nodeStates"` - LocalID string `json:"localID"` - ClusterName string `json:"clusterName"` + State string `json:"state"` + Nodes []*pilosa.Node `json:"nodes"` + LocalID string `json:"localID"` + ClusterName string `json:"clusterName"` } func hash(s string) string { From 8a3d93fe78847ae32f2a1684ccc1f0790d59cb77 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Thu, 1 Oct 2020 10:45:08 -0500 Subject: [PATCH 16/53] Move TransactionList endpoint to UI namespace --- http/handler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/http/handler.go b/http/handler.go index c91adef68..98634111d 100644 --- a/http/handler.go +++ b/http/handler.go @@ -381,8 +381,6 @@ func newRouter(handler *Handler) http.Handler { router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET").Name("GetSchema") router.HandleFunc("/schema", handler.handlePostSchema).Methods("POST").Name("PostSchema") router.HandleFunc("/status", handler.handleGetStatus).Methods("GET").Name("GetStatus") - router.HandleFunc("/transaction", handler.handleGetTransactionList).Methods("GET").Name("GetTransactionList") - router.HandleFunc("/transaction/", handler.handleGetTransactionList).Methods("GET").Name("GetTransactionList") router.HandleFunc("/transaction", handler.handlePostTransaction).Methods("POST").Name("PostTransaction") router.HandleFunc("/transaction/", handler.handlePostTransaction).Methods("POST").Name("PostTransaction") router.HandleFunc("/transaction/{id}", handler.handleGetTransaction).Methods("GET").Name("GetTransaction") @@ -393,6 +391,8 @@ func newRouter(handler *Handler) http.Handler { router.HandleFunc("/version", handler.handleGetVersion).Methods("GET").Name("GetVersion") router.HandleFunc("/ui/usage", handler.handleGetUsage).Methods("GET").Name("GetUsage") + router.HandleFunc("/ui/transaction", handler.handleGetTransactionList).Methods("GET").Name("GetTransactionList") + router.HandleFunc("/ui/transaction/", handler.handleGetTransactionList).Methods("GET").Name("GetTransactionList") // /internal endpoints are for internal use only; they may change at any time. // DO NOT rely on these for external applications! From 415f7d20b898a319f512594f96dd68dcf57224eb Mon Sep 17 00:00:00 2001 From: Nia Weiss Date: Thu, 1 Oct 2020 12:08:45 -0400 Subject: [PATCH 17/53] add field parameter error handling to Distinct --- executor.go | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/executor.go b/executor.go index aa94cb3ce..919a2ea76 100644 --- a/executor.go +++ b/executor.go @@ -1074,14 +1074,16 @@ func (e *executor) executeDistinct(ctx context.Context, qcx *Qcx, index string, span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeDistinct") defer span.Finish() - field := c.Args["field"] - if field == "" { - return SignedRow{}, fmt.Errorf("plugin operation %s(): field required", c.Name) + field, hasField, err := c.StringArg("field") + if err != nil { + return SignedRow{}, errors.Wrap(err, "loading field option in Distinct query") + } else if !hasField { + return SignedRow{}, fmt.Errorf("missing field option in Distinct query") } // Execute calls in bulk on each remote node and merge. mapFn := func(ctx context.Context, shard uint64) (_ interface{}, err error) { - return e.executeDistinctShard(ctx, qcx, index, c, shard) + return e.executeDistinctShard(ctx, qcx, index, field, c, shard) } // Merge returned results at coordinating node. @@ -1098,7 +1100,7 @@ func (e *executor) executeDistinct(ctx context.Context, qcx *Qcx, index string, return SignedRow{}, err } other, _ := result.(SignedRow) - other.field = field.(string) + other.field = field return other, nil } @@ -1385,11 +1387,15 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, qcx *Qcx, index s // executeDistinctShard executes a Distinct call on a single shard, yielding // a SignedRow of the values found. -func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (result SignedRow, err error) { +func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index string, fieldName string, c *pql.Call, shard uint64) (result SignedRow, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeDistinctShard") defer span.Finish() idx := e.Holder.Index(index) + field := e.Holder.Field(index, fieldName) + if field == nil { + return SignedRow{}, ErrFieldNotFound + } var filter *Row var filterBitmap *roaring.Bitmap @@ -1406,13 +1412,6 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str } } - fieldName, _ := c.Args["field"].(string) - - field := e.Holder.Field(index, fieldName) - if field == nil { - return result, nil - } - bsig := field.bsiGroup(fieldName) if bsig == nil { return result, nil From aec11ff713f6fde328e5b4bec9dcba97a067c3d1 Mon Sep 17 00:00:00 2001 From: Seebs Date: Mon, 14 Sep 2020 15:40:19 -0500 Subject: [PATCH 18/53] Update cache entries when setting a row even in shards with no new data The logic assumes that the lack of a corresponding rowSegment means that there's no changes, but that's not true -- we just deleted all the existing data! Update to match clearRow behavior better. Also, add a corresponding test case for this. Also, change references to 'defaultSnapshotQueue' to use [fragment].holder.SnapshotQueue, because defaultSnapshotQueue was the queueless queue, but holders were getting a snapshot queue, meaning that "awaiting" a snapshot could result in moving on and closing the holder before the actual snapshot queue finished snapshotting. --- fragment.go | 36 +++++++++++++++++++----------------- fragment_internal_test.go | 29 +++++++++++++++++++++-------- 2 files changed, 40 insertions(+), 25 deletions(-) diff --git a/fragment.go b/fragment.go index 122688ec5..33d0f3bfd 100644 --- a/fragment.go +++ b/fragment.go @@ -798,26 +798,28 @@ func (f *fragment) unprotectedSetRow(tx Tx, row *Row, rowID uint64) (changed boo // From the given row, get the rowSegment for this shard. seg := row.segment(f.shard) - if seg == nil { - return changed, nil - } - - // Put each container from rowSegment to fragment storage. - citer, _ := seg.data.Containers.Iterator(f.shard << shardVsContainerExponent) - for citer.Next() { - k, c := citer.Value() - if err := tx.PutContainer(f.index, f.field, f.view, f.shard, headContainerKey+(k%(1< Date: Tue, 29 Sep 2020 15:52:56 -0500 Subject: [PATCH 19/53] retrun confirmDownRetries to 10, tourney mac compatible At some point the cluster code was modified to do 120 tries to confirm if a node was down which is a bit excessive for production. My understanding is that this was done to help trigger or fix a problem during testing which is hopefully no longer relevant. Also changed the tournament script to use what I think are more standard bash-isms that work on mac. Please confirm this still works on Linux as well. --- cluster.go | 2 +- tournament.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cluster.go b/cluster.go index f31a819e5..10bc2b72d 100644 --- a/cluster.go +++ b/cluster.go @@ -62,7 +62,7 @@ const ( resizeJobActionAdd = "ADD" resizeJobActionRemove = "REMOVE" - defaultConfirmDownRetries = 120 + defaultConfirmDownRetries = 10 defaultConfirmDownSleep = 1 * time.Second ) diff --git a/tournament.sh b/tournament.sh index 2b2a559f8..31c3d42c3 100755 --- a/tournament.sh +++ b/tournament.sh @@ -8,5 +8,6 @@ for i in rbf lmdb roaring rbf_lmdb rbf_roaring lmdb_rbf lmdb_roaring roaring_rbf roaring_lmdb ; do echo "$(date) starting ${i}, output to tourna.log.${i}" echo "***=== ${i} ====================*** $(date)" &> tourna.log.${i} - PILOSA_TXSRC=${i} make testv-race &>> tourna.log.${i} + PILOSA_TXSRC=${i} make testv-race 2>&1 > tourna.log.${i} done + From fe425a84c0d5807b19604d946a9183658ea09aec Mon Sep 17 00:00:00 2001 From: "Jason E. Aten" Date: Wed, 23 Sep 2020 17:56:39 -0500 Subject: [PATCH 20/53] pilosa-fsck: scan and repair of pilosa backups - the -fix flag repairs replication errors by copying from the primary. - the -fixkeys flag repairs any string key translation issues. - make pilosa-fsck installs pilosa-fsck and builds release-pilosa-fsck.COMMIT.GOOS.tar.gz release tarbar --- .gitignore | 1 + Makefile | 7 +- boltdb/translate.go | 798 ++++++++++++++- boltdb/translate_test.go | 125 ++- cluster.go | 173 +++- cluster_internal_test.go | 28 +- cmd/pilosa-chk/chk.go | 6 +- cmd/pilosa-fsck/Makefile | 36 + cmd/pilosa-fsck/fsck.go | 952 ++++++++++++++++++ cmd/pilosa-fsck/fsck_test.go | 365 +++++++ .../release-pilosa-fsck/.gitignore | 1 + .../release-pilosa-fsck/backups.tar.gz | Bin 0 -> 248477 bytes .../release-pilosa-fsck/example.sh | 21 + cmd/pilosa-fsck/vprint.go | 177 ++++ go.mod | 1 + holder.go | 9 +- index.go | 167 ++- license.exceptions | 2 + mock/translator.go | 20 +- server.go | 2 +- server/server.go | 8 + server/server_test.go | 12 +- test/cluster.go | 2 +- translate.go | 91 +- utils_internal_test.go | 58 +- 25 files changed, 2993 insertions(+), 69 deletions(-) create mode 100644 cmd/pilosa-fsck/Makefile create mode 100644 cmd/pilosa-fsck/fsck.go create mode 100644 cmd/pilosa-fsck/fsck_test.go create mode 100644 cmd/pilosa-fsck/release-pilosa-fsck/.gitignore create mode 100644 cmd/pilosa-fsck/release-pilosa-fsck/backups.tar.gz create mode 100755 cmd/pilosa-fsck/release-pilosa-fsck/example.sh create mode 100644 cmd/pilosa-fsck/vprint.go diff --git a/.gitignore b/.gitignore index 7a41479b9..bee19753e 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ vendor build *~ lattice +release-pilosa-fsck.*.*.tar.gz diff --git a/Makefile b/Makefile index 52d246c64..085cef0b8 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,7 @@ testv-race: topt-race testvsub-race # find which test is hung/deadlocked. # testvsub: - set -e; for i in ctl http pg pql rbf roaring server sql txkey; do \ + set -e; for i in boltdb ctl http pg pql rbf roaring server sql txkey; do \ echo; echo "___ testing subpkg $$i"; \ cd $$i; pwd; \ go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v -timeout 60m || break; \ @@ -64,7 +64,7 @@ testvsub: done testvsub-race: - set -e; for i in ctl http pg pql rbf roaring server sql txkey; do \ + set -e; for i in boltdb ctl http pg pql rbf roaring server sql txkey; do \ echo; echo "___ testing subpkg $$i -race"; \ cd $$i; pwd; \ go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v -race -timeout 60m || break; \ @@ -201,6 +201,9 @@ pilosa-keydump: pilosa-chk: go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-chk +pilosa-fsck: + cd ./cmd/pilosa-fsck && make install && make release + # Run Pilosa tests inside Docker container docker-test: docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) ./... diff --git a/boltdb/translate.go b/boltdb/translate.go index 78e81ed7a..34d1d270d 100644 --- a/boltdb/translate.go +++ b/boltdb/translate.go @@ -17,9 +17,12 @@ package boltdb import ( "bytes" "context" + "fmt" "io" + "io/ioutil" "os" "path/filepath" + "sort" "sync" "time" @@ -27,8 +30,13 @@ import ( "github.com/pilosa/pilosa/v2" "github.com/pkg/errors" "github.com/zeebo/blake3" + + "runtime/pprof" ) +var _ = ioutil.TempFile +var _ = pprof.StartCPUProfile + var ( // ErrTranslateStoreClosed is returned when reading from an TranslateEntryReader // and the underlying store is closed. @@ -90,6 +98,10 @@ type TranslateStore struct { Path string } +func (s *TranslateStore) GetStorePath() string { + return s.Path +} + // NewTranslateStore returns a new instance of TranslateStore. func NewTranslateStore(index, field string, partitionID, partitionN int) *TranslateStore { return &TranslateStore{ @@ -485,7 +497,7 @@ func findKeyByID(bkt *bolt.Bucket, id uint64) string { return string(boltKey) } -func (s *TranslateStore) ComputeTranslatorSummary() (sum *pilosa.TranslatorSummary, err error) { +func (s *TranslateStore) ComputeTranslatorSummaryRows() (sum *pilosa.TranslatorSummary, err error) { sum = &pilosa.TranslatorSummary{} hasher := blake3.New() @@ -524,3 +536,787 @@ func (s *TranslateStore) ComputeTranslatorSummary() (sum *pilosa.TranslatorSumma sum.Checksum = string(buf[:]) return sum, nil } + +func (s *TranslateStore) ComputeTranslatorSummaryCols(partitionID int, topo *pilosa.Topology) (sum *pilosa.TranslatorSummary, err error) { + sum = &pilosa.TranslatorSummary{} + hasher := blake3.New() + + if partitionID != s.partitionID { + panic(fmt.Sprintf("inconsistent partitionID arg %v with TranslateStore.paritionID %v", partitionID, s.partitionID)) + } + firstPrimary := topo.PrimaryNodeIndex(partitionID) + + err = s.db.View(func(tx *bolt.Tx) error { + + bkt := tx.Bucket(bucketKeys) // key -> id + if bkt == nil { + panic("bucketKeys not found") + } + + cur := bkt.Cursor() + for k, v := cur.First(); k != nil; k, v = cur.Next() { + input := append(k, v...) + //vv("55555 ComputeTranslatorSummaryCols(partitionID=%v, path='%v'), k='%v', v=%x", partitionID, s.Path, string(k), v) + _, _ = hasher.Write(input) + sum.KeyCount++ + } + + bkt = tx.Bucket(bucketIDs) // id -> key + if bkt == nil { + panic("bucketIDs not found") + } + + cur = bkt.Cursor() + for k, v := cur.First(); k != nil; k, v = cur.Next() { + + // should the primary be the same for each key in this partition? + id := btou64(k) + shard := id / pilosa.ShardWidth + + ks := string(v) + primary := topo.GetPrimaryForColKeyTranslation(s.index, ks) + if firstPrimary < 0 { + firstPrimary = primary + } else { + if primary != firstPrimary { + panic(fmt.Sprintf("primary (%v) != firstPrimary (%v); key='%v', id=%v, shard=%v; partitionID=%v", primary, firstPrimary, ks, id, shard, partitionID)) + } + } + + // Verify the invariant that the primaries agree. Just a sanity check. + primaryForShard := topo.GetPrimaryForShardReplication(s.index, shard) + if primaryForShard != firstPrimary { + panic(fmt.Sprintf("primaryForShard (%v) != firstPrimary (%v); key='%v', id=%v, shard=%v; partitionID=%v", primaryForShard, firstPrimary, ks, id, shard, partitionID)) + } + + input := append(k, v...) + //vv("55555 ComputeTranslatorSummaryCols(partitionID=%v, path='%v'), idBucket id=%x key='%v'", partitionID, s.Path, id, ks) + _, _ = hasher.Write(input) + sum.IDCount++ + } + return nil + }) + if err != nil { + return nil, err + } + + sum.PrimaryNodeIndex = firstPrimary + + var buf [16]byte + _, _ = hasher.Digest().Read(buf[0:]) + sum.Checksum = string(buf[:]) + return sum, nil +} + +func (s *TranslateStore) KeyWalker(walk func(key string, col uint64)) error { + return s.db.View(func(tx *bolt.Tx) error { + bkt := tx.Bucket(bucketKeys) + if bkt == nil { + panic("bucketKeys not found") + } + cur := bkt.Cursor() + for k, v := cur.First(); k != nil; k, v = cur.Next() { + walk(string(k), btou64(v)) + } + return nil + }) +} +func (s *TranslateStore) IDWalker(walk func(key string, col uint64)) error { + return s.db.View(func(tx *bolt.Tx) error { + bkt := tx.Bucket(bucketIDs) + if bkt == nil { + panic("bucketIDs not found") + } + cur := bkt.Cursor() + for k, v := cur.First(); k != nil; k, v = cur.Next() { + walk(string(v), btou64(k)) + } + return nil + }) +} + +// call s.notifyWrite() when done +func (s *TranslateStore) SetFwdRevMaps(tx *bolt.Tx, fwd map[string]uint64, rev map[uint64]string) (err error) { + + localTx := false + if tx == nil { + localTx = true + tx, err = s.db.Begin(true) + if err != nil { + return err + } + defer func() { + _ = tx.Rollback() + }() + } + + // reinitialize buckets + err = tx.DeleteBucket(bucketKeys) + if err != nil { + return err + } + err = tx.DeleteBucket(bucketIDs) + if err != nil { + return err + } + if _, err := tx.CreateBucketIfNotExists(bucketKeys); err != nil { + return err + } else if _, err := tx.CreateBucketIfNotExists(bucketIDs); err != nil { + return err + } + + key2id := tx.Bucket(bucketKeys) + for k, v := range fwd { + err := key2id.Put([]byte(k), u64tob(v)) + if err != nil { + return err + } + } + id2key := tx.Bucket(bucketIDs) + for k, v := range rev { + err := id2key.Put(u64tob(k), []byte(v)) + if err != nil { + return err + } + } + if localTx { + return tx.Commit() + } + return nil +} + +func (s *TranslateStore) GetFwdRevMaps(tx *bolt.Tx) (fwd map[string]uint64, rev map[uint64]string, err error) { + fwd = make(map[string]uint64) + rev = make(map[uint64]string) + + key2id := tx.Bucket(bucketKeys) + + err = key2id.ForEach(func(k, v []byte) error { + fwd[string(k)] = btou64(v) + return nil + }) + if err != nil { + return + } + + id2key := tx.Bucket(bucketIDs) + err = id2key.ForEach(func(k, v []byte) error { + rev[btou64(k)] = string(v) + return nil + }) + return +} + +//var vv = pilosa.VV + +// helpers for repair + +// muint64 holds multiple unit64 +type muint64 struct { + slc []uint64 +} + +func (m *muint64) String() (s string) { + for _, e := range m.slc { + s += fmt.Sprintf("%x, ", e) + } + return +} + +// mstring holds multiple strings +type mstring struct { + slc []string +} + +func (m *mstring) String() (s string) { + for _, e := range m.slc { + s += e + "," + } + return +} + +func addToProblemKeys(problemKeys map[string]*muint64, k string, v uint64, noValue bool) { + mu, already := problemKeys[k] + if !already { + mu = &muint64{} + problemKeys[k] = mu + } + if !noValue { + mu.slc = append(mu.slc, v) + } +} +func addToProblemIDs(problemIDs map[uint64]*mstring, k uint64, v string, noValue bool) { + mu, already := problemIDs[k] + if !already { + mu = &mstring{} + problemIDs[k] = mu + } + if !noValue { + mu.slc = append(mu.slc, v) + } +} + +// only actually apply the fixes if applyKeyRepairs is true. +// if anything changed, return changed == true. +func (s *TranslateStore) RepairKeys(topo *pilosa.Topology, verbose, applyKeyRepairs bool) (changed bool, err error) { + // strategy: get the full set of keys; the domain keys from + // the forward key->id mapping, and the range keys from the reverse id->key mapping. + // Then march through them and make sure they are mapped correctly. + // At the moment we do try to reuse dangling IDs instead of making + // new ones. This might not always be possible, but we hope for + // now that it suffices b/c it minimizes the amount of fragment + // re-write we may have to do. + + /* + // ============ profiling =============== + fd, err := ioutil.TempFile(".", "cpu.prof") + if err != nil { + panic(err) + } + _ = pprof.StartCPUProfile(fd) + defer func() { + pprof.StopCPUProfile() + fd.Close() + }() + // ============ end profiling =============== + */ + + tx, err := s.db.Begin(true) + if err != nil { + return false, err + } + defer func() { + _ = tx.Rollback() + }() + fwd, rev, err := s.GetFwdRevMaps(tx) + if err != nil { + return false, err + } + + // place to store the correct stuff. + // + // fwd2, rev2: new, repaired versions. + // INVAR: they only contain (correct) invertible mappings. + fwd2 := make(map[string]uint64) + rev2 := make(map[uint64]string) + + // and a place to store the problems. + problemKeys := make(map[string]*muint64) + problemIDs := make(map[uint64]*mstring) + +fwdscan: + for k, v := range fwd { + _, already := fwd2[k] + if already { + // k has already been repaired. don't worry about further. + continue fwdscan + } else { + // if its already invertible, then just keep it, no need to repair it + + // INVAR: k is not in fwd2 (at least not yet). + rkey, ok := rev[v] + if !ok { + // k -> v -> X + addToProblemIDs(problemIDs, v, k, false) + addToProblemKeys(problemKeys, k, v, false) + continue fwdscan + } + if rkey == k { + // yay. a good, invertible, mapping. no repair needed. + if k == "" { + panic("bad empty key") + } + fwd2[k] = v + rev2[v] = k + continue fwdscan + } + + // some kind of problem. + // what kind? + // Define problemKey as: 2nd key mapping to id already in fwd2. + // Define problemID as: 2nd ID mapping to key already in fwd2. + + // k -> v -> rkey, and rkey != k. + v2, ok := fwd[rkey] + if ok { + // k -> v -> rkey -> v2, where v2 ?= v + if v2 == v { + // k -> v -> rkey -> v + // just have a problemKey in k. + + if rkey == "" { + panic("bad empty rkey") + } + fwd2[rkey] = v + rev2[v] = rkey + addToProblemKeys(problemKeys, k, v, true) + continue fwdscan + } + // this is i = 1 test case. :) + + // k -> v -> rkey -> v2, where v != v2, and k != rkey. + addToProblemKeys(problemKeys, k, v, false) + addToProblemIDs(problemIDs, v, rkey, false) + continue fwdscan + } else { + // k -> v -> rkey -> X(nil), and rkey != k. + addToProblemKeys(problemKeys, k, v, false) + addToProblemKeys(problemKeys, rkey, 0, true) + addToProblemIDs(problemIDs, v, rkey, false) + } + } + } +revscan: + for id, key := range rev { + k1, already := rev2[id] + _ = k1 + if already { + // fine, already there. + continue + } + + // if its already invertible, keep it. + rid, ok := fwd[key] + if !ok { + // id -> key -> X + addToProblemKeys(problemKeys, key, 0, true) + addToProblemIDs(problemIDs, id, key, false) + continue revscan + } + if rid == id { + // id -> key -> id. good. but should have been added to fwd2/rev2 above. + panic("should have been added to fwd2/rev2 above!") + + } else { + // id -> key -> rid, where id != rid + // so rid -> ? + keyr, ok := rev[rid] + if !ok { + // id -> key -> rid -> X, where id != rid + addToProblemKeys(problemKeys, key, rid, false) + addToProblemKeys(problemKeys, key, id, false) + addToProblemIDs(problemIDs, rid, key, false) + continue revscan + } + if keyr == key { + // id -> key -> rid -> key. So rid is correct and id is dangling. + // + // Heuristic: ASSUME here, that the 2 consistent links key->rid->key are correct, + // and that the single id -> key is in the wrong. This DOESN'T HAVE + // TO BE THE CASE. + + if key == "" { + panic("bad empty key") + } + rev2[rid] = key + fwd2[key] = rid + addToProblemIDs(problemIDs, id, "", true) + } else { + // this is test case i = 0. Must handle it. + + // id -> key -> rid -> keyr, id != rid, keyr != key. + id2, ok := fwd[keyr] + if ok && id2 == rid { + // id -> key -> rid -> keyr -> rid, id != rid, keyr != key. + // so rid -> keyr -> rid is good. + + if keyr == "" { + panic("bad empty keyr") + } + fwd2[keyr] = rid + rev2[rid] = keyr + // and id -> key -> rid is bad, b/c id != rid. + addToProblemKeys(problemKeys, key, rid, false) + addToProblemIDs(problemIDs, id, key, false) + continue revscan + } + // one of these 3 cases holds. all have the same treatment. + // 1) id2 == id: id -> key -> rid -> keyr -> id2; id != rid, rid != id2, keyr != key. + // 2) id2 != id: id -> key -> rid -> keyr -> id2; id != rid, rid != id2, id2 != id, keyr != key. + // 3) !ok: id -> key -> rid -> keyr -> X, id != rid, keyr != key. + addToProblemIDs(problemIDs, id, key, false) + addToProblemKeys(problemKeys, key, rid, false) + addToProblemIDs(problemIDs, rid, keyr, false) + } + } + + } + //vv("problemKeys = '%v'", problemKeys) + //vv("problemIDs = '%v'", problemIDs) + + newIDs := make(map[uint64]bool) + + // assign new IDs to any problemKeys; but first + // try to reuse already allocated IDs that are just dangling. +loopProblemKeys: + for key, ids := range problemKeys { + // first try a minor repair, maybe it was just mssing from rev + // and we can avoid allocate another id. + + // sanity check + v2, already := fwd2[key] + if already { + panic(fmt.Sprintf("should not get here since fwd2 is only correct invertibles: key='%v', v2='%x'", key, v2)) + } + // INVAR: we have no correct mapping for key in fwd2. + + // treat the danglers as "suggestions" for the correction. + for k, id := range ids.slc { + _ = k + _, already = rev2[id] + if !already { + // is this correct? + // id is not in rev2, and key is not in fwd2. + // therefore, we can add them both and maintain consistency. + + //vv("add %v to fwd2", key) + if key == "" { + panic("bad empty key") + } + fwd2[key] = id + rev2[id] = key + continue loopProblemKeys + } + } + // INVAR: key -> ? don't know. We didn't find a usable suggestion for the id. + + // yes, we get here. We have key. We are looking for a suitable id for it. + + // can we get a usable id from the problemIDs? + found := false + suggestions: + for idp, mkeyp := range problemIDs { + for _, candk := range mkeyp.slc { + //vv("checking problemIDs, ipd=%x, candk='%v'; candk==key is %v", idp, candk, candk == key) + if candk == key { + // we have a suggestion from problemIDs that idp might work, doing key -> idp. + // Validate that this is possible. + k2, already := rev2[idp] + _ = k2 + if already { + //vv("idp is already in rev2: idp=%v, k2=%v", idp, k2) + continue suggestions + } + // idp works. put it in the correct set. + if key == "" { + panic("bad empty key") + } + rev2[idp] = key + fwd2[key] = idp + found = true + break suggestions + } + } + } + if !found { + id2 := pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN) + //vv("could not minor repair, allocating new id2 = %v instead", id2) + newIDs[id2] = true + + if key == "" { + panic("bad empty key") + } + fwd2[key] = id2 + rev2[id2] = key + } + } // end problemKeys + + //for id, keys := range problemIDs { + //} + + if verbose { + reportIfGainedOrLostIDs(s, fwd, fwd2, rev, rev2, newIDs) + reportIfGainedOrLostKeys(s, fwd, fwd2, rev, rev2) + } + + adds, changes, changeIDs, err := makeStringKeyChanges(verbose, applyKeyRepairs, tx, s, topo, fwd, fwd2, rev, rev2, newIDs) + if err != nil { + return false, err + } + _, _, _ = adds, changes, changeIDs + + //vv("changedIDs = '%#v'", changeIDs) + if len(adds) > 0 || len(changes) > 0 || len(changeIDs) > 0 || len(newIDs) > 0 { + changed = true + } + + //vv("newIDs = '%#v'", newIDs) + //vv("fwd2 = '%#v'", fwd2) + //vv("rev2 = '%#v'", rev2) + + err = tx.Commit() + if err == nil { + s.notifyWrite() + } + return changed, err +} + +func reportIfGainedOrLostIDs(s *TranslateStore, fwd, fwd2 map[string]uint64, rev, rev2 map[uint64]string, newIDs map[uint64]bool) { + // get all IDs ever mentioned + before := make(map[uint64]bool) + after := make(map[uint64]bool) + for _, id := range fwd { + before[id] = true + } + for _, id := range fwd2 { + if !newIDs[id] { + after[id] = true + } + } + for id := range rev { + before[id] = true + } + for id := range rev2 { + if !newIDs[id] { + after[id] = true + } + } + nb := len(before) + na := len(after) + if nb != na { + fmt.Printf("# needs-repair: Num ID before %v != Num ID after %v, for boltdb = '%v'. before counts(fwd/rev) = %v/%v. after repair counts(fwd2/rev2) = %v/%v\n", nb, na, s.Path, len(fwd), len(rev), len(fwd2), len(rev2)) + } + if len(newIDs) > 0 { + fmt.Printf("# needs-repair: adding newIDs '%#v', for boltdb = '%v'. before counts(fwd/rev) = %v/%v. after repair counts(fwd2/rev2) = %v/%v\n", newIDs, s.Path, len(fwd), len(rev), len(fwd2), len(rev2)) + } +} +func reportIfGainedOrLostKeys(s *TranslateStore, fwd, fwd2 map[string]uint64, rev, rev2 map[uint64]string) { + // get all IDs ever mentioned + nb := len(fwd) + na := len(fwd2) + if nb != na { + diffAB := mapDiffStrings(fwd, fwd2) + diffBA := mapDiffStrings(fwd2, fwd) + + fmt.Printf("# needs-repair: Num Keys before != Num Keys after, for boltdb = '%v'. before counts(fwd/rev) = %v/%v. after repair counts(fwd2/rev2) = %v/%v. fwd - fwd2 = '%#v'; fwd2-fwd = '%#v'\n", s.Path, len(fwd), len(rev), len(fwd2), len(rev2), diffAB, diffBA) + } +} + +// return A - B +func mapDiffStrings(mapA, mapB map[string]uint64) (r []string) { + for a := range mapA { + _, ok := mapB[a] + if !ok { + r = append(r, a) + } + } + sort.Strings(r) + return +} + +type BeforeAfterKeyChange struct { + BeforeID uint64 + AfterID uint64 +} + +type BeforeAfterIDChange struct { + IsDelete bool + IsAdd bool + BeforeString string + AfterString string +} + +// do the minimal state update. +// fwd2 is the "after" map, all string keys repaired. +func makeStringKeyChanges( + verbose bool, + applyKeyRepairs bool, + tx *bolt.Tx, + s *TranslateStore, + topo *pilosa.Topology, + fwd, fwd2 map[string]uint64, + rev, rev2 map[uint64]string, + newIDs map[uint64]bool, +) ( + adds map[string]uint64, + changeKeys map[string]*BeforeAfterKeyChange, + changeIDs map[uint64]*BeforeAfterIDChange, + err error, +) { + //vv("makeStringKeyChanges called") + + //vv("fwd2 = '%#v'", fwd2) + //vv("rev2 = '%#v'", rev2) + //vv("fwd = '%#v'", fwd) + //vv("rev = '%#v'", rev) + + var action string + if applyKeyRepairs { + action = "applying " + } + + // addition of string key + adds = make(map[string]uint64) + + // change of the mapping of key -> id. + changeKeys = make(map[string]*BeforeAfterKeyChange) + + // changes to bucketIDs + changeIDs = make(map[uint64]*BeforeAfterIDChange) + + localTx := false + if applyKeyRepairs && tx == nil { + localTx = true + tx, err = s.db.Begin(true) + if err != nil { + return + } + defer func() { + //vv("tx.Rollback happening") + _ = tx.Rollback() + }() + } + + key2id := tx.Bucket(bucketKeys) + id2key := tx.Bucket(bucketIDs) + + // make a copy of rev2 that we can delete from, to see if + // any additions left in rev2 need to be added after all of + // rev is analyzed. + rev2cp := make(map[uint64]string) + for id, k := range rev2 { + rev2cp[id] = k + } + + // first we clean up any stale IDs from id2key. Then the fwd2 pass + // that follows will write to both key2id and id2key. + for id, key := range rev { + //vv("makeStringKeyChanges on rev2: id=%x -> key='%v'", id, key) + key2, ok := rev2[id] + if !ok { + changeIDs[id] = &BeforeAfterIDChange{IsDelete: true} + if verbose { + fmt.Printf("# %vkey-translation-delete-id: (id %x -> %v). Remaining for that key: ('%v' -> %x)\n", action, id, key, key, fwd2[key]) + } + if applyKeyRepairs { + u := u64tob(id) + err = id2key.Delete(u) + if err != nil { + return + } + } + continue + } + delete(rev2cp, id) + + if key2 != key { + u := u64tob(id) + k := []byte(key2) + changeIDs[id] = &BeforeAfterIDChange{ + BeforeString: key, + AfterString: key2, + } + if verbose { + fmt.Printf("# %vkey-translation-update-id: (id %x -> %v). fwd2 for that key: ('%v' -> %x)\n", action, id, key2, key2, fwd2[key2]) + } + if applyKeyRepairs { + err = id2key.Put(u, k) + if err != nil { + return + } + } + } + } + // anything leftover in rev2cp is stuff that is new, only + // in rev2 and not in rev. It needs to be added. + for id, key2 := range rev2cp { + u := u64tob(id) + k := []byte(key2) + changeIDs[id] = &BeforeAfterIDChange{ + IsAdd: true, + //BeforeString: left empty + AfterString: key2, + } + if verbose { + fmt.Printf("# %vkey-translation-add-id: (id %x -> %v). Fwd for that key: ('%v' -> %x)\n", action, id, key2, key2, fwd2[key2]) + } + if applyKeyRepairs { + err = id2key.Put(u, k) + if err != nil { + return + } + } + } + + // We assume here that fwd2 is a super-set of fwd. No string keys + // should be deleted in the repair. Confirm that. + for key, id := range fwd { + _, ok := fwd2[key] + if !ok { + panic(fmt.Sprintf("fwd2 is missing a string key from fwd. key='%v' -> id='%x'", key, id)) + } + } + + for key2, id2 := range fwd2 { + //vv("makeStringKeyChanges on fwd2, key2='%v', id2=%x", key2, id2) + isPrimary := false + if topo != nil { + primary := topo.GetPrimaryForColKeyTranslation(s.index, key2) + isPrimary = s.partitionID == primary + } + _ = isPrimary + id, ok := fwd[key2] + if !ok { + adds[key2] = id2 + + u2 := u64tob(id2) + k2 := []byte(key2) + if verbose { + fmt.Printf("# %vkey-translation-new-key: ('%v' -> %x) added: isPrimary: %v\n", action, key2, id2, isPrimary) + } + if applyKeyRepairs { + err = key2id.Put(k2, u2) + if err != nil { + return + } + err = id2key.Put(u2, k2) + if err != nil { + return + } + } + continue + } + if id != id2 { + changeKeys[key2] = &BeforeAfterKeyChange{ + BeforeID: id, + AfterID: id2, + } + if verbose { + fmt.Printf("# %vkey-translation-change-id: ('%v' -> %x) changes to ('%v' -> %x); isPrimary: %v\n", action, key2, id, key2, id2, isPrimary) + } + if applyKeyRepairs { + u2 := u64tob(id2) + k2 := []byte(key2) + + err = key2id.Put(k2, u2) + if err != nil { + return + } + err = id2key.Put(u2, k2) + if err != nil { + return + } + } + } + } + + if localTx { + err = tx.Commit() + } + return +} + +func (s *TranslateStore) DumpBolt(label string) { + + fmt.Printf("dumping bolt %v : path='%v'\n", label, s.Path) + + _ = s.KeyWalker(func(key string, col uint64) { + fmt.Printf("keyWalker: key '%v' -> col '%x'\n", key, col) + }) + _ = s.IDWalker(func(key string, col uint64) { + fmt.Printf("idWalker: id '%x' -> key '%v'\n", col, key) + }) + + fmt.Printf("DONE with dumping bolt %v; path='%v'\n", label, s.Path) + +} diff --git a/boltdb/translate_test.go b/boltdb/translate_test.go index 8c3915f00..1e0803b1b 100644 --- a/boltdb/translate_test.go +++ b/boltdb/translate_test.go @@ -28,6 +28,8 @@ import ( "github.com/pilosa/pilosa/v2/boltdb" ) +//var vv = pilosa.VV + func TestTranslateStore_TranslateKey(t *testing.T) { s := MustOpenNewTranslateStore() defer MustCloseTranslateStore(s) @@ -533,8 +535,7 @@ func TestCryptoHashPerKey(t *testing.T) { } // done with setup - - sum, err := s.ComputeTranslatorSummary() + sum, err := s.ComputeTranslatorSummaryCols(0, pilosa.NewTopology(&pilosa.Jmphasher{}, pilosa.DefaultPartitionN, 1, nil)) if err != nil { panic(err) } @@ -558,3 +559,123 @@ func TestCryptoHashPerKey(t *testing.T) { } } + +func TestTranslateStore_RepairNonInvertibleStringKeyTranslation(t *testing.T) { + + const N = 6 + // before repair + var fwd [N]map[string]uint64 + var rev [N]map[uint64]string + + // after repair + var fwd2 [N]map[string]uint64 + var rev2 [N]map[uint64]string + + // case 0: forward is messed up (unlikely but check for it anyway, be sure we can repair) + // "key0" -> id 0 // correct. + // "key1" -> id 0 // wrong. after Repair, should see key1 -> 1 (0xec0002) + // + // id 0 -> "key0" // correct + // id 1 -> "key1" // correct + // + fwd[0] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00001} + rev[0] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"} + fwd2[0] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00002} + rev2[0] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"} + + // case 1: reverse is messed up (we have seen this in the past) + // "key0" -> id 0 // correct + // "key1" -> id 1 // correct + // + // id 0 -> "key0" // correct. + // id 1 -> "key0" // wrong. after Repair, should see id 1 -> "key1" + // + fwd[1] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00002} + rev[1] = map[uint64]string{0xec00001: "key0", 0xec00002: "key0"} + fwd2[1] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00002} + rev2[1] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"} + + // case 2: only present in reverse. + fwd[2] = map[string]uint64{} + rev[2] = map[uint64]string{0xec00001: "key0"} + fwd2[2] = map[string]uint64{"key0": 0xec00001} + rev2[2] = map[uint64]string{0xec00001: "key0"} + + // case 3: same thing. with camoflage. + fwd[3] = map[string]uint64{"key1": 0xec00002} + rev[3] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"} + fwd2[3] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00002} + rev2[3] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"} + + // case 4: only present in forward. + + fwd[4] = map[string]uint64{"key0": 0xec00001} + rev[4] = map[uint64]string{} + fwd2[4] = map[string]uint64{"key0": 0xec00001} + rev2[4] = map[uint64]string{0xec00001: "key0"} + + // case 5: same thing. with camoflage. + fwd[5] = map[string]uint64{"key0": 0xec00001} + rev[5] = map[uint64]string{0xec00002: "key1"} + fwd2[5] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00002} + rev2[5] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"} + + // case 6: we had an id, but b/c of the fix, that id is no longer used. + // now that id might still be used in the fragment for a column, + // and so we will need to remove that id/column from the fragment. + // encapsulated: "did it affect the state of the fields?" + + for i := 0; i < 5; i++ { + //println("i = ", i) + s := MustOpenNewTranslateStore() + defer MustCloseTranslateStore(s) + + if err := s.SetFwdRevMaps(nil, fwd[i], rev[i]); err != nil { + t.Fatal(err) + } + + if err := verifyState("setup", i, s, fwd[i], rev[i]); err != nil { + t.Fatal(err) + } + + var topo *pilosa.Topology + verbose := false + applyKeyRepairs := true + changed, err := s.RepairKeys(topo, verbose, applyKeyRepairs) + if err != nil { + t.Fatal(err) + } + if !changed { + t.Fatalf("expected changes!") + } + + if err := verifyState("afterRepair", i, s, fwd2[i], rev2[i]); err != nil { + t.Fatal(err) + } + } +} + +func verifyState(label string, i int, s *boltdb.TranslateStore, fwd map[string]uint64, rev map[uint64]string) error { + + // verify the setup + const writable = true + for key, expectID := range fwd { + id, err := s.TranslateKey(key, !writable) + if err != nil { + return err + } + if id != expectID { + return fmt.Errorf("fwd %v problem. i=%v, for key '%v', expected %x, observed %x", label, i, key, expectID, id) + } + } + for id, expectKey := range rev { + key, err := s.TranslateID(id) + if err != nil { + return err + } + if key != expectKey { + return fmt.Errorf("rev %v problem. i=%v, for id '%x', expected %v, observed %v", label, i, id, expectKey, key) + } + } + return nil +} diff --git a/cluster.go b/cluster.go index 10bc2b72d..9059c8732 100644 --- a/cluster.go +++ b/cluster.go @@ -260,7 +260,7 @@ type cluster struct { // nolint: maligned // newCluster returns a new instance of Cluster with defaults. func newCluster() *cluster { return &cluster{ - Hasher: &jmphasher{}, + Hasher: &Jmphasher{}, partitionN: DefaultPartitionN, ReplicaN: 1, @@ -978,12 +978,13 @@ func (c *cluster) translationNodes(to *cluster) (map[string][]*translationResize return m, nil } -// shardPartition returns the partition that a shard belongs to. -func (c *cluster) shardPartition(index string, shard uint64) int { - return shardPartition(index, shard, c.partitionN) +// shardPartition returns the shard-partition that a shard belongs to. +// NOTE: this is DIFFERENT from the key-partition +func (c *cluster) shardToShardPartition(index string, shard uint64) int { + return shardToShardPartition(index, shard, c.partitionN) } -func shardPartition(index string, shard uint64, partitionN int) int { +func shardToShardPartition(index string, shard uint64, partitionN int) int { var buf [8]byte binary.BigEndian.PutUint64(buf[:], shard) @@ -994,12 +995,13 @@ func shardPartition(index string, shard uint64, partitionN int) int { return int(h.Sum64() % uint64(partitionN)) } -// keyPartition returns the partition that a key belongs to. -func (c *cluster) keyPartition(index, key string) int { - return keyPartition(index, key, c.partitionN) +// keyPartition returns the key-partition that a key belongs to. +// NOTE: the key-partition is DIFFERENT from the shard-partition. +func (topo *Topology) KeyPartition(index, key string) int { + return keyToKeyPartition(index, key, topo.PartitionN) } -func keyPartition(index, key string, partitionN int) int { +func keyToKeyPartition(index, key string, partitionN int) int { // Hash the bytes and mod by partition count. h := fnv.New64a() _, _ = h.Write([]byte(index)) @@ -1009,7 +1011,7 @@ func keyPartition(index, key string, partitionN int) int { // idPartition returns the partition that an id belongs to. func (c *cluster) idPartition(index string, id uint64) int { - return shardPartition(index, id/ShardWidth, c.partitionN) + return shardToShardPartition(index, id/ShardWidth, c.partitionN) } // ShardNodes returns a list of nodes that own a fragment. Safe for concurrent use. @@ -1021,7 +1023,7 @@ func (c *cluster) ShardNodes(index string, shard uint64) []*Node { // shardNodes returns a list of nodes that own a fragment. unprotected func (c *cluster) shardNodes(index string, shard uint64) []*Node { - return c.partitionNodes(c.shardPartition(index, shard)) + return c.partitionNodes(c.shardToShardPartition(index, shard)) } // KeyNodes returns a list of nodes that own a fragment. Safe for concurrent use. @@ -1033,7 +1035,7 @@ func (c *cluster) KeyNodes(index, key string) []*Node { // keyNodes returns a list of nodes that own a key. unprotected func (c *cluster) keyNodes(index, key string) []*Node { - return c.partitionNodes(c.keyPartition(index, key)) + return c.partitionNodes(c.Topology.KeyPartition(index, key)) } // ownsShard returns true if a host owns a fragment. @@ -1077,8 +1079,14 @@ func (c *cluster) partitionNodes(partitionID int) []*Node { } // Determine primary owner node. - nodeIndex := c.Hasher.Hash(uint64(partitionID), nodeN) - + if c.Topology == nil { + c.Topology = NewTopology(c.Hasher, c.partitionN, c.ReplicaN, c) + } + nodeIndex := c.Topology.PrimaryNodeIndex(partitionID) + if nodeIndex < 0 { + // no nodes anyway + return nil + } // Collect nodes around the ring. nodes := make([]*Node, 0, replicaN) for i := 0; i < replicaN; i++ { @@ -1109,11 +1117,66 @@ func (c *cluster) unprotectedPrimaryPartitionNode(partition int) *Node { return nil } +func (topo *Topology) IsPrimary(nodeID string, partitionID int) bool { + primary := topo.PrimaryNodeIndex(partitionID) + return nodeID == topo.nodeIDs[primary] +} + +func (topo *Topology) PrimaryNodeIndex(partitionID int) (nodeIndex int) { + n := len(topo.nodeIDs) + if n == 0 { + if topo.cluster != nil { + n = len(topo.cluster.nodes) + } + } + nodeIndex = topo.Hasher.Hash(uint64(partitionID), n) + return +} + +func (topo *Topology) GetNonPrimaryReplicas(partitionID int) (nonPrimaryReplicas []string) { + + primary := topo.PrimaryNodeIndex(partitionID) + nodeN := len(topo.nodeIDs) + + // Collect nodes around the ring. + for i := 1; i < nodeN; i++ { + nodeID := topo.nodeIDs[(primary+i)%nodeN] + if i < topo.ReplicaN { + nonPrimaryReplicas = append(nonPrimaryReplicas, nodeID) + } + } + return +} + +// the map replicaNodeIDs[nodeID] will have a true value for the primary nodeID, and false for others. +func (topo *Topology) GetReplicasForPrimary(primary int) (replicaNodeIDs, nonReplicas map[string]bool) { + if primary < 0 { + // no nodes anyway + return + } + replicaNodeIDs = make(map[string]bool) + nonReplicas = make(map[string]bool) + + nodeN := len(topo.nodeIDs) + + // Collect nodes around the ring. + for i := 0; i < nodeN; i++ { + nodeID := topo.nodeIDs[(primary+i)%nodeN] + if i < topo.ReplicaN { + // mark true if primary + replicaNodeIDs[nodeID] = (i == 0) + } else { + nonReplicas[nodeID] = false + } + } + return +} + // containsShards is like OwnsShards, but it includes replicas. func (c *cluster) containsShards(index string, availableShards *roaring.Bitmap, node *Node) []uint64 { var shards []uint64 _ = availableShards.ForEach(func(i uint64) error { - p := c.shardPartition(index, i) + p := c.shardToShardPartition(index, i) // Determine the nodes for partition. nodes := c.partitionNodes(p) for _, n := range nodes { @@ -1133,10 +1196,10 @@ type Hasher interface { } // jmphasher represents an implementation of jmphash. Implements Hasher. -type jmphasher struct{} +type Jmphasher struct{} // Hash returns the integer hash for the given key. -func (h *jmphasher) Hash(key uint64, n int) int { +func (h *Jmphasher) Hash(key uint64, n int) int { b, j := int64(-1), int64(0) for j < int64(n) { b = j @@ -1202,7 +1265,7 @@ func (c *cluster) waitForStarted() error { c.logger.Printf("%v wait for joining to complete", c.Node.ID) <-c.joining - c.logger.Printf("joining has completed") + c.logger.Printf("joining has completed. I am NodeID '%v'", c.Node.ID) } return nil } @@ -1853,6 +1916,8 @@ func (n nodeIDs) ContainsID(id string) bool { } // Topology represents the list of hosts in the cluster. +// Topology now encapsulates all knowledge needed to +// determine the primary node in the replication scheme. type Topology struct { mu sync.RWMutex nodeIDs []string @@ -1862,14 +1927,51 @@ type Topology struct { // nodeStates holds the state of each node according to // the coordinator. Used during startup and data load. nodeStates map[string]string + + // moved Hasher, PartitionN and ReplicaN + // from cluster for standalone use and comprehension: + + // Hashing algorithm used to assign partitions to nodes. + Hasher Hasher + // The number of partitions in the cluster. + PartitionN int + // The number of replicas a partition has. + ReplicaN int + + // can be nil + cluster *cluster } -func newTopology() *Topology { +// NewTopology creates a Topology. +// +// The arguments and members hasher, partitionN, and +// replicaN were refactored out of struct cluster +// to allow pilosa-fsck to load a Topology from +// backup and then compute primaries standalone -- without starting a cluster. +// As pilosa-fsck operates on all backups at once from +// a single cpu, starting a full cluster isn't possible. +// +// The hasher is the Hashing algorithm used to assign partitions to nodes. +// The cluster c should be provided if possible by pilosa code; +// the pilosa-fsck utility won't be able to provide it. +// +// For the cluster size N, the topology gives preference to +// len(t.nodeIDs) before falling back on len(c.nodes). +// +func NewTopology(hasher Hasher, partitionN int, replicaN int, c *cluster) *Topology { return &Topology{ + Hasher: hasher, + PartitionN: partitionN, + ReplicaN: replicaN, nodeStates: make(map[string]string), + cluster: c, } } +func (t *Topology) GetNodeIDs() []string { + return t.nodeIDs +} + // ContainsID returns true if id matches one of the topology's IDs. func (t *Topology) ContainsID(id string) bool { t.mu.RLock() @@ -1933,7 +2035,7 @@ func (t *Topology) encode() *internal.Topology { func (c *cluster) loadTopology() error { buf, err := ioutil.ReadFile(filepath.Join(c.Path, ".topology")) if os.IsNotExist(err) { - c.Topology = newTopology() + c.Topology = NewTopology(c.Hasher, c.partitionN, c.ReplicaN, c) return nil } else if err != nil { return errors.Wrap(err, "reading file") @@ -1943,7 +2045,7 @@ func (c *cluster) loadTopology() error { if err := proto.Unmarshal(buf, &pb); err != nil { return errors.Wrap(err, "unmarshalling") } - top, err := decodeTopology(&pb) + top, err := DecodeTopology(&pb, c.Hasher, c.partitionN, c.ReplicaN, c) if err != nil { return errors.Wrap(err, "decoding") } @@ -1954,7 +2056,6 @@ func (c *cluster) loadTopology() error { // saveTopology writes the current topology to disk. unprotected. func (c *cluster) saveTopology() error { - if err := os.MkdirAll(c.Path, 0777); err != nil { return errors.Wrap(err, "creating directory") } @@ -2461,6 +2562,27 @@ func (c *cluster) translateIndexKeys(ctx context.Context, indexName string, keys return ids, nil } +// The boltdb key translation stores are partitioned, designated by partitionIDs. These +// are shared between replicas, and one node is the primary for +// replication. So with 4 nodes and 3-way replication, each node has 3/4 of +// the translation stores on it. +func (topo *Topology) GetPrimaryForColKeyTranslation(index, key string) (primary int) { + partitionID := topo.KeyPartition(index, key) + return topo.PrimaryNodeIndex(partitionID) +} + +// should match cluster.go:1033 cluster.ownsShard(nodeID, index, shard) +// return Nodes(c.shardNodes(index, shard)).ContainsID(nodeID) +func (t *Topology) GetPrimaryForShardReplication(index string, shard uint64) int { + n := len(t.nodeIDs) + if n == 0 { + return -1 + } + partition := uint64(shardToShardPartition(index, shard, t.PartitionN)) + nodeIndex := t.Hasher.Hash(partition, n) + return nodeIndex +} + func (c *cluster) translateIndexKeySet(ctx context.Context, indexName string, keySet map[string]struct{}, writable bool) (map[string]uint64, error) { keyMap := make(map[string]uint64) @@ -2472,7 +2594,7 @@ func (c *cluster) translateIndexKeySet(ctx context.Context, indexName string, ke // Split keys by partition. keysByPartition := make(map[int][]string, c.partitionN) for key := range keySet { - partitionID := c.keyPartition(indexName, key) + partitionID := c.Topology.KeyPartition(indexName, key) keysByPartition[partitionID] = append(keysByPartition[partitionID], key) } @@ -2651,12 +2773,13 @@ func encodeTopology(topology *Topology) *internal.Topology { } } -func decodeTopology(topology *internal.Topology) (*Topology, error) { +// the cluster c is optional but give it if you have it. +func DecodeTopology(topology *internal.Topology, hasher Hasher, partitionN, replicaN int, c *cluster) (*Topology, error) { if topology == nil { return nil, nil } - t := newTopology() + t := NewTopology(hasher, partitionN, replicaN, c) t.clusterID = topology.ClusterID t.nodeIDs = topology.NodeIDs sort.Slice(t.nodeIDs, diff --git a/cluster_internal_test.go b/cluster_internal_test.go index 003e7f668..a9ec930be 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -381,7 +381,7 @@ func TestCluster_Partition(t *testing.T) { c := newCluster() c.partitionN = partitionN - partitionID := c.shardPartition(index, shard) + partitionID := c.shardToShardPartition(index, shard) if partitionID < 0 || partitionID >= partitionN { t.Errorf("partition out of range: shard=%d, p=%d, n=%d", shard, partitionID, partitionN) } @@ -411,7 +411,7 @@ func TestHasher(t *testing.T) { {0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}}, } { for i, v := range tt.bucket { - hasher := &jmphasher{} + hasher := &Jmphasher{} if got := hasher.Hash(tt.key, i+1); got != v { t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v) } @@ -1053,3 +1053,27 @@ func TestCluster_confirmNodeDownDown(t *testing.T) { t.Errorf("expected node to be down") } } + +func TestCluster_GetNonPrimaryReplicas(t *testing.T) { + + c := newCluster() + c.ReplicaN = 3 + topo := NewTopology(c.Hasher, c.partitionN, c.ReplicaN, c) + c.Topology = topo + nNodes := 4 + for i := 0; i < nNodes; i++ { + nodeID := fmt.Sprintf("node%d", i) + c.nodes = append(c.nodes, &Node{ + ID: nodeID, + URI: NewTestURI("http", fmt.Sprintf("host%d", i), uint16(0)), + }) + c.Topology.addID(nodeID) + } + + partitionID := 256 + nonPrimes := topo.GetNonPrimaryReplicas(partitionID) + m := len(nonPrimes) + if m != c.ReplicaN-1 { + t.Fatalf("expected 2 non primes, got %v", m) + } +} diff --git a/cmd/pilosa-chk/chk.go b/cmd/pilosa-chk/chk.go index f0d5d9258..7e315700c 100644 --- a/cmd/pilosa-chk/chk.go +++ b/cmd/pilosa-chk/chk.go @@ -76,8 +76,10 @@ func main() { final := pilosa.NewAllTranslatorSummary() const verbose = true + const checkKeys = false + const applyKeyRepairs = false for _, idx := range holder.Indexes() { - asum, err := idx.ComputeTranslatorSummary(verbose) + asum, err := idx.ComputeTranslatorSummary(verbose, checkKeys, applyKeyRepairs, nil, "fake-nodeID") if err != nil { log.Fatal(err) } @@ -103,7 +105,7 @@ func main() { fmt.Printf("==============================\n") fmt.Printf("index: %v\n", idx.Name()) fmt.Printf("==============================\n") - idx.WriteFragmentChecksums(os.Stdout, showBits, showOpsLog) + idx.WriteFragmentChecksums(os.Stdout, showBits, showOpsLog, nil, verbose) } } } diff --git a/cmd/pilosa-fsck/Makefile b/cmd/pilosa-fsck/Makefile new file mode 100644 index 000000000..1b1dcf14c --- /dev/null +++ b/cmd/pilosa-fsck/Makefile @@ -0,0 +1,36 @@ +.PHONY: install build release + +CLONE_URL=github.com/pilosa/pilosa +VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) +LATTICE_COMMIT := $(shell git -C lattice rev-parse --short HEAD 2>/dev/null) +VARIANT = Molecula +VERSION_ID = $(VERSION)-$(GOOS)-$(GOARCH) +BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))) +BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH) +BUILD_TIME := $(shell date -u +%FT%T%z) +SHARD_WIDTH = 20 +COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD) +LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Variant=$(VARIANT) -X github.com/pilosa/pilosa/v2.Commit=$(COMMIT) -X github.com/pilosa/pilosa/v2.LatticeCommit=$(LATTICE_COMMIT)" +GOOS = $(shell go env GOOS) + +# Install pilosa-fsck +install: + go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) + +# Compile pilosa-fsck +build: + go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) + +REL = release-pilosa-fsck.$(COMMIT).$(GOOS) + +release: + mkdir $(REL) + cd release-pilosa-fsck; tar cf - . |(cd ../$(REL); tar xf - ) + go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) -o $(REL)/pilosa-fsck + tar cf - $(REL) | gzip > $(REL).tar.gz + rm -rf $(REL) + mv $(REL).tar.gz ../.. + +clean: + find . -name pilosa-fsck | xargs rm -f + rm -f release-pilosa-fsck*.tar.gz diff --git a/cmd/pilosa-fsck/fsck.go b/cmd/pilosa-fsck/fsck.go new file mode 100644 index 000000000..daf5b7b69 --- /dev/null +++ b/cmd/pilosa-fsck/fsck.go @@ -0,0 +1,952 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/dustin/go-humanize" + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/boltdb" + "github.com/pilosa/pilosa/v2/internal" + "github.com/pilosa/pilosa/v2/server" + "github.com/pkg/errors" + "github.com/zeebo/blake3" +) + +// pilosa-fsck : +// an external customer tool (originally for Q2) to do 2 jobs: +// Given a set of cluster backups (and their .id and .topology files) +// mounted on the same file system, we can: +// 1) scan for fragment differences between the primary and its replicas (default); or +// 2) repair those differences by overwriting the replcas with the primary fragments (if -fix is given). +// +// pilosa-chk is deliberately NOT a part of pilosa so that it can run without +// forcing a customer to upgrade or downgrade their installed version. + +// FsckConfig configures the dumpcols() and/or read() runs. +type FsckConfig struct { + Fix bool // -fix + FixCol bool // -fixcol + + Colkeydump bool // -col + + // -col column key dump only options: + Dir string + Index string + PartitionID int + ShowHeader bool + ShowKey bool + ShowID bool + + // not flags, just the Args() left after all other flags. Should be the list + // of pilosa (holder) directories for the cluster. + Dirs []string + + Verbose bool // -v + Quiet bool // -q + + // manual workaround for not having PilosaConfigPath, if really need be. + ReplicaN int // -replicas + PilosaConfigPath string // -config + + topo *pilosa.Topology +} + +// call DefineFlags before myflags.Parse() +func (cfg *FsckConfig) DefineFlags(fs *flag.FlagSet) { + fs.BoolVar(&cfg.Fix, "fix", false, "(warning: alters the backed-up node images on disk) copy primary data to replicas to create a consistent cluster. Implies -fixcol") + fs.BoolVar(&cfg.FixCol, "fixcol", false, "(warning: alters the backed-up node images on disk) repair string key translation tables. Skip repair of index data.") + fs.BoolVar(&cfg.Verbose, "v", false, "be very verbose during analysis") + fs.BoolVar(&cfg.Quiet, "q", false, "be very quiet") + + fs.IntVar(&cfg.ReplicaN, "replicas", 0, "(required) manually entered replicaN; the number of replicas maintained in the cluster. Must be the same as the [cluster] 'replicas = R' entry in the pilosa.conf file for the cluster.") + + fs.StringVar(&cfg.PilosaConfigPath, "config", "", "(required: -replicas or -config, with -config preferred) path to the pilosa.conf for the cluster (e.g. /etc/pilosa.conf)") + + fs.Usage = func() { + fmt.Fprintf(os.Stderr, "pilosa-fsck version: %v\n\n", pilosa.VersionInfo()) + fmt.Fprintf(os.Stderr, `Use: pilosa-fsck -replicas R {-fix} {-fixcol} {-q} {-v} /backup/1/.pilosa /backup/2/.pilosa ... /backup/N/.pilosa + + -fix + (warning: alters the backed-up node images on disk) copy primary data to replicas to create a consistent cluster. Implies -fixcol + + -fixcol + (warning: alters the backed-up node images on disk) repair string key translation tables. Skip repair of index data. + + -replicas R + (required) R is a positive integer, giving the replicaN or replicator factor for the cluster. This is + the number of replicas maintained in the cluster. Must be the same as the + [cluster] 'replicas = R' entry shared across all the pilosa.conf files on each node. + + -v + be very verbose during analysis + -q + be very quiet during analysis and repair + +`) + /* + key translation dump options, usually only used by developers debugging key translation: + + -col + (optional) display column keys (very long output) + -dir string + (optional; requires -col), one pilosa data dir to read (default "/home/ubuntu/.pilosa") + -header + (optional; requires -col), display header + -id + (optional; requires -col), dump reverse mapping id->key + -index string + (optional; requires -col), index name (default "i") + -key + (optional; requires -col), dump forward mapping key->id + -partition int + (optional; requires -col), partition id to dump + + */ + fmt.Fprintf(os.Stderr, ` +Welcome to pilosa-fsck. This is a scan and repair +tool that is modeled after the classic unix file +system utility fsck. + +WARNING: DO NOT RUN ON A LIVE SYSTEM. + +The most important point to remember is that analysis +and repair must be done *offline*. + +Just as fsck must be run on an unmounted disk, +pilosa-fsck must be run on a backup. It must +not be run on the directories where a live Pilosa system +is serving queries. Instead, take a backup first. +A backup is a set of N cluster-node directories that have been +copied from your live system. They must all +be visible and mounted on one filesystem together. + +pilosa-fsck can be run in scan-mode (without -fix or -fixcol), +or in repair-mode with -fix (or -fixcol). The console output +supplies a shell script documenting the analysis +and showing what data changes would be made. If a +fix has been requested, those fixes will have +been applied during the run. The output then serves +as documentation of what has been updated. If +a fix has not been requested (in other words, neither +-fix nor -fixcol was given) then no changes will +have been made to the backups. The fragment level +sync can be completed next by running the script +if you wish. The -fixcol fixes can only be +applied by doing a -fixcol run of pilosa-fsck. + +REQUIRED COMMAND LINE ARGUMENTS + +The paths to all the top-level pilosa +directories in a cluster must be given on the command +line. The -replicas R flag is also always required. It +must be correct for your cluser. Here R is the same as +the [cluster] stanza "replicas = R" line from your +pilosa.conf. + +Example: + +Suppose you are ready to run pilosa-fsck: +you have taken a backup of your four node Pilosa +cluster and stored it all on one filesystem with +all nodes visible and uncompressed. This +is a pre-requisite to running pilosa-fsck. +Let's suppose we have replication R = 3 set. +In this example, have stored our backed-up directories in + +/backup/molecula + +and the four node backups are in +subdirectories node1/ node2/ node3/ node4/ under this: + +/backup/molecula/node1/ +/backup/molecula/node1/.pilosa/.id +/backup/molecula/node1/.pilosa/.topology +/backup/molecula/node1/.pilosa/myindex + +/backup/molecula/node2/ +/backup/molecula/node2/.pilosa/.id +/backup/molecula/node2/.pilosa/.topology +/backup/molecula/node2/.pilosa/myindex + +/backup/molecula/node3/ +/backup/molecula/node3/.pilosa/.id +/backup/molecula/node3/.pilosa/.topology +/backup/molecula/node3/.pilosa/myindex + +/backup/molecula/node4/ +/backup/molecula/node4/.pilosa/.id +/backup/molecula/node4/.pilosa/.topology +/backup/molecula/node4/.pilosa/myindex + +NOTE: your .pilosa directories need not be named .pilosa. They can +be something else, such as when the -d flag to pilosa server was used. +The .id and .topology and index directories must be found directly underneath. + +Then a typical invocation to scan a cluster backup for issues: + +$ cd /backup/molecula/ +$ pilosa-fsck -replicas 3 node1/.pilosa node2/.pilosa node3/.pilosa node4/.pilosa + +A typical invocation to repair the replication in the same backup: + +$ pilosa-fsck -replicas 3 -fix node1/.pilosa node2/.pilosa node3/.pilosa node4/.pilosa + +In both cases, the .id and .topology files must +be present in the backups. + +KEY REPAIR NOTE + +While the output of pilosa-fsck wihtout -fix or -fixcol +gives a script showing the index fragment repair operations +that can be applied by (cp/rm) shell commands subsequently, +this script alone is an incomplete repair. It does not +address string key tranlsation repairs. For a complete repair, +a run of pilosa-fsck with the -fixcol or -fix flags +will be required. + +A note about the -fixcol key translation repairs: these are fine +grained operations on the internal databases that do not have +corresponding (cp/rm) shell commands. Therefore a run of pilosa-fsck +with the -fix or -fixcol flag is required to repair these. +`) + } +} + +// call c.ValidateConfig() after myflags.Parse() +func (c *FsckConfig) ValidateConfig() error { + if c.Fix { + c.FixCol = true + } + if c.ReplicaN == 0 && c.PilosaConfigPath == "" { + return fmt.Errorf("must supply -replicas with the replica count from your pilosa.conf (positive integer count)") + } + + if c.ReplicaN == 0 && c.PilosaConfigPath != "" { + + if !FileExists(c.PilosaConfigPath) { + return fmt.Errorf(" -config path '%v' does not exist", c.PilosaConfigPath) + } + by, err := ioutil.ReadFile(c.PilosaConfigPath) + if err != nil { + return fmt.Errorf("error: could not read the -config path '%v': '%v'", c.PilosaConfigPath, err) + } + srvcfg, err := server.ParseConfig(string(by)) + if err != nil { + //vv("warning: -config path '%v' problem, could not parse toml: '%v'", c.PilosaConfigPath, err) + + // fall back to manual parsing of config + lines := strings.Split(string(by), "\n") + clusterStart := -1 + for i, line := range lines { + if strings.Contains(line, `[cluster]`) { + clusterStart = i + } + if i > clusterStart { + if strings.Contains(line, "replicas") { + split := strings.Split(line, "=") + ns := strings.TrimSpace(split[1]) + n, err := strconv.Atoi(ns) + if err != nil { + return fmt.Errorf("error: could not parse the replicaN from line %v in -config path '%v' (%v): '%v'", i+1, c.PilosaConfigPath, line, err) + } + c.ReplicaN = n + } + } + } + } else { + c.ReplicaN = srvcfg.Cluster.ReplicaN + } + if c.ReplicaN == 0 { + return fmt.Errorf("error: -config path '%v' did not list the Replica count: cannot be 0. See the [cluster] section, the 'replicas = R' line.", c.PilosaConfigPath) + } + //vv("c.ReplicaN = %v", c.ReplicaN) + } + return nil +} + +var ProgramName = "pilosa-fsck" + +func main() { + + myflags := flag.NewFlagSet(ProgramName, flag.ContinueOnError) + cfg := &FsckConfig{} + cfg.DefineFlags(myflags) + + err := myflags.Parse(os.Args[1:]) + if err != nil { + fmt.Fprintf(os.Stderr, "\n%v\n", err.Error()) + os.Exit(1) + } + err = cfg.ValidateConfig() + if err != nil { + fmt.Fprintf(os.Stderr, "%s error: %s\n", ProgramName, err) + os.Exit(1) + } + dirs := myflags.Args() + nDir := len(dirs) + if nDir <= 0 && !cfg.Colkeydump { + fmt.Fprintf(os.Stderr, "error: %v command line arguments missing error: provide all of the top-level pilosa directories for the cluster as command line arguments.\n", ProgramName) + os.Exit(1) + } + + cmdline := strings.Join(os.Args, " ") + + // make sure all the dir are distinct + dup := make(map[string]bool) + for _, dir := range dirs { + if dup[dir] { + fmt.Fprintf(os.Stderr, "%v error: duplicate data directory '%v' given in command line '%v'. Each backup directory must be distinct.\n", ProgramName, dir, cmdline) + os.Exit(1) + } else { + dup[dir] = true + } + } + + fmt.Fprintf(os.Stdout, "#!/bin/bash\n\n# pilosa-fsck version: %v\n", pilosa.VersionInfo()) + cwd, err := os.Getwd() + if err != nil { + fmt.Fprintf(os.Stderr, "error: could not read current dir: '%v'\n", err) + os.Exit(1) + } + fmt.Fprintf(os.Stdout, "# cwd: %v\n", cwd) + fmt.Fprintf(os.Stdout, "# command line: %v\n", cmdline) + t0 := time.Now() + fmt.Fprintf(os.Stdout, "# started at %v\n\n", t0.Format(RFC3339MsecTz0)) + defer func() { + fmt.Fprintf(os.Stdout, "# finished at %v (elapsed %v)\n\n", time.Now().Format(RFC3339MsecTz0), time.Since(t0)) + }() + cfg.Dirs = dirs + + _, err = cfg.Run() + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } +} + +func (cfg *FsckConfig) Run() (fixNeeded bool, err error) { + + if cfg.Colkeydump { + cfg.dumpcols() + } + + perNodeIndexMaps, clusterNodes, ats, err := cfg.read() + if err != nil { + return false, err + } + + if cfg.FixCol { + err := cfg.RepairTranslationStores(ats) + if err != nil { + return false, fmt.Errorf("error fixing key translation stores with cfg.RepairTranslationStores(): '%v'\n", err) + } + } + + //vv("perNodeIndexMaps='%#v', clusterNodes='%#v'", perNodeIndexMaps, clusterNodes) + + fixme, reports, err := cfg.analyze(clusterNodes, perNodeIndexMaps, ats) + if err != nil { + return false, fmt.Errorf("error in FsckConfig.analyze(): '%v'", err) + } + fixNeeded = ats.RepairNeeded || fixme + for _, report := range reports { + fmt.Printf("%v\n", report) + } + if len(reports) == 0 { + fmt.Fprintf(os.Stderr, "pilosa-fsck: no index found to analyze. cmdline was: %v\n", strings.Join(os.Args, " ")) + } + return +} + +var _ = (&FsckConfig{}).dumpAts + +func (cfg *FsckConfig) dumpAts(ats *pilosa.AllTranslatorSummary) { + fmt.Printf("# dumpAts: RepairNeeded=%v\n", ats.RepairNeeded) + for _, sum := range ats.Sums { + fmt.Printf("# sum = '%#v'\n", sum) + } + +} + +type group struct { + elem []*pilosa.TranslatorSummary + partitionID int +} + +func (g *group) String() (s string) { + for i, e := range g.elem { + s += fmt.Sprintf("partition %v, group elem [%v] out of %v: %v\n", g.partitionID, i, len(g.elem), e.String()) + } + return +} + +func (cfg *FsckConfig) RepairTranslationStores(ats *pilosa.AllTranslatorSummary) (err error) { + + verbose := cfg.Verbose + + m := make(map[int]*group) + for _, sum := range ats.Sums { + if !sum.IsColKey { + continue + } + grp := m[sum.PartitionID] + if grp == nil { + grp = &group{ + partitionID: sum.PartitionID, + } + m[sum.PartitionID] = grp + } + grp.elem = append(grp.elem, sum) + } + + for partitionID, group := range m { + _ = partitionID + prim := -1 + keyCount := 0 + for k, e := range group.elem { + if e.IsPrimary { + prim = k + } + keyCount += e.KeyCount + } + if prim == -1 { + panic(fmt.Sprintf("no primary found for group '%v'", group.String())) + } + + primary := group.elem[prim] + primaryChecksum := primary.Checksum + for _, e := range group.elem { + if e.IsPrimary { + continue + } + // is e a replica? not necessarily! have to check. + if !e.IsReplica { + //if verbose { + // since this will happen even on a fix point, where it is already empty, + // we don't report it again. + //fmt.Printf("# non-replica should have no data: creating an empty translation store here at '%v'\n", e.StorePath) + //} + err := os.RemoveAll(e.StorePath) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("RepairTranslationStores() os.RemoveAll(e.StorePath='%v')", e.StorePath)) + } + store, err := boltdb.OpenTranslateStore(e.StorePath, e.Index, e.Field, e.PartitionID, pilosa.DefaultPartitionN) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("RepairTranslationStores() create empty boldtdb: boltdb.OpenTranslateStore e.StorePath='%v'", e.StorePath)) + } + err = store.Close() + if err != nil { + return errors.Wrap(err, fmt.Sprintf("RepairTranslationStores() closing empty boltdb at path '%v'", e.StorePath)) + } + continue + } + // INVAR: e is a replica for this paritionID. + // Copy from primary if checksums are different. + if e.Checksum != primaryChecksum { + from := group.elem[prim].StorePath + dest := e.StorePath + if verbose { + fmt.Printf("# e.Checksum '%v' != primaryChecksum '%v': copying from primary translation store '%v' -> '%v'\n", e.Checksum, primaryChecksum, from, dest) + } + err := cp(from, dest) + if err != nil { + return fmt.Errorf("error: could not copy from primary '%v' to replica translation store '%v': '%v' ... try to keep going...\n", from, dest, err) + } + } + } + } + return nil +} + +func (cfg *FsckConfig) dumpcols() { + + verbose := cfg.Verbose + quiet := cfg.Quiet + _, _ = verbose, quiet + + dir := cfg.Dir + index := cfg.Index + partitionID := cfg.PartitionID + showKey := cfg.ShowKey + showID := cfg.ShowID + + if !quiet { + fmt.Printf("# dumpcols: opening dir '%v'... this may take a few minutes...\n", dir) + } + holder := pilosa.NewHolder(dir, nil) + holder.OpenTranslateStore = boltdb.OpenTranslateStore + err := holder.Open() + if err != nil { + log.Fatal(err) + } + if cfg.ShowHeader { + fmt.Println("# columnKey columId") + } + id_key := make(map[uint64]string) + key_id := make(map[string]uint64) + for _, idx := range holder.Indexes() { + fmt.Printf("# Looking '%v'\n", idx.Name()) + if idx.Name() == index { + store := idx.TranslateStore(partitionID) + fmt.Printf("# Key By ID partitionID = %v\n", partitionID) + err := store.KeyWalker(func(key string, col uint64) { + key_id[key] = col + if showKey { + fmt.Printf("# '%v' %v shard: %v partition: %v\n", key, col, col/pilosa.ShardWidth, partitionID) + } + }) + panicOn(err) + } + } + for _, idx := range holder.Indexes() { + if idx.Name() == index { + store := idx.TranslateStore(partitionID) + //fmt.Printf("# ID ByKey\n") + err := store.IDWalker(func(key string, col uint64) { + id_key[col] = key + if showID { + fmt.Printf("# '%v' %v\n", key, col) + } + }) + panicOn(err) + } + } + fmt.Printf("# k: %d i: %d\n", len(key_id), len(id_key)) + fmt.Println("id_key") + for k, v := range id_key { + l, ok := key_id[v] + if ok { + if k != l { + fmt.Printf("# X: %v %v %v\n", k, l, v) + } + } else { + fmt.Printf("# key not in id %v\n", v) + } + } + fmt.Println("key_id") + for k, v := range key_id { + l, ok := id_key[v] + if ok { + if k != l { + fmt.Printf("# T: %v %v %v\n", k, l, v) + } + } else { + fmt.Printf("# id not in key %v\n", v) + } + } +} + +func (cfg *FsckConfig) read() (perNodeIndexMaps []map[string]*pilosa.IndexFragmentSummary, clusterNodes []string, final *pilosa.AllTranslatorSummary, err error) { + + final = pilosa.NewAllTranslatorSummary() + + dirs := cfg.Dirs + for _, dir := range dirs { + idx2frag, nodeID, atsNode, err := cfg.readOneDir(dir) + if err != nil { + return nil, nil, nil, err + } + final.Append(atsNode) + clusterNodes = append(clusterNodes, nodeID) + perNodeIndexMaps = append(perNodeIndexMaps, idx2frag) + } + return +} + +func (cfg *FsckConfig) readOneDir(dir string) (idx2frag map[string]*pilosa.IndexFragmentSummary, nodeID string, atsNode *pilosa.AllTranslatorSummary, err error) { + + verbose := cfg.Verbose + quiet := cfg.Quiet + + if !quiet { + fmt.Printf("# opening dir '%v'... this may take a few minutes... fixcol=%v\n\n", dir, cfg.FixCol) + } + + jmphasher := &pilosa.Jmphasher{} + partitionN := pilosa.DefaultPartitionN + replicaN := cfg.ReplicaN + topo, err := loadTopology(dir, jmphasher, partitionN, replicaN) + if err != nil { + return nil, "", nil, err + } + cfg.topo = topo + //vv("topo = '%#v'", topo) + nodeIDs := topo.GetNodeIDs() + //vv("nodeIDs = '%#v'", nodeIDs) + nNodes := len(nodeIDs) + nDir := len(cfg.Dirs) + if nDir != nNodes { + return nil, "", nil, fmt.Errorf("command line had %v directories (%#v) but the .topology had %v nodes (%#v)", nDir, cfg.Dirs, nNodes, nodeIDs) + } + + holder := pilosa.NewHolder(dir, nil) + holder.OpenTranslateStore = boltdb.OpenTranslateStore + + nodeID, err = holder.LoadNodeID() + panicOn(err) + //vv("nodeID = '%v'", nodeID) + err = holder.Open() + + if err != nil { + log.Fatal(err) + } + + if !quiet { + fmt.Printf("\n# calculating hashes of row and column key translation maps on data from dir '%v'...\n", dir) + } + var indexes []*pilosa.Index + + const checkKeys = true + atsNode = pilosa.NewAllTranslatorSummary() + for _, idx := range holder.Indexes() { + asum, err := idx.ComputeTranslatorSummary(verbose, checkKeys, cfg.FixCol, topo, nodeID) + if err != nil { + log.Fatal(err) + } + atsNode.Append(asum) + indexes = append(indexes, idx) + } + atsNode.Sort() + + hasher := blake3.New() + if !quiet { + fmt.Printf("\n# summary of col/row translations in dir: %v:\n", dir) + } + for _, sum := range atsNode.Sums { + if !quiet { + fmt.Printf("# index: %v partitionID: %v blake3-%v keyCount: %v idCount: %v\n", sum.Index, sum.PartitionID, sum.Checksum, sum.KeyCount, sum.IDCount) + } + _, _ = hasher.Write([]byte(sum.Checksum)) + } + + var buf [16]byte + _, _ = hasher.Digest().Read(buf[0:]) + + if !quiet { + fmt.Printf("# all-checksum = blake3-%x\n", buf) + } + + // fragment analysis + + showBits := false + showOpsLog := false + idx2frag = make(map[string]*pilosa.IndexFragmentSummary) // on this node. + for _, idx := range indexes { + if verbose { + fmt.Printf("# ==============================\n") + fmt.Printf("# index: %v\n", idx.Name()) + fmt.Printf("# ==============================\n") + } + frgsum := idx.WriteFragmentChecksums(os.Stdout, showBits, showOpsLog, topo, verbose) + frgsum.Dir = dir + frgsum.NodeID = nodeID + idx2frag[idx.Name()] = frgsum + } + + _ = holder.Close() + + //vv("idx2frag = '%v'", idx2frag) // tons of output. see 1234.out.full for examaple. + + return +} + +// from cluster.go:1924 +func loadTopology(holderDir string, hasher pilosa.Hasher, partitionN, replicaN int) (*pilosa.Topology, error) { + + buf, err := ioutil.ReadFile(filepath.Join(holderDir, ".topology")) + if err != nil { + return nil, err + } + + var pb internal.Topology + err = proto.Unmarshal(buf, &pb) + if err != nil { + return nil, err + } + + return pilosa.DecodeTopology(&pb, hasher, partitionN, replicaN, nil) +} + +func (cfg *FsckConfig) analyze(clusterNodes []string, perNodeIndexMaps []map[string]*pilosa.IndexFragmentSummary, ats *pilosa.AllTranslatorSummary) (fixNeeded bool, reports []string, err error) { + + verbose := cfg.Verbose + quiet := cfg.Quiet + _, _ = verbose, quiet + + allIndex := make(map[string]bool) + for _, mp := range perNodeIndexMaps { + for index := range mp { + allIndex[index] = true + } + } + if !quiet { + vv("allIndex = '%#v'", allIndex) + } + for index := range allIndex { + if !quiet { + vv("on index '%v'", index) + } + nodes2fragsum := make(map[string]*pilosa.IndexFragmentSummary) + for _, mp := range perNodeIndexMaps { + sum := mp[index] + if sum == nil { + continue + } + nodes2fragsum[sum.NodeID] = sum + } + fixme, report, err := cfg.analyzeThisIndex(index, nodes2fragsum, ats) + if err != nil { + return false, reports, fmt.Errorf("error in analyze of index '%v': '%v'", index, err) + } + fixNeeded = fixNeeded || fixme + reports = append(reports, report) + } + return fixNeeded, reports, nil +} + +func (cfg *FsckConfig) analyzeThisIndex( + index string, + nodes2fragsum map[string]*pilosa.IndexFragmentSummary, + ats *pilosa.AllTranslatorSummary, +) (fixNeeded bool, report string, err error) { + + verbose := cfg.Verbose + quiet := cfg.Quiet + _, _ = verbose, quiet + + var removedBytes int64 + var copiedBytes int64 + var changedFiles int64 + var totalFiles int64 + var overwrittenBytes int64 + var totalBytes int64 + + if !quiet { + vv("top of analyzeThisIndex(index='%v'); len of nodes2fragsum = %v; nodes2fragsum='%#v'", + index, len(nodes2fragsum), nodes2fragsum) + } + + for node, sum := range nodes2fragsum { + if !quiet { + fmt.Printf("# on node '%v'\n", node) + } + // do they disagree on who is the primary? + // for each fragment, do they disagree on the checksum? + + // Q: which nodes are supposed to have data, and which + // nodes are not supposed to have data? + + // loopFragSum: + for relpath, fragsum := range sum.RelPath2fsum { + fragsum.NodeID = node + totalFiles++ + //vv("checking %v on node %v", relpath, node) + + replicas, nonReplicas := cfg.topo.GetReplicasForPrimary(fragsum.Primary) + _, _ = replicas, nonReplicas + //vv("replicas = '%#v'", replicas) + //vv("nonReplicas = '%#v'", nonReplicas) + + err := cfg.verifyReplicasAvailable(replicas, nonReplicas, nodes2fragsum, fragsum) + if err != nil { + return fixNeeded, "", err + } + + // find the primary's checksum + primaryChecksum := "" + var primaryFragSum *pilosa.FragSum + for node, isPrimary := range replicas { + if isPrimary { + primarySum := nodes2fragsum[node] + primaryFragSum = primarySum.RelPath2fsum[relpath] + if primaryFragSum == nil { + + // This seems clear indication that we have the topology wrong. + // When the topology is right, there are NO errors of this kind. + // + msg := fmt.Sprintf("# ugh. BAD. Stopping because any fix will be wrong. We see wrong -replica %v param, OR the .id files are mis-assigned with respect to the topology file. Could not find primary FragSum for relpath = '%v'. replicas = '%#v', nonReplicas = '%#v'\n", cfg.ReplicaN, relpath, replicas, nonReplicas) + vv(msg) + fmt.Fprintf(os.Stderr, "%v\n", msg) + panic(msg) // stop. the fixes are going to be wrong. + } else { + primaryChecksum = primaryFragSum.Checksum + primaryFragSum.NodeID = node + primaryFragSum.ScanDone = true + } + break + } + } + if primaryChecksum == "" { + return fixNeeded, "", fmt.Errorf("could not find primary replica??? replicas='%#v', nodes2fragsum='%v'; for fragsum='%#v'", replicas, nodes2fragsum, fragsum) + } + + // is this a non-replica? + _, isNon := nonReplicas[fragsum.NodeID] + if isNon { + removedBytes += FileSize(fragsum.AbsPath) + changedFiles++ + + //vv("yes, is nonReplica: fragsum.NodeID='%v'", fragsum.NodeID) + if !quiet { + fmt.Printf("rm %v #### REPAIR REMOVE data from non-replica at node '%v' (fragsum='%#v') vs. primary (%#v)\n\n", fragsum.AbsPath, node, fragsum, primaryFragSum) + } + if cfg.Fix { + err := os.Remove(fragsum.AbsPath) + if err != nil { + return fixNeeded, "", fmt.Errorf("error removing non-replica extra fragment '%v': '%v'", fragsum.AbsPath, err) + } + } + } else { + presz := FileSize(fragsum.AbsPath) + totalBytes += presz + + checksum := fragsum.Checksum + if checksum != primaryChecksum { + copiedBytes += FileSize(primaryFragSum.AbsPath) + changedFiles++ + overwrittenBytes += presz + + if !quiet { + fmt.Printf("cp %v %v #### REPAIR OVERWRITE replica at node '%v' (%#v) from primary '%v' (%#v)\n", primaryFragSum.AbsPath, fragsum.AbsPath, node, fragsum, primaryFragSum.NodeID, primaryFragSum) + } + if cfg.Fix { + err := cp(primaryFragSum.AbsPath, fragsum.AbsPath) + if err != nil { + return fixNeeded, "", fmt.Errorf("error copying from '%v' to '%v': '%v'", + primaryFragSum.AbsPath, fragsum.AbsPath, err) + } + } + } + } + fragsum.ScanDone = true + } + } + nDir := len(nodes2fragsum) + + keyCount, idCount := cfg.getKeyIDCounts(ats) + + fixNeeded = changedFiles > 0 || ats.RepairNeeded + var actionTaken string + var wouldBe string + if cfg.Fix || cfg.FixCol { + if fixNeeded { + actionTaken = "*REPAIRS WERE MADE TO THE BACKUPS*" + wouldBe = "sync repairs made:" + } else { + wouldBe = "" + actionTaken = "NO REPAIR NEEDED." + } + } else { + if fixNeeded { + wouldBe = "sync actions that would be taken under -fix:" + actionTaken = "*REPAIRS NEEDED BUT WERE NOT APPLIED* ; pilosa-fsck -fix and -fixcol were omitted." + } else { + wouldBe = "" + actionTaken = "NO REPAIR NEEDED." + } + } + var fragUpdate string + if changedFiles > 0 { + fragUpdate = fmt.Sprintf(` +# %v +# copied bytes: %v +# file bytes overwritten: %v +# new bytes added: %v +# new bytes is %0.01f%% of %v total bytes +# removed %v bytes from non-replicas +# changed file count %v (%0.01f%%; total files=%v) +# +`, wouldBe, humanize.Comma(copiedBytes), humanize.Comma(overwrittenBytes), humanize.Comma(copiedBytes-overwrittenBytes), 100*float64(copiedBytes-overwrittenBytes)/float64(totalBytes), humanize.Comma(totalBytes), humanize.Comma(removedBytes), changedFiles, 100*float64(changedFiles)/float64(totalFiles), humanize.Comma(totalFiles)) + } + + report = fmt.Sprintf(` +# ======================================================== +# pilosa-fsck final report +# +# run with -fix: %v +# -fixcol: %v +# +# index examined: '%v' +# +# nodes examined: %v +# -replicas %v replication factor used +# +# feature data examined: %v bytes +# feature files examined: %v files +# +# key-translation-stores examined: %v +# key-count: %v over all replicas +# id-count: %v over all replicas +# +# %v +# %v +# ======================================================== +`, + cfg.Fix, cfg.FixCol, index, nDir, cfg.ReplicaN, humanize.Comma(totalBytes), humanize.Comma(totalFiles), humanize.Comma(int64(nDir*pilosa.DefaultPartitionN)), humanize.Comma(int64(keyCount)), humanize.Comma(int64(idCount)), actionTaken, fragUpdate) + return +} + +func (cfg *FsckConfig) verifyReplicasAvailable(replicas, nonReplicas map[string]bool, nodes2fragsum map[string]*pilosa.IndexFragmentSummary, fragsum *pilosa.FragSum) error { + for node := range replicas { + if nodes2fragsum[node] == nil { + return fmt.Errorf("error: node '%v' needed for a replica set was not availabe. Did you give ALL the directories for your cluster on the command line at once? In nodes2fragsum '%#v' (replicas: '%#v'; non-replicas '%#v') for fragsum '%v'", node, nodes2fragsum, replicas, nonReplicas, fragsum) + } + } + return nil +} + +func cp(fromPath, toPath string) (err error) { + tmpTo := toPath + ".fsck.tmp" + toFd, err := os.Create(tmpTo) + if err != nil { + return err + } + defer toFd.Close() + fromFd, err := os.Open(fromPath) + if err != nil { + return err + } + defer fromFd.Close() + + _, err = io.Copy(toFd, fromFd) + if err != nil { + return err + } + err = toFd.Close() + if err != nil { + return err + } + return os.Rename(tmpTo, toPath) +} + +func (cfg *FsckConfig) getKeyIDCounts(ats *pilosa.AllTranslatorSummary) (keyCount, idCount int) { + for _, sum := range ats.Sums { + keyCount += sum.KeyCount + idCount += sum.IDCount + } + return +} diff --git a/cmd/pilosa-fsck/fsck_test.go b/cmd/pilosa-fsck/fsck_test.go new file mode 100644 index 000000000..b8002f78f --- /dev/null +++ b/cmd/pilosa-fsck/fsck_test.go @@ -0,0 +1,365 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "io/ioutil" + "reflect" + "strconv" + "testing" + "time" + + "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/boltdb" + "github.com/pilosa/pilosa/v2/hash" + "github.com/pilosa/pilosa/v2/http" + "github.com/pilosa/pilosa/v2/server" + "github.com/pilosa/pilosa/v2/test" +) + +func Test_Repair(t *testing.T) { + + // a) setup 1 primary + 3 replicas of disagree-ing cluster dirs. + + nNodes := 4 + nReplicas := 3 + + name := t.Name() + var nodeid []string + for i := 0; i < nNodes; i++ { + // work around a bug in the test.MustRunCluster that corrupts + // the .topology file if we only join name with one "_" underscore. + nodeid = append(nodeid, name+"__"+strconv.Itoa(i)) + } + + c := test.MustRunCluster(t, nNodes, + []server.CommandOption{ + server.OptCommandServerOptions( + pilosa.OptServerNodeID(nodeid[0]), + pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), + pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerReplicaN(nReplicas), + )}, + []server.CommandOption{ + server.OptCommandServerOptions( + pilosa.OptServerNodeID(nodeid[1]), + pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), + pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerReplicaN(nReplicas), + )}, + []server.CommandOption{ + server.OptCommandServerOptions( + pilosa.OptServerNodeID(nodeid[2]), + pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), + pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerReplicaN(nReplicas), + )}, + []server.CommandOption{ + server.OptCommandServerOptions( + pilosa.OptServerNodeID(nodeid[3]), + pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), + pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerReplicaN(nReplicas), + )}, + ) + // note: do not defer c.Close() here. We manually close below. + + var nodes []*test.Command + var dirs []string + for i := 0; i < nNodes; i++ { + nd := c.GetNode(i) + nodes = append(nodes, nd) + dirs = append(dirs, nd.Server.Holder().Path()) + } + + ctx := context.Background() + + index := "rick" + fieldName := "f" + + idx, err := nodes[0].API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true, TrackExistence: true}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + if idx.CreatedAt() == 0 { + t.Fatal("index createdAt is empty") + } + + field, err := nodes[0].API.CreateField(ctx, index, fieldName, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100)) + if err != nil { + t.Fatalf("creating field: %v", err) + } + if field.CreatedAt() == 0 { + t.Fatal("field createdAt is empty") + } + + rowID := uint64(1) + timestamp := int64(0) + + // Generate some keyed records. + rowIDs := []uint64{} + timestamps := []int64{} + N := 10 + for i := 1; i <= N; i++ { + rowIDs = append(rowIDs, rowID) + timestamps = append(timestamps, timestamp) + } + + // Keys are sharded so ordering is not guaranteed. + colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"} + + colKeys = colKeys[:N] + + // Import data with keys to the coordinator (node0) and verify that it gets + // translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher) + req := &pilosa.ImportRequest{ + Index: index, + IndexCreatedAt: idx.CreatedAt(), + Field: fieldName, + FieldCreatedAt: field.CreatedAt(), + + // even though this says Shard: 0, that won't matter. The column keys + // get hashed and that decides the actual shard. + Shard: 0, + RowIDs: rowIDs, + ColumnKeys: colKeys, + Timestamps: timestamps, + } + + qcx := nodes[0].API.Txf().NewQcx() + + if err := nodes[0].API.Import(ctx, qcx, req); err != nil { + t.Fatal(err) + } + panicOn(qcx.Finish()) + + pql := fmt.Sprintf("Row(%s=%d)", fieldName, rowID) + + // Query node0. + if res, err := nodes[0].API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil { + t.Fatal(err) + } else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) { + t.Fatalf("expected colKeys='%#v'; observed column keys: %#v", colKeys, keys) + } + + // Query node1. + if err := test.RetryUntil(5*time.Second, func() error { + if res, err := nodes[1].API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil { + return err + } else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) { + return fmt.Errorf("unexpected column keys: %#v", keys) + } + return nil + }); err != nil { + t.Fatal(err) + } + + // end of setup. + + // partitionID in use: 6, 31, 57, 133, 185, 235 + targetPartition := 31 // which partitionID we mess with. + targetNode := nodes[0] // this is the first replica. + // 0 first replica + // 1 second replica + // 2 -- not a replica + // 3 primary + + cfg := &FsckConfig{ + Fix: false, + FixCol: false, + Quiet: true, + //Verbose: true, + ReplicaN: nReplicas, + Dirs: dirs, + } + panicOn(cfg.ValidateConfig()) + + // for this test, mess up a replica that is not the primary. + + h := targetNode.API.Holder() + idx = h.Index(index) + store := idx.TranslateStore(targetPartition) + fwd, rev := getFwdRev(store, targetPartition) + //vv("targetPartition=%v, store.PartitionID=%v, before corruption, fwd='%#v', rev='%#v'", targetPartition, store.PartitionID, fwd, rev) + + // # fsck_test.go:288 2020-10-01T13:39:57.718995-05:00 partition 31, key 'col5' -> db00001 + presz := len(rev) + delete(rev, fwd["col5"]) + postsz := len(rev) + + if postsz == presz { + panic("did not delete any key!") + } + + bolt := store.(*boltdb.TranslateStore) + //vv("pre corruption, bolt = '%v'", fileChecksum(bolt.Path)) + //bolt.DumpBolt("pre-corruption") + + if err := bolt.SetFwdRevMaps(nil, fwd, rev); err != nil { + t.Fatal(err) + } + //vv("post corruption, bolt = '%v'", fileChecksum(bolt.Path)) + //bolt.DumpBolt("post-corruption") + + //fwd3, rev3 := getFwdRev(store, targetPartition) + //vv("after corruption, fwd='%#v', rev='%#v'", fwd3, rev3) + + for _, nd := range nodes { + nd.Command.Close() + } + //panicOn(bolt.Open()) + //bolt.DumpBolt("post-corruption, after Close. bolt:") + //bolt.Close() + + //chksums := getChecksums(dirs, cfg, targetPartition) + //vv("post corruption, pre repair chksums = '%#v'", chksums) + + // first we check that the corruption can be detected + // by our test with the checksums. + + chk, err := check(dirs, cfg, targetPartition) + _ = chk + //vv("pre-fix, chk='%v'; err='%v'", chk, err) + + if err == nil { + panic("expected to see checksums not match! but no corruption detected.") + } + + // b) running in reporting mode only should report that a fix is needed. + fixNeeded, err := cfg.Run() + panicOn(err) + if !fixNeeded { + panic("fix should be needed now, before repair") + } + + // c) run the fix. + cfg.Fix = true + cfg.FixCol = true + fixNeeded, err = cfg.Run() + panicOn(err) + if !fixNeeded { + panic("fix should be marked needed if repair was made") + } + + // d) check that the replicas all look like the primary. + + //chksums = getChecksums(dirs, cfg, targetPartition) + //vv("after repair chksums = '%#v'", chksums) + + chk, err = check(dirs, cfg, targetPartition) + _ = chk + //vv("chk = '%v' after repair; err='%v'", chk, err) + panicOn(err) + + // e) run again, should see no fix needed. + fixNeeded, err = cfg.Run() + panicOn(err) + if fixNeeded { + panic("should see no fix needed after the prior repair") + } + +} + +func getFwdRev(store pilosa.TranslateStore, partitionID int) (fwd map[string]uint64, rev map[uint64]string) { + fwd = make(map[string]uint64) + rev = make(map[uint64]string) + _ = store.KeyWalker(func(key string, col uint64) { + //vv("partition %v, key '%v' -> %x", partitionID, key, col) + fwd[key] = col + }) + _ = store.IDWalker(func(key string, col uint64) { + //vv("partition %v, id %x -> '%v'", partitionID, col, key) + rev[col] = key + }) + return +} + +func check(dirs []string, cfg *FsckConfig, targetPartition int) (chksum string, err error) { + + firstChecksum := "" + firstDir := "" + quiet := cfg.Quiet + defer func() { + cfg.Quiet = quiet + }() + cfg.Quiet = true + for i := range dirs { + dir := dirs[i] + _, _, ats, err := cfg.readOneDir(dir) + panicOn(err) + + for _, s := range ats.Sums { + if s.PartitionID != targetPartition { + continue + } + if s.IsPrimary || s.IsReplica { + chksum := s.Checksum + if firstChecksum == "" { + firstChecksum = chksum + firstDir = dir + } else { + if chksum != firstChecksum { + return chksum, fmt.Errorf("bolt chksum on node %v '%v' disagrees with '%v' on '%v'", dir, chksum, firstChecksum, firstDir) + } + } + } + } + } + return firstChecksum, nil +} + +var _ = getChecksums + +func getChecksums(dirs []string, cfg *FsckConfig, targetPartition int) (chksum []string) { + + for i := range dirs { + dir := dirs[i] + _, _, ats, err := cfg.readOneDir(dir) + panicOn(err) + + for _, s := range ats.Sums { + if s.PartitionID != targetPartition { + continue + } + chksum = append(chksum, s.Checksum) + } + } + return +} + +/* on shardwidth 20 +# fsck_test.go:211 2020-09-30T17:19:05.823278-05:00 partition 6, key 'col2' -> dc00001 +# fsck_test.go:214 2020-09-30T17:19:05.823309-05:00 partition 6, id dc00001 -> 'col2' +# fsck_test.go:211 2020-09-30T17:19:05.823430-05:00 partition 31, key 'col5' -> db00001 +# fsck_test.go:214 2020-09-30T17:19:05.823447-05:00 partition 31, id db00001 -> 'col5' +# fsck_test.go:211 2020-09-30T17:19:05.823970-05:00 partition 57, key 'col10' -> 5d00001 +# fsck_test.go:214 2020-09-30T17:19:05.823998-05:00 partition 57, id 5d00001 -> 'col10' +# fsck_test.go:211 2020-09-30T17:19:05.827007-05:00 partition 133, key 'col7' -> d900001 +# fsck_test.go:214 2020-09-30T17:19:05.827071-05:00 partition 133, id d900001 -> 'col7' +# fsck_test.go:211 2020-09-30T17:19:05.827549-05:00 partition 185, key 'col3' -> dd00001 +# fsck_test.go:214 2020-09-30T17:19:05.827573-05:00 partition 185, id dd00001 -> 'col3' +# fsck_test.go:211 2020-09-30T17:19:05.827792-05:00 partition 235, key 'col9' -> d700001 +# fsck_test.go:214 2020-09-30T17:19:05.827809-05:00 partition 235, id d700001 -> 'col9' +*/ + +var _ = fileChecksum + +func fileChecksum(path string) string { + by, err := ioutil.ReadFile(path) + panicOn(err) + return hash.Blake3sum16(by) +} diff --git a/cmd/pilosa-fsck/release-pilosa-fsck/.gitignore b/cmd/pilosa-fsck/release-pilosa-fsck/.gitignore new file mode 100644 index 000000000..a08586f1c --- /dev/null +++ b/cmd/pilosa-fsck/release-pilosa-fsck/.gitignore @@ -0,0 +1 @@ +pilosa-fsck diff --git a/cmd/pilosa-fsck/release-pilosa-fsck/backups.tar.gz b/cmd/pilosa-fsck/release-pilosa-fsck/backups.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..28b08adbdd6fc719c85ba9f15ccc3010fca43eb8 GIT binary patch literal 248477 zcmd>nX+RU#8n&&y#k!%bidH3V6{h?N~OT2!p4sHjvB zVnM-*5EUfIHYy?_Vgy7~gs_J_KnPi9&UYq=T7gNw+wb15lbatJk`OXE=e*1FzVGue zf(8v}+vC9;Ft8ctxy~<(9P4rVeCvT9r-dA_-8*d1^|6k%wpYJ*EY3I{Z8qSYxj&rK z3z%7x{dpFn@wanf)7H$)>3IH|**Tjz4!*=7Fw9}=sdwIe@0W8&7C8@{qx(+Rwl&{H zB!`{;(xLtLQSG?j?6BCCC6Ygurl$Jw1-_Ra%DPf<>xHvIehOMU#H`?t(xm$HoYF~O zCr)YFy*9piTJYKii{~!pZd=0x24C7$9+B<#z4%glx_pvLwv2J9L`md?+_KsJ4VWSQ zf(7(i%+`lp2r6O$W`!$3#AY>FJj08@d@lkwVC6I^iFFa1+7+%`tEKQ2mHB+o?l&)D zy%iA{&ipx8u5(t<;)wz$BUFZSgQg%$?3ILQrqQoePdGR599A_3FdWt0U6cgU9ty$b zP@M>5JIEGth-Oc;fCqNW`3g)CDorhY@v1h^$!DGt<$aMf3W=} zR=K|A1X05O)qA0dgK~a_&Wmc90u&%X2RvlT2^~FWKH%&uTNbCjE^42x5uP+5QabmWzMW{P%ov2QNE@}>DRWjc#ArwF)LDu;Jqn6Vc@c&hU z0<=Ak&}sXXk2(OHCqxyda-87_EIhr+S~f5zhJqBM@h92oIhQcNCGPTp4hBM!PSp}P z7)=mWSWSi<9#D_`N(2eIK(dqvKE{OwWmQP?rQl2u9T(1H)q(@s7-;A~K&hAyjPz!~ zkKbu-sjvg3sN7B=zQrc;0os;hsnD}b<0n-&3yBQ0D-^WyceouKFkrwZcM&)y z3{--1PK@B$t-_<)uH5-w!zX|EXfhiV)#=xWQ00~d@PW3E_WPkAOCR2?&>JF@2o2{C z5c}*|>~luvU^wf^onpLFk#=+6r~87FkIc!3uNuIglrCVD1MLswx55j^2VUN8de*xu zqPMM`UY%QU>D0G@B_Y?s9~{7~J9Ki}&lw4cUwo6sbfj-E|l&E#)(dbKdM+C)MZZjT&olQ^Bp-pDX@ti9Tj%V>QOu{2N z_;~oN;UnQ6mY&au@izd>a;;71r>067P+}P}A+b`xAVN#WeF9%`<=108(7-+VF1-IV zrp=GJaRqsAL!}9U*I5$sM3&Au2LZ!?;25!a_a(#?i-F|(j=arcq0gs@CSPDWNVMT2 zJvTqL0~fgzvw2?R(1f+QyS+?4P0QW!l<^6Ad;+K~PFD8Idmezcw%+)A-FMsqn8*tom(flh*g;tcmoB0+bk3Wr5~(%SjTl}F(hy%u3#!9 z^L2M`4d4OoN^JsXCy|G;T3d>P?8^#7SzGKyXo0Df+5l(oUOEv2oVGvsz(Sse)acFt zMCdFW7f#!L0lu$dFp9|xyb)!*5J&(4h!WrXdvRO84h2xa7qjI=Y$jmdk^pW?2p=d@ z_${D1PTYF6L&3lA;16`%^jd7wS`=`@cSSZt@~>I>1C)(VW&$fekSLHb_@G6E17}4u z2EZ|2zd9k9#X~dq!Jm@-JNN*7WJ0tFfyMD$fbA0k*HS%z^UHk*v3+a%m?|P z<$XTjCrdDp6VPg4kKd?LuuaiSIl+G1gk7`Ek$@TdmsX%+439;{>jySlfjWQ;65l4y zJAILH#jVQ|lp~5fRN|Vv_)EAsDnf2I^L=Gt*Tgpbg0xKlM4+nw1n(ta(!$F!n-)VFw+N(gSNOsvzs2xt8&pnLE%>qfR z+KQAL+T>lr;2t-gbG5tpaP)zg>ixZ~t9?seAaCh)(zMueV{+;?nzx`mH-y zO~t+J|2DA@q$sRXvzlCl^2QWpPT8}Hva(6#HsBXgI!*|&2mpL=rID1v4^DFMFu}4*X^;_xYJN zgYg|Qgf;(T_^n8PU7pD~X#npvC$zBuYyQAh9Ib=Vq27B_8n#-lFJok4;G5r;OMl85 zR5TR6s%_Y9U!oER$Ida*#|`WGTH*hGfqftGP&=@D&BMGJuSXG{hNt$3Hsta_?JzcI8OXL5GvXGlzB)SB@p21N~B>$vOrMlvEl>xiXBs+ z>6XVwlOH$vQ)9l}Z=@OYu=5H=G4BwJxXl08#S&Rg5NLWRNbJZ!b`+AYWbM1c=Cc>& z`hC?9T7r`XuII*&Eolk93< zt5FD$aSnL!=4-hojT|rQ-Q|GbuH|F`dmM+c0`$39W1#Y$t5Sir3ek3cA^hHlnTn8o zh*E+{8GHC3R2l&N@!Tdt8}(io3Elx;NwZMUlFb~eB(U4>L(kjk*;MvpEgxKa*dYYx zBjgIcom7#J<|-}dmyt_ z?;8nRTZF(qk2GQu@(-1R(oNka4q80tE(t0^kqFXYDxi5Q@qa%K11;DY5bN!6LP4PW z!6t()t3CWH&^N#%`D{no)>UX$2@@12U|?bs$h>=rk7kVcaW#VwA^su>a}5w*CoVdH zMpR)18gIN`!zKfd5sk}<+(PzvcKNrA$cMaF#;-=0Dg&&d14`t9ma*=J7yOBg^4E3` z#O7Py69DPD73AjI@rCL{*A1*84cZl>+X`G^S;zeIX&ruFGJAB)@J9csw%O!vW$GIX zm@EQqQ^hUhBjl}ZqE8udG3<1%*SwT%pA=P!&o@s`+=!lB|D;J8l2r6esvE!G*tNuY z?e4;oma2N$d@>MDg2AZ$oeSOtFb;0gQTom3`kc`-cYsnD!RP#~NCWXjKn%=3Z-|R5S zMQ;C*og^@zBjcO>e+2mO58BLyuezx-m?sNmgTHE##7l{3xc~O*HxKBWv6bV<&^j}t zpsIXf`hiPlJ-50|u*ukF%rQN<&W#9M$0dGTqw`@SazIpF6yA_kBzra{a1i_yeuWN2 z3E?0RpS+x4)D6__6+&t&kOO`WxH)|7(}|P$%mgqAXhD-)HnLpEcbx}(Og^8G@nFau zOJsXhBe8Q;rsp4X+*b+5U;XTT_$$)ol}kW}2m}~LLp$Dc+ar9`)MZ|r^zB`kfXp~E zoTInd@XRnpbj>w~9QJu{HP@InIy;>2J*-z;#dU#!EzSi_>gSW)8Z$Z>-q z&{ll0ukS6(=t76e(&fSM^Ib7oo{6mo%Lp}2`*%=hzl?}=ruLGcxe2_(qm<+TTdJI690!=NMyPiQabSE8ksQXt{411xQJGxk(pZ5K4 zOD{@}tQk!@9>3q)&Z06I_-mQ#E_+V3RpI5Qy?n^p7kV1zLjYbU`Ds>-cbZF4R7j?a zjyaEQ9X8rNcEc!H?v&a2qNvf?@%gS9YiYfb{0rmx(bta&T2d{8R?1U1OfW%L^knBj zDGLmq+iSvpU8%I?{$!H65I6x zor9j;ho}PrZ@ok7!udYedtIY1hp6s2Fl=(G)oD3On1^8Ul8k0%4?Nhium$GSPj#l$ zfQ{fF+GYT7OwI*u3|!Q5_(NzVpWY#uT;PTWg471Uj0%|v9cwAaW;L7-f_(qzvYxjBOjn0w;H4&^Zr zC?4>a!y_AXS0RDlr7}`MD`y;9hgAsH{qdQg%zlEo4d-KCoOYG%Sx{LX2c+iTxIxd~ z%i(l}c&n7J1ulR&w@a z5-4CNl_WLQfsN=vXa;3_a-xG2Y1LV!FYWJ__XJr)@MN|fP=R~q_avcB$UOj)ogX~+ zNfeUs2bhEQ2y+9`yKY6iu#{9}!b#zV&%T}bSh3>rAu^@TT;=r;Qz}74{!iU|RrPO$ zQD=8)u-$QI@z*hAG0$>K7NLLRa!I=$Wa6s3dKo1|((+ zoYe$J*YwMk54T^WwSDCpX2R{tT)(y8JXvY$oqT#|f`MJY@0EQ13BX(#ogyB>$~@Hz zdv__52XEZ))kQG&VX&2JjJB0v>$(Y3pc!bh@U={Hf#sCIK{E6=ZLhv+Dq}mm#f)xi zfL0+bB+iyr1)7>ea8o5Ta$I_DXTFvKta8mGuwCM&;i`|+|2*E^TS!dF!c*)LG$ z!4hQoVG`==xn+LADP`Cc%^PR+zFoZI()7raWsb1Mej?9jB6=eE5Vd`6S)c*-)jnB9 z1qr3Eh7~o@I*x7M^>2FMUd@goDj?dMtbW6Kb$qY*oJM}qemnIw#!k$}DxSpAxuQ_x z89|wc7-+y&H7-?BRfZzj8gPvZ@)nUkvd=0t4Cx$_*1^i{zYu=!ZWJ0-O_n$T7d0{6 zLHg&DxfZ_tnH@j3EL&I{n$iyzM57{~vQ-jJ1<>+>LiAmn`|4YvQM9w`ge$A3YSIUC zY0N9qfNLz2m-Pz_koANhk1?8%XO&US-BQ+g___TR;6iQY}}GU|G*aS;78 zEzIZ{rG1UuYW?E5lIE$_`3*bzJCdqGre+u(C-?t>Or*h)I(ijB7^}~6Esa9-vyKha zf!$?INVbHflq<=E*GP95OV2);x(cfqp@14U2?uECzmj?~>LaNUxt7P){_qba+2icZ zf*qA~Z%|u#?Fsct&J3o1Be#<;HCku;r3R!R=iRSST30jJxz%^45{)V$PBTiWij%Ua zk2C?}*-i-z_!az0`y)9jn>v@&1SjdFrfMgcb_l&iZ>Y|qT|rs36PSHWe-~L8l|gA( zqskz=0R5W>)ikPKGu8z6k#?PODW~K`dx?xVztg!{z{%%-AP#5{Qupta#8zv-ktF^X zItN!~QZ^|qHSW9o+L|E`&KtjIUkYO12xJUPqLC@NhF<-1t1FXib;YL=w=ksyH1dfo zw3F9EN}@o^h#WD4U=;yT{bts=R&)*6DC7h;Jqy4_85$ly;ij+ln_(~D_ zczrbSbLGeAO)Ij^?7d2F)<2V;|6wc+~B^d!?6GE30SD@d&GnszhKPDo>kJv z(CI2`H>&C3bU8UZH>f&g9X)=yg6qO%IGFG^jUN zWK!m1nn79+(kxMPD$O*YpQ&)Anzhsjn8fEhs?e@5{U_KULi_&aTxdQiNDFvIW~0C_ zG@ly@Q8(sfssX58uEkZ#Hi|&JXVlth)u2c+RB5gfMtF*RvHmmD-111sLUBd6;;#-yOB$_Ku3SqQF)(*W^^H?gkxqR7{Xd3u;&}t*Qj1pV zYCzeDUrV`$21_*|7gJ^?QhOK!4H!ts%@wKv4@CXpvNYNvV{b2g}1N6!v@+OH$V zQ?pki=%1hL+O>tMpHoLz?C#&~p(kSlwKFF?@n`Ki{>o$_q);C)R|95z< z)i7KBtO=~TAs+t=&~1Afb=k;F1Ik88D3>}N6KVj*$Q0@s_{dldC|66=3=}WbfMo2` zYt#qQ8xl6qeZHp9m~vpjyWnAeE|Nz4B89RtN<2@8{uq!&q#6npBl;ViY5D4Ca*GcA z!#driAW;I+^Nobjzgd)GKOdX^^k(hsAFe$AY>t`r6pXQW=hhzvPN?|7D0W_F&i+Z9 zX$Ln|#wUN(u*kjq_89MB-s2Bc-nC2~Jak&|p+$l_);qqrI&=J(gt&^Gy5-tSW(&vV zzi11%y%aRcK|xvpj5>R6H(l!`?^kcLogUKAmU`;J3d5pnp%2Uh+;-UrA$z|^ve4q}XJX>D2FLXtqy)potL0E3VZz3*(!0AWg8<l^_J*xr)S^PMc-O{uX@C{{ZG1e`$SZ9Td62w_Uqv`s+H8GPpE}`0=Sq zSa3yIZvzEjObs_}Pzk{N-*49E_FH={(y73bpPZK#x}B5_ zX&Gf!?r;~%6*)ob{_#ZS^-H@7!gHW*XIyTSe40zX2yrQbY@bY$?K9Hqq081_o{qq% zo9)v~%QDT0U_5-u_PO+iUkYN9S(>j>P;-U~YG=VKwoi1QAA0>I+h>=#)E{hwazzR# zSLEI!S7e^m5p4(MisUa2x~t@hv&E z9%I1vSc6TGzB}ULU3t0sG?kZIXDKQ$gj%P}GxYIi^_8 zZF?ke3x}ef3X;^c&2N{7k*{CoVt%Dc%*SBqPtPqyV5e!fm`_n|Z5H0pUgpv*emAv7 zRH=L==A(dOK5;!_KFDpMLK_lemqTJ~&MRWrigB9C9()bA0r~i4ulV@8ARoV74R-(Vbc_6Y;Tux>LL(YjjA!S9$Kr zLoAg*J6%Xk*PdJ3czMRHl(18|kPzGDlf;7jq~=dV1wWi!U#x)Ig=NnvJ&0q3emRjh z{^ETzS6wKQ!I>V^7?QG(2U0AdNQOTtlHp~#BrU?klzCRT^C0|_H+3d1Xo-=bi(iUl zG}Lrd5}i^PLzPHIG8D|JP!)`t;^l9d;TonT)zcl_3Xr^#Q6w!Eh69~T0}a?(wd=?Q9q-e7uc`sBW*)*S>1|UBTh)M&nvg`E zFDs`yj**^e8sHr38lF(q>Y_p*13&$%A2KzCtQJkE@fcArrK-zz7|LHu=en!1i1JK2 zXn>1!Dxd~hcNb~^&^ks<6rc$hq-9Y&RaL<332f7&eW4C)SG??l=(9y260%v*J%Q}7 zQge;mXz6%UddKB0Nr~#9Jv9JF7PbrM+$`yc2Y=@h!SQ`hM1 zM`}Rbd{!ZK%nH?{>E9gStZpSC!wBs=AZ8KN%>k?l^sipIsPU1cARo=ESB0&~)V0%FLkklq;yA>?;#Rjn1ZhyH45eS0Ln7zsm5pTXuPA zkeoaj(0i0O$tZ&QJnBL1dfK1Ih8Mgc}>0}(29B}!io*4?c{ zw}D_N{1q1i7||uNy{W(n4-5dinUG0sYXMjk-^K=Lbil377=^dW;j^%N+LeepCJa;}547$9`1B9xK4YW?Y(#GURn7=@F+0iV88y6K)Gxda zzYyU3iOcmqhVS+tIz%~OSd7A}*@KV%FU;LdbM{n&acRQb_H(?1dKp)1LK7MgbEv&3 zR|6^t=S9>F_ZR378_4vijW9_tOZUjHMoqu~gIn77gRJCI6Dh!Wf$qzAzU(2jVGS|U zfF9N4KHWJ?JvXf2(8|M#P1tO6*d1{nxix1A(-}dbI$e3~PWfeO6L`ouP`aZ?_o?C`tK0*5&Uk#8Kv2OgQBm{RZ2lv z<4JB_x`(hQwg5E)F{zr+IMIqw`ph6rAj+~ZvZMrh3?NZ1?a5r0wonJLTv*T$!7?4fiHbL{R?~zh&tm6TVQCvl1_2dy|DM5Tc_iphO6kDS3bW_XPl5S2Z|@1{9>PZF#=XNPxAHlipc=CTvEV8j z_k2rXV*en>Esa!Rgbdw#O~x9O{xnR8hZ#AvEdpu>{n4SiPeL;6H^Zu7*ov zi$+}KHR{8wrwK9?$^Qf06F?nR`ViWZFna*v3q;i^ zrwtB93@!8f)gvo%=;QnM-W$Ar=&pi$nbXXEy}Zk$3#cX-w?D)w6k?VVeX<;~Fk;imqmZL7mq4xRJcwi=f^gCa}XpW6L?dDUk%2NkmQ zqqMEsXLMYQ@|v0%Xe{WdDDE6wUEt`zx#W2DtV2xhHjnFw1xahqVChf<{{o0mri8w@;Ng-?pg7*k3VU8<`}tt zLA5kwCN92J@)h^crbj33mpo`aU78W@e6n`4>4cM;3;1_8n=QE)y%-RDG1vIUPj}n(?3Wr#KW?zK9vU`pM&PruU1^?)N3Nwm2n@J3Q(l^KSzk2oP?4|= z@46J1|I@@AYm)?>aCu~kNe(;3bpx1tc*qA(+N3&&zn()VOZa%#K}+&Vbn7qA!WIH; z5h+&Jsimw9#Xu1QRc14e7JW^A6(gXOn+Qs|mCJ+Sk8mOVu(X#dXOY2eER>V43~wLc zw!73|1XrAXWP4+x)AgqBe{VW>#&8o;CFOST_RmDC2ryXl-c*fRyubaYc^o^NYim{- z6ge(TTWcHbXcGP)BxuV7v{1NJi#fIIvqQ^PZ>gP=w$4T3l5BlCRJL=~X)|aXyl-y9fm zy_Q=CHnK>S8e$Ko?2w@H>R?Qo<+k<%_<{*jYD60esE@S{kVXGUs6Oaf@=|?JG_qVM zynY6quRH0ei{f8Z$&p+)OaDdj9uc4Ip144?qFJSnrEBf=knc}`EUP~Wy2Z}XwX73g z`g%-W!{)HdTkP_-oxHp?#Cuo4;cBaE*U+|wh4E2i7i_j2o|snaA>1;4v-cTotGvp#BDk+`1 zi2UeY=Ms~ZPhcNEDUtPv(H6=i2rE?Cu7fUKIRmyf?D+yp*F7Mm>zWXFa&L;DUQO#u zy&5mm+X_5mk;wHfRLOG2K7a8ZUQBVq>ir1;Tgngm&n3Uise^g4G#)^uxqlK` z9CbH3H{HP|chi!LZ$pZkx7i;#A>J|1YIc~L(gHEi&RmgdGb$kF!TpUM+!w87JECVA zh$_#`3?dfIo^k)y9L?FkK#c(&sxemS8Am~RR%?G4o890kW%(8tZ%H~@eNne0$kN~H zMP#+r3vL1!dw2gnXolOfT@&n3qDjc|y)&KkCEB~6oaRG$Nf{|GDIZ95%QV2NGIafp?2gla-}20^EIk_Sv)*8l*u>5EDB{LXC4L;M^C1){%+bMaIN&l4rf3-%PI^h> zNA~Mj+X+S;sis-4EzL7PK`cT@gOCZar{ME9Cqrq}I#L={%z#D;I#Ld5^9WtjjVO2;>q!YE@UMR6rn z{9xS~z1T!qsG$Gu&Fx?^s26pLrr6&TqXVhJ2eacV+ErTD|C_fX&lvd7E>reAp7u}TDU7W8ize-vN<*LbWz>{ z`}~B{#rpQV>7tsbaXIWxEj2MxsOH2-6R6$kOigj89dWm)cQ4FlXdNSURPT_RaE`l- zc6#28B5|)x>A}k+(F?}M$R@$He0Fd{uk+c6rCna610;FHKP6|T+*?`pv2JOZ0{Q0X z&qwOU801y9Do};+K)3?^Ee79sH3Nutq-21=9H)nfcs>R|0}sO`z%%o1J`ZG~@SFjJ zP@r?hIjhy{VYL27As+;6ln@4rFC^f{`dGY^xupU7HW4JS2|Ifk^FnrT7OOpOw<4e( z!7B6o*m%fH=a%H90?RK>lLjrC=9qNlftYUlU7?#Tb8}y1p~opZfph>ytL@nlB&zW}d~B zXX)qV`7PGZi(;EQ*$xfd+)r5ZxGA)$U=wS}kywzgU0hp-ng$V}i8#CJMv9OChKYIL z>es`u6+Z`s*6x(x?3gkIHnT`ym-MErItd`ypbKSZ?@o#|Rf0jNh=;L<<4Q&jzYXq8 zaXl*CedDPzAABt>MG3s}N}C8Bsx6mE5}Ej{lBWs@vs|X&uw(7(gji0Sqy^l|XtU!3 zA;1C7;my?Q`nmn40#mxp3sW}bxu)GMS0MRD5-@4vP`DW@*=!35>!Gd}&<<+>hl`^T&kEt6mbW?o6Y*$?n*+EDF^ z0mbH40v*)~*jz0FM;7}44E|C*P+D^N6Vp>C%VaRU~C2n$3g59sb!0zu2~JaE*XyvXjqS%I@B;;%nKy^>*$fefI~ ze*q9zZ5#*O_0`L8^e#XK<0yQj1xN^xqWd*IJU3hoR5e*cKybSIF9`Ji~a zxGk&nby$?qrUiF0&(158pJq51-u1eF+gF*D4Q$s<=DM{whTNH{Hm9??5znDfFX2v>u5)lq z_M}6BflME)S7G8g5COMN*AkWzCP3dxBcDcMEvvDrFHH#rOtA z`YVIm{J}?vhzX29=a z&yRz7l3BGw;9dBw24f{%0~QEHId1^S9z5`2>WJ3xIm->n@9*sAzww0Wmib@&Qa(|l z-ScgdPdt1Lz}de6z-8BB5jNqU$=sGcKRi6O9~_y5JlCDYt#p$|WL9VNa@H;-@?e9d zZqP{PB@eAO8zT$%WF!9guGf&G$|H>U3m$$)sR+Akhb&XZBNm-*JYk>-YD~=D7lH-R)2!9QeXV}BGZcv zda!a9^0lIL4;fr`P89Y6;H){!1oI$S?E-=cVi`m@<5FHe@Ee00Y*ipRqATE)$-ME} zWWpnsegi1*793GJFgr{UkpiqkiBfQCOi_W4{1}+3Y(YQ^ct@#ZWb>787|vnchBKfA z`gcawI|yKP#gS$Z105SB0u0aLqrsKJQgGb2(lrx8xEK+S0oJ17sUVM`7oCe`zVJMf zAwk37C)kM1d_mgGv=a0Qd_K@T9caBYnMU;l5OX(FFGIVns1RDPfL2g-5CehaOGvBn zjs$Q-u4y=9H;}A`@e64%C!Pjq=|Wvt=5a^`i~!QC4Ob{YX+rxsC12QWsf=m}AzD35 zevS|1(3B~lDLbbG<2mhn6vzztfuN-oc|3xV(bO)G1{nJZjLpp|2a4qF#`?gJXtI`) zFL){h@Uq`9me98GpS}a_Jz_w9x{X4hI}7ku_gKp-0@S=2{0yxh`RpTyTG1bmg5 zpSTF*j1;~yyBjfpwS27I2<8*r#@gsI;42WT2tKr=X+n@C@E;D3cL3@GwwjeoBp_O9 zz}p1gmySylz(A-&e#M&|P&XWO$(V|K9C`2n$o0E$W%#dawB3amNMtKzp?6W!1RZBt zcjP1qix8o5z7|??+Mx7W$?0B!Qv?Q&31eydUpOYzfm3%)bi^Aq(FX$0@|e2G%Y;W2 zq=D|at=#%L;hgza!s(b**)IepgptL`0OS7hunkQ}X8oQ0?Nba&=N;CuuW>E+CpyJ> zuhHt(`yfs9dFRshN)e}=iTar++qzyCfJGt6WT|3tOq(D7_F;Ih&2PU%>g%lixyTgl zl~pf)Bdean=6i>S1-_SI)Jpc>_1qRmye^3*zEu)EfiE@bxB83g=O5bjD7Y$%qa60% zvU!IaFaC1}ncS+AHmd_j|#7-0gUdPDIKahnmgn4zy2sd(%RwWP=R# z)II2+XX-~3E1H5ifsAt9S9J%JRcK2tL(Zs3;X2jKcO<7x7v;3P^HaGXxL3=PZui)1q)n*RKo5vbl zdplT9Onz|+cnbE`yH{lim>C!$^^sP#yza>m=sA?Z`1v-vo!^M!vt0A ze%*5N;KW-kCtY#P{gxO=OM8DAq7lKtu-==t2VLpjv|0Ui5z%Tnk^Fu;h+tNfxM|(5 zRh>qD*ncha`_IHcL-VrUwm?0LNkV0#vSFXp--r&b%zZGV{n~0&%puB{ARCp6U#0UI z=uA)q`DB}t2{gz>p%o_2XhP27)vO}lHeM8vrrzY^oPuYL-lr}|42HK`S+#k#Sh$#u zeb`~>XGorETQ3&l2|7sljXo@pq$p4TCNHCE zfqXMJCY=y|i$l2sq9&YF8&i;Nki7%M!A+n3gQETKy0l5)sIOKza-&D-U(;WyjrCFO zrd1d$f5sW!Uq_(oGAGoiUCsZup!a_!5WcLh<}&;32BcMlXpinxm%H`TwM(yE>*PUw znUUt2aG9(Px~%?M0Liq*I;NkNh5r!B`k(Q&okbK)aoyQle@!U#I-94dlhiYkXldIF zG@w^8f?iiScQKixUNzw*-+Q~oT$3Ei%o%DN%6X0On=@yr3HCJOGK~>se+_>mu)Ar^ zm;H3B(h9$pP)D?5hxgY*lptm>cURr^Iv(mg?rzY`4nv_SDwo&39$)gT1!dlod?KW6 zGCLs$$<$?e`M#P+Q_3vXk(IXztGzN^H?Dy9yfVpBZ?e`oZ#L3vmS^%CvpoNo1xj*4 zAh)RB)J>I#AMhT927V*7>(w?-Cy1=ZE{pmq4{aP3+h=8)R?ftXCGBr8i>ySL9XHd2 zV_7Tfw|d4?0c0QdH@)+h&gBQ=v(iTx7>VsbBOeztm4QsqQ$Z_G8{i*em}SKrYX763 zs1b-tcE63f-Kai1&Gbk05xlrxqt#vV?{_?nPJVV;o~nDSUl4F?mnMWv4O=nOSB1s5 z$J$km4Ra1^0;=CBVovOLUOvG!$9|YV|k$t3iz_ zj=wP|<92HZj`cZGNH2U|q^5x*T_b`6rO!@kuwm}us?IjL14rlC0_u}FyigO`RB}0o zDDknUhcsa=`37p>k$$JQ=@kG~s>!IwXg~mDfml&iTFAyK4d^X@Rzckis2RSbvxuNh zz*NJ`ee%T7Da<=@p?~CB1p;ZvW!Co3r_ghm+E|C-#a>)CbXv^sPE#geS4V4tkv&b_ zP;=E}0{v@)=ap*u$~S8S0&P27P4F{RBW6DABPk)v?BoysR&nqjQX9&BpW4Lkn9&bq z&8s>e9hy*QHmRA$($NS2ra%pJ^Ukuwtokh8KYD7Pcp;fU(k`;fFNhGEpx|lr#H^MxdO?Sn6Ku{we8cg^WFGqb?0@9H|LO8=?;8 ztkH;8>m82}$|j6?1{yI2A*)ccXEkE#85ys}0DYwqi)oLkrak)9!%#I29W&F2h8mf= znOF6VsB}nBcdY53Pqn*0sz1>WT8$50V|i02Zs%Z2P2YM6bZWT$w6NdAt$Px}Oamq% z&a+2T#qChN1W_-UZBEnuXrvB- zra;_?_MIj*cCvoU_T*SgSli#(uzz!?1$;>kwRrr08ERpvp;t3KsHV_QO2hT$%3HO@ z+U?(B8UD`@E-dq)JjvNRp(GLQ#~^Y6I|GzBPh7(Y*s#{^@PfTI28NeyM~*$G|8-6I z=f`|TzyGe}ouRnfGzMnr3eE9HG%C{A*A2~lX7@oM<|LWJWs3VBzr6*hEn*CR}n+;XZYptomLEJ6hdw~=k=3Z zp4*4makm!jDv7^6VtIzkoy-a3FAUi)R;F4y=??DfO7Be!we}RLMP>!H$Tr#^RcVp6 zHhL>_k-$JCntMtRurwxnjz5EI&cWbJ-?N>oMtNflaiQOgL%xBC4|eGVDH>=~z3u8B zruRyJCc;avgKF;?1`l*@{IuyeNM)9gz}96(20t|;KS{{Lu#`B$l{2uwsW!050L3%$ zx#=?RvkoLpWZ)pNpX7W7yay@8)A`)FB!R3lJM>PuBhZxhtaq%QhmpHHT5p#yv!8U2+r_)?U9~jZ-KDDcs`~{}|WkHGa*P>PDxPsn9(#4m)g(89eP$W=0Z&$ZS zpdAm21Xfn^>YZz4=Z-qf{g$88QqbfI88{YA7-d#2G|yPT6FH2M3`aXzHaApqhqiYmIVUBV>f z{)!*HM54tR-3phDvQ`VyPBJv=yTizose;4$Q?_g?H!F>JJzkOWj`?o|lwCPkg|yY< zmv*USV2;FP$H*tS@?@?qB}A|wK79_eChFIXnfIoorLuhdyiGUhklWE}>GrAwIhHZ} z%h*$l1WYMbbmg?w3EX_TMJ*qT%0jwc64kQ!cU45SlEM6T&#;ku2B@LG#_2` z1R-kM+L_$pw<;%TY>cqFt6psQb>fuq!w*dnN}?UY1l|b!ZR=3z9Vo1hF=KeNRNI)| z{zEyRdb7Iwyf9K0C~Rs~vW_3y1zC!hl}C#BFA{JDZ=E;Xre+^y`5ia&aK}5P9e#Ia zlY5Z8`^lXRV$^ry!Id+R^{A{#(h^*m&#Uh27A(3bNYiV1N%o0n9#N5f3P)#F3l~fs zAeJcIIv33zVc2pxH?B0%=|+?7#zDV3K$Y3y@QvH{-}YgZ#VpYNmyd5b0w3tPb*pZ~ z--klU<*~Wjx~|MLR|zG1ij9f$d?7=5xZE~UyYaU%S)KkbbTfz*PPUF&%dBfk1>(Z= zdmZ|%_+G%Jwf872M~1WdGp%FbRgUe9hCY(uJN$e87SdR{yq09kRyBFD;4mB+wUxi{s`VDjc z4JkPw8zCLZFKvWrGZ-x&XltNRh&CLA z2lLy>XPC3o1=>RJ#a=>4Y+eK(=(%<4V)lJBrr}%N`u%FiJfNw_O{lHJGWz#b)0)B9 zx}cfY;+kS}LIqPjNM&N_`76=?AZJy`sL`S_Um{ma7Th#K{2Oh0-8GiL;8I8s)T2L}SOja}k4_>Rh+I-Fr3CGH(NS@E3!` zsUsD;$X_rR^+eRP=z_QYqf$|C*^F&A)1f-LDO5)f+`Ob)9X(Fat`~q#Ao?-oIP38@ zqNOIS6A5~UR^Oecjyib0*0guYHEgtkDn#}+I07xQSJ%JktNtMe@>&+Vp!B|I*~Fqv zzhwrOMwjp*(btZ}`iIi_!9Grr0$iW=oApH9vs+Y2tHJy5B55C69RQ0dGdge6!}K?; z-YiP@(bP%*#5@dCS5RfGIssD-`Jxq|#paYs@qF9*S&NFbynP>{Kc{t}#;&#dWm9x4 zhlt0d4|vRn@>!)RpLd74yqMaAb^9mp^C>&%zl8h+oxx2q_wYjF zQS~j}06x%j>sI8r{#i)&E+6*vxMx|{CVbNL%rWvsu}B({#Gcq+x})-uQ}X^+BJj^J zuH!m~Qv*p8zHDcB(HktY2&(KcuBD|HdV?%F0UJk2h?}4^j?N(+zGEk43KF#JWBLQw z7Hb{Vq#%ygFBAiTD~Ha-i1T3V1WMDhX%2Lh)R{I7@Fe70?N>-k3tpc6Hd1f!euf4F zJ%tO5C<{xaD>T6PG*nQ-*jdoR(!J(lp}+(BNkse<=?-D8<2AzcOwgq^J*&P{2W9>9 z%Pt+koz(l)8~VyDMh274b>ZA5`owyj4NH$paSro%R2R zq@u(XPwLqIGVDZSE5UL)+bg*}pk7LQ4d|t^cn)&ZoJ)I6hzFtmB^5B8J&6RSDNBX_ z`~;?kp$^Y12%BK>)ZO=@2k#t|mX%mOO&PDrbsG3`EPXDlX-aR#7pC)vTIZ#(N zjTtN4gl*0Pg)nuYD zNY_R?z>t5hrRtwiNojxpSATtDs+QblbJ|x@8nhGpgTPd^c!!~K4Zy%U75#ON$h!8z z2J*lFO=vi1P?Lse1on;+BI=wn>uC(GaByUgy zqiY2A+O8Ez(MW1K^AOo?uvJLwt~CNtvs)$6^lv~!7poiA9iaiePIHYZyJa=rsWHu+V<$BI{Nq)!ZIW1l!MO&yA#|rdDuG6UM## zc9L^@vaS`S$KGbQP>p*T3t&r*S{aIGHbuE@d}4~do(T@$Hx0r=Qf8VvaLJFfYSW&QskA3KlSQ?vT& z3k)B2p}pxizIzIJs{b*RrxuDVYVduiMxZ8HsG8CLV;V4saDu32Py>CY6X}l)1)yrG zo0>3+fX4LKEUV%MH|8nmKZ-C74D1i&BWH}YgH-s}GYZ~Is#?+Cq`gE*+RI14%k?0@ z<)S)Wv3sR1&;dgs$I+nM3dI%hh5l23V-H9k%*^OjLJ%hb-E`Y{r2-f-uxgUMICH%h zk5LW+-^KZO-i{uAB=~>DsJqd*>3tK!TOcNs`HpOk+WEKJL~w74=D|jv{e!p?8+NKA z)`1+&^k=(L2es&HKnQ6O)NcscO(h7M-s#SJd7@U0K`zpOYb54VFb1*ogLXaeB~VyVW74AX$XZd@mIHPrO2q6C3HRfLy6$#7rReBOQH)K9;E zzGB5*lge{fwe|{shmv4Rmbt$_uI1MIzm(lOr}e?2&Z_Va9l!ZS-}b()_@dUhg};;8s8lW(Z&yzdF;)e4tJmMbhCxvNdkV&8P zb0|y`G<~7=2XBjg7)Mun+0M#y%>3=e`P^MTY5IY=+m=Rtf&$GwbdwnRVG);hnXJ!t`~C3S0y{b{Z`#YC3n|hU0)M%Ad)VV8>e6x_S!1vyk z7>X>|X&iTY^E~FaE`^RuGB%vda$Go(cY+CJTc+W8YzgI3W}nl^F5|c19+&QmcaF}n zUU;dlsqsk4L;-Tc8H&^;S3!}w#2APV*Fn1Gtg@o&cfv`!=JXD6k_PRSa*?)04 zu0x8{C5(lyuNVkL>Sl*MHlFr2U32^41J(A|XNJf&?FyNm*U%C*VR~GH*J%Xlo?TWkgHFP0u_^#1Yrz#V4`4x46=<$goro+BH}QN$TBR#0K+iMa_{+{JAkZn z=k@yM>)tn?Pl>D^?m55nTfU2veW{+OZSaWw8gDb3Io7upLOoe%QNh>$FqJjgM=C@~p!a#BqZTet$K#%AqYD+SJ7e{UlCq z#%&qQQ0FBdGDE{{`MJ0l{%?kX8KpsK&Wv3a6FiY@x!^}quM)8$-mw<+pl8kOPJh$Q zH9o0h#nL6X`B!_;EQu_N6{ZhKcjP~p-fTd$V zPDtu|WXp4KvUz=^ym9Hm*XkP&O6wJqaT`!2v;p<8GcLd<=|Ys2>9{!|ARhWE+AU|= zvwvw?3G!S7pf!0S^i{mLeE0W;OZf>2wB|?yy~iKGUG+4=ChVXi1Uo~4lR&*v4#ET8 z?{~H~Exv|t=8o}gilFEeJPfUjY3cE*b3fs#j))y`$pHE)8ltbdZsLcfrkO{4CtyuX z!+jM!a^U9=qd{Lq&%&76nQI1EWS(|1n#RAf`@((wkZ>ah8@7JUVh8Koq-uNUIcVjJ zlo9V&)Sm*ur@9{F*RiU-d5em7p2=Ed%!v%f%}y^dCwM37yBC|$CfJn=L18+%Lr>yk z9glKBU^=f-)%5AP#>)3jM}PTgL21cUSJMZ`K>>7daLIuV4o=onYJ=`4#Wx0|$K?)R zfg4tN#f8L{pYL)4eU>c~izEnRtiPY>$Mf;L=UGT zbD@L72HxuOhe_`yB-qZWNpU=}2zPMUz65^$?GezyVY|_f-X9KhIjWj6CO7Fzhj*np z)$dxLFBVw5|M8aM)&+{q&N7z9{uxqe9?$BGejqcEg)8=qfEoMOP$l{Ey;}{{Gh+*IvBYAO-yNO}g) z`-f75h7`hg^t^5_At+$#IljL1==5*JXxhg?Ym{l_em>^7zveN2yXs8FLu+Xi$3j=h z-Q^-UFepP8!4zmd2rX|X(h}&vA4m_jR0(!$#Vv22zg;cWw{+3ih<;evVX`HqL@tVgRI&Dj$tP0lsbAWa6H2&U|k(=|EUNC&u z*!jlAibeN!+z31El$VHG-o_#oTDO2E?L`?BNF!npEWd&%k$^^`QK_n_St1*9cNaFK z5SQIbsFsF(CE;!&ckZhqJeq!IEL=T-rDV&06t-~V^uIv5RK#o7RdxqcpzLnvmk}S# zIw+As{)5>h{=>Hk3EI?8fK)*f&{I+R?QO#c{HSicgyk~JCL(-qj zn9>czVK)3DD&#*F?EirsTa+J|NjN|%gsfrmkI53{;{SzZDnl!DTqab=3}Q-~baafa z>i>Zn{I8PC?!T_1E^($pltn54(FjQljm*eDcu3+sCvI_KGbqrlF7ngS|AX?Bq#u+$ zu_n%aEFMu|vP*}k5hC!6cg3k^q85_RJSE0vMBU0Y(EO;G?bSi<_Lg;!j>N~70u@QM zAyP(sjaV24rmN^aTryXeMU9?1V$XTYR?HTg|0$V4B}R=B;=9t{m;!etRxTpku%(rx zZO39p5$XpxVur=-LLe|>y!+tT+vMC2!B~5!1yXhpf2`@&ZuTmImqpF0^+>I z*11p!wnk#b+NLWwuQ}7F z?#g`5M?}-ral&S#g@0Q3;n-S^8cmF?Dr2iHpE6Mi5Q~s4Vt^($iV)fk6$jAODPXi$ zE(c)%6`untGi72BSIK+Jh8++s2&bO|2UDM9wTZ{3V~`4I2{4gbAunRlWpZHIsV@v& z&N#*k!j9F^{TG480-&NrgZGup6(BgV;hG3Ij#gj`E7WKlc!&V$&+m?DQGwMvrq>}- zIx0n^8LvIep<|p8J2EdrKxLyv!2tBcZ*r~%Ybex>H57R-!!e|t_lE=0Rwa_nw}LPeXVmeb(|9qA8{ z{Sq}Q9sy7>{EVL^Ah;Xp&_{Xj@J0WNky(8`qbKX>k3eE$teSX%^C_L}s7~F498zE-Yl)WCG zi*p=s(CGTM(p;Yft5@;q6>3oFlldNZ`_*E6XPTeToW}bB-({_m(D3(kzgG5varZrq zOk?1)H6ec&ULOYT+}H*E?Xy`6%2<(>t)CAyfBYmJsDY8%KD)l@fSa#0u8ld?KIB^b zn*E!>D`IbagV8}Q+<=V4ItVTpCXyp!JxmM?(*R44|3qW~*0j^;p+Hs((BMKa4K)l0 zx%M|za(enLAb^%_cf)xg85IEr{BP$|IrssBseN%U_22n8nEK)XF!c`-wg%96dFWqd zKS1M_Eck0_85+})>UlVitNs}QNE06N zHfTZ~ABMap1PcelTf|@{R!axSXJbHkdbmQ#?zFul!XRoe_D4Dhz7FqbLsZI?pzQ-Sc+!~lVr7(f$l@|dI{!{6zo=TWl_veO#L$_POufR_e zD^d6z8)3YY!}lN~6>uT`DC`0vaMkc9VJZI3b{{@qc$?51EpdKxyn?G{i1fNnp2ZY<8 zm6`yFB+lc9-6;NU0IGhT8wPnGAYr2DkESjE5l7;0qZb;*;<8;+ESkitL#t?PJH?^k*JW~C598@;Ed0akhwB2#40xY1o}=n2FhaagCN_A zY#JwnqE9KRgKSktiJp(Oj&8`TM)DyT2Gt`*5)FbO>lQ!u!=ZVQlqkE53Ur`Rcw|)N z8JPPxb9D&~kdf>;u;sj~3?rg?n(@=gN8KKDF0XQlFOm^ZdWh^7$?V2vVj-8GX9ObI zBoi7KcqS18Vlf4225b^w!4=mA0*wC3N?8MsWVMbXDKaxA9h!PjyC-b7_B?$7l&>q4 z=u`|K;&}e^)Q;_6Cv8F1JS7b)1=WZuAL&s(t5XZwpde8;Z*K#zXfL8~EESc5%4(5+ zg0CKi)`{Hbs{;VPZdXR?!itKi&uC2KNqKbEqVg(s=N>q;UOlct6oAs!-oYeY5(&4R zCbXeQ1B!GB!0;3p$Pq|7aord_{R(M-9L!t;(vhWa^_hIH{Hj;>Zg5Jdiy1oEO(%4> z%NAvT?A<14i}K%7Wg3{@*UNN6%}IZl)e2obYG`mo`W|(r4g!c6s?(E=05T|KC90@M zB^Ba;I@e& zwwy?*kp-D9L!y5F!pfq9N;Us1!cr|UyNdiN!7bu)lnQHXSLP5;8y=gjAfL9ldg)3 zj3nj_1LUcFggWGg?n1*X(n)6yOz*7Hb;vDJW0gtG#Hl)5Wd+DXPLN4KVc;_?jfD)jf*$Z`|*r)*c4*M5zZZ~T_ z2?bCGPV;6r>C$eYM%vRvtla{w9R6Qyj{w~lBnnKfo4=Nx{A{xGtLZ6Ix5STpedC08 z4zg!_m^o_rQPnVWW!2g<3;z-I*#hQ=4Tr46AD!G(b1dC9`F-BX`{&PnwqeqKGut03 zj~M3JDaI-ulrzS%{HO2v4o+vgt-tK>L2a+~mw!UbbR_P=E0=lD6^rYAG_@YOVhOuk zu{QO(Vj1?jVl`qo?YaAPq>s;sISknW-98_;;u|}mvPe&#jD>r)u#GJfd3^}Z#a&YC z`dB}y^~2+!k(+V1k=tdcj{9Vz#(#%nz>L&esqxa2?74F~L^?H9< z;{V2Jz20B9?s%#r_PllpgTnb0wNN;p1BLSujd1?Qy~6ofy~6nkD%^g(=her5M+Jj6 zZBn=YPC`PJZBaX?=cHRhTwIoLS|--t9{|V2F*}idiWO_1E0(Cccl`sS`l|D~OpRTg zkDs^s({cEC-PXkUNng7<6~C$GX>I9_^}Y50(+AiCh)0Q_UEP!;N?&L^7!HjG{h{$7 zsxcm1+{buu?F&|Zdb1XtgXW}N?;rlG+paD>?KFK#bx)2lc##m&pjbZ$Wb`{8$lnX!@zivb0^|7rnlTC; z!&I~^fJj?S{qc|FpOC97QRKtlfRLwgl6dj-cf4?48uaviUO2yTq4S%F8y;OM>gW8X zQGgx|`YAx0;|kE>-3rhTgN~!L!B2mb>3e-${6q-LaD*{$?6O`@w1&C(0((8t==Pn^ z#qa*Ghaza%Bgj}?-VQB$n!7A}=Ji_k==WOoxPzW0!ar~Kaq;88DO7g1i(h$(JG7nD z>me|OL)*#2>F@iXr~27W(qRZB`yRKF?(8154f!m?1IDI776I|Iytv%uvc{{>HrS9q zV%jYDi*2!*W1Zf|s}KRVGN+ZRP43Udk9n8-QZ)V`dfk`Q+bT^ln`Of{BG504Eq&Fs z821aKYy84ud;P+gy?$Y(C~k(*^XlVOco??U^t!zY589Wu&dL0_JCPnbD4X%25&Zx- zgK`G6No9B2q;7=fn#n&fG>YY`vdm})zJ@>b_M8Z#roM{hsJP?c0cqpn=w45>(*r!w z4xNy6nijmi8eIj=H4C7*CLNkuz`pW1)elh;^EQ^fM6s$$LNfpJaVcV{ORo>O7Q%ANii-jUH#sp%=+$$N5WSKacY(f|@}}B4nQ>oV&7nk*X}B9>LAM)Y19XC2@lfM!7nd6<9zGWL zw(E=ui52$ow&Q{SYR=0r=>gUcy-0n0JVW{?`6R!m+pR4ffFiSGPY3FEgYs@iEG^IS z7>feW(&VnLXGw+B&R0h%sxcQLdnwx1(<9ZA+CgwVHo8B7fOMCL;R}8il|#=&?bS9u z7&hY%Mlu|O&hWf$XZW?5cO8nq^7+==!yRq!Z1cN46VJUlZbiw0Hqc=*E@F#cDyEim z1o=M~Bxav{1Ag$a4&4!7sVZpO{r8F$830h0nH!P@4vxQ{F-4^|=9gUz$X|OVIMlA3+RuU7Z`>fc2-4*|$S&=>^}pR40(fuejDLplFTe+N zJ`}7>I}i+l5L|utZro00`cs=+GpE=Ea0(|rK%oB3eiaqd4j&SDtL3P(X!x|)t zsyY|O43^U2O?!D<$U(uiI}_GsvVKXJ48P#%=o2#aqOgZ%v+J7D5crOs*Hf?c*{xT% zp3n1Ja*Gd@;C9%J4JmOm+tA!#g3U%Pvk2=mTn~lU0zCSPk&-SR^K~?a3}N?cpu;-Z z+Sv6n>=xmZG*m*mEa~erVA$;eGPqffpUE}f!U?OPnQvcL!T~+$zkVOHOjq}7FfEI8 z-LKbuh=Z#cv^g2buOnOB`NXYZ6xRx-_r|>EWFXod!0xkp3|) zts`Edbj|o2q_Qx5a~~HMvfUmEr<87d&d_aI(O`l;>FVpm(jI*Ipr)6|EHS*+f;TmX z^tsMi5Ac)LGyr#*&`16MNE$GD#aHe#A8bXXphE<8{FMl%VD^k|BanSSj?%D0#UQaz zpk6Hm2S7jwVw&KjhplU;$4$hlfU(IGLjxsEGNzTs=ppEXXB-^Vu*M6abs!`Eq2hti zLiqRFo;X2YhS9t7UAi9twN;*K{3D^GC5mhnICBXS^rwvJNp#_#)3W!jjNF{R^n&5L zgGOgDU~-RC?%-H;<8pYdtUgsm+$j5piaxmaFu~)~a@eNV-~W2!+KV@pVgOxlf%a7ySwK7l z&zYgK>kW&|$%CdepY&HQ_%F9Cd{7+|xBUiF$)6#&2BTdD!CD5yj2H?OJK&3M=Ecw* zwjrcSN!NRtCG6H{A9PzuWH644hoJ9~uF`?N4_z`=TH5-OJ@^QmU(wemgmihww@uv& zPlInAA82YtfeiUbWz8Q7|FnDO6~A zp6$3n3==EF%`XhqyuMa?q-!FzL!xY#Bk<7OK)*AV6f_NiEz(D41*RjF zpy|_djg{}6j{fq~g3=PQjUi$}WxAjmIt!4$KGNJOXg?7O*O?lKzk2!$7SzzEGK%>D z1wzkj_1*MKnDQLrc@Dz-Q3DPaw#Sz+!fLo)#61%d@LVur+&FCl`735C-%PCuL4h>Q znX!us(?KP_$B9o%A!0tA^bMXodJ%DhC(?-u+f}G_yD|(RD=YTR92H#wR&P!Sh+ld7 zirsRiJ?UyK#tQ9-YTWKR2%GUoQoFxT%}yKVbPZD=&@jDxmOwheSo{)j=ZB% zT?gwvd<@VoK3P`u83~A&gDy4?s8j)C8m|iH8D?$s;?pWXz~7*^xrV0J)2x3wnE;hi z&_?+5q0BHo<|nLPa3$SA*qeFJFQ@)>_)5+^p4L%K7dHax$B-Ve!Tznsx^L)4rao#E zD2K+*NBam>kgk!LdG&vR=JG_9+ne+bMSIqmo{&BrT$4imW0^=lO2>?uL+t8jbYaUQ`FoMSP6>eES$s*W=kY^!PRi`fHQF67?j6Q$*@1?bQ1HCCEruZCA|D z3jdSffQml|jB8x&wzaiVDg7_YnI!gzln=`4vb;YxE|9JPmbC05p>#XfWoM67_Fz5$}or0an=7wT&CFeXSq zH5g82zJrgmc6fDH&kltWx+eg46PY{HM%Kg;Hr$2+jY&)wP!`&?k)49n#@8X7z)zY> ze&K`{;@#R(^tZ^L4}79N;pArAmcbkv@f_(1#m%r2P7+eXfaE%kn8Xd)72Uf zQ=m1FLrh_%0zTjGI&mM%n171=DP4uGnt8z=nNa^lG!Pc*V$Q!tg*j}ZKt!zhg;GKD zYikTKU@eLR4T_181Al!H&=?^`Jcqp-NP!|Gxl_lqd2%}xnkRc};I(7KXf8^mJUzBE zL(lFq>ZmXcamM(>prs{sR0uuCbUjNs1rBAAzb*&}lwcu!muC$^JoOpEzKnDT5FgZy z5COuV1n0X#!`kyZ6m1z{7y0$A{-qtjjw6D-znw^dy4l%KM-SZr&hN-R+WLgR zaYLg4I)s}k*|eX|b_5Y;1BHAF%(aRzI`N<&=Mn`pb&8+_K^?2wn>Td&>bJ2ys*_D} z%{g#ihgR9QgPr!s#xmV?rdP;og|T1hAn%MwIBO>R#HXlHuH#77{`6n(fRupz3edjt zv~2Xwci(=G9hF`f`TE0qcmFnO+v^8+R9go|eYoXT!oD4sju@R;Z96t&?(h|_D&OQf zZj71z_0IIS?HQAk7Js>9@vjxi4V}a9-`kgHcI1`ljhC%mSgc<+t@l~J^KbknuaX}( zX8iE!$ni3>6QKd(8ry0+izUna1I)z77aiYI@wLkm$V9olYLrwFk^H0ghrgBP_Vbwu zSXQSuN0#wXY*L}a`TM!zq$`{Szh+wkg+LtcO=8m-ZxY*x8U9a z$+_KyP3t3)=LW(?ng_4l!~N!S!)Gh6jF7oGh{kbiI)`(e(cC%vTz6#LElzp8$|ANb z$evb#C{uIa#YL<=-5#C~Zv!K+C1Ws5nPx4h(Li0UBy8m=4bF}_U^ax4Y^#K&;L}T-JK|iQARmPW*O%>Q4)CJ>XQ!9p=1{ z70dPhp;V^!lZHL;8d@0>X`kn|!X*E!qSJ4SEF{NkrIk&%tJ(_tdji2K@8&NJG-W=0 z;#g90gZFjSWrK&syAQ7`KVPL+9xJ19uI8L*onHivJ z=90gdu`4XoOIC@-1^e!w1s~D@OG}qq^8k}G;+fY%b3d#sJjjzTP1HEv$|GdcA4 z_s)l(c;xUXzI@?*dKx4jm|D&nboLshTq6J^EjvKXoWI@Jw!A|@O zd?(aOO`SJM&s<6j?q^vy*6BBzO@sHjkDa2E7F-DCJ8h;roD2x5PM(s>aq}&EP+E6& zf<@xI;-$8Uq4AmdF5m6heO`X=Mx95r&DQVY^M8#nY(Z15#P3GN=i05h9G9>BDs^El zE!Db_Ztf5?y2>IlW%p#mr3H)(WeZlEyJ`jQ#gZBwol^cCNaPQ9&6q?p2hR&~(!~lv zsLEL*+Z=R;&zY7y5q{!r^Alp#B#mry?);2gq`YI8kp#MyZRV{mf0VQ$;m2UhlGsfr z{%8m)Vr3EU+Qg|jlvt-)5J4Dgr4Wgdbz3gDl5ifim}V)v6uGEz>a&z-$kP?qKGuJR_{cXJ-T%uomD1Gn3gY`}hOKBYb75^KP_2%R`FFQ@Q;-Mbm zeQsLT~cly^{AL|R}y?rvb zHEm}>`?91A`|}pusqRHlbMMVJw`x2kXH`q* zjBvp{C9m67DzZ*YfqIiqTSrM+Y1~!A3RG%Ww-Uo~T-nuA`x>h3Zq|SL!{{F_81Hn% zxvmF-@A&!8arx5Vjdq!9KUcxZ*Ol2;s}j%F?{_=@jw&WuS5$R|NQwwk zguQpF9yEtmB#?X4&da*3?MiWYXcZI{vffoZdYf;2+SU$z*J&lkeA2G6zZM)+9~C5H zRUtQ17c;7z9lvUwRs1|BUQa?lT>izIxNo4T$(Q-o3ZOiGY6s)%xP3Awpg^G1Xba*e z&+dMn&pK)Oh$kAr+t&zYr8Us@HW(*>1MR^oYst^_Z!j0`9iAMVKK)gaw7Fp4I(3bh zlPunAuK%Fb-ftOvrZYzJCD}^7nL9u@U*nIU2mKM~ay7jxv0(dQNKLJiMrky2G+&V| zg-WA{tZXm*(w^7TaRix@u@HSL;#^eS^4(i&>tzRTdftMTlK0yf4wHl|_=}kZOaCO* zzqs+*@1Q%%;2~nxoxs4qQi2#J_sAkT%g<#|snWk^iMEhD5#D;7( zTvqL?3R8N0>SD6FZtdRMQ;#R%)+Oc9i#Z~z@UVct9X?#%&zooEG3>x6{PC zu%!#7?LN1jq?bb@?mga#3xZh24KWssTDx+^tECkr&w>@r0>ZRu?!5(+h(Qa3{zr-$ zt@$C|S@FJL@rUxmzsikLkR?<6R?pO=ARDgm+3{4ddGk--zzqBL^yxQiV%5lD zgRiS@cD5*-*o!Jw@kLqXw)62qefx1gtoqV5_=&?h=9!7Oi{FNCnK3F?VO1$3E_rk4 zrxTQaf0hz?)7AR0*_9O$@43x1lCIw$glGHBcHi?QG1dj^{+VUoOvArL_iN?FHOR^z z#haH#oO*C$m$^A-GrDVufz8?CJ=2i0{_-Z`_AV5~r$j9l@ql=p4wG`ynTkmyzfqDm zJtqvcWgDU?(LhjcPb02UC^BObvQl)clN!O+!c$Jvs}C%3r$T!-?;c?>MvWe}Rl2B& zI0nt9MicYfW`Clr_NO#fc^O(HAp10*bxTJCqQ*3TH=8&D&4SA$|A5I#sv`}`w3#6B zuW`JreUL(iVSbygnW(89CC=u4;ys&jk_vmaJ~l#YbFUx@B2XO0|B=-F%@tpH&x}kc z;h_U38apnjxTgerU0u{)_>0_M4 zIsc8JsM5uguzhroVW6{BPmZyuzpj~tzRrb`A#4?~c>==5=TZsY;3a?Kvps#kj-5M& z8V|IYcmnJV4=5FyNTs^AEj9Y}?E+ofc04tT1cRTTjF1BnRkyxXKb7Kj$Md zs}Yn|6^3R&l3ATw7xlsWorO_-_@Wsh=O7yQ?5W2w19SFESaH&q-b#cXF3$}rE^hw8 zG5c`3Kky7ew?*H|$zLk+1BuxF)C?fCiF4c)ucB{|E(nyOMq)!WPXc&q1sB1P%6;1~ zHCrjAtJR>D4;YE1GAU1wgDQB82enDkGG0dK($ZA5idW~NR>Wn>cpzv&qe2DD)k&c7 zGAA~hEtOy%8^2SCIxs9fiHCMb0&+yB*Vl{m6`sx}lg~2&6MH5<{$T5>?eU(a4P`&@ z-U;`^Fn04dyH_EZ%4JVSZ~LNA0y6zj%rN}~{Kjsk!Uv+*2wqbRaOO&Bu{Fr!c?@}0 zlY5%CVLnI+0Ra&Y=%`ZwYqqbnOoc$?WY$x|qg)IHZ5Z1h@O>kSB#3;6ln*4l$H*(5 zoeKQ|3F42bJd63j9Wdp&V)SQ4Sz`lJrXTKvmG2ZtKm@N7MPyP*Yy=}$1mu+x6Z=CM zuQ4rvkx~wS=uR0Qyn(U{4dW57tE@D@4!bK)bge=;{>^+aOA^}*R1<2CVgW$L0ogUv zHqJp5V8o8B%aF%g8x%G`9m1|omF|Mk+R!r~z&i>7wQ)?u8&kH{oWCJlz`J%L_jBBt;(y~P1&=X`$*;njhB zxl?fs6Scg<1X4+@cfA;)HMhfESkt%z=}dgU@;r5o2i$M$ql3-cxTW>y{6V@Ncz!q0 z7un+pk9DR9+=2%t6@c<)FM!BP;YmU{*JQvg$tLU;kYtN)g0sQdwBdjU_&`oKxiPJd zlM~SlBsX}qiNI8ia<+GXR3PVz0K)P_QIL4+*X z0Ke%peE;UsPIzKj7vPW3yf1+du&{mQKLT3}F)YWwRTB`&rMz+!F>D9uE#Yz?Qb`dx z$UFeREnzDL-Hm{!5XgT1b%>LAmiPw<1dsWKDRRAKO(%9 zc|i>1^REtvj?RsA5Rkw8whh(>^zQiVfD$ZM^;ZlOv-quv3q?DaBh*?%JLGrt;BJCY zZUxqQ$_~DHd)*{lf<;u#PANzqRBI@a_^b?hl@*c*2@dZlP6MB2j+4rs)9&B1)tF=E!zCo14wR5sD-yXtYPcjo zO~3zUyULi3D)So?k%|}Sc=P*UYMX+m&;**6aVPvlpJc&b7OvW(!c}|rn62*1ZorcJt+15#iBh|^!jMZ8aTcmfxA!T%81?7-aI%G)S3|Zkv(B{Q@}RsRlD@q? zE+olLO$YQWWiao`gY%%ku3co{0;UCLcc=g}n2+VZS61^@!9vD)MXicge9Hqk6`-Q+ z=#p__X2XVxYFmFG>JamkT%+kS{pfUcKual5hl62iwIrK~N`cA(Fth04i$Ea#4NS>F z5<@)j4~?Y**48Y8jsVcGU$8s{!%FiKP_@0Q(998?Qz_^9w(!*n^0<^@lp|@vBqmD020gxFfOeHB94Hi00tb8-WB6q;#71YyzpVjI|jp`3s(i;*KPtx zGse?v19WbI95h#%A8SH@96P9X+PX_vgd#Kyjeab|=6Ga_KuA)M@Ylny6->wMPl1nu z{AB4TTAA|C1$>_OxTN^0Ky6!!V41aIW=Az5Z?JO2vVoEhDi4mi8^TkAW7ii|7vJPt z^LXEbj@)4S^KSOai3cJa+`M}9rEsPSB-B&Winbd zE@;x?bsP^z_j&yjr!HNav3+!H%*0Ft10_z;c*O619a%T|g|^n!@Bu<;MH20WX1E1Z zO7qQ}TY=u=ZXBeXK(>WOC255q;8G|zLf5x8+h?5^q@9r71cg4fefSwzGL7SpECEfy zG~{uw3<})6JN&Dx7;CRk+Icfl>+*&8Lp(d7SXiX?J>Mii86RSbMax%rwulZI%{(Hp$jm zWSyk-h;GmMmxXp>lI8+UJp(V$mF6h66BE$r*!5n0KW+iYR*#zr^w3v$h_pD7mw_q8 zz0!Bt>fo6iE~eqp8O#(hnPAPI8;fED^R&)#je|^_?KgD^8%O#k36Sclw}h}7+khKW4PyI0C^G5F%H_!`jY!CBswmzE-D089yBV5*SN zS*pj-vn2Oyp5#8*k%CmXf*6%W* zYU9vKVwxIGAFttZ<3!B+h^i`=*6~6{tSv=>d{3K&UmQ>6Y=5OJw3G=lUdWsHeGisY zOCKRnr3#+QYE@$+{Vsq3^l9rziJZp0fGjhZF^DHQy$?^)>j4i_gZM6?qMnroEbf)3 zG#;RV7vnIvTYD`bM5Acf6^tykz%rXt;LYMXa|fDxz)y+hsJV21^^RVFEp|8vP|RG5 z(Uimehbqh!v~}ie)Dr$1z0B5T@v^XEYMj|=5ZvMqT*DQ{Rj)i7sNPt*rjL4~Sl!46 z|7KaPQK)PcY6UDNJcC+d>DqAfE~T&@UklO1m7VRXAO#yycV_p>Q&y}1deU{r)%5le zYOKJPhqWmiIlYR8dQ$pVeih1KxHNcm`XPtW5ei&>#>@b3K&+6me;xkIK)t_&V|^q? z`HE5wAkdTQS0~|WV(o(7y*p4G4zJL6Cu&6$3lhNOUr!u|S$%80Twf}}I$HyJB@HuR ztEDzzqynka>qNsj{%DP?lrDLpfk&%FURPwuXkFV?fhm=GZHVUa-cg~sF14bQBfcJ_ z;5ed>f@7pKkq-Xd@&#ApbY^HL-%o<_aL@5}-;m^cvfVIEyR`P;(*rjI-6+MIEh*a>!7O= zdk$(Y_zX*Lq^m7mP%vq`OwUL*wG>&Y%td46PdOie6yK+8OkB5jrKzo{vEkU(&^evq z9kWAeuRAH`XHA8h{J=YAe7#2carrQ;R#Si9UN=G0mc)j2ANQGjJq0pZujSv(h(9iY z`lJNdMtp<6KTXRe)Fo%hIy_Wto_jUUi#YeMP~s&GY&^V-aCIYH>l7_MHCZcflw{Ye zTQOTl>eNP+)G0{2m{`~K1DRWth*ik~t<-?{M*RA5*M?LjUEXh{b@O^o44t%li#F&n z$#TRzPwldXd!Vjv;0-8*HzcWrWc5wGN0!#TE5v71W%(4@yR)rL2~(%d?*w}C+7><> zq)Sh-#3{K>qfGFV3IEE619eBcx@X>0saT!0Gk7!=YpGV<5VAefO)0b?Xfc6UX>^jfBQ{?p$%+EmgUYaxS_R;uhC-2gkealboVyS^_jmw zTowqLlZ13~UMoYm{Tulicq3nGsh$I}fr5@n*dPh`bho%&)xkBqbPR$_WXSj-x(|+( zA{DaUGP}&%Tl!=iqEO6fdOwVR;*ffx2!|<+ydza#$JTxL82mr=9`0*;4}_PGF8$6& zuxUED0S-0kbqqDG;X_8~`tmZm{Pi8$C&Y_ZV7vAtZaEMQ&PoK6o?9qCy+So@6PLTd_nd+fdVD zNu)~GjMbri@8WZ~`%u5?zcjP~bYFR5@#;6t1HF$jql}zWhb7zJ_Q~vMmVfj0=~J8M?y857SVno`RjLlB6m7jN^h8c zC%NiWvC?O|{@3CDkA5jZ3$Lr|>~_>&GB@729ly6*j@|LZN+iHiqVbi%FFyRm`$07I zWW2+Lk7KhNJ(f-tew@!(N*nWQ-sa3;x--|*kr#dQJ8?vsbr4ILvb`d68?;2a!HS#P z<}vAJQAS;IbgFC&5N+LjtFEAGQXz(0Ijy&ofzhfhL z68iq7FpXI~Ei+;CFV^&Z&l<43pZT|6zV5b>GxeuMIO@7{O{FPlkb`K?eo!FMZ;~;; zmR$Y#-Qr4j(+j6qvAYc~1iw~QeWfCoqn3`5>|I#n9mgt;3eR#0^)xJg$)aQdBts z4kV8ln(TdIEd>Y6A;=)x=XP;kO*UHJDrD`!9iR5UT`jFtLv&2M24bLsgZKDuxYKp| z9f?XW{64P&;=~ghUv(|!hoz>`-#SlQ_1=Q%55?=RY6Ob?H+DYYDN#@->}UI9-0pkK zb@}{|JU1_)&7SCR=dum8(Lb-ebw%DfC&EwVnS8!2V-wf8&{~v|pB}pJyFC_~!=MruC$fIj;MD1-SFvSBGc*05O-Gi^2y69K#2mD=pw6 zo(7E#^C(u+FXEW4>jo`u2NeepA3tum18$un7^(1EiDL^IG>9&2qyqc0dlLOTxJ89! zzbj3Y!%1{w+UoN5q<0gVgTJb=3?8*%gzVP%MvcQBZ9(?DpKmL; zw`=Ki1~;UvZqL?^vx>coZ4w#|tMlkhO`^iB_oOz$SPSFBf<+QWf?e1|2kxc`-ljoY zof!eS*P57_3^G5(CYzsRrdv07SNteQu8U<0YJ~aAKT4X_=n*)J?PNE`a>NOVI$i{VD>R%4_&qf*|8+d$$@WQ0FMJ$O+O%s$ z+J*C(yUgD@84~Y9LyB1LTz|`b^Z2Ij;gf{zF6kM*x1giQE}@NGb<^%`Vfqms6S!HG zz%l+k{Y>v^mYaED3$EBcdvp>4IUo*3wRSB8|Cqm@9rU8tQ3^!NlG)O=a7H9E)XiGU;CGtHoZ6t z{-#?jMp&miW3&f2GUPKoFOyq-sq|G@f`SEj+;utbXMPb!l4@`vpCb?fbF&-i(_z_p z80-j9l&T=o$9?+@meS!(d%0zHTeRhge;renf9<{me*4>cqeKdO9-w9R!yT%>@@Fqj z*t2`-g|Uf(y!~#=7W(G-8BLyJFEhugMGrc(T^|t^*NphmS%HoMo=@hLZSA$L(>9}D z-neZ3jpH&b7wX$Dc|Q6daHXBq&pkRi{amxM=i%X#>GoT4crB>(!{L>(BXJ@50jlDv zbm4uC4ek27;%o)P;@NA!l^qbFX5Wq5f(sVOX)7)unY0W&yKp!jnMm+$6OC?mQ>k3 zaY)e`cnqsx90X|ssdY#S``ShfLv@bjrNIeKDFbYV$EIt9 znd>fZ3M4$q=UmX!k3Na?Y*?`{tzO{aEk6jINAIRZMBNoQJp4#H3t|9B|0rV|_B9gK zS^5qJIRc*da59(66LPfXJDPI&!)VgqlCxc00MTB}2)Af8&LDS67l;twkqp?{A$ztn zd!dQe`=ev&qJkis@#HT+W)~FcxIQeXaD5a%A+9C2e9+p#u2@ax_Kx$D3gR`Mw<{DWpJr_1;GuU-RX^idsmX4aDi%tqv6EWO@8VjQlVQdiJOzk&#GdbPEzloV)5f zPvE8?G9NO}L3V-tS}QFWb(nz^c#JoQAEOusR??RsTu0*8$K5v4&(UK-v^NFdR^>~# zkpDQwfNJ9BXmFbJsd*kxM^8YX3OxZX&;=X(`Wg9Au8OaSfC8RDW_TspOllZbUYdY- zJRqRJc)*2Clm+mm5Ufo0OlQJ19ch5`1SFf>xX9V4Ct-O-0fK}T_a<@S!5W`K_6kyA zEWn5m=Nl*xa9CXJlVuG(!iWkzNTm}okS<~Y1y-rF_0+mb#;^bKUNub%(S=rmY+D5- z08s9H6tvTdK_jSh0*(Fg*oSaD_JiO~8sGvk0<9$CfC4G&W=c%M#2vf@&_Xvt!Gz2N zy@#l6BYm1$tn!gPs~!BOB|ro zo4Et9yQn=Fv^#-r16N0ABxE5K!3+wP@1WoQcUcCU5>5=5M4ZCc?a?-cKKv{uJgkf`2z!6Kp7d-(D=~*K zC(4>Y{%My*8gafr1&SK*&(=o6gS;j)wmUxRk}yb3HJqhACkl0Sj-)~p&q}&q>qUz5 zoAo1XC@?u_rt7GugUH{G&683-t~z6UV$h4=H2M2mXH$}{X(2T!+79EBXV|JFJ?JIk z5iu9W@dMKHM2ezbU|wpfiiRR_Q=uFCKs!;S3;#+5f^*R<0RjHinc>z(fg@Q@kI)u) zx^zI?0x7V{CRW-DB-&@@66?Vt8C2+xgzX{j2SAE5`R@>C?Z8RA2NcOAzv{qj7lNK8 z&iVOvqPE5ki9|Ar5FI26Domj1^@JIZi5ckuAy$HGgcnHGbMi9{o36e{>V_7QvYDf; zy&l{^PcV2q?inUHhat+~dLTpElYo1FakpEzxSQBg9Sj6O$^#-$4Ai{m-I`Kk!66m1 z6zQaeVMrjObVLIdC!>wUq=z$oT0|B=bmI;}55hao2t9F3D-%NzBmQNu$gB$FBDloc z-zt6mFN1M!bbwlo0NPW-Q3`)tXzhr`(+eu%NY`Psj^We(T@fD#)GACD$;+7F_krbp zRQ}}vQbVWn_^bnU^fk zbs}$2phFU|Q!6$P-ja$>$p0K;K(da+H7|<%iK`9LwfCE z4_~e45g)dTg%G<=cE#HwGwmbY%03Y_)OA&};_l>%uiModJY z0v$1VFR_GZRPpI&O`?pWWEP=*;;dcbU!$~5iN1$(@APsgAA`^c3S_Mc9cV2>7c-iZTV@|({L^aZo4McY+#cHAd0!E!T6U<-O@85h zF@zC`UXuQr7DFLce?y@RG4n9^J%&QbolcETeJ#G?Lq&aU%%TL};HBK*m*WlPr^VP~ zOMgQ;U#vv_`wZz!9O-=x=@^;n7v4~s+t^v7+LiTkc=K698`8PO_cx?l1sYVp$B-_@ zB2BieugzCUKD)15n|>GhQT~!-=Ct@GIrjHWk|^v&zsEOewu^g;|N6c*Up|Wdh7j{{ z?hPV(Io^U=jFZwf_cu<8L}cjiF-|Ia#?-N4yVS19yP`|4rHUpo0%xc=M+&4DW)Z4oG@5 z3M?dlkP#`w4{{bIQXUT-*(>zGB7NNXq*{B%gV!OE`gn7<@Z~rR{z=7_m1)i+{7Akq z#tlRfp>br$dLq6r^37oOc`B{t2i?024I?>7WGc z3Rw0x$b8Cbxa4eH0J0V1W=e3f7!BhCIVeN`Xh+f?_j7cwki-MrYFhPnwJKgls|N7x z>-0N=qHH;mrWWn}%)h$>+vjqnyGO-BX{HAtOvNMwbO+|7c|;?B=2)66A|q_Ak9^3N`&z3ex8Yx( z2lo>W{FCMgn3eVjj59VgylZl23)P|)qE4QojSkxeULAwSL8{AGRm0ISGn+*T?H~wQ zhVs;0^f|A*a>6T+UTbKw_ldO>99Z-LqzEgXmn45LbAN7H~a8gL9jwkH~>0}6qYLnU3wjym>pdo1eh@->hkfF0)h5G*RM0>Ql3 z;kOuhN2T5fX3l;Neoilhf4|L~(1xA?0kUBVde<%yLv_sOBUbttzXku^^XeRfR(!s# z;NGsK(;3{5vbsH6JI*ThF1AT%NK%G)^k(}Sl-z)hm{_P`une)W@!tr~;Ty+mVDhhX zoVZE3;vngCoAL06c1CTqb1hcWFY1kN1J?~&+zu)ZAU=NFaEDjnM+GC{MT6fh*Lg4H zqDha?deL;L?zmB51Q_+0FhgvtB!Ba^Xs1dW_MnSvrBOj2Vy6m{u`eF$lOhD)@YAtI zsJJ5_`H-`ot{`mrSr}_n0>}*_u}bRDx=6hCW_0lZNZw#5Gd9x(lz}7pZhwjAq{hjp zrHM^Mg^D!lP4c(=_$#FBD>uIrN2FN?v6Ly>`yrmTp7z*^JNQ-UZHpUP)52mcntPqO z>ny4VLO=mfLJ4qTZ(r%ztL55Hg-Z^lKj zToU%sY<69|3WKRw&+CY&STg3ZOkjlXBXHpsOrNqje`AtZHq6;d86qh)8E*$9@Lw0*#G3g+vJ zO)t)Zv8BaggmtsX#u{^D?>Rmr7riCFs}OZ1a}UV0eU<34X{;>{z#mRp*~G zCOaFey}8#_bHz}gB~(V&{v;SK`DO-tK?%(dV%Robn zH@+0d?LP`}@rh^qifxbsfJx@CZ8s3qHCRDgS+L=H*u9Q0UGDblA z(m;M3DbU|>)=_|@!wu^K$c=}I0dCq)C64)_1T(=3oYMhO(|Ej)o{LFLI9))S3S}Wl zjy8OvDAhVw$N54Q+=*1^M`F4F-x})-k{8V`?!-*RsA4LxqPWFcJCd4NNSD79*<;UM z;@IOx1v*XBRbr4`kPRi8%iw=uo(__A#2zZtT*;2tw8kEd!dJxH3M_BxWipSN_vjim zYfLpsjx!8Mv)K-KZ&6fvlv> zss=k&J*}^9tHjznS=~tW@okC&*cW%jB?ttrA3?|W^N#wH8g^l*4pEqhS-3kE$y1&J z>dy&^w&;>(pg`_g&Adk3U*-$!)MH3~Bvn%*wJ`KFi7PeGgOjw5(wj6|e|Z#OhhPdP zt+RBz!JJXIo&o|!&f0rCc;^)=iL`duR;=E<$Va;J^cDZ*4xe&Ten~>gX=VQ9y31Em6B{0p8;AW_&=}-CDo5GXj^pxwfT?_MG*Rv`0TxczUZd z_goEZoIkMgiy4^8*#T+eL01_)JyNz_8Xr$n(5JyosCjiR0_;&iRp_vBlV1HlezSD$ literal 0 HcmV?d00001 diff --git a/cmd/pilosa-fsck/release-pilosa-fsck/example.sh b/cmd/pilosa-fsck/release-pilosa-fsck/example.sh new file mode 100755 index 000000000..79fd4cf11 --- /dev/null +++ b/cmd/pilosa-fsck/release-pilosa-fsck/example.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set +x +export PATH=.:${PATH} + +# unpack the sample Molecula Pilosa cluster. +tar xf backups.tar.gz + + +# check if repair is needed. +pilosa-fsck -replicas 3 backups/node0/pilosa backups/node1/pilosa backups/node2/pilosa backups/node3/pilosa + + +# yes, so do the repairs. This can be done first (only) as well. +# +pilosa-fsck -fix -replicas 3 backups/node0/pilosa backups/node1/pilosa backups/node2/pilosa backups/node3/pilosa + + +# check again if you like +# +pilosa-fsck -replicas 3 backups/node0/pilosa backups/node1/pilosa backups/node2/pilosa backups/node3/pilosa diff --git a/cmd/pilosa-fsck/vprint.go b/cmd/pilosa-fsck/vprint.go new file mode 100644 index 000000000..83b1681f7 --- /dev/null +++ b/cmd/pilosa-fsck/vprint.go @@ -0,0 +1,177 @@ +// home: https://github.com/glycerine/vprint +// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved. +// License: MIT +// +// MIT License +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package main + +import ( + "fmt" + "io" + "os" + "path" + "runtime" + "runtime/debug" + "sync" + "time" +) + +const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00" +const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00" + +// for tons of debug output +var VerboseVerbose bool = false + +// convience functions for . import +var pp = PP +var vv = VV + +var panicOn = PanicOn + +func init() { + // keeper linter happy + _ = pp + _ = vv +} + +func PanicOn(err error) { + if err != nil { + panic(err) + } +} + +func PP(format string, a ...interface{}) { + if VerboseVerbose { + TSPrintf(format, a...) + } +} + +func VV(format string, a ...interface{}) { + TSPrintf(format, a...) +} + +func AlwaysPrintf(format string, a ...interface{}) { + TSPrintf(format, a...) +} + +var tsPrintfMut sync.Mutex + +// time-stamped printf +func TSPrintf(format string, a ...interface{}) { + tsPrintfMut.Lock() + Printf("# %s %s ", FileLine(3), ts()) + Printf(format+"\n", a...) + tsPrintfMut.Unlock() +} + +// get timestamp for logging purposes +func ts() string { + return time.Now().Format(RFC3339UsecTz0) +} + +// so we can multi write easily, use our own printf +var OurStdout io.Writer = os.Stdout + +// Printf formats according to a format specifier and writes to standard output. +// It returns the number of bytes written and any write error encountered. +func Printf(format string, a ...interface{}) (n int, err error) { + return fmt.Fprintf(OurStdout, format, a...) +} + +func FileLine(depth int) string { + _, fileName, fileLine, ok := runtime.Caller(depth) + var s string + if ok { + s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine) + } else { + s = "" + } + return s +} + +func stack() string { + return string(debug.Stack()) +} + +func FileExists(name string) bool { + fi, err := os.Stat(name) + if err != nil { + return false + } + if fi.IsDir() { + return false + } + return true +} + +func DirExists(name string) bool { + fi, err := os.Stat(name) + if err != nil { + return false + } + if fi.IsDir() { + return true + } + return false +} + +func FileSize(name string) int64 { + fi, err := os.Stat(name) + if err != nil { + return 0 + } + return fi.Size() +} + +// Caller returns the name of the calling function. +func Caller(upStack int) string { + // elide ourself and runtime.Callers + target := upStack + 2 + + pc := make([]uintptr, target+2) + n := runtime.Callers(0, pc) + + f := runtime.Frame{Function: "unknown"} + if n > 0 { + frames := runtime.CallersFrames(pc[:n]) + for i := 0; i <= target; i++ { + contender, more := frames.Next() + if i == target { + f = contender + } + if !more { + break + } + } + } + return f.Function +} + +// happy linter: +var _ = DirExists +var _ = FileExists +var _ = Caller +var _ = stack +var _ = RFC3339MsecTz0 +var _ = RFC3339UsecTz0 +var _ = AlwaysPrintf +var _ = FileSize diff --git a/go.mod b/go.mod index 33b1d4d59..0425a1b03 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect github.com/davecgh/go-spew v1.1.1 github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361 + github.com/dustin/go-humanize v1.0.0 github.com/glycerine/lmdb-go v1.9.32 github.com/go-ole/go-ole v1.2.4 // indirect github.com/gogo/protobuf v1.2.1 diff --git a/holder.go b/holder.go index 5d8691a92..bece8baf2 100644 --- a/holder.go +++ b/holder.go @@ -1218,7 +1218,7 @@ func (h *Holder) setFileLimit() { } } -func (h *Holder) loadNodeID() (string, error) { +func (h *Holder) LoadNodeID() (string, error) { idPath := path.Join(h.path, ".id") h.Logger.Printf("load NodeID: %s", idPath) if err := os.MkdirAll(h.path, 0777); err != nil { @@ -1227,7 +1227,9 @@ func (h *Holder) loadNodeID() (string, error) { nodeIDBytes, err := ioutil.ReadFile(idPath) if err == nil { - return strings.TrimSpace(string(nodeIDBytes)), nil + nodeid := strings.TrimSpace(string(nodeIDBytes)) + h.Logger.Printf("I am NodeID: %s", nodeid) + return nodeid, nil } if !os.IsNotExist(err) { return "", errors.Wrap(err, "reading file") @@ -1237,6 +1239,7 @@ func (h *Holder) loadNodeID() (string, error) { if err != nil { return "", errors.Wrap(err, "writing file") } + h.Logger.Printf("I am NodeID: %s", nodeID) return nodeID, nil } @@ -1728,7 +1731,7 @@ func (s *holderSyncer) readIndexTranslateReader(rd TranslateEntryReader) { } // Apply replication to store. - store := idx.TranslateStore(s.Cluster.keyPartition(entry.Index, entry.Key)) + store := idx.TranslateStore(s.Cluster.Topology.KeyPartition(entry.Index, entry.Key)) if err := store.ForceSet(entry.ID, entry.Key); err != nil { s.Holder.Logger.Printf("cannot force set index translation data: %d=%q", entry.ID, entry.Key) return diff --git a/index.go b/index.go index 4e7d3459a..f2c045726 100644 --- a/index.go +++ b/index.go @@ -33,6 +33,7 @@ import ( "github.com/pilosa/pilosa/v2/stats" "github.com/pilosa/pilosa/v2/testhook" "github.com/pkg/errors" + "github.com/zeebo/blake3" "golang.org/x/sync/errgroup" ) @@ -734,6 +735,19 @@ func (idx *Index) SliceOfShards(field, view, viewPath string) (sliceOfShards []u type AllTranslatorSummary struct { Sums []*TranslatorSummary + + RepairNeeded bool +} + +func (ats *AllTranslatorSummary) Checksum() string { + ats.Sort() + hasher := blake3.New() + for _, sum := range ats.Sums { + _, _ = hasher.Write([]byte(sum.Checksum)) + } + var buf [16]byte + _, _ = hasher.Digest().Read(buf[0:]) + return fmt.Sprintf("blake3-%x", buf) } func NewAllTranslatorSummary() *AllTranslatorSummary { @@ -741,6 +755,7 @@ func NewAllTranslatorSummary() *AllTranslatorSummary { } func (ats *AllTranslatorSummary) Append(b *AllTranslatorSummary) { ats.Sums = append(ats.Sums, b.Sums...) + ats.RepairNeeded = ats.RepairNeeded || b.RepairNeeded } func (ats *AllTranslatorSummary) Sort() { @@ -761,57 +776,158 @@ func (ats *AllTranslatorSummary) Sort() { if a.PartitionID > b.PartitionID { return false } - return a.Field < b.Field + if a.Field < b.Field { + return true + } + if a.Field > b.Field { + return false + } + return a.NodeID < b.NodeID }) } // sums is only guaranteed to be sorted by (index, PartitionID, field) iff err returns nil -func (i *Index) ComputeTranslatorSummary(verbose bool) (ats *AllTranslatorSummary, err error) { +func (i *Index) ComputeTranslatorSummary(verbose, checkKeys, applyKeyRepairs bool, topo *Topology, nodeID string) (ats *AllTranslatorSummary, err error) { i.mu.RLock() defer i.mu.RUnlock() ats = &AllTranslatorSummary{} - fmt.Printf("\nindex: %v\n=================\n", i.name) + if verbose { + fmt.Printf("\n# index: %v\n# =================\n", i.name) + } for _, fld := range i.fields { - sum, err := fld.translateStore.ComputeTranslatorSummary() + sum, err := fld.translateStore.ComputeTranslatorSummaryRows() if err != nil { return ats, err } sum.Field = fld.name sum.Index = i.Name() sum.Checksum = hash.Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, fld.name, i.Name()))) - + sum.IsColKey = false if verbose { - fmt.Printf("row blake3-%v keyN: %5v idN: %5v field: '%v'\n", sum.Checksum, sum.KeyCount, sum.IDCount, fld.name) + fmt.Printf("# row blake3-%v keyN: %5v idN: %5v field: '%v'\n", sum.Checksum, sum.KeyCount, sum.IDCount, fld.name) } ats.Sums = append(ats.Sums, sum) } - - fmt.Printf("====================\n") + if verbose { + fmt.Printf("# ====================\n") + } for partitionID, store := range i.translateStores { - sum, err := store.ComputeTranslatorSummary() + if checkKeys { + prim := topo.PrimaryNodeIndex(partitionID) + primID := topo.nodeIDs[prim] + + // note: we fix irrespective of nodeID == primID now, so that we + // get a fine grain report of what maps were off. + + if verbose { + // This is pilosa-fsck output, not regular log. + fmt.Printf("# doing repair of keys on nodeID '%v', and primID '%v'\n", nodeID, primID) + } + changed, err := store.RepairKeys(topo, verbose, applyKeyRepairs) + if err != nil { + return nil, errors.Wrap(err, "ComputeTranslatorSummary() call to store.Repair()") + } + if changed { + ats.RepairNeeded = true + } + } + + // key repair has to be above, because we compute the checksum below. + + sum, err := store.ComputeTranslatorSummaryCols(partitionID, topo) if err != nil { return ats, err } if sum == nil { - // probably one of the Noop stores + // probably one of the Noop stores from the tests. continue } + sum.IsColKey = true sum.PartitionID = partitionID sum.Index = i.Name() + sum.StorePath = store.GetStorePath() + sum.NodeID = nodeID + sum.IsPrimary = topo.IsPrimary(nodeID, partitionID) + + replicas := topo.GetNonPrimaryReplicas(partitionID) + for _, replica := range replicas { + if nodeID == replica { + sum.IsReplica = true + break + } + } sum.Checksum = hash.Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, partitionID, i.Name()))) if verbose { - fmt.Printf("col blake3-%v keyN: %10v idN: %10v paritionID: %03v \n", sum.Checksum, sum.KeyCount, sum.IDCount, partitionID) + // This is not regular index logging. This is output of the pilosa-fsck tool. + // So it must be printing straight to stdout. + fmt.Printf("# col blake3-%v keyN: %10v idN: %10v paritionID: %03v primary: %03v\n", sum.Checksum, sum.KeyCount, sum.IDCount, partitionID, sum.PrimaryNodeIndex) } + ats.Sums = append(ats.Sums, sum) } return } -func (idx *Index) WriteFragmentChecksums(w io.Writer, showBits, showOps bool) { +// returned by WriteFragmentChecksums +type IndexFragmentSummary struct { + Dir string + NodeID string + Index string + IndexPath string + Frg []*FragSum + + RelPath2fsum map[string]*FragSum +} + +func (ifs *IndexFragmentSummary) String() (s string) { + s = fmt.Sprintf(`&pilosa.IndexFragmentSummary{ + Dir: '%v' + NodeID: '%v' + Index: '%v' + IndexPath: '%v' +`, ifs.Dir, ifs.NodeID, ifs.Index, ifs.IndexPath) + for _, frg := range ifs.Frg { + s += frg.String() + "\n" + } + s += "}\n" + return +} + +// used in IndexFragmentSummary +type FragSum struct { + AbsPath string + RelPath string + + // critically, NodeID is how pilosa-fsck figures out if this + // fragment should be deleted if it is on a node it should not be. + NodeID string + + Index string + Field string + View string + Shard uint64 + Hotbits int + Checksum string + Primary int + + ScanDone bool // pilosa-fsck will set this once done to avoid repairing multiple times. +} + +func (fsum *FragSum) String() (s string) { + return fmt.Sprintf("%#v", fsum) +} + +// if verbose, then print to w. +func (idx *Index) WriteFragmentChecksums(w io.Writer, showBits, showOps bool, topo *Topology, verbose bool) (sum *IndexFragmentSummary) { + sum = &IndexFragmentSummary{ + Index: idx.name, + IndexPath: idx.path, + RelPath2fsum: make(map[string]*FragSum), + } paths, err := listFilesUnderDir(idx.path, false, "", true) panicOn(err) index := idx.name @@ -822,14 +938,37 @@ func (idx *Index) WriteFragmentChecksums(w io.Writer, showBits, showOps bool) { continue // ignore .meta paths } abspath := idx.path + sep + relpath + primary := topo.GetPrimaryForShardReplication(index, shard) checksum, hotbits := RoaringFragmentChecksum(abspath, index, field, view, shard) - fmt.Fprintf(w, "frg blake3-%v field: '%v' view: '%v' shard: %3v hotbits: %10v\n", checksum, field, view, shard, hotbits) + if verbose { + fmt.Fprintf(w, "# frg blake3-%v field: '%v' view: '%v' shard: %3v hotbits: %10v primary:%03v\n", checksum, field, view, shard, hotbits, primary) + } + fsum := &FragSum{ + AbsPath: abspath, + RelPath: relpath, + Index: index, + Field: field, + View: view, + Shard: shard, + Hotbits: hotbits, + Checksum: checksum, + Primary: primary, + } + sum.Frg = append(sum.Frg, fsum) + _, already := sum.RelPath2fsum[relpath] + if already { + panic(fmt.Sprintf("relpath '%v' was already present!?!", relpath)) + } + sum.RelPath2fsum[relpath] = fsum n++ } if n == 0 { - fmt.Fprintf(w, "empty index '%v'", idx.path) + if verbose { + fmt.Fprintf(w, "empty index '%v'", idx.path) + } } + return } func (idx *Index) Txf() *TxFactory { diff --git a/license.exceptions b/license.exceptions index 5fb7846e8..2c8c4162f 100644 --- a/license.exceptions +++ b/license.exceptions @@ -21,3 +21,5 @@ ./synthload/vprint.go ./proto/vdsm/vdsm.proto ./proto/vdsm/vdsm.pb.go +./cmd/pilosa-fsck/vprint.go + diff --git a/mock/translator.go b/mock/translator.go index e577baf12..6afd0d48e 100644 --- a/mock/translator.go +++ b/mock/translator.go @@ -37,7 +37,10 @@ type TranslateStore struct { EntryReaderFunc func(ctx context.Context, offset uint64) (pilosa.TranslateEntryReader, error) } -func (s *TranslateStore) ComputeTranslatorSummary() (sum *pilosa.TranslatorSummary, err error) { +func (s *TranslateStore) ComputeTranslatorSummaryRows() (sum *pilosa.TranslatorSummary, err error) { + return +} +func (s *TranslateStore) ComputeTranslatorSummaryCols(partitionID int, topo *pilosa.Topology) (sum *pilosa.TranslatorSummary, err error) { return } @@ -93,6 +96,14 @@ func (s *TranslateStore) ReadFrom(r io.Reader) (int64, error) { return 0, nil } +func (s *TranslateStore) RepairKeys(topo *pilosa.Topology, verbose, applyKeyRepairs bool) (changed bool, err error) { + return +} + +func (s *TranslateStore) GetStorePath() string { + return "" +} + var _ pilosa.TranslateEntryReader = (*TranslateEntryReader)(nil) type TranslateEntryReader struct { @@ -107,3 +118,10 @@ func (r *TranslateEntryReader) Close() error { func (r *TranslateEntryReader) ReadEntry(entry *pilosa.TranslateEntry) error { return r.ReadEntryFunc(entry) } + +func (s *TranslateStore) KeyWalker(walk func(key string, col uint64)) error { + panic("TODO") +} +func (s *TranslateStore) IDWalker(walk func(key string, col uint64)) error { + panic("TODO") +} diff --git a/server.go b/server.go index a011ab0ba..be0a41aa3 100644 --- a/server.go +++ b/server.go @@ -579,7 +579,7 @@ func (s *Server) loadNodeID() string { if s.nodeID != "" { return s.nodeID } - nodeID, err := s.holder.loadNodeID() + nodeID, err := s.holder.LoadNodeID() if err != nil { s.logger.Printf("loading NodeID: %v", err) return s.nodeID diff --git a/server/server.go b/server/server.go index 077c8a55c..180fd4334 100644 --- a/server/server.go +++ b/server/server.go @@ -37,6 +37,7 @@ import ( "golang.org/x/sync/errgroup" + "github.com/pelletier/go-toml" "github.com/pilosa/pilosa/v2" "github.com/pilosa/pilosa/v2/boltdb" "github.com/pilosa/pilosa/v2/encoding/proto" @@ -622,3 +623,10 @@ func (f *filteredWriter) Write(p []byte) (n int, err error) { } return len(p), nil } + +// ParseConfig parses s into a Config. +func ParseConfig(s string) (Config, error) { + var c Config + err := toml.Unmarshal([]byte(s), &c) + return c, err +} diff --git a/server/server_test.go b/server/server_test.go index ca1ea0e3e..270fdcaf5 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -31,7 +31,6 @@ import ( "testing" "time" - "github.com/pelletier/go-toml" "github.com/pilosa/pilosa/v2" "github.com/pilosa/pilosa/v2/http" "github.com/pilosa/pilosa/v2/pql" @@ -338,7 +337,7 @@ func TestMain_MinMaxFloat(t *testing.T) { // Ensure the host can be parsed. func TestConfig_Parse_Host(t *testing.T) { - if c, err := ParseConfig(`bind = "local"`); err != nil { + if c, err := server.ParseConfig(`bind = "local"`); err != nil { t.Fatal(err) } else if c.Bind != "local" { t.Fatalf("unexpected host: %s", c.Bind) @@ -347,7 +346,7 @@ func TestConfig_Parse_Host(t *testing.T) { // Ensure the data directory can be parsed. func TestConfig_Parse_DataDir(t *testing.T) { - if c, err := ParseConfig(`data-dir = "/tmp/foo"`); err != nil { + if c, err := server.ParseConfig(`data-dir = "/tmp/foo"`); err != nil { t.Fatal(err) } else if c.DataDir != "/tmp/foo" { t.Fatalf("unexpected data dir: %s", c.DataDir) @@ -600,13 +599,6 @@ func GenerateSetCommands(n int, rand *rand.Rand) []SetCommand { return cmds } -// ParseConfig parses s into a Config. -func ParseConfig(s string) (server.Config, error) { - var c server.Config - err := toml.Unmarshal([]byte(s), &c) - return c, err -} - // MustMarshalJSON marshals v into a string. Panic on error. func MustMarshalJSON(v interface{}) string { buf, err := json.Marshal(v) diff --git a/test/cluster.go b/test/cluster.go index 41b1cea13..c71e3e68f 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -248,7 +248,7 @@ func newCluster(tb testing.TB, size int, opts ...[]server.CommandOption) (*Clust commandOpts = opts[i%len(opts)] } m := NewCommandNode(tb, i == 0, commandOpts...) - err := ioutil.WriteFile(path.Join(m.Config.DataDir, ".id"), []byte(name+"_"+strconv.Itoa(i)), 0600) + err := ioutil.WriteFile(path.Join(m.Config.DataDir, ".id"), []byte(name+"__"+strconv.Itoa(i)), 0600) if err != nil { return nil, errors.Wrap(err, "writing node id") } diff --git a/translate.go b/translate.go index 101a854a3..0f8169e5c 100644 --- a/translate.go +++ b/translate.go @@ -17,6 +17,7 @@ package pilosa import ( "context" "encoding/json" + "fmt" "io" "io/ioutil" "sync" @@ -90,7 +91,15 @@ type TranslateStore interface { // the read payload. ReadFrom(io.Reader) (int64, error) - ComputeTranslatorSummary() (sum *TranslatorSummary, err error) + ComputeTranslatorSummaryRows() (sum *TranslatorSummary, err error) + ComputeTranslatorSummaryCols(partitionID int, topo *Topology) (sum *TranslatorSummary, err error) + + KeyWalker(walk func(key string, col uint64)) error + IDWalker(walk func(key string, col uint64)) error + + RepairKeys(topo *Topology, verbose, applyKeyRepairs bool) (changed bool, err error) + + GetStorePath() string } // TranslatorSummary is returned, for example from the boltdb string key translators, @@ -101,6 +110,14 @@ type TranslatorSummary struct { // ParitionID is filled for column keys PartitionID int + NodeID string + StorePath string + IsPrimary bool + IsReplica bool + + // PrimaryNodeIndex indexes into the cluster []node array to find the primary + PrimaryNodeIndex int + // Field is filled for row keys Field string @@ -112,6 +129,41 @@ type TranslatorSummary struct { // IDCount has the number of ID->Key mappings IDCount int + + // false for RowIDs, true for string-Key column IDs. + IsColKey bool +} + +func (s *TranslatorSummary) String() string { + return fmt.Sprintf(` +TranslatorSummary{ + Index : %v + PartitionID: %v + NodeID : %v + StorePath : %v + IsPrimary : %v + IsReplica : %v + PrimaryNodeIndex: %v + Field : %v + Checksum: %v + KeyCount: %v + IDCount : %v + IsColKey: %v +} +`, + s.Index, + s.PartitionID, + s.NodeID, + s.StorePath, + s.IsPrimary, + s.IsReplica, + s.PrimaryNodeIndex, + s.Field, + s.Checksum, + s.KeyCount, + s.IDCount, + s.IsColKey, + ) } // OpenTranslateStoreFunc represents a function for instantiating and opening a TranslateStore. @@ -127,7 +179,7 @@ func GenerateNextPartitionedID(index string, prev uint64, partitionID, partition // Try to use the next ID if it is in the same partition. // Otherwise find ID in next shard that has a matching partition. for id := prev + 1; ; id += ShardWidth { - if shardPartition(index, id/ShardWidth, partitionN) == partitionID { + if shardToShardPartition(index, id/ShardWidth, partitionN) == partitionID { return id } } @@ -304,6 +356,30 @@ func NewInMemTranslateStore(index, field string, partitionID, partitionN int) *I } } +func (s *InMemTranslateStore) GetStorePath() string { + return "" +} + +// KeyWalker executes walk for every pair in the database +func (s *InMemTranslateStore) KeyWalker(walk func(key string, col uint64)) error { + s.mu.RLock() + defer s.mu.RUnlock() + for id, key := range s.keysByID { + walk(key, id) + } + return nil +} + +// IDWalker executes walk for every pair in the database +func (s *InMemTranslateStore) IDWalker(walk func(key string, col uint64)) error { + s.mu.RLock() + defer s.mu.RUnlock() + for key, id := range s.idsByKey { + walk(key, id) + } + return nil +} + var _ OpenTranslateStoreFunc = OpenInMemTranslateStore // OpenInMemTranslateStore returns a new instance of InMemTranslateStore. @@ -312,9 +388,18 @@ func OpenInMemTranslateStore(rawurl, index, field string, partitionID, partition return NewInMemTranslateStore(index, field, partitionID, partitionN), nil } -func (s *InMemTranslateStore) ComputeTranslatorSummary() (sum *TranslatorSummary, err error) { +func (s *InMemTranslateStore) ComputeTranslatorSummaryRows() (sum *TranslatorSummary, err error) { panic("TODO") } + +func (s *InMemTranslateStore) ComputeTranslatorSummaryCols(partitionID int, topo *Topology) (sum *TranslatorSummary, err error) { + panic("TODO") +} + +func (s *InMemTranslateStore) RepairKeys(topo *Topology, verbose, applyKeyRepairs bool) (changed bool, err error) { + panic("TODO") +} + func (s *InMemTranslateStore) Close() error { return nil } diff --git a/utils_internal_test.go b/utils_internal_test.go index 91f2c0745..195c53cef 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -70,7 +70,7 @@ func NewTestCluster(tb testing.TB, n int) *cluster { c.ReplicaN = 1 c.Hasher = NewTestModHasher() c.Path = path - c.Topology = newTopology() + c.Topology = NewTopology(c.Hasher, c.partitionN, c.ReplicaN, c) for i := 0; i < n; i++ { c.nodes = append(c.nodes, &Node{ @@ -137,6 +137,15 @@ func (t *ClusterCluster) CreateIndex(name string) error { return nil } +func (t *ClusterCluster) CreateIndexWithOpt(name string, opt IndexOptions) error { + for _, c := range t.Clusters { + if _, err := c.holder.CreateIndexIfNotExists(name, opt); err != nil { + return err + } + } + return nil +} + func (t *ClusterCluster) CreateField(index, field string, opts FieldOption) error { for _, c := range t.Clusters { idx, err := c.holder.CreateIndexIfNotExists(index, IndexOptions{}) @@ -272,7 +281,8 @@ func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*cluster, error) c.ReplicaN = 1 c.Hasher = NewTestModHasher() c.Path = path - c.Topology = newTopology() + c.partitionN = DefaultPartitionN + c.Topology = NewTopology(c.Hasher, c.partitionN, c.ReplicaN, c) c.holder = h c.Node = node c.Coordinator = t.common.Nodes[0].ID // the first node is the coordinator @@ -515,3 +525,47 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error node := instr.Coordinator return bcast{t: t}.SendTo(node, complete) } + +var _ = NewTestClusterWithReplication // happy linter + +func NewTestClusterWithReplication(tb testing.TB, nNodes, nReplicas, partitionN int) (c *cluster, cleaner func()) { + path, err := testhook.TempDir(tb, "pilosa-cluster-") + if err != nil { + panic(err) + } + + // holder + h := NewHolder(path, nil) + + // cluster + availableShardFileFlushDuration.Set(100 * time.Millisecond) + c = newCluster() + c.holder = h + c.ReplicaN = nReplicas + c.Hasher = &Jmphasher{} + c.Path = path + c.partitionN = partitionN + c.Topology = NewTopology(c.Hasher, c.partitionN, c.ReplicaN, c) + + for i := 0; i < nNodes; i++ { + nodeID := fmt.Sprintf("node%d", i) + c.nodes = append(c.nodes, &Node{ + ID: nodeID, + URI: NewTestURI("http", fmt.Sprintf("host%d", i), uint16(0)), + }) + c.Topology.addID(nodeID) + } + + c.Node = c.nodes[0] + c.Coordinator = c.nodes[0].ID + c.SetState(ClusterStateNormal) + + if err := c.holder.Open(); err != nil { + panic(err) + } + + return c, func() { + c.holder.Close() + c.close() + } +} From 9dacbccf8d5bd9d84d28f7b1847401e8865c3859 Mon Sep 17 00:00:00 2001 From: "Jason E. Aten" Date: Sun, 4 Oct 2020 22:50:59 -0500 Subject: [PATCH 21/53] DESIGN.md for pilosa-fsck - Orient users to pilosa-fsck by giving an overview of its operations. --- cmd/pilosa-fsck/fsck.go | 75 ++--- cmd/pilosa-fsck/release-pilosa-fsck/DESIGN.md | 256 ++++++++++++++++++ 2 files changed, 278 insertions(+), 53 deletions(-) create mode 100644 cmd/pilosa-fsck/release-pilosa-fsck/DESIGN.md diff --git a/cmd/pilosa-fsck/fsck.go b/cmd/pilosa-fsck/fsck.go index daf5b7b69..31afe3dc9 100644 --- a/cmd/pilosa-fsck/fsck.go +++ b/cmd/pilosa-fsck/fsck.go @@ -79,7 +79,7 @@ type FsckConfig struct { func (cfg *FsckConfig) DefineFlags(fs *flag.FlagSet) { fs.BoolVar(&cfg.Fix, "fix", false, "(warning: alters the backed-up node images on disk) copy primary data to replicas to create a consistent cluster. Implies -fixcol") fs.BoolVar(&cfg.FixCol, "fixcol", false, "(warning: alters the backed-up node images on disk) repair string key translation tables. Skip repair of index data.") - fs.BoolVar(&cfg.Verbose, "v", false, "be very verbose during analysis") + //fs.BoolVar(&cfg.Verbose, "v", false, "be very verbose during analysis") fs.BoolVar(&cfg.Quiet, "q", false, "be very quiet") fs.IntVar(&cfg.ReplicaN, "replicas", 0, "(required) manually entered replicaN; the number of replicas maintained in the cluster. Must be the same as the [cluster] 'replicas = R' entry in the pilosa.conf file for the cluster.") @@ -88,44 +88,20 @@ func (cfg *FsckConfig) DefineFlags(fs *flag.FlagSet) { fs.Usage = func() { fmt.Fprintf(os.Stderr, "pilosa-fsck version: %v\n\n", pilosa.VersionInfo()) - fmt.Fprintf(os.Stderr, `Use: pilosa-fsck -replicas R {-fix} {-fixcol} {-q} {-v} /backup/1/.pilosa /backup/2/.pilosa ... /backup/N/.pilosa + fmt.Fprintf(os.Stderr, `Use: pilosa-fsck -replicas R {-fix} {-q} /backup/1/.pilosa /backup/2/.pilosa ... /backup/N/.pilosa -fix - (warning: alters the backed-up node images on disk) copy primary data to replicas to create a consistent cluster. Implies -fixcol - - -fixcol - (warning: alters the backed-up node images on disk) repair string key translation tables. Skip repair of index data. + (warning: alters the backed-up node images on disk) copy primary data to replicas to create a consistent cluster. -replicas R (required) R is a positive integer, giving the replicaN or replicator factor for the cluster. This is the number of replicas maintained in the cluster. Must be the same as the [cluster] 'replicas = R' entry shared across all the pilosa.conf files on each node. - -v - be very verbose during analysis -q be very quiet during analysis and repair `) - /* - key translation dump options, usually only used by developers debugging key translation: - - -col - (optional) display column keys (very long output) - -dir string - (optional; requires -col), one pilosa data dir to read (default "/home/ubuntu/.pilosa") - -header - (optional; requires -col), display header - -id - (optional; requires -col), dump reverse mapping id->key - -index string - (optional; requires -col), index name (default "i") - -key - (optional; requires -col), dump forward mapping key->id - -partition int - (optional; requires -col), partition id to dump - - */ fmt.Fprintf(os.Stderr, ` Welcome to pilosa-fsck. This is a scan and repair tool that is modeled after the classic unix file @@ -144,19 +120,10 @@ A backup is a set of N cluster-node directories that have been copied from your live system. They must all be visible and mounted on one filesystem together. -pilosa-fsck can be run in scan-mode (without -fix or -fixcol), -or in repair-mode with -fix (or -fixcol). The console output -supplies a shell script documenting the analysis -and showing what data changes would be made. If a -fix has been requested, those fixes will have -been applied during the run. The output then serves -as documentation of what has been updated. If -a fix has not been requested (in other words, neither --fix nor -fixcol was given) then no changes will -have been made to the backups. The fragment level -sync can be completed next by running the script -if you wish. The -fixcol fixes can only be -applied by doing a -fixcol run of pilosa-fsck. +pilosa-fsck can be run in scan-mode (without -fix), +or in repair-mode with -fix. The console output +supplies a log documenting the analysis +and showing what data changes would have been made. REQUIRED COMMAND LINE ARGUMENTS @@ -218,20 +185,17 @@ $ pilosa-fsck -replicas 3 -fix node1/.pilosa node2/.pilosa node3/.pilosa node4/. In both cases, the .id and .topology files must be present in the backups. -KEY REPAIR NOTE +Without -fix, no modifications will be made to the backups. Only +by running with -fix will repairs be made. The user can safely +always run with -fix to repair only if needed. -While the output of pilosa-fsck wihtout -fix or -fixcol -gives a script showing the index fragment repair operations -that can be applied by (cp/rm) shell commands subsequently, -this script alone is an incomplete repair. It does not -address string key tranlsation repairs. For a complete repair, -a run of pilosa-fsck with the -fixcol or -fix flags -will be required. +A zero error code will be returned to the shell if no repairs were needed. -A note about the -fixcol key translation repairs: these are fine -grained operations on the internal databases that do not have -corresponding (cp/rm) shell commands. Therefore a run of pilosa-fsck -with the -fix or -fixcol flag is required to repair these. +A zero error code will be also be returned to the shell if +repairs were needed and they were accomplished under -fix. + +A non-zero error code indicates that repairs were needed but +were not made. `) } } @@ -295,6 +259,7 @@ func main() { myflags := flag.NewFlagSet(ProgramName, flag.ContinueOnError) cfg := &FsckConfig{} cfg.DefineFlags(myflags) + cfg.Verbose = true err := myflags.Parse(os.Args[1:]) if err != nil { @@ -341,11 +306,15 @@ func main() { }() cfg.Dirs = dirs - _, err = cfg.Run() + fixNeeded, err := cfg.Run() if err != nil { fmt.Fprintf(os.Stderr, "error: %v\n", err) os.Exit(1) } + if fixNeeded && !cfg.Fix { + fmt.Fprintf(os.Stderr, "# pilosa-fsck exiting with non-zero error code because a repair is needed, but -fix was not given.\n") + os.Exit(1) + } } func (cfg *FsckConfig) Run() (fixNeeded bool, err error) { diff --git a/cmd/pilosa-fsck/release-pilosa-fsck/DESIGN.md b/cmd/pilosa-fsck/release-pilosa-fsck/DESIGN.md new file mode 100644 index 000000000..9a0c270b8 --- /dev/null +++ b/cmd/pilosa-fsck/release-pilosa-fsck/DESIGN.md @@ -0,0 +1,256 @@ +Design for pilosa-fsck +====================== + +Problem Background +------------------ + +Molecula Pilosa provides replication for fault-tolerance within a Pilosa cluster. + +Three kinds of data are replicated: Roaring bitmap data, Column-Key translation data, +and Row-Key translations are replicated. Only the first two, Roaring data and Column-Key +translation are relevant here. Broadly, the Roaring bitmap data +forms the central features -- the bits -- of a large, sparse bitmap matrix. +The Column-Keys are the labels for the columns at the top margin of this matrix. + +For speed, the Roaring bitmap data is stored separately from the +Key-translation data. The Roaring data is stored in sharded files +within a directory heirarchy under PILOSA-DATA-DIR/index_name/field_name/... +The Key translation data is stored in sharded BoltDB databases with +in the PILOSA-DATA-DIR/index_name/_key directory. + +The current approach to Roaring file replication involves an +eventually consistent mechanism that uses an Anti-Entropy agent to +fix partial or incomplete replication from the primary shard to all +replica shards. + +Unfortunately, the Anti-Entropy agent approach has proved inadequate on two +fronts. First, it does not provide for immediately consistent reads in the +event that the primary is lost. Second, the Anti-Entropy agent itself experienced +out-of-memory issues that have not yet been resolved. + +Therefore, work is now underway to replace this replication +approach with a more consistent design. + +However, in the meantime, for our customers in production with Molecula +Pilosa, we wish to provide a means to re-establish correct replication. + +The pilosa-fsck tool can therefore been seen as a temporary, stop-gap +measure to address immediate issues while the cluster replication +mechanism is replaced. + +The second factor motivating the creation of pilosa-fsck was the discovery +of a bug in the Key-translation process. Unfortunately this was a hard +to reproduce bug, since it happened only on the customer's premises. +It happened only after running the system for a long time, with a +large amount of data, and with various eccentric node failures +and recoveries. + +However, we were able to reproduce a plausible explanation. +Non-primary replicas were creating keys when they should have been +forwarding the request to the primary. Correcting this bug is impetus +for the release of the v2.1.4 version of Molecula Pilosa. + +A fine point here: since we were not able to precisely reproduce the customer's +issue in the development environment, we cannot guarantee with 100% +certainty that we have actually addressed the bug that the customer +was seeing. + +Therefore we also desired an additional insurance +policy. We wished to be able to empower customers to pro-actively discover any +future Key-translation issues that happen in their on-premise system. + +To do this, we proposed providing select customers with the pilosa-fsck +tool which can analyze their offline backups for issues. + +Optionally, these issues can also be repaired in-place in the +offline backup on which pilosa-fsck is run. +The -fix flag repairs both kinds of replication issues. + +Solution Approach: mechanism of action +-------------------------------------- + +The pilosa-fsck is run offline on a full set of backups taken from +all nodes in a Pilosa cluster. It runs on a single computer that +must be separate from the production or staging Pilosa environments. + +When run, pilosa-fsck analyzes the differences between the +primary and its replicas. Both the Roaring +files and the Key translation databases are analyzed. +The computer running pilosa-fsck must have the same or more +memory as the Pilosa nodes in the cluster, as it will +"pretend" to be each Pilosa node in turn. However, as each +node's backup is closed before the next node's backup is +read, we do not require substantially more memory than a single +production node. Short Blake3 cryptographic checksums are +computed for each Roaring fragment and each Key translation +database. These are held in memory (and printed to the log) +for comparing nodes. This comparison forms the heart of +the consistency checks, and is the basis for any subsequent +repair. + +We recommend capturing both stdout and stderr to a log. +Use `&> log` or `2>&1 > log` at the end of the +pilosa-fsck invocation to save a log of the run to disk. + +In a typical cluster, the Replication factor R may be less +than the number nodes N in the cluster. For example, while +N may be 4, the R may be only 3. In this example, within +each replicated shard, one node will be the primary for +that shard, two nodes will be "regular" non-primary replicas, and one +node will be a non-replica. Note that the designation +of primary changes for different Roaring shard within an index, +even on a single node. + +The essence of the the -fix repair operation that pilosa-fsck +can do is this: it will copy from the primary to the +the non-primary replicas. Further, it will remove data from +any non-replica node if it was mistakenly present. + +The pilosa-fsck output log will contain +a sequence of command line 'cp' and 'rm' commands. +These commands are merely a record (with +accompanying justifcation in the comment following the +command) of what actions would be performed to repair +the Roaring file data. + +Only with -fix will the repair actions actually happen +during the pilosa-fsck run. + +The Key translation repairs must be made to the BoltDB +databases directly, and so cannot be represented by +simple cp/rm commands. Therefore if any key-translation +repairs are indicated, and if the user wishes to +make those repairs to the backups, then the -fix +flag must be used. + +If only the Roaring files needs repair, then it suffices +to chmod +x and run the output log, which is in the +form of a bash shell script with comments. + +Details: running pilosa-fsck +---------------------------- + +Errors in invocation are reported on stderr and the program will exit with a non-zero +error code if invocation errors are present. A non-zero error code +is returned if a repair is needed. + +The log of the run is printed to stdout. + +The -h flag to pilosa-fsck prints a summary of its operation +and a guide to laying out the backup directories in preparation. + +The help is reproduced below. + +~~~ +jaten@twg-pilosa0 ~/pilosa/cmd/pilosa-fsck (fsck-design) $ pilosa-fsck -h +pilosa-fsck version: Molecula Pilosa v2.2.1-43-g61047fde (Oct 5 2020 11:51AM, 61047fde) + +Use: pilosa-fsck -replicas R {-fix} {-q} /backup/1/.pilosa /backup/2/.pilosa ... /backup/N/.pilosa + + -fix + (warning: alters the backed-up node images on disk) copy primary data to replicas to create a consistent cluster. + + -replicas R + (required) R is a positive integer, giving the replicaN or replicator factor for the cluster. This is + the number of replicas maintained in the cluster. Must be the same as the + [cluster] 'replicas = R' entry shared across all the pilosa.conf files on each node. + + -q + be very quiet during analysis and repair + + +Welcome to pilosa-fsck. This is a scan and repair +tool that is modeled after the classic unix file +system utility fsck. + +WARNING: DO NOT RUN ON A LIVE SYSTEM. + +The most important point to remember is that analysis +and repair must be done *offline*. + +Just as fsck must be run on an unmounted disk, +pilosa-fsck must be run on a backup. It must +not be run on the directories where a live Pilosa system +is serving queries. Instead, take a backup first. +A backup is a set of N cluster-node directories that have been +copied from your live system. They must all +be visible and mounted on one filesystem together. + +pilosa-fsck can be run in scan-mode (without -fix), +or in repair-mode with -fix. The console output +supplies a log documenting the analysis +and showing what data changes would have been made. + +REQUIRED COMMAND LINE ARGUMENTS + +The paths to all the top-level pilosa +directories in a cluster must be given on the command +line. The -replicas R flag is also always required. It +must be correct for your cluser. Here R is the same as +the [cluster] stanza "replicas = R" line from your +pilosa.conf. + +Example: + +Suppose you are ready to run pilosa-fsck: +you have taken a backup of your four node Pilosa +cluster and stored it all on one filesystem with +all nodes visible and uncompressed. This +is a pre-requisite to running pilosa-fsck. +Let's suppose we have replication R = 3 set. +In this example, have stored our backed-up directories in + +/backup/molecula + +and the four node backups are in +subdirectories node1/ node2/ node3/ node4/ under this: + +/backup/molecula/node1/ +/backup/molecula/node1/.pilosa/.id +/backup/molecula/node1/.pilosa/.topology +/backup/molecula/node1/.pilosa/myindex + +/backup/molecula/node2/ +/backup/molecula/node2/.pilosa/.id +/backup/molecula/node2/.pilosa/.topology +/backup/molecula/node2/.pilosa/myindex + +/backup/molecula/node3/ +/backup/molecula/node3/.pilosa/.id +/backup/molecula/node3/.pilosa/.topology +/backup/molecula/node3/.pilosa/myindex + +/backup/molecula/node4/ +/backup/molecula/node4/.pilosa/.id +/backup/molecula/node4/.pilosa/.topology +/backup/molecula/node4/.pilosa/myindex + +NOTE: your .pilosa directories need not be named .pilosa. They can +be something else, such as when the -d flag to pilosa server was used. +The .id and .topology and index directories must be found directly underneath. + +Then a typical invocation to scan a cluster backup for issues: + +$ cd /backup/molecula/ +$ pilosa-fsck -replicas 3 node1/.pilosa node2/.pilosa node3/.pilosa node4/.pilosa + +A typical invocation to repair the replication in the same backup: + +$ pilosa-fsck -replicas 3 -fix node1/.pilosa node2/.pilosa node3/.pilosa node4/.pilosa + +In both cases, the .id and .topology files must +be present in the backups. + +Without -fix, no modifications will be made to the backups. Only +by running with -fix will repairs be made. The user can safely +always run with -fix to repair only if needed. + +A zero error code will be returned to the shell if no repairs were needed. + +A zero error code will be also be returned to the shell if +repairs were needed and they were accomplished under -fix. + +A non-zero error code indicates that repairs were needed but +were not made. + +~~~ From 05d4b664df8a657e58f556b1daf144a526596329 Mon Sep 17 00:00:00 2001 From: "Jason E. Aten" Date: Mon, 5 Oct 2020 13:29:29 -0500 Subject: [PATCH 22/53] groom --- cmd/pilosa-fsck/fsck.go | 13 +++-- cmd/pilosa-fsck/release-pilosa-fsck/DESIGN.md | 58 +++++++++---------- 2 files changed, 34 insertions(+), 37 deletions(-) diff --git a/cmd/pilosa-fsck/fsck.go b/cmd/pilosa-fsck/fsck.go index 31afe3dc9..d0a307c92 100644 --- a/cmd/pilosa-fsck/fsck.go +++ b/cmd/pilosa-fsck/fsck.go @@ -116,7 +116,7 @@ Just as fsck must be run on an unmounted disk, pilosa-fsck must be run on a backup. It must not be run on the directories where a live Pilosa system is serving queries. Instead, take a backup first. -A backup is a set of N cluster-node directories that have been +A backup is a set of N Pilosa data directories that have been copied from your live system. They must all be visible and mounted on one filesystem together. @@ -127,8 +127,8 @@ and showing what data changes would have been made. REQUIRED COMMAND LINE ARGUMENTS -The paths to all the top-level pilosa -directories in a cluster must be given on the command +The paths to all the top-level Pilosa +data directories in a cluster must be given on the command line. The -replicas R flag is also always required. It must be correct for your cluser. Here R is the same as the [cluster] stanza "replicas = R" line from your @@ -171,16 +171,17 @@ subdirectories node1/ node2/ node3/ node4/ under this: NOTE: your .pilosa directories need not be named .pilosa. They can be something else, such as when the -d flag to pilosa server was used. -The .id and .topology and index directories must be found directly underneath. +The .id file, the .topology file, and the index directories must be +found directly underneath. Then a typical invocation to scan a cluster backup for issues: $ cd /backup/molecula/ -$ pilosa-fsck -replicas 3 node1/.pilosa node2/.pilosa node3/.pilosa node4/.pilosa +$ pilosa-fsck -replicas 3 node1/.pilosa node2/.pilosa node3/.pilosa node4/.pilosa &> log A typical invocation to repair the replication in the same backup: -$ pilosa-fsck -replicas 3 -fix node1/.pilosa node2/.pilosa node3/.pilosa node4/.pilosa +$ pilosa-fsck -replicas 3 -fix node1/.pilosa node2/.pilosa node3/.pilosa node4/.pilosa &> log In both cases, the .id and .topology files must be present in the backups. diff --git a/cmd/pilosa-fsck/release-pilosa-fsck/DESIGN.md b/cmd/pilosa-fsck/release-pilosa-fsck/DESIGN.md index 9a0c270b8..4fbaffe31 100644 --- a/cmd/pilosa-fsck/release-pilosa-fsck/DESIGN.md +++ b/cmd/pilosa-fsck/release-pilosa-fsck/DESIGN.md @@ -7,16 +7,16 @@ Problem Background Molecula Pilosa provides replication for fault-tolerance within a Pilosa cluster. Three kinds of data are replicated: Roaring bitmap data, Column-Key translation data, -and Row-Key translations are replicated. Only the first two, Roaring data and Column-Key -translation are relevant here. Broadly, the Roaring bitmap data +and Row-Key data are replicated. Only the first two, Roaring data and Column-Key +data are relevant here. Broadly, the Roaring bitmap data forms the central features -- the bits -- of a large, sparse bitmap matrix. The Column-Keys are the labels for the columns at the top margin of this matrix. For speed, the Roaring bitmap data is stored separately from the -Key-translation data. The Roaring data is stored in sharded files +Key data. The Roaring data is stored in sharded files within a directory heirarchy under PILOSA-DATA-DIR/index_name/field_name/... -The Key translation data is stored in sharded BoltDB databases with -in the PILOSA-DATA-DIR/index_name/_key directory. +The Key translation data is stored in sharded BoltDB databases within +the PILOSA-DATA-DIR/index_name/_key directory. The current approach to Roaring file replication involves an eventually consistent mechanism that uses an Anti-Entropy agent to @@ -26,13 +26,15 @@ replica shards. Unfortunately, the Anti-Entropy agent approach has proved inadequate on two fronts. First, it does not provide for immediately consistent reads in the event that the primary is lost. Second, the Anti-Entropy agent itself experienced -out-of-memory issues that have not yet been resolved. +out-of-memory issues that have yet to be resolved. Therefore, work is now underway to replace this replication approach with a more consistent design. However, in the meantime, for our customers in production with Molecula Pilosa, we wish to provide a means to re-establish correct replication. +Thus even in the event of a node failure followed by a read from a replica, the +returned read will be correct. The pilosa-fsck tool can therefore been seen as a temporary, stop-gap measure to address immediate issues while the cluster replication @@ -40,7 +42,7 @@ mechanism is replaced. The second factor motivating the creation of pilosa-fsck was the discovery of a bug in the Key-translation process. Unfortunately this was a hard -to reproduce bug, since it happened only on the customer's premises. +to reproduce bug. It happened only on the customer's premises. It happened only after running the system for a long time, with a large amount of data, and with various eccentric node failures and recoveries. @@ -48,7 +50,7 @@ and recoveries. However, we were able to reproduce a plausible explanation. Non-primary replicas were creating keys when they should have been forwarding the request to the primary. Correcting this bug is impetus -for the release of the v2.1.4 version of Molecula Pilosa. +for the v2.1.4 release of Molecula Pilosa. A fine point here: since we were not able to precisely reproduce the customer's issue in the development environment, we cannot guarantee with 100% @@ -57,13 +59,14 @@ was seeing. Therefore we also desired an additional insurance policy. We wished to be able to empower customers to pro-actively discover any -future Key-translation issues that happen in their on-premise system. +future Key-translation issues that happen in their on-premise systems. To do this, we proposed providing select customers with the pilosa-fsck tool which can analyze their offline backups for issues. Optionally, these issues can also be repaired in-place in the offline backup on which pilosa-fsck is run. + The -fix flag repairs both kinds of replication issues. Solution Approach: mechanism of action @@ -80,7 +83,7 @@ The computer running pilosa-fsck must have the same or more memory as the Pilosa nodes in the cluster, as it will "pretend" to be each Pilosa node in turn. However, as each node's backup is closed before the next node's backup is -read, we do not require substantially more memory than a single +opened, we do not require substantially more memory than a single production node. Short Blake3 cryptographic checksums are computed for each Roaring fragment and each Key translation database. These are held in memory (and printed to the log) @@ -98,7 +101,7 @@ N may be 4, the R may be only 3. In this example, within each replicated shard, one node will be the primary for that shard, two nodes will be "regular" non-primary replicas, and one node will be a non-replica. Note that the designation -of primary changes for different Roaring shard within an index, +of primary changes for different Roaring shards within an index, even on a single node. The essence of the the -fix repair operation that pilosa-fsck @@ -116,34 +119,26 @@ the Roaring file data. Only with -fix will the repair actions actually happen during the pilosa-fsck run. -The Key translation repairs must be made to the BoltDB -databases directly, and so cannot be represented by -simple cp/rm commands. Therefore if any key-translation -repairs are indicated, and if the user wishes to -make those repairs to the backups, then the -fix -flag must be used. - -If only the Roaring files needs repair, then it suffices -to chmod +x and run the output log, which is in the -form of a bash shell script with comments. Details: running pilosa-fsck ---------------------------- Errors in invocation are reported on stderr and the program will exit with a non-zero error code if invocation errors are present. A non-zero error code -is returned if a repair is needed. +is returned if a repair is needed and -fix was not given. + +A -fix run will return a zero error code to the shell, if the fix was +successfully made; or if no fix was required. The log of the run is printed to stdout. The -h flag to pilosa-fsck prints a summary of its operation -and a guide to laying out the backup directories in preparation. +and a guide to laying out the backup directories. The help is reproduced below. ~~~ -jaten@twg-pilosa0 ~/pilosa/cmd/pilosa-fsck (fsck-design) $ pilosa-fsck -h -pilosa-fsck version: Molecula Pilosa v2.2.1-43-g61047fde (Oct 5 2020 11:51AM, 61047fde) +$ pilosa-fsck version: Molecula Pilosa v2.2.1-43-g9dacbccf (Oct 5 2020 1:28PM, 9dacbccf) Use: pilosa-fsck -replicas R {-fix} {-q} /backup/1/.pilosa /backup/2/.pilosa ... /backup/N/.pilosa @@ -172,7 +167,7 @@ Just as fsck must be run on an unmounted disk, pilosa-fsck must be run on a backup. It must not be run on the directories where a live Pilosa system is serving queries. Instead, take a backup first. -A backup is a set of N cluster-node directories that have been +A backup is a set of N Pilosa data directories that have been copied from your live system. They must all be visible and mounted on one filesystem together. @@ -183,8 +178,8 @@ and showing what data changes would have been made. REQUIRED COMMAND LINE ARGUMENTS -The paths to all the top-level pilosa -directories in a cluster must be given on the command +The paths to all the top-level Pilosa +data directories in a cluster must be given on the command line. The -replicas R flag is also always required. It must be correct for your cluser. Here R is the same as the [cluster] stanza "replicas = R" line from your @@ -227,16 +222,17 @@ subdirectories node1/ node2/ node3/ node4/ under this: NOTE: your .pilosa directories need not be named .pilosa. They can be something else, such as when the -d flag to pilosa server was used. -The .id and .topology and index directories must be found directly underneath. +The .id file, the .topology file, and the index directories must be +found directly underneath. Then a typical invocation to scan a cluster backup for issues: $ cd /backup/molecula/ -$ pilosa-fsck -replicas 3 node1/.pilosa node2/.pilosa node3/.pilosa node4/.pilosa +$ pilosa-fsck -replicas 3 node1/.pilosa node2/.pilosa node3/.pilosa node4/.pilosa &> log A typical invocation to repair the replication in the same backup: -$ pilosa-fsck -replicas 3 -fix node1/.pilosa node2/.pilosa node3/.pilosa node4/.pilosa +$ pilosa-fsck -replicas 3 -fix node1/.pilosa node2/.pilosa node3/.pilosa node4/.pilosa &> log In both cases, the .id and .topology files must be present in the backups. From bb0d8bea1410878867cc489b4c660398c31d97ad Mon Sep 17 00:00:00 2001 From: Nia Weiss Date: Mon, 5 Oct 2020 15:37:05 -0400 Subject: [PATCH 23/53] generate errors for all unsupported types in Rows calls --- executor.go | 6 +++--- executor_test.go | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/executor.go b/executor.go index 919a2ea76..89376914f 100644 --- a/executor.go +++ b/executor.go @@ -2592,10 +2592,8 @@ func (e *executor) executeRowsShard(ctx context.Context, qcx *Qcx, index string, // in order to represent `Rows` for the field. var views = []string{viewStandard} - // Handle `int` and `time` fields. switch f.Type() { - case FieldTypeInt: - return nil, errors.New("int fields not supported by Rows() query") + case FieldTypeSet, FieldTypeMutex: case FieldTypeTime: var err error @@ -2657,6 +2655,8 @@ func (e *executor) executeRowsShard(ctx context.Context, qcx *Qcx, index string, // Determine the views based on the specified time range. views = viewsByTimeRange(viewStandard, fromTime, toTime, q) } + default: + return nil, errors.Errorf("%s fields not supported by Rows() query", f.Type()) } start := uint64(0) diff --git a/executor_test.go b/executor_test.go index d173ecd1b..e379daf7d 100644 --- a/executor_test.go +++ b/executor_test.go @@ -4851,6 +4851,8 @@ func TestExecutor_Execute_Query_Error(t *testing.T) { defer c.Close() c.CreateField(t, "i", pilosa.IndexOptions{}, "general") c.CreateField(t, "i", pilosa.IndexOptions{}, "integer", pilosa.OptFieldTypeInt(-1000, 1000)) + c.CreateField(t, "i", pilosa.IndexOptions{}, "decimal", pilosa.OptFieldTypeDecimal(2)) + c.CreateField(t, "i", pilosa.IndexOptions{}, "bool", pilosa.OptFieldTypeBool()) tests := []struct { query string @@ -4884,6 +4886,18 @@ func TestExecutor_Execute_Query_Error(t *testing.T) { query: "GroupBy(Rows(integer), prev=-1)", error: "unknown arg 'prev'", }, + { + query: "Rows(integer)", + error: "int fields not supported by Rows() query", + }, + { + query: "Rows(decimal)", + error: "decimal fields not supported by Rows() query", + }, + { + query: "Rows(bool)", + error: "bool fields not supported by Rows() query", + }, } for i, test := range tests { From 675373f0f324834446531a5631de8e1688d1d5a3 Mon Sep 17 00:00:00 2001 From: Travis Date: Mon, 5 Oct 2020 14:38:57 -0500 Subject: [PATCH 24/53] adjust some wording and typos in the fsck design doc --- cmd/pilosa-fsck/release-pilosa-fsck/DESIGN.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/pilosa-fsck/release-pilosa-fsck/DESIGN.md b/cmd/pilosa-fsck/release-pilosa-fsck/DESIGN.md index 4fbaffe31..598896c8d 100644 --- a/cmd/pilosa-fsck/release-pilosa-fsck/DESIGN.md +++ b/cmd/pilosa-fsck/release-pilosa-fsck/DESIGN.md @@ -36,14 +36,14 @@ Pilosa, we wish to provide a means to re-establish correct replication. Thus even in the event of a node failure followed by a read from a replica, the returned read will be correct. -The pilosa-fsck tool can therefore been seen as a temporary, stop-gap +The pilosa-fsck tool can therefore be seen as a temporary, stop-gap measure to address immediate issues while the cluster replication mechanism is replaced. The second factor motivating the creation of pilosa-fsck was the discovery of a bug in the Key-translation process. Unfortunately this was a hard -to reproduce bug. It happened only on the customer's premises. -It happened only after running the system for a long time, with a +to reproduce bug. It happened only on the customer's premises, +and only after running the system for a long time, with a large amount of data, and with various eccentric node failures and recoveries. @@ -58,7 +58,7 @@ certainty that we have actually addressed the bug that the customer was seeing. Therefore we also desired an additional insurance -policy. We wished to be able to empower customers to pro-actively discover any +policy. We wished to be able to empower customers to proactively discover any future Key-translation issues that happen in their on-premise systems. To do this, we proposed providing select customers with the pilosa-fsck @@ -96,10 +96,10 @@ Use `&> log` or `2>&1 > log` at the end of the pilosa-fsck invocation to save a log of the run to disk. In a typical cluster, the Replication factor R may be less -than the number nodes N in the cluster. For example, while +than the number of nodes N in the cluster. For example, while N may be 4, the R may be only 3. In this example, within each replicated shard, one node will be the primary for -that shard, two nodes will be "regular" non-primary replicas, and one +that shard, two nodes will be non-primary replicas, and one node will be a non-replica. Note that the designation of primary changes for different Roaring shards within an index, even on a single node. @@ -127,7 +127,7 @@ Errors in invocation are reported on stderr and the program will exit with a non error code if invocation errors are present. A non-zero error code is returned if a repair is needed and -fix was not given. -A -fix run will return a zero error code to the shell, if the fix was +A -fix run will return a zero error code to the shell if the fix was successfully made; or if no fix was required. The log of the run is printed to stdout. From fd87e8d2f3bf4ee02517894436819a6dab32e17e Mon Sep 17 00:00:00 2001 From: Jason Aten Date: Sun, 4 Oct 2020 16:17:54 -0500 Subject: [PATCH 25/53] follow suggestion on https://github.com/molecula/idk/issues/224, does it fix Q2 delete consumer issue? - apparently not, but this might still be worth doing. --- server/grpc.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/server/grpc.go b/server/grpc.go index f378fd403..7d12073ca 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -882,6 +882,8 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe return nil } cols = limitedCols + } else { + return errors.Errorf("expected 1 result for inspect query; got %d from %q", len(resp.Results), req.Query) } } @@ -924,6 +926,9 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64ArrayVal{Uint64ArrayVal: &pb.Uint64Array{Vals: ids.Rows}}}) colAdded++ } + } else { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: nil}) } case "mutex": @@ -955,6 +960,10 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: nil}) } + + } else { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: nil}) } case "int": @@ -1028,6 +1037,9 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: nil}) } + } else { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: nil}) } } @@ -1052,6 +1064,9 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: nil}) } + } else { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: nil}) } case "bool": @@ -1083,6 +1098,9 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: nil}) } + } else { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: nil}) } case "time": From 5e2a14400a3cba60aef395a18dbeb400339377b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Mon, 5 Oct 2020 13:08:59 +0200 Subject: [PATCH 26/53] Add an extra else branches --- server/grpc.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/server/grpc.go b/server/grpc.go index 7d12073ca..47dcf6a07 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -925,6 +925,9 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64ArrayVal{Uint64ArrayVal: &pb.Uint64Array{Vals: ids.Rows}}}) colAdded++ + } else { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: nil}) } } else { rowResp.Columns = append(rowResp.Columns, @@ -960,7 +963,6 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: nil}) } - } else { rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: nil}) @@ -1000,7 +1002,13 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe value = vals[0] exists = true } + } else { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: nil}) } + } else { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: nil}) } } else { value, exists, err = field.StringValue(tx, id) @@ -1103,7 +1111,7 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe &pb.ColumnResponse{ColumnVal: nil}) } - case "time": + default: rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: nil}) } From 2f9c51349a0f641d3e56686437e733b39201aca0 Mon Sep 17 00:00:00 2001 From: Jason Aten Date: Mon, 5 Oct 2020 19:59:32 -0500 Subject: [PATCH 27/53] correct repair -> analysis in pilosa-fsck logging --- index.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.go b/index.go index f2c045726..56adbd91b 100644 --- a/index.go +++ b/index.go @@ -824,7 +824,7 @@ func (i *Index) ComputeTranslatorSummary(verbose, checkKeys, applyKeyRepairs boo if verbose { // This is pilosa-fsck output, not regular log. - fmt.Printf("# doing repair of keys on nodeID '%v', and primID '%v'\n", nodeID, primID) + fmt.Printf("# doing analysis of keys on nodeID '%v', and primID '%v'\n", nodeID, primID) } changed, err := store.RepairKeys(topo, verbose, applyKeyRepairs) if err != nil { From c82a1593ff1db64dc4c70d6461c50e8931e1f6a3 Mon Sep 17 00:00:00 2001 From: Jason Aten Date: Mon, 5 Oct 2020 20:21:51 -0500 Subject: [PATCH 28/53] api usage should ignore tx stores. fixes #919 - avoid deadlock in txfactory_internal_test w Qcx --- api.go | 3 +++ txfactory_internal_test.go | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/api.go b/api.go index 018eb8091..9f4190535 100644 --- a/api.go +++ b/api.go @@ -812,6 +812,9 @@ func (api *API) Usage() (map[string]int64, int64, error) { if !file.IsDir() { continue } + if api.holder.Txf().IsTxDatabasePath(file.Name()) { + continue + } fullName := path.Join(dirName, file.Name()) indexSizes[file.Name()], err = diskUsage(fullName) if err != nil { diff --git a/txfactory_internal_test.go b/txfactory_internal_test.go index dd8225aa7..d172099d5 100644 --- a/txfactory_internal_test.go +++ b/txfactory_internal_test.go @@ -95,9 +95,9 @@ func Test_TxFactory_Qcx_query_context(t *testing.T) { // allow all goro to finish before Closing the lmdb.env, otherwise // we will crash as the goroutines making Tx will try to use the env // after it is closed. It can take quite a while. - // one writer might be blocking the other... so ask for only N-1 at first. - barrier.BlockUntil(N - 1) - //barrier.BlockUntil(N) + // one writer might be blocking the other... so ask for only N-2 at first + // to avoid deadlock. + barrier.BlockUntil(N - 2) barrier.UnblockReaders() time.Sleep(1 * time.Second) } From 6ddeec2ee84afc44ed8f1608e548bd1ae2eb52b0 Mon Sep 17 00:00:00 2001 From: Jason Aten Date: Tue, 6 Oct 2020 08:19:53 -0500 Subject: [PATCH 29/53] print time at end of fsck run, even if we return 1 to shell --- cmd/pilosa-fsck/fsck.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/pilosa-fsck/fsck.go b/cmd/pilosa-fsck/fsck.go index d0a307c92..587485762 100644 --- a/cmd/pilosa-fsck/fsck.go +++ b/cmd/pilosa-fsck/fsck.go @@ -309,10 +309,12 @@ func main() { fixNeeded, err := cfg.Run() if err != nil { + fmt.Fprintf(os.Stdout, "# finished at %v (elapsed %v)\n\n", time.Now().Format(RFC3339MsecTz0), time.Since(t0)) fmt.Fprintf(os.Stderr, "error: %v\n", err) os.Exit(1) } if fixNeeded && !cfg.Fix { + fmt.Fprintf(os.Stdout, "# finished at %v (elapsed %v)\n\n", time.Now().Format(RFC3339MsecTz0), time.Since(t0)) fmt.Fprintf(os.Stderr, "# pilosa-fsck exiting with non-zero error code because a repair is needed, but -fix was not given.\n") os.Exit(1) } From cdd10a26f06c3db2dbacc9cd39e1d0b63bb4cbc6 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Tue, 22 Sep 2020 13:15:29 -0600 Subject: [PATCH 30/53] Refactor RBF to use immutable list of WAL segments. Previously, RBF shared a list of WAL segments between the DB & Tx. However, this increased the need for mutexes to access the data. WAL segments are effectively immutable on-disk so the list of segments has been refactored so that changes to the segment list are done via copy-on-write which allows read transactions to access segment data without a mutex. The database checkpointing can remove early, unused segments and there is an update/add check to make sure that Tx segments pushed back to the DB do not include removed segments. --- rbf/db.go | 364 +++++++++++++++--------------------------------- rbf/db_test.go | 5 +- rbf/tx.go | 182 +++++++++++++++++++++--- rbf/wal.go | 311 ++++++++++++++++++----------------------- rbf/wal_test.go | 20 +-- 5 files changed, 427 insertions(+), 455 deletions(-) diff --git a/rbf/db.go b/rbf/db.go index 1732eead5..e807956ec 100644 --- a/rbf/db.go +++ b/rbf/db.go @@ -41,12 +41,14 @@ const ( type DB struct { data []byte // mmap data file *os.File // file descriptor - segments []*WALSegment // write-ahead log rootRecords []*RootRecord // cached root records pageMap *immutable.Map // pgno-to-WALID mapping txs map[*Tx]struct{} // active transactions opened bool // true if open + wcache []byte // wal write cache + segments []WALSegment // write-ahead log + mu sync.RWMutex // general mutex rwmu sync.Mutex // mutex for restricting single writer exclmu sync.RWMutex // mutex for locking out everyone but a single writer @@ -71,6 +73,7 @@ func NewDB(path string) *DB { db := &DB{ txs: make(map[*Tx]struct{}), pageMap: immutable.NewMap(&uint32Hasher{}), + wcache: make([]byte, MaxWALSegmentFileSize+PageSize), Path: path, MaxSize: DefaultMaxSize, } @@ -84,12 +87,10 @@ func (db *DB) DataPath() string { // WALPath returns the path to the WAL directory. func (db *DB) WALPath() string { - return filepath.Join(db.Path, "wal") } func CreateDirIfNotExist(path string) { - dir := filepath.Dir(path) if _, err := os.Stat(dir); os.IsNotExist(err) { err = os.MkdirAll(dir, 0755) @@ -102,7 +103,6 @@ func CreateDirIfNotExist(path string) { // Open opens a database with the file specified in Path. // Creates a new file if one does not already exist. func (db *DB) Open() (err error) { - db.mu.Lock() defer db.mu.Unlock() @@ -143,7 +143,7 @@ func (db *DB) Open() (err error) { // Open write-ahead log & checkpoint to the end since no transactions are open. if err := db.openWALSegments(); err != nil { return fmt.Errorf("wal open: %w", err) - } else if err := db.checkpoint(true); err != nil { + } else if err := db.checkpoint(true, &nopLocker{}); err != nil { return fmt.Errorf("checkpoint: %w", err) } @@ -171,27 +171,62 @@ func (db *DB) openWALSegments() error { } // Truncate everything after the last successful meta page. - if walID, err := db.findLastWALMetaPage(); err != nil { + if walID, err := findLastWALMetaPage(db.segments); err != nil { return err - } else if err := db.truncateWALAfter(walID); err != nil { + } else if db.segments, err = truncateWALAfter(db.segments, walID); err != nil { return err } return nil } -// checkpoint copies pages from WAL segments into the main DB file. This can +// updateWALSegment updates or adds a segment. +func (db *DB) updateWALSegment(s WALSegment) { + segments := make([]WALSegment, len(db.segments), len(db.segments)+1) + copy(segments, db.segments) + + // Find the matching segment using the path. + segment := walSegmentByPath(segments, s.Path) + + // Update existing segment if it already exists. + // Otherwise append segment to the end. + if segment != nil { + *segment = s + } else { + assert(len(segments) == 0 || segments[len(segments)-1].MinWALID < s.MinWALID) + segments = append(segments, s) + } + + // Replace DB segment list. + db.segments = segments +} + +// Checkpoint copies pages from WAL segments into the main DB file. This can // only copy pages that aren't in use by an active transaction. The page map // is rebuilt as well for all WAL pages still in use. // // If exclusive is true, all WAL writes are flushed to disk. -func (db *DB) checkpoint(exclusive bool) error { - if !db.opened { +func (db *DB) Checkpoint() error { + return db.checkpoint(false, &db.mu) +} + +// checkpoint moves WAL segments to the main DB file. +// +// Note that mu should db.mu when called through the external API but it +// can be &nopLocker if called under lock. +func (db *DB) checkpoint(exclusive bool, mu sync.Locker) error { + // Obtain a snapshot of WAL segments at the start. + mu.Lock() + opened := db.opened + segments := db.segments + mu.Unlock() + + if !opened { return nil } // Determine last checkpointed WAL ID. - page, err := db.readPage(nil, 0) + page, err := db.readDBPage(0) if err != nil { return err } @@ -206,7 +241,7 @@ func (db *DB) checkpoint(exclusive bool) error { pageMap := immutable.NewMap(&uint32Hasher{}) for { // Determine last page of transaction. - metaWALID, err := db.findNextWALMetaPage(walID) + metaWALID, err := findNextWALMetaPage(segments, walID) if err == io.EOF { break } else if err != nil { @@ -217,7 +252,7 @@ func (db *DB) checkpoint(exclusive bool) error { for ; walID <= metaWALID; walID++ { canCheckpoint := exclusive || minActiveWALID == 0 || walID <= minActiveWALID - page, err := db.readWALPage(walID) + page, err := readWALPage(segments, walID) if err != nil { return err } @@ -242,13 +277,13 @@ func (db *DB) checkpoint(exclusive bool) error { // Ensure we actually read the bitmap data in when we checkpoint. // NOTE: The walID variable is incremented above in the pgno check. if isBitmapHeader { - if page, err = db.readWALPage(walID); err != nil { + if page, err = readWALPage(segments, walID); err != nil { return err } } // Write page data into main db file. - if err := db.writePage(pgno, page); err != nil { + if err := db.writeDBPage(pgno, page); err != nil { return err } @@ -261,98 +296,59 @@ func (db *DB) checkpoint(exclusive bool) error { // Remove WAL segments that have been checkpointed. if maxCheckpointedWALID != 0 { - for len(db.segments) > 0 { - segment := db.segments[0] + for _, segment := range segments { if segment.MaxWALID() > maxCheckpointedWALID { break } - segpath := segment.Path() - if err := segment.Close(); err != nil { - return err - } else if err := os.Remove(segpath); err != nil { + if err := func() error { + mu.Lock() + defer mu.Unlock() + return db.removeWALSegment(segment.Path) + }(); err != nil { return err } - db.segments, db.segments[0] = db.segments[1:], nil } } // Ensure all segments are flushed and there is no remapped pages. if exclusive { + mu.Lock() assert(len(db.segments) == 0) assert(pageMap.Len() == 0) + mu.Unlock() } db.pageMap = pageMap return nil } -// truncateWALAfter removes all pages in the WAL after walID. -func (db *DB) truncateWALAfter(walID int64) error { - for i := len(db.segments) - 1; i >= 0; i-- { - segment := db.segments[i] - if segment.MaxWALID() <= walID { - break - } - - // Drop entire segment if all pages are after WAL ID. - if walID < segment.MinWALID() { +// removeWALSegment closes and deletes the segment with the given path. +// +// The DB's segment list is entirely replaced so that transactions with +// a reference to the old list can continue to use it without a lock. +func (db *DB) removeWALSegment(path string) error { + newSegments := make([]WALSegment, 0, len(db.segments)) + for _, segment := range db.segments { + // Close and remove if path matches. + if segment.Path == path { if err := segment.Close(); err != nil { return err - } else if err := os.Remove(segment.Path()); err != nil { + } else if err := os.Remove(segment.Path); err != nil { return err } - db.segments, db.segments[i] = db.segments[:len(db.segments)-1], nil continue } - // If we only remove some of the WAL pages then truncate and exit - // since segments before this will retain all their pages. - return segment.TruncateAfter(walID) + // Otherwise append to new slice of segments. + newSegments = append(newSegments, segment) } + + // Replace entire slice of segments. + db.segments = newSegments return nil } -func (db *DB) findNextWALMetaPage(walID int64) (metaWALID int64, err error) { - maxWALID := db.maxWALID() - - for ; walID <= maxWALID; walID++ { - // Read page data from WAL and return if it is a meta page. - page, err := db.readWALPage(walID) - if err != nil { - return walID, err - } else if IsMetaPage(page) { - return walID, nil - } - - // Skip over next page if this is a bitmap header. - if IsBitmapHeader(page) { - walID++ - } - } - - return -1, io.EOF -} - -func (db *DB) findLastWALMetaPage() (walID int64, err error) { - if len(db.segments) == 0 { - return 0, nil - } - - var maxMetaWALID int64 - maxWALID := db.maxWALID() - for walID := db.minWALID(); walID <= maxWALID; walID++ { - if page, err := db.readWALPage(walID); err != nil { - return walID, err - } else if IsBitmapHeader(page) { - walID++ // skip next page for bitmap headers - } else if IsMetaPage(page) { - maxMetaWALID = walID // save max meta WAL ID - } - } - return maxMetaWALID, nil -} - // minActiveWALID returns the lowest WAL ID in use by any active transaction. // Returns 0 if no transactions are active. func (db *DB) minActiveWALID() int64 { @@ -365,147 +361,8 @@ func (db *DB) minActiveWALID() int64 { return walID } -// ActiveWALSegment returns the most recent WAL segment. -func (db *DB) ActiveWALSegment() *WALSegment { - db.mu.RLock() - defer db.mu.RUnlock() - return db.activeWALSegment() -} - -func (db *DB) activeWALSegment() *WALSegment { - if len(db.segments) == 0 { - return nil - } - return db.segments[len(db.segments)-1] -} - -// MinWALID returns the lowest WAL ID available in the WAL. -func (db *DB) MinWALID() int64 { - db.mu.RLock() - defer db.mu.RUnlock() - return db.minWALID() -} - -func (db *DB) minWALID() int64 { - if len(db.segments) == 0 { - return 0 - } - return db.segments[0].MinWALID() -} - -// MaxWALID returns the highest WAL ID available in the WAL. -func (db *DB) MaxWALID() int64 { - db.mu.RLock() - defer db.mu.RUnlock() - return db.maxWALID() -} - -func (db *DB) maxWALID() int64 { - - if len(db.segments) == 0 { - return 0 - } - s := db.segments[len(db.segments)-1] - return s.MaxWALID() -} - -// WALPageN returns the number of pages across all segments. -func (db *DB) WALPageN() int64 { - db.mu.RLock() - defer db.mu.RUnlock() - - var n int64 - for _, s := range db.segments { - n += int64(s.PageN()) - } - return n -} - -// SyncWAL flushes the active segment to disk. -func (db *DB) SyncWAL() error { - if s := db.ActiveWALSegment(); s != nil { - return s.Sync() - } - return nil -} - -// readWALPage reads a single page at the given WAL ID. -func (db *DB) readWALPage(walID int64) ([]byte, error) { - // TODO(BBJ): Binary search for segment. - for _, s := range db.segments { - if walID >= s.MinWALID() && walID <= s.MaxWALID() { - return s.ReadWALPage(walID) - } - } - return nil, fmt.Errorf("cannot find segment containing WAL page: %d", walID) -} - -func (db *DB) writeWALPage(page []byte, isMeta bool) (walID int64, err error) { - if err := db.ensureWritableWALSegment(); err != nil { - return 0, err - } - return db.activeWALSegment().WriteWALPage(page, isMeta) -} - -func (db *DB) writeBitmapPage(pgno uint32, page []byte) (walID int64, err error) { - - if err := db.ensureWritableWALSegment(); err != nil { - return 0, err - } - - // Write header page for next bitmap page. - buf := make([]byte, PageSize) - writePageNo(buf[:], pgno) - writeFlags(buf[:], PageTypeBitmapHeader) - // TODO(BBJ): Write checksum. - if _, err := db.activeWALSegment().WriteWALPage(buf, false); err != nil { - return 0, fmt.Errorf("write bitmap header: %w", err) - } - - // Write the bitmap page and return its WALID. - return db.activeWALSegment().WriteWALPage(page, false) -} - -func (db *DB) ensureWritableWALSegment() error { - if s := db.activeWALSegment(); s != nil && s.Size() < MaxWALSegmentFileSize { - return nil - } - return db.addWALSegment() -} - -// addWALSegment appends a new, writable segment and closing an existing segments for write. -func (db *DB) addWALSegment() error { - - // If we have a current active WAL segment then close it and start the - // next segment from the next WAL ID. If there is no existing WAL segments, - // read the last checkpointed WAL ID from the DB and start after that. - var base int64 - if s := db.activeWALSegment(); s != nil { - base = s.MaxWALID() + 1 - if err := s.CloseForWrite(); err != nil { - return err - } - } else { - page, err := db.readPage(db.pageMap, 0) - if err != nil { - return err - } - base = readMetaWALID(page) + 1 - } - - // Create new segment file. - s := NewWALSegment(filepath.Join(db.WALPath(), FormatWALSegmentPath(base))) - if err := s.Open(); err != nil { - return fmt.Errorf("add wal segment: %w", err) - } - db.segments = append(db.segments, s) - - return nil -} - // Close closes the database. func (db *DB) Close() (err error) { - // TODO(bbj): Add wait group to hang until last Tx is complete. // Wait for writer lock. @@ -542,12 +399,12 @@ func (db *DB) Close() (err error) { // closeWALSegments closes the WAL and all its segments. func (db *DB) closeWALSegments() (err error) { - for _, s := range db.segments { if e := s.Close(); e != nil && err == nil { err = e } } + db.segments = nil return err } @@ -613,7 +470,6 @@ func (db *DB) HasData(requireOneHotBit bool) (hasAnyRecords bool, err error) { // Size returns the size of the database & WAL, in bytes. func (db *DB) Size() (int64, error) { - db.mu.RLock() defer db.mu.RUnlock() @@ -621,38 +477,27 @@ func (db *DB) Size() (int64, error) { if err != nil { return 0, err } - return db.walSize() + fi.Size(), nil + return walSize(db.segments) + fi.Size(), nil } // WALSize returns the size of all WAL segments, in bytes. func (db *DB) WALSize() int64 { - db.mu.RLock() defer db.mu.RUnlock() - return db.walSize() -} - -func (db *DB) walSize() int64 { - - var sz int64 - for _, s := range db.segments { - sz += s.Size() - } - return sz + return walSize(db.segments) } // WALSegments returns the WAL segments currently on the DB. -// This should only be used for debugging & testing purposes. -func (db *DB) WALSegments() []*WALSegment { - +func (db *DB) WALSegments() []WALSegment { db.mu.RLock() defer db.mu.RUnlock() - return db.segments + other := make([]WALSegment, len(db.segments)) + copy(other, db.segments) + return other } // init initializes a new database file. func (db *DB) init() error { - if err := db.initMetaPage(); err != nil { return fmt.Errorf("meta: %w", err) } else if err := db.initRootRecordPage(); err != nil { @@ -713,6 +558,10 @@ func (db *DB) BeginWithExclusiveLock() (_ *Tx, err error) { } func (db *DB) begin(writable, exclusive bool) (_ *Tx, err error) { + if exclusive { + assert(writable) // exclusive transactions must be writable + } + if exclusive { db.exclmu.Lock() } else { @@ -749,7 +598,7 @@ func (db *DB) begin(writable, exclusive bool) (_ *Tx, err error) { // Flush all WAL writes to disk before an exclusive writer so that we can // work directly with the on-disk database. if exclusive { - if err := db.checkpoint(true); err != nil { + if err := db.checkpoint(true, &nopLocker{}); err != nil { cleanup() return nil, err } @@ -762,10 +611,21 @@ func (db *DB) begin(writable, exclusive bool) (_ *Tx, err error) { writable: writable, exclusive: exclusive, } + if writable { + tx.wcache = db.wcache[:0] + } + + // Copy list of WAL segments so they can be altered by the tx. + // Add last segment to the list of segments that will be updated/added. + if len(db.segments) != 0 { + tx.segments = make([]WALSegment, len(db.segments)) + copy(tx.segments, db.segments) + tx.updatedSegmentPaths = []string{tx.segments[len(tx.segments)-1].Path} + } // Copy meta page into transaction's buffer. // This page is only written at the end of a dirty transaction. - page, err := db.readPage(db.pageMap, 0) + page, err := db.readMetaPage() if err != nil { tx.Rollback() return nil, err @@ -802,8 +662,8 @@ func (db *DB) removeTx(tx *Tx) error { // Write pages from WAL to DB. // TODO(bbj): Move this to an async goroutine. if tx.writable { - if err := db.checkpoint(false); err != nil { - return err + if err := db.checkpoint(false, &nopLocker{}); err != nil { + return fmt.Errorf("checkpoint: %w", err) } } @@ -824,21 +684,25 @@ func (db *DB) Check() error { return tx.Check() } -// writePage writes a page to the data file. -func (db *DB) writePage(pgno uint32, page []byte) error { +// writeDBPage writes a page to the data file. +func (db *DB) writeDBPage(pgno uint32, page []byte) error { _, err := db.file.WriteAt(page, int64(pgno)*PageSize) return err } -func (db *DB) readPage(pageMap *immutable.Map, pgno uint32) ([]byte, error) { - // Check if page is currently in WAL. - if pageMap != nil { - if walID, ok := pageMap.Get(pgno); ok { - return db.readWALPage(walID.(int64)) - } - } - - // Otherwise read from the data file. +func (db *DB) readDBPage(pgno uint32) ([]byte, error) { offset := int64(pgno) * PageSize return db.data[offset : offset+PageSize], nil } + +func (db *DB) readMetaPage() ([]byte, error) { + if walID, ok := db.pageMap.Get(uint32(0)); ok { + return readWALPage(db.segments, walID.(int64)) + } + return db.readDBPage(0) +} + +type nopLocker struct{} + +func (*nopLocker) Lock() {} +func (*nopLocker) Unlock() {} diff --git a/rbf/db_test.go b/rbf/db_test.go index af0552c6f..a06cd26f6 100644 --- a/rbf/db_test.go +++ b/rbf/db_test.go @@ -112,10 +112,11 @@ func TestDB_Recovery(t *testing.T) { tx1.Rollback() // Close database & truncate WAL to remove commit page & bitmap data page. - segment := db.ActiveWALSegment() + segments := db.WALSegments() + segment := segments[len(segments)-1] if err := db.Close(); err != nil { t.Fatal(err) - } else if err := os.Truncate(segment.Path(), segment.Size()-(2*rbf.PageSize)); err != nil { + } else if err := os.Truncate(segment.Path, segment.Size()-(2*rbf.PageSize)); err != nil { t.Fatal(err) } diff --git a/rbf/tx.go b/rbf/tx.go index 5be536221..16e5505d0 100644 --- a/rbf/tx.go +++ b/rbf/tx.go @@ -17,9 +17,9 @@ import ( "fmt" "io" "math" + "os" + "path/filepath" "sort" - - //"strconv" "strings" "sync" @@ -33,15 +33,19 @@ var _ = txkey.ToString // Tx represents a transaction. type Tx struct { - mu sync.RWMutex - db *DB // parent db - meta [PageSize]byte // copy of current meta page - walID int64 // max WAL ID at start of tx - rootRecords []*RootRecord // read-only cache of root records - pageMap *immutable.Map // mapping of database pages to WAL IDs - writable bool // if true, tx can write - exclusive bool // if true, tx writes directly to db file (no wal) - dirty bool // if true, changes have been made + mu sync.RWMutex + db *DB // parent db + segments []WALSegment // copy of WAL segments + updatedSegmentPaths []string // updated or added segment paths + meta [PageSize]byte // copy of current meta page + walID int64 // max WAL ID at start of tx + rootRecords []*RootRecord // read-only cache of root records + pageMap *immutable.Map // mapping of database pages to WAL IDs + writable bool // if true, tx can write + exclusive bool // if true, tx writes directly to db file (no wal) + dirty bool // if true, changes have been made + + wcache []byte // write cache // If Rollback() has already completed, don't do it again. // Note db == nil means that commit has already been done. @@ -76,7 +80,7 @@ func (tx *Tx) Commit() error { if tx.dirty { if err := tx.writeMetaPage(MetaPageFlagCommit); err != nil { return err - } else if err := tx.db.SyncWAL(); err != nil { + } else if err := tx.flushWALWriter(); err != nil { return err } @@ -89,6 +93,11 @@ func (tx *Tx) Commit() error { tx.db.mu.Lock() tx.db.rootRecords = tx.rootRecords tx.db.pageMap = tx.pageMap + for _, path := range tx.updatedSegmentPaths { + segment := walSegmentByPath(tx.segments, path) + assert(segment != nil) + tx.db.updateWALSegment(*segment) + } tx.db.mu.Unlock() } @@ -112,11 +121,12 @@ func (tx *Tx) Rollback() { // TODO(bbj): Invalidate DB if rollback fails. Possibly attempt reopen? - // Remove all WAL pages that have been written by this transaction. if tx.dirty { - if err := tx.db.truncateWALAfter(tx.walID); err != nil { - panic(err) + if _, err := truncateWALAfter(tx.segments, tx.walID); err != nil { + panicOn(err) } + tx.segments = nil + tx.updatedSegmentPaths = nil } // Disconnect transaction from DB. @@ -915,11 +925,28 @@ func (tx *Tx) readPage(pgno uint32) ([]byte, error) { return tx.meta[:], nil } + // Verify page number requested is within current size of database. pageN := readMetaPageN(tx.meta[:]) if pgno > pageN { return nil, fmt.Errorf("rbf: page read out of bounds: pgno=%d max=%d", pgno, pageN) } - return tx.db.readPage(tx.pageMap, pgno) + + // Check if page is remapped. + if walID, ok := tx.pageMap.Get(pgno); ok { + walID64 := walID.(int64) + + // Read from write cache if not yet flushed to disk. + maxWALID := activeWALSegment(tx.segments).MaxWALID() + if walID64 > maxWALID { + offset := (walID64 - maxWALID - 1) * PageSize + return tx.wcache[offset : offset+PageSize], nil + } + + // Otherwise return remapped page from WAL segment. + return readWALPage(tx.segments, walID64) + } + + return tx.db.readDBPage(pgno) } func (tx *Tx) writePage(page []byte) error { @@ -928,11 +955,11 @@ func (tx *Tx) writePage(page []byte) error { // If we are running in exclusive mode, directly write page to database. if tx.exclusive { - return tx.db.writePage(readPageNo(page), page) + return tx.db.writeDBPage(readPageNo(page), page) } // Write page to WAL and obtain position in WAL. - walID, err := tx.db.writeWALPage(page, false) + walID, err := tx.writeWALPage(page, false) if err != nil { return err } @@ -948,11 +975,11 @@ func (tx *Tx) writeBitmapPage(pgno uint32, page []byte) error { // If we are running in exclusive mode, directly write page to database. if tx.exclusive { - return tx.db.writePage(pgno, page) + return tx.db.writeDBPage(pgno, page) } // Write bitmap to WAL and obtain WAL position of the actual page data (not the prefix page). - walID, err := tx.db.writeBitmapPage(pgno, page) + walID, err := tx.writeBitmapWALPage(pgno, page) if err != nil { return err } @@ -968,11 +995,11 @@ func (tx *Tx) writeMetaPage(flag uint32) error { // If we are running in exclusive mode, directly write page to database. if tx.exclusive { - return tx.db.writePage(0, tx.meta[:]) + return tx.db.writeDBPage(0, tx.meta[:]) } // Write page to WAL and obtain position in WAL. - walID, err := tx.db.writeWALPage(tx.meta[:], true) + walID, err := tx.writeWALPage(tx.meta[:], true) if err != nil { return err } @@ -1562,3 +1589,114 @@ func (tx *Tx) ImportRoaringBits(name string, itr roaring.RoaringIterator, clear } return } + +func (tx *Tx) flushWALWriter() error { + // Ignore if we have no data in the write cache. + if len(tx.wcache) == 0 { + return nil + } + + // Determine active WAL segment. + assert(len(tx.segments) != 0) + segment := &tx.segments[len(tx.segments)-1] + + // Open write handle to active segment. + w, err := os.OpenFile(segment.Path, os.O_WRONLY, 0666) + if err != nil { + return fmt.Errorf("open wal segment write handle: %w", err) + } + defer w.Close() + + // Flush cache to writer. + if _, err := w.WriteAt(tx.wcache, int64(segment.PageN)*PageSize); err != nil { + return fmt.Errorf("write wal segment: %w", err) + } else if err := w.Sync(); err != nil { + return fmt.Errorf("sync wal segment: %w", err) + } else if err := w.Close(); err != nil { + return fmt.Errorf("close wal segment: %w", err) + } + + // Increase the size of the last WAL segment & clear cache. + assert(len(tx.wcache)%PageSize == 0) + segment.PageN += len(tx.wcache) / PageSize + tx.wcache = tx.wcache[:0] + + return nil +} + +func (tx *Tx) writeWALPage(page []byte, isMeta bool) (walID int64, err error) { + if err := tx.ensureWritableWALSegment(); err != nil { + return 0, err + } + + // Determine next WAL ID from cached meta page. + walID = readMetaWALID(tx.meta[:]) + 1 + + // Update WAL ID on cached meta page. + writeMetaWALID(tx.meta[:], walID) + + // Append write to write buffer. + tx.wcache = append(tx.wcache, page...) + + return walID, nil +} + +func (tx *Tx) writeBitmapWALPage(pgno uint32, page []byte) (walID int64, err error) { + if err := tx.ensureWritableWALSegment(); err != nil { + return 0, err + } + + // Write header page for next bitmap page. + buf := make([]byte, PageSize) + writePageNo(buf[:], pgno) + writeFlags(buf[:], PageTypeBitmapHeader) + // TODO(BBJ): Write checksum. + if _, err := tx.writeWALPage(buf, false); err != nil { + return 0, fmt.Errorf("write bitmap header: %w", err) + } + + // Write the bitmap page and return its WALID. + return tx.writeWALPage(page, false) +} + +func (tx *Tx) ensureWritableWALSegment() error { + writeCacheSize := int64(len(tx.wcache)) + if len(tx.segments) != 0 && activeWALSegment(tx.segments).Size()+writeCacheSize < MaxWALSegmentFileSize { + return nil + } + return tx.addWALSegment() +} + +// addWALSegment appends a new, writable segment and closing an existing segments for write. +func (tx *Tx) addWALSegment() error { + if err := tx.flushWALWriter(); err != nil { + return err + } + + // If we have a current active WAL segment then close it and start the + // next segment from the next WAL ID. If there is no existing WAL segments, + // read the last checkpointed WAL ID from the DB and start after that. + var base int64 + if len(tx.segments) != 0 { + base = activeWALSegment(tx.segments).MaxWALID() + 1 + } else { + page, err := tx.readPage(0) + if err != nil { + return err + } + base = readMetaWALID(page) + 1 + } + + // Create new segment file. + s := NewWALSegment(filepath.Join(tx.db.WALPath(), FormatWALSegmentPath(base))) + if err := s.Open(); err != nil { + return fmt.Errorf("add wal segment: %w", err) + } + + // Track all segments that need to be added back to DB. + // The DB can remove segments in the background so we don't want to replace. + tx.segments = append(tx.segments, s) + tx.updatedSegmentPaths = append(tx.updatedSegmentPaths, s.Path) + + return nil +} diff --git a/rbf/wal.go b/rbf/wal.go index 7a0518d96..3a26c3bf6 100644 --- a/rbf/wal.go +++ b/rbf/wal.go @@ -16,9 +16,9 @@ package rbf import ( "fmt" + "io" "os" "path/filepath" - "sync" "syscall" "github.com/pilosa/pilosa/v2/syswrap" @@ -26,66 +26,40 @@ import ( // WALSegment represents a single file in the WAL. type WALSegment struct { - mu sync.RWMutex - minWALID int64 // base WALID; calculated from path - path string // path to file - w *os.File // write handle - data []byte // read-only mmap data - writeCache []byte // write buffer - pageN int // number of written pages + Path string // path to file + MinWALID int64 // base WALID; calculated from path + PageN int // number of written pages + + data []byte // read-only mmap data } // NewWALSegment returns a new instance of WALSegment for a given path. -func NewWALSegment(path string) *WALSegment { - return &WALSegment{ - path: path, +func NewWALSegment(path string) WALSegment { + return WALSegment{ + Path: path, } } -// Path returns the path the segment was initialized with. -func (s *WALSegment) Path() string { return s.path } - -// MinWALID returns the initial WAL ID of the segment. Only available after Open(). -func (s *WALSegment) MinWALID() int64 { - s.mu.RLock() - defer s.mu.RUnlock() - return s.minWALID -} - // MaxWALID returns the maximum WAL ID of the segment. Only available after Open(). -func (s *WALSegment) MaxWALID() int64 { - s.mu.RLock() - defer s.mu.RUnlock() - return s.minWALID + int64(s.pageN) - 1 -} - -// PageN returns the number of pages in the segment. -func (s *WALSegment) PageN() int { - s.mu.RLock() - defer s.mu.RUnlock() - return s.pageN +func (s WALSegment) MaxWALID() int64 { + return s.MinWALID + int64(s.PageN) - 1 } // Size returns the current size of the segment, in bytes. -func (s *WALSegment) Size() int64 { - s.mu.RLock() - defer s.mu.RUnlock() - return int64(s.pageN) * PageSize +func (s WALSegment) Size() int64 { + return int64(s.PageN) * PageSize } func (s *WALSegment) Open() (err error) { - s.mu.Lock() - defer s.mu.Unlock() - // Extract base WAL ID and validate path. - if s.minWALID, err = ParseWALSegmentPath(s.path); err != nil { + if s.MinWALID, err = ParseWALSegmentPath(s.Path); err != nil { return err } // Determine file size & create if necessary. var sz int64 - if fi, err := os.Stat(s.path); os.IsNotExist(err) { - if f, err := os.OpenFile(s.path, os.O_RDWR|os.O_CREATE, 0666); err != nil { + if fi, err := os.Stat(s.Path); os.IsNotExist(err) { + if f, err := os.OpenFile(s.Path, os.O_RDWR|os.O_CREATE, 0666); err != nil { return fmt.Errorf("touch wal segment file: %w", err) } else if err := f.Close(); err != nil { return fmt.Errorf("close touched wal segment file: %w", err) @@ -97,10 +71,10 @@ func (s *WALSegment) Open() (err error) { } // Determine page count & truncate if a partial page is written. - s.pageN = int(sz / PageSize) + s.PageN = int(sz / PageSize) if sz%PageSize != 0 { - sz = int64(s.pageN * PageSize) - if err := os.Truncate(s.path, sz); err != nil { + sz = int64(s.PageN * PageSize) + if err := os.Truncate(s.Path, sz); err != nil { return fmt.Errorf("truncate wal segment file: %w", err) } } @@ -113,7 +87,7 @@ func (s *WALSegment) Open() (err error) { } // Open file as a read-only memory map. - if f, err := os.OpenFile(s.path, os.O_RDONLY, 0666); err != nil { + if f, err := os.OpenFile(s.Path, os.O_RDONLY, 0666); err != nil { return fmt.Errorf("open wal segment file: %w", err) } else if s.data, err = syswrap.Mmap(int(f.Fd()), 0, int(mmapSize), syscall.PROT_READ, syscall.MAP_SHARED); err != nil { f.Close() @@ -127,12 +101,6 @@ func (s *WALSegment) Open() (err error) { // Close closes the write handle and the read-only mmap. func (s *WALSegment) Close() error { - s.mu.Lock() - defer s.mu.Unlock() - - if err := s.closeForWrite(); err != nil { - return err - } if s.data != nil { if err := syswrap.Munmap(s.data); err != nil { return err @@ -142,144 +110,141 @@ func (s *WALSegment) Close() error { return nil } -// CloseForWrite closes the write handle, if initialized. -func (s *WALSegment) CloseForWrite() error { - s.mu.Lock() - defer s.mu.Unlock() - return s.closeForWrite() -} - -func (s *WALSegment) closeForWrite() error { - // Ensure write buffer is flushed out. - if err := s.sync(); err != nil { - return err - } - s.writeCache = nil - - // Close underlying file writer. - if s.w != nil { - if err := s.w.Close(); err != nil { - return err - } - s.w = nil - } - return nil -} - // ReadWALPage reads a single page at the given WAL ID. func (s *WALSegment) ReadWALPage(walID int64) ([]byte, error) { - s.mu.RLock() - defer s.mu.RUnlock() - // Ensure requested ID is contained in this file. - if walID < s.minWALID || walID > s.minWALID+int64(s.pageN) { - return nil, fmt.Errorf("wal segment page read out of range: id=%d base=%d pageN=%d", walID, s.minWALID, s.pageN) + if walID < s.MinWALID || walID > s.MinWALID+int64(s.PageN) { + return nil, fmt.Errorf("wal segment page read out of range: id=%d base=%d pageN=%d", walID, s.MinWALID, s.PageN) } - offset := (walID - s.minWALID) * PageSize - - // If offset is within write buffer, return from write buffer. - writeBufferOffset := int64((s.pageN * PageSize) - len(s.writeCache)) - if offset >= writeBufferOffset { - buf := s.writeCache[offset-writeBufferOffset:] - return buf[:PageSize:PageSize], nil - } - - // Otherwise return from on-disk mmap. + offset := (walID - s.MinWALID) * PageSize return s.data[offset : offset+PageSize], nil } -// WriteWALPage writes a single page to the WAL segment and returns its WAL identifier. -func (s *WALSegment) WriteWALPage(page []byte, isMeta bool) (walID int64, err error) { - assert(len(page) == PageSize) // invalid page size - - s.mu.Lock() - defer s.mu.Unlock() - - // Initialize write file handle if not yet initialized. - if s.w == nil { - if s.w, err = os.OpenFile(s.path, os.O_WRONLY, 0666); err != nil { - return 0, fmt.Errorf("open wal segment write handle: %w", err) +func walSegmentByPath(segments []WALSegment, path string) *WALSegment { + for _, segment := range segments { + if segment.Path == path { + return &segment } } - - // Determine current WAL position. - walID = s.minWALID + int64(s.pageN) - - // Write WAL ID if this is a meta page. - if isMeta { - writeMetaWALID(page, walID) - // TODO: Write meta page checksum - } - - // Append write to write buffer & increment page count. - if s.writeCache == nil { - s.writeCache = make([]byte, 0, MaxWALSegmentFileSize+PageSize) - } - s.writeCache = append(s.writeCache, page...) - s.pageN++ - - return walID, nil -} - -// TruncateAfter removes all pages after a given WAL ID. -func (s *WALSegment) TruncateAfter(walID int64) error { - s.mu.Lock() - defer s.mu.Unlock() - - // Ensure this is a partial truncation. Full truncation of a segment - // should be performed by the DB since it needs to remove the segment. - assert(walID > s.minWALID) - - // Update to new page size. - newPageN := int((walID - s.minWALID) + 1) // new page count of segment - truncPageN := s.pageN - newPageN // number of pages removed - s.pageN = newPageN - - // Check to see if we are only truncating from the write cache. - writeCachePageN := len(s.writeCache) / PageSize - if truncPageN <= int(writeCachePageN) { - s.writeCache = s.writeCache[:(writeCachePageN-truncPageN)*PageSize] - return nil - } - - // Clear write cache. - s.writeCache = s.writeCache[:0] - - // Remove on disk pages. - return os.Truncate(s.path, int64(s.pageN)*PageSize) -} - -// Flush flushes the write buffer to the OS cache. -func (s *WALSegment) Flush() error { - s.mu.Lock() - defer s.mu.Unlock() - return s.flush() -} - -func (s *WALSegment) flush() error { - if _, err := s.w.WriteAt(s.writeCache, int64((s.pageN*PageSize)-len(s.writeCache))); err != nil { - return fmt.Errorf("wal segment write: %w", err) - } - s.writeCache = s.writeCache[:0] return nil } -// Sync flushes the write buffer and invokes a file sync to flush data to disk. -func (s *WALSegment) Sync() error { - s.mu.Lock() - defer s.mu.Unlock() - return s.sync() +func activeWALSegment(segments []WALSegment) WALSegment { + if len(segments) == 0 { + return WALSegment{} + } + return segments[len(segments)-1] } -func (s *WALSegment) sync() error { - if s.w == nil { - return nil +func minWALID(segments []WALSegment) int64 { + if len(segments) == 0 { + return 0 } - if err := s.flush(); err != nil { - return err + return segments[0].MinWALID +} + +func maxWALID(segments []WALSegment) int64 { + if len(segments) == 0 { + return 0 } - return s.w.Sync() + s := segments[len(segments)-1] + return s.MaxWALID() +} + +func walSize(segments []WALSegment) int64 { + var sz int64 + for _, s := range segments { + sz += s.Size() + } + return sz +} + +// readWALPage reads a single page at the given WAL ID. +func readWALPage(segments []WALSegment, walID int64) ([]byte, error) { + // TODO(BBJ): Binary search for segment. + for _, s := range segments { + if walID >= s.MinWALID && walID <= s.MaxWALID() { + return s.ReadWALPage(walID) + } + } + return nil, fmt.Errorf("cannot find segment containing WAL page: %d", walID) +} + +func findNextWALMetaPage(segments []WALSegment, walID int64) (metaWALID int64, err error) { + maxWALID := maxWALID(segments) + + for ; walID <= maxWALID; walID++ { + // Read page data from WAL and return if it is a meta page. + page, err := readWALPage(segments, walID) + if err != nil { + return walID, err + } else if IsMetaPage(page) { + return walID, nil + } + + // Skip over next page if this is a bitmap header. + if IsBitmapHeader(page) { + walID++ + } + } + + return -1, io.EOF +} + +func findLastWALMetaPage(segments []WALSegment) (walID int64, err error) { + if len(segments) == 0 { + return 0, nil + } + + var maxMetaWALID int64 + maxWALID := maxWALID(segments) + for walID := minWALID(segments); walID <= maxWALID; walID++ { + if page, err := readWALPage(segments, walID); err != nil { + return walID, err + } else if IsBitmapHeader(page) { + walID++ // skip next page for bitmap headers + } else if IsMetaPage(page) { + maxMetaWALID = walID // save max meta WAL ID + } + } + return maxMetaWALID, nil +} + +// truncateWALAfter removes all pages in the WAL after walID. +func truncateWALAfter(segments []WALSegment, walID int64) ([]WALSegment, error) { + var newSegments []WALSegment + + for i := range segments { + segment := &segments[i] + + // Append entire segment if WAL range entirely before target WAL ID. + if walID > segment.MaxWALID() { + newSegments = append(newSegments, *segment) + continue + } + + // If we only remove some of the WAL pages then truncate and append. + if segment.MinWALID < walID { + newSegment := *segment + newSegment.PageN = int((walID - newSegment.MinWALID) + 1) + + if err := os.Truncate(newSegment.Path, int64(newSegment.PageN)*PageSize); err != nil { + return segments, err + } + newSegments = append(newSegments, newSegment) + continue + } + + // Drop entire segment if all pages are after WAL ID. + if err := segment.Close(); err != nil { + return segments, err + } else if err := os.Remove(segment.Path); err != nil { + return segments, err + } + } + + return newSegments, nil } // FormatWALSegmentPath returns a path for a WAL segment using a WAL ID. diff --git a/rbf/wal_test.go b/rbf/wal_test.go index b41e7e792..59463d435 100644 --- a/rbf/wal_test.go +++ b/rbf/wal_test.go @@ -15,10 +15,10 @@ package rbf_test import ( - "bytes" - "encoding/hex" + // "bytes" + // "encoding/hex" "io/ioutil" - "math/rand" + // "math/rand" "os" "path/filepath" "testing" @@ -30,9 +30,9 @@ func TestWALSegment_Open(t *testing.T) { t.Run("OK", func(t *testing.T) { s := MustOpenWALSegment(t, 10) defer MustCloseWALSegment(t, s) - if got, want := s.MinWALID(), int64(10); got != want { + if got, want := s.MinWALID, int64(10); got != want { t.Fatalf("Base()=%d, want %d", got, want) - } else if got, want := s.PageN(), 0; got != want { + } else if got, want := s.PageN, 0; got != want { t.Fatalf("PageN()=%d, want %d", got, want) } }) @@ -40,6 +40,7 @@ func TestWALSegment_Open(t *testing.T) { // TODO(BBJ): Test open w/ partially written pages. } +/* func TestWALSegment_WritePage(t *testing.T) { rand := rand.New(rand.NewSource(0)) s := MustOpenWALSegment(t, 10) @@ -84,6 +85,7 @@ func TestWALSegment_WritePage(t *testing.T) { t.Fatal("unexpected second page") } } +*/ func TestFormatWALSegmentPath(t *testing.T) { if got, want := rbf.FormatWALSegmentPath(1234), "00000000000004d2.wal"; got != want { @@ -107,6 +109,7 @@ func TestParseWALSegmentPath(t *testing.T) { }) } +/* func BenchmarkWALSegment_WriteWALPage(b *testing.B) { b.Run("8KB", func(b *testing.B) { benchmarkWALSegment_WriteWALPage(b, 8*(1<<10)) }) b.Run("16KB", func(b *testing.B) { benchmarkWALSegment_WriteWALPage(b, 16*(1<<10)) }) @@ -147,9 +150,10 @@ func benchmarkWALSegment_WriteWALPage(b *testing.B, flushSize int) { b.SetBytes(rbf.MaxWALSegmentFileSize) } +*/ // MustOpenWALSegment opens a WAL segment in a temporary path. Fails on error. -func MustOpenWALSegment(tb testing.TB, walID int64) *rbf.WALSegment { +func MustOpenWALSegment(tb testing.TB, walID int64) rbf.WALSegment { tb.Helper() dir, err := ioutil.TempDir("", "") @@ -169,11 +173,11 @@ func MustOpenWALSegment(tb testing.TB, walID int64) *rbf.WALSegment { } // MustCloseWALSegment closes s. Fails on error. -func MustCloseWALSegment(tb testing.TB, s *rbf.WALSegment) { +func MustCloseWALSegment(tb testing.TB, s rbf.WALSegment) { tb.Helper() if err := s.Close(); err != nil { tb.Fatal(err) - } else if err := os.Remove(s.Path()); err != nil { + } else if err := os.Remove(s.Path); err != nil { tb.Fatal(err) } } From d826cec4b395a77db7d026ea24f528b41317b0f1 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Mon, 28 Sep 2020 13:57:33 -0600 Subject: [PATCH 31/53] fix RBF WAL segment reference error --- rbf/tx.go | 6 ++---- rbf/wal.go | 13 ++++++++++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/rbf/tx.go b/rbf/tx.go index 16e5505d0..98bd999d4 100644 --- a/rbf/tx.go +++ b/rbf/tx.go @@ -1660,15 +1660,13 @@ func (tx *Tx) writeBitmapWALPage(pgno uint32, page []byte) (walID int64, err err } func (tx *Tx) ensureWritableWALSegment() error { + // Ignore if we still have space in the write cache. writeCacheSize := int64(len(tx.wcache)) if len(tx.segments) != 0 && activeWALSegment(tx.segments).Size()+writeCacheSize < MaxWALSegmentFileSize { return nil } - return tx.addWALSegment() -} -// addWALSegment appends a new, writable segment and closing an existing segments for write. -func (tx *Tx) addWALSegment() error { + // Flush write cache out to file before adding new segment. if err := tx.flushWALWriter(); err != nil { return err } diff --git a/rbf/wal.go b/rbf/wal.go index 3a26c3bf6..bb9b53157 100644 --- a/rbf/wal.go +++ b/rbf/wal.go @@ -122,9 +122,9 @@ func (s *WALSegment) ReadWALPage(walID int64) ([]byte, error) { } func walSegmentByPath(segments []WALSegment, path string) *WALSegment { - for _, segment := range segments { - if segment.Path == path { - return &segment + for i := range segments { + if segments[i].Path == path { + return &segments[i] } } return nil @@ -247,6 +247,13 @@ func truncateWALAfter(segments []WALSegment, walID int64) ([]WALSegment, error) return newSegments, nil } +func DumpWALSegments(segments []WALSegment) { + fmt.Printf("WAL (%d segments)\n", len(segments)) + for i, s := range segments { + fmt.Printf("[%d] WALIDs=(%d-%d) PageN=%d\n", i, s.MinWALID, s.MaxWALID(), s.PageN) + } +} + // FormatWALSegmentPath returns a path for a WAL segment using a WAL ID. func FormatWALSegmentPath(walID int64) string { return fmt.Sprintf("%016x.wal", walID) From 3429421148ce8f6e27ae1a4db789b294cb21784f Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Mon, 28 Sep 2020 14:46:27 -0600 Subject: [PATCH 32/53] Fix RBF checkpoint off-by-one WAL ID issue --- rbf/db.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rbf/db.go b/rbf/db.go index e807956ec..737d7793b 100644 --- a/rbf/db.go +++ b/rbf/db.go @@ -250,7 +250,7 @@ func (db *DB) checkpoint(exclusive bool, mu sync.Locker) error { // Loop over pages in the transaction. for ; walID <= metaWALID; walID++ { - canCheckpoint := exclusive || minActiveWALID == 0 || walID <= minActiveWALID + canCheckpoint := exclusive || minActiveWALID == 0 || walID < minActiveWALID page, err := readWALPage(segments, walID) if err != nil { From c51ba69c09051acdbd1dec034d69f74c3041db4c Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Wed, 30 Sep 2020 08:39:46 -0600 Subject: [PATCH 33/53] Fix RBF sync calls --- rbf/db.go | 5 +++++ rbf/rbf.go | 16 ++++++++++++++++ rbf/wal.go | 6 +++--- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/rbf/db.go b/rbf/db.go index 737d7793b..9ff40a204 100644 --- a/rbf/db.go +++ b/rbf/db.go @@ -294,6 +294,11 @@ func (db *DB) checkpoint(exclusive bool, mu sync.Locker) error { } } + // Ensure WAL pages are fully copied & synced to DB file. + if err := db.file.Sync(); err != nil { + return fmt.Errorf("db file sync: %w", err) + } + // Remove WAL segments that have been checkpointed. if maxCheckpointedWALID != 0 { for _, segment := range segments { diff --git a/rbf/rbf.go b/rbf/rbf.go index f152e3499..a512fe237 100644 --- a/rbf/rbf.go +++ b/rbf/rbf.go @@ -648,3 +648,19 @@ func RowValues(b []uint64) []uint64 { // _, file, line, _ := runtime.Caller(skip + 1) // return fmt.Sprintf("%s:%d", file, line) // } + +// truncate truncates the file at path to sz bytes. File must exist. +func truncate(path string, sz int64) error { + f, err := os.OpenFile(path, os.O_WRONLY, 0666) + if err != nil { + return fmt.Errorf("open file: %w", err) + } + defer f.Close() + + if err := f.Truncate(sz); err != nil { + return fmt.Errorf("truncate: %w", err) + } else if err := f.Sync(); err != nil { + return fmt.Errorf("sync: %w", err) + } + return f.Close() +} diff --git a/rbf/wal.go b/rbf/wal.go index bb9b53157..14e64a107 100644 --- a/rbf/wal.go +++ b/rbf/wal.go @@ -74,8 +74,8 @@ func (s *WALSegment) Open() (err error) { s.PageN = int(sz / PageSize) if sz%PageSize != 0 { sz = int64(s.PageN * PageSize) - if err := os.Truncate(s.Path, sz); err != nil { - return fmt.Errorf("truncate wal segment file: %w", err) + if err := truncate(s.Path, sz); err != nil { + return fmt.Errorf("truncate wal file: %w", err) } } @@ -229,7 +229,7 @@ func truncateWALAfter(segments []WALSegment, walID int64) ([]WALSegment, error) newSegment := *segment newSegment.PageN = int((walID - newSegment.MinWALID) + 1) - if err := os.Truncate(newSegment.Path, int64(newSegment.PageN)*PageSize); err != nil { + if err := truncate(newSegment.Path, int64(newSegment.PageN)*PageSize); err != nil { return segments, err } newSegments = append(newSegments, newSegment) From c978e2242edb1b6d47876b91272845261f7ed00f Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Wed, 30 Sep 2020 09:51:16 -0600 Subject: [PATCH 34/53] Skip some RBF tests during race detection --- fragment_internal_test.go | 8 ++++++-- rbf/rbf_norace.go => race/norace.go | 6 +++--- rbf/rbf_race.go => race/race.go | 6 +++--- rbf/cursor_test.go | 9 +++++---- rbf/tx_test.go | 5 +++-- server/cluster_test.go | 8 ++++++-- server/server_test.go | 11 +++++++++++ 7 files changed, 37 insertions(+), 16 deletions(-) rename rbf/rbf_norace.go => race/norace.go (86%) rename rbf/rbf_race.go => race/race.go (87%) diff --git a/fragment_internal_test.go b/fragment_internal_test.go index b33862d54..a47f9fa4b 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -33,13 +33,13 @@ import ( "testing" "testing/quick" - "golang.org/x/sync/errgroup" - "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa/v2/pql" + "github.com/pilosa/pilosa/v2/race" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/testhook" "github.com/pkg/errors" + "golang.org/x/sync/errgroup" ) // Test flags @@ -450,6 +450,10 @@ func TestFragment_SetValue(t *testing.T) { }) t.Run("QuickCheck", func(t *testing.T) { + if os.Getenv("PILOSA_TXSRC") == "rbf" && race.Enabled { + t.Skip("race detection enabled, skipping for rbf") + } + if err := quick.Check(func(bitDepth uint, bitN uint64, values []uint64) bool { // Limit bit depth & maximum values. bitDepth = (bitDepth % 62) + 1 diff --git a/rbf/rbf_norace.go b/race/norace.go similarity index 86% rename from rbf/rbf_norace.go rename to race/norace.go index 15667ee93..6626c8380 100644 --- a/rbf/rbf_norace.go +++ b/race/norace.go @@ -14,7 +14,7 @@ // +build !race -package rbf +package race -// RaceEnabled is true if the -race flag is enabled. -const RaceEnabled = false +// Enabled is true if the -race flag is enabled. +const Enabled = false diff --git a/rbf/rbf_race.go b/race/race.go similarity index 87% rename from rbf/rbf_race.go rename to race/race.go index e178a736b..a6fe765bc 100644 --- a/rbf/rbf_race.go +++ b/race/race.go @@ -14,7 +14,7 @@ // +build race -package rbf +package race -// RaceEnabled is true if the -race flag is enabled. -const RaceEnabled = true +// Enabled is true if the -race flag is enabled. +const Enabled = true diff --git a/rbf/cursor_test.go b/rbf/cursor_test.go index 40f2756f0..401117d06 100644 --- a/rbf/cursor_test.go +++ b/rbf/cursor_test.go @@ -23,6 +23,7 @@ import ( "strings" "testing" + "github.com/pilosa/pilosa/v2/race" "github.com/pilosa/pilosa/v2/rbf" "github.com/pilosa/pilosa/v2/roaring" ) @@ -76,7 +77,7 @@ func TestCursor_FirstNext_Quick(t *testing.T) { t.Skip("-short enabled, skipping") } else if is32Bit() { t.Skip("32-bit build, skipping quick check tests") - } else if rbf.RaceEnabled { + } else if race.Enabled { t.Skip("race detection enabled, skipping") } @@ -198,7 +199,7 @@ func TestCursor_LastPrev_Quick(t *testing.T) { t.Skip("-short enabled, skipping") } else if is32Bit() { t.Skip("32-bit build, skipping quick check tests") - } else if rbf.RaceEnabled { + } else if race.Enabled { t.Skip("race detection enabled, skipping") } @@ -312,7 +313,7 @@ func TestCursor_Union(t *testing.T) { t.Skip("-short enabled, skipping") } else if is32Bit() { t.Skip("32-bit build, skipping quick check tests") - } else if rbf.RaceEnabled { + } else if race.Enabled { t.Skip("race detection enabled, skipping") } @@ -393,7 +394,7 @@ func TestCursor_Intersect(t *testing.T) { t.Skip("-short enabled, skipping") } else if is32Bit() { t.Skip("32-bit build, skipping quick check tests") - } else if rbf.RaceEnabled { + } else if race.Enabled { t.Skip("race detection enabled, skipping") } diff --git a/rbf/tx_test.go b/rbf/tx_test.go index defd0ac23..7e723aa8a 100644 --- a/rbf/tx_test.go +++ b/rbf/tx_test.go @@ -21,6 +21,7 @@ import ( "testing" "time" + "github.com/pilosa/pilosa/v2/race" "github.com/pilosa/pilosa/v2/rbf" "github.com/pilosa/pilosa/v2/txkey" ) @@ -218,7 +219,7 @@ func TestTx_Add_Quick(t *testing.T) { t.Skip("-short enabled, skipping") } else if is32Bit() { t.Skip("32-bit build, skipping quick check tests") - } else if rbf.RaceEnabled { + } else if race.Enabled { t.Skip("race detection enabled, skipping") } @@ -257,7 +258,7 @@ func TestTx_AddRemove_Quick(t *testing.T) { t.Skip("-short enabled, skipping") } else if is32Bit() { t.Skip("32-bit build, skipping quick check tests") - } else if rbf.RaceEnabled { + } else if race.Enabled { t.Skip("race detection enabled, skipping") } diff --git a/server/cluster_test.go b/server/cluster_test.go index f1a73a921..7b17a253d 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -25,9 +25,9 @@ import ( "testing" "time" - "github.com/pilosa/pilosa/v2/server" - "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/race" + "github.com/pilosa/pilosa/v2/server" "github.com/pilosa/pilosa/v2/test" "golang.org/x/sync/errgroup" ) @@ -634,6 +634,10 @@ func TestCluster_GossipMembership(t *testing.T) { } func TestClusterResize_RemoveNode(t *testing.T) { + if os.Getenv("PILOSA_TXSRC") == "rbf" && race.Enabled { + t.Skip("race detection enabled, skipping for rbf") + } + cluster := test.MustRunCluster(t, 3) defer cluster.Close() m0 := cluster.GetNode(0) diff --git a/server/server_test.go b/server/server_test.go index 270fdcaf5..02af30e59 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -34,6 +34,7 @@ import ( "github.com/pilosa/pilosa/v2" "github.com/pilosa/pilosa/v2/http" "github.com/pilosa/pilosa/v2/pql" + "github.com/pilosa/pilosa/v2/race" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/server" "github.com/pilosa/pilosa/v2/test" @@ -51,6 +52,8 @@ func init() { // nolint: gochecknoinits func TestMain_Set_Quick(t *testing.T) { if testing.Short() { t.Skip("short") + } else if os.Getenv("PILOSA_TXSRC") == "rbf" && race.Enabled { + t.Skip("race detection enabled, skipping for rbf") } for i := 0; i < 100; i++ { @@ -947,6 +950,10 @@ func TestMain_ImportTimestampNoStandardView(t *testing.T) { } func TestClusterQueriesAfterRestart(t *testing.T) { + if os.Getenv("PILOSA_TXSRC") == "rbf" && race.Enabled { + t.Skip("race detection enabled, skipping for rbf") + } + cluster := test.MustRunCluster(t, 3) defer cluster.Close() cmd1 := cluster.GetNode(1) @@ -1171,6 +1178,10 @@ func TestClusterExhaustingConnectionsImport(t *testing.T) { } func TestClusterMinMaxSumDecimal(t *testing.T) { + if os.Getenv("PILOSA_TXSRC") == "rbf" && race.Enabled { + t.Skip("race detection enabled, skipping for rbf") + } + cluster := test.MustRunCluster(t, 3) defer cluster.Close() cmd := cluster.GetNode(0) From 3852ac79c4eac7c6df7a7d7124e18be627a0c25d Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Thu, 1 Oct 2020 09:47:32 -0600 Subject: [PATCH 35/53] Add rbf.DB.TxN() function and test check --- rbf/db.go | 7 +++++++ rbf/rbf_test.go | 2 ++ 2 files changed, 9 insertions(+) diff --git a/rbf/db.go b/rbf/db.go index 9ff40a204..5bbf117e9 100644 --- a/rbf/db.go +++ b/rbf/db.go @@ -100,6 +100,13 @@ func CreateDirIfNotExist(path string) { } } +// TxN returns the number of active transactions. +func (db *DB) TxN() int { + db.mu.RLock() + defer db.mu.RUnlock() + return len(db.txs) +} + // Open opens a database with the file specified in Path. // Creates a new file if one does not already exist. func (db *DB) Open() (err error) { diff --git a/rbf/rbf_test.go b/rbf/rbf_test.go index 8b4768cd5..e4ea450e2 100644 --- a/rbf/rbf_test.go +++ b/rbf/rbf_test.go @@ -86,6 +86,8 @@ func MustCloseDB(tb testing.TB, db *rbf.DB) { tb.Helper() if err := db.Check(); err != nil && err != rbf.ErrClosed { tb.Fatal(err) + } else if n := db.TxN(); n != 0 { + tb.Fatalf("db still has %d active transactions; must closed before closing db", n) } else if err := db.Close(); err != nil && err != rbf.ErrClosed { tb.Fatal(err) } else if err := os.RemoveAll(db.Path); err != nil { From cf208cfaa32d7c2c64472cedc5140cae42f197bc Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Fri, 2 Oct 2020 09:53:05 -0600 Subject: [PATCH 36/53] Increase test http client timeout The timeout was increased to allow additional time for RBF to process a lot of individual `Set()` commands in `TestMain_RecalculateHashes`. --- test/pilosa.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pilosa.go b/test/pilosa.go index be0aaa1fc..d7fdf7aaa 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -302,7 +302,7 @@ func Do(t *testing.T, method, urlStr string, body string) *httpResponse { // set a timeout instead of allowing gohttp.Defaultclient to // potentially hang forever. hc := &gohttp.Client{ - Timeout: time.Second * 10, + Timeout: time.Second * 30, } resp, err := hc.Do(req) From 265452cf3dcac45662a0c189cf02ab0ddb7c24d4 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Mon, 5 Oct 2020 10:14:03 -0600 Subject: [PATCH 37/53] Add rbf.SyncEnabled --- rbf/db.go | 2 +- rbf/rbf.go | 14 +++++++++++++- rbf/tx.go | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/rbf/db.go b/rbf/db.go index 5bbf117e9..970c66b0c 100644 --- a/rbf/db.go +++ b/rbf/db.go @@ -302,7 +302,7 @@ func (db *DB) checkpoint(exclusive bool, mu sync.Locker) error { } // Ensure WAL pages are fully copied & synced to DB file. - if err := db.file.Sync(); err != nil { + if err := fsync(db.file); err != nil { return fmt.Errorf("db file sync: %w", err) } diff --git a/rbf/rbf.go b/rbf/rbf.go index a512fe237..b504e4fbc 100644 --- a/rbf/rbf.go +++ b/rbf/rbf.go @@ -91,6 +91,11 @@ var ( // Debug is just a temporary flag used for debugging. var Debug bool +// Testing constants. +const ( + SyncEnabled = true +) + // Magic32 returns the magic bytes as a big endian encoded uint32. func Magic32() uint32 { return binary.BigEndian.Uint32([]byte(Magic)) @@ -659,8 +664,15 @@ func truncate(path string, sz int64) error { if err := f.Truncate(sz); err != nil { return fmt.Errorf("truncate: %w", err) - } else if err := f.Sync(); err != nil { + } else if err := fsync(f); err != nil { return fmt.Errorf("sync: %w", err) } return f.Close() } + +func fsync(f *os.File) error { + if !SyncEnabled { + return nil + } + return f.Sync() +} diff --git a/rbf/tx.go b/rbf/tx.go index 98bd999d4..c2647b20f 100644 --- a/rbf/tx.go +++ b/rbf/tx.go @@ -1610,7 +1610,7 @@ func (tx *Tx) flushWALWriter() error { // Flush cache to writer. if _, err := w.WriteAt(tx.wcache, int64(segment.PageN)*PageSize); err != nil { return fmt.Errorf("write wal segment: %w", err) - } else if err := w.Sync(); err != nil { + } else if err := fsync(w); err != nil { return fmt.Errorf("sync wal segment: %w", err) } else if err := w.Close(); err != nil { return fmt.Errorf("close wal segment: %w", err) From 644969e6a94cdbd231108dac7a3ea28a12c9e5a6 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Tue, 6 Oct 2020 09:27:48 -0600 Subject: [PATCH 38/53] Add log.* and tourna.* to gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index bee19753e..84fdc73f4 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ build *~ lattice release-pilosa-fsck.*.*.tar.gz +/log.* +/tourna.log.* \ No newline at end of file From 7b64abbb53d109df1ea93259811d1799fe522eed Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Tue, 6 Oct 2020 09:34:17 -0600 Subject: [PATCH 39/53] Reduce default RBF DB size to 4GB; remove race skips --- fragment_internal_test.go | 5 ----- rbf/os.go | 2 +- server/cluster_test.go | 5 ----- server/server_test.go | 11 ----------- 4 files changed, 1 insertion(+), 22 deletions(-) diff --git a/fragment_internal_test.go b/fragment_internal_test.go index a47f9fa4b..8514f7284 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -35,7 +35,6 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa/v2/pql" - "github.com/pilosa/pilosa/v2/race" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/testhook" "github.com/pkg/errors" @@ -450,10 +449,6 @@ func TestFragment_SetValue(t *testing.T) { }) t.Run("QuickCheck", func(t *testing.T) { - if os.Getenv("PILOSA_TXSRC") == "rbf" && race.Enabled { - t.Skip("race detection enabled, skipping for rbf") - } - if err := quick.Check(func(bitDepth uint, bitN uint64, values []uint64) bool { // Limit bit depth & maximum values. bitDepth = (bitDepth % 62) + 1 diff --git a/rbf/os.go b/rbf/os.go index 7b2627f11..a76a550ef 100644 --- a/rbf/os.go +++ b/rbf/os.go @@ -19,4 +19,4 @@ package rbf // DefaultMaxSize is the default mmap size and therefore the maximum allowed // size of the database. The size can be increased by updating the DB.MaxSize // and reopening the database. This setting mainly affects virtual space usage. -const DefaultMaxSize = 100 * (1 << 30) // 100GB +const DefaultMaxSize = 4 * (1 << 30) diff --git a/server/cluster_test.go b/server/cluster_test.go index 7b17a253d..1cbda5ffc 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -26,7 +26,6 @@ import ( "time" "github.com/pilosa/pilosa/v2" - "github.com/pilosa/pilosa/v2/race" "github.com/pilosa/pilosa/v2/server" "github.com/pilosa/pilosa/v2/test" "golang.org/x/sync/errgroup" @@ -634,10 +633,6 @@ func TestCluster_GossipMembership(t *testing.T) { } func TestClusterResize_RemoveNode(t *testing.T) { - if os.Getenv("PILOSA_TXSRC") == "rbf" && race.Enabled { - t.Skip("race detection enabled, skipping for rbf") - } - cluster := test.MustRunCluster(t, 3) defer cluster.Close() m0 := cluster.GetNode(0) diff --git a/server/server_test.go b/server/server_test.go index 02af30e59..270fdcaf5 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -34,7 +34,6 @@ import ( "github.com/pilosa/pilosa/v2" "github.com/pilosa/pilosa/v2/http" "github.com/pilosa/pilosa/v2/pql" - "github.com/pilosa/pilosa/v2/race" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/server" "github.com/pilosa/pilosa/v2/test" @@ -52,8 +51,6 @@ func init() { // nolint: gochecknoinits func TestMain_Set_Quick(t *testing.T) { if testing.Short() { t.Skip("short") - } else if os.Getenv("PILOSA_TXSRC") == "rbf" && race.Enabled { - t.Skip("race detection enabled, skipping for rbf") } for i := 0; i < 100; i++ { @@ -950,10 +947,6 @@ func TestMain_ImportTimestampNoStandardView(t *testing.T) { } func TestClusterQueriesAfterRestart(t *testing.T) { - if os.Getenv("PILOSA_TXSRC") == "rbf" && race.Enabled { - t.Skip("race detection enabled, skipping for rbf") - } - cluster := test.MustRunCluster(t, 3) defer cluster.Close() cmd1 := cluster.GetNode(1) @@ -1178,10 +1171,6 @@ func TestClusterExhaustingConnectionsImport(t *testing.T) { } func TestClusterMinMaxSumDecimal(t *testing.T) { - if os.Getenv("PILOSA_TXSRC") == "rbf" && race.Enabled { - t.Skip("race detection enabled, skipping for rbf") - } - cluster := test.MustRunCluster(t, 3) defer cluster.Close() cmd := cluster.GetNode(0) From 2440413c49b0ec4e919aa6c05c7e9cf0057362d9 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Tue, 6 Oct 2020 10:29:17 -0600 Subject: [PATCH 40/53] Remove test skips based on race detector --- race/norace.go | 20 -------------------- race/race.go | 20 -------------------- rbf/cursor_test.go | 9 --------- rbf/tx_test.go | 5 ----- 4 files changed, 54 deletions(-) delete mode 100644 race/norace.go delete mode 100644 race/race.go diff --git a/race/norace.go b/race/norace.go deleted file mode 100644 index 6626c8380..000000000 --- a/race/norace.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build !race - -package race - -// Enabled is true if the -race flag is enabled. -const Enabled = false diff --git a/race/race.go b/race/race.go deleted file mode 100644 index a6fe765bc..000000000 --- a/race/race.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build race - -package race - -// Enabled is true if the -race flag is enabled. -const Enabled = true diff --git a/rbf/cursor_test.go b/rbf/cursor_test.go index 401117d06..9f134a92e 100644 --- a/rbf/cursor_test.go +++ b/rbf/cursor_test.go @@ -23,7 +23,6 @@ import ( "strings" "testing" - "github.com/pilosa/pilosa/v2/race" "github.com/pilosa/pilosa/v2/rbf" "github.com/pilosa/pilosa/v2/roaring" ) @@ -77,8 +76,6 @@ func TestCursor_FirstNext_Quick(t *testing.T) { t.Skip("-short enabled, skipping") } else if is32Bit() { t.Skip("32-bit build, skipping quick check tests") - } else if race.Enabled { - t.Skip("race detection enabled, skipping") } const n = 10000 @@ -199,8 +196,6 @@ func TestCursor_LastPrev_Quick(t *testing.T) { t.Skip("-short enabled, skipping") } else if is32Bit() { t.Skip("32-bit build, skipping quick check tests") - } else if race.Enabled { - t.Skip("race detection enabled, skipping") } const n = 10000 @@ -313,8 +308,6 @@ func TestCursor_Union(t *testing.T) { t.Skip("-short enabled, skipping") } else if is32Bit() { t.Skip("32-bit build, skipping quick check tests") - } else if race.Enabled { - t.Skip("race detection enabled, skipping") } QuickCheck(t, func(t *testing.T, rand *rand.Rand) { @@ -394,8 +387,6 @@ func TestCursor_Intersect(t *testing.T) { t.Skip("-short enabled, skipping") } else if is32Bit() { t.Skip("32-bit build, skipping quick check tests") - } else if race.Enabled { - t.Skip("race detection enabled, skipping") } QuickCheck(t, func(t *testing.T, rand *rand.Rand) { diff --git a/rbf/tx_test.go b/rbf/tx_test.go index 7e723aa8a..2d5505a4a 100644 --- a/rbf/tx_test.go +++ b/rbf/tx_test.go @@ -21,7 +21,6 @@ import ( "testing" "time" - "github.com/pilosa/pilosa/v2/race" "github.com/pilosa/pilosa/v2/rbf" "github.com/pilosa/pilosa/v2/txkey" ) @@ -219,8 +218,6 @@ func TestTx_Add_Quick(t *testing.T) { t.Skip("-short enabled, skipping") } else if is32Bit() { t.Skip("32-bit build, skipping quick check tests") - } else if race.Enabled { - t.Skip("race detection enabled, skipping") } QuickCheck(t, func(t *testing.T, rand *rand.Rand) { @@ -258,8 +255,6 @@ func TestTx_AddRemove_Quick(t *testing.T) { t.Skip("-short enabled, skipping") } else if is32Bit() { t.Skip("32-bit build, skipping quick check tests") - } else if race.Enabled { - t.Skip("race detection enabled, skipping") } QuickCheck(t, func(t *testing.T, rand *rand.Rand) { From 3b7758a4f2a101dc27720b6d814e9df48b8025e9 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Tue, 6 Oct 2020 10:36:57 -0600 Subject: [PATCH 41/53] Add comment explaining rbf.DB.checkpoint() args --- rbf/db.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/rbf/db.go b/rbf/db.go index 970c66b0c..f60b12081 100644 --- a/rbf/db.go +++ b/rbf/db.go @@ -219,8 +219,12 @@ func (db *DB) Checkpoint() error { // checkpoint moves WAL segments to the main DB file. // -// Note that mu should db.mu when called through the external API but it -// can be &nopLocker if called under lock. +// Note that mu should db.mu when called through DB.Checkpoint() but it +// can be &nopLocker if called under lock. The external API will be used +// to periodically checkpoint outside of a transaction and the locking +// must be used only in the beginning (to obtain the segment list) and at +// the end (when removing old segments from the list). If the entire function +// were to obtain a lock then it would block all new read & write transactions. func (db *DB) checkpoint(exclusive bool, mu sync.Locker) error { // Obtain a snapshot of WAL segments at the start. mu.Lock() From 83f230bbfb5a5b34a8d258a7c81fc37b0593d1df Mon Sep 17 00:00:00 2001 From: Jason Aten Date: Tue, 6 Oct 2020 11:56:34 -0500 Subject: [PATCH 42/53] run CI on rbf and rbf_lmdb --- .circleci/config.yml | 14 +++++++++++++- Makefile | 9 ++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0f4fdb943..86814df27 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -198,12 +198,24 @@ workflows: resource_class: xlarge requires: - setup + - test: + name: test-txstore-rbf + test_make_target: test-txstore-rbf + resource_class: xlarge + requires: + - setup + - test: + name: test-txstore-rbf_lmdb + test_make_target: test-txstore-rbf_lmdb + resource_class: xlarge + requires: + - setup - test: name: test-shardwidth-22 shard_width: "22" resource_class: large requires: - - setup + - setup - cluster-tests: requires: - setup diff --git a/Makefile b/Makefile index 085cef0b8..058a88847 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race +.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race test-txstore-rbf_lmdb test-txstore-rbf CLONE_URL=github.com/pilosa/pilosa VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) @@ -410,3 +410,10 @@ install-gometalinter: GO111MODULE=off go get -u github.com/alecthomas/gometalinter GO111MODULE=off gometalinter --install GO111MODULE=off go get github.com/remyoudompheng/go-misc/deadcode + +test-txstore-rbf: + PILOSA_TXSRC=rbf $(MAKE) testv-race + +test-txstore-rbf_lmdb: + PILOSA_TXSRC=rbf_lmdb $(MAKE) testv-race + From a63c6ee2b209b186a408dd679b03c3d6de0ad649 Mon Sep 17 00:00:00 2001 From: Jason Aten Date: Tue, 6 Oct 2020 16:47:05 -0500 Subject: [PATCH 43/53] pilosa-fsck: multi-index counts and repairs. - multiple indexes repaired at once could crosstalk. Fixed. - the counts of keys and ids are now broken down by index. --- boltdb/translate.go | 2 +- cluster.go | 17 +++ cmd/pilosa-fsck/fsck.go | 157 ++++++++++++---------- cmd/pilosa-fsck/fsck_test.go | 251 +++++++++++++++++++++++------------ 4 files changed, 272 insertions(+), 155 deletions(-) diff --git a/boltdb/translate.go b/boltdb/translate.go index 34d1d270d..f5d2681ea 100644 --- a/boltdb/translate.go +++ b/boltdb/translate.go @@ -579,7 +579,7 @@ func (s *TranslateStore) ComputeTranslatorSummaryCols(partitionID int, topo *pil firstPrimary = primary } else { if primary != firstPrimary { - panic(fmt.Sprintf("primary (%v) != firstPrimary (%v); key='%v', id=%v, shard=%v; partitionID=%v", primary, firstPrimary, ks, id, shard, partitionID)) + panic(fmt.Sprintf("s.index='%v' primary (%v) != firstPrimary (%v); key='%v', id=%v, shard=%v; partitionID=%v; topo='%v'", s.index, primary, firstPrimary, ks, id, shard, partitionID, topo.String())) } } diff --git a/cluster.go b/cluster.go index 9059c8732..23c86ac7b 100644 --- a/cluster.go +++ b/cluster.go @@ -1968,6 +1968,23 @@ func NewTopology(hasher Hasher, partitionN int, replicaN int, c *cluster) *Topol } } +func (t *Topology) String() string { + return fmt.Sprintf(` +&pilosa.Topology{ + nodeIDs: %v, + clusterID: %v, + nodeStates: %v, + PartitionN: %v, + ReplicaN: %v, +} +`, + t.nodeIDs, + t.clusterID, + t.nodeStates, + t.PartitionN, + t.ReplicaN, + ) +} func (t *Topology) GetNodeIDs() []string { return t.nodeIDs } diff --git a/cmd/pilosa-fsck/fsck.go b/cmd/pilosa-fsck/fsck.go index 587485762..2bb6b892e 100644 --- a/cmd/pilosa-fsck/fsck.go +++ b/cmd/pilosa-fsck/fsck.go @@ -22,6 +22,7 @@ import ( "log" "os" "path/filepath" + "sort" "strconv" "strings" "time" @@ -376,77 +377,96 @@ func (g *group) String() (s string) { return } +func indexesFromAts(ats *pilosa.AllTranslatorSummary) (indexes []string) { + indexMap := make(map[string]bool) + for _, sum := range ats.Sums { + if !indexMap[sum.Index] { + indexMap[sum.Index] = true + indexes = append(indexes, sum.Index) + } + } + sort.Strings(indexes) + return +} + func (cfg *FsckConfig) RepairTranslationStores(ats *pilosa.AllTranslatorSummary) (err error) { verbose := cfg.Verbose - m := make(map[int]*group) - for _, sum := range ats.Sums { - if !sum.IsColKey { - continue - } - grp := m[sum.PartitionID] - if grp == nil { - grp = &group{ - partitionID: sum.PartitionID, - } - m[sum.PartitionID] = grp - } - grp.elem = append(grp.elem, sum) - } + // group by index first. then repair. + indexes := indexesFromAts(ats) - for partitionID, group := range m { - _ = partitionID - prim := -1 - keyCount := 0 - for k, e := range group.elem { - if e.IsPrimary { - prim = k - } - keyCount += e.KeyCount - } - if prim == -1 { - panic(fmt.Sprintf("no primary found for group '%v'", group.String())) - } + for _, index := range indexes { - primary := group.elem[prim] - primaryChecksum := primary.Checksum - for _, e := range group.elem { - if e.IsPrimary { + m := make(map[int]*group) + for _, sum := range ats.Sums { + + if !sum.IsColKey || sum.Index != index { continue } - // is e a replica? not necessarily! have to check. - if !e.IsReplica { - //if verbose { - // since this will happen even on a fix point, where it is already empty, - // we don't report it again. - //fmt.Printf("# non-replica should have no data: creating an empty translation store here at '%v'\n", e.StorePath) - //} - err := os.RemoveAll(e.StorePath) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("RepairTranslationStores() os.RemoveAll(e.StorePath='%v')", e.StorePath)) + grp := m[sum.PartitionID] + if grp == nil { + grp = &group{ + partitionID: sum.PartitionID, } - store, err := boltdb.OpenTranslateStore(e.StorePath, e.Index, e.Field, e.PartitionID, pilosa.DefaultPartitionN) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("RepairTranslationStores() create empty boldtdb: boltdb.OpenTranslateStore e.StorePath='%v'", e.StorePath)) - } - err = store.Close() - if err != nil { - return errors.Wrap(err, fmt.Sprintf("RepairTranslationStores() closing empty boltdb at path '%v'", e.StorePath)) - } - continue + m[sum.PartitionID] = grp } - // INVAR: e is a replica for this paritionID. - // Copy from primary if checksums are different. - if e.Checksum != primaryChecksum { - from := group.elem[prim].StorePath - dest := e.StorePath - if verbose { - fmt.Printf("# e.Checksum '%v' != primaryChecksum '%v': copying from primary translation store '%v' -> '%v'\n", e.Checksum, primaryChecksum, from, dest) + grp.elem = append(grp.elem, sum) + } + + for partitionID, group := range m { + _ = partitionID + prim := -1 + keyCount := 0 + for k, e := range group.elem { + if e.IsPrimary { + prim = k } - err := cp(from, dest) - if err != nil { - return fmt.Errorf("error: could not copy from primary '%v' to replica translation store '%v': '%v' ... try to keep going...\n", from, dest, err) + keyCount += e.KeyCount + } + if prim == -1 { + panic(fmt.Sprintf("no primary found for group '%v'", group.String())) + } + + primary := group.elem[prim] + primaryChecksum := primary.Checksum + for _, e := range group.elem { + if e.IsPrimary { + continue + } + // is e a replica? not necessarily! have to check. + if !e.IsReplica { + //if verbose { + // since this will happen even on a fix point, where it is already empty, + // we don't report it again. + //fmt.Printf("# non-replica should have no data: creating an empty translation store here at '%v'\n", e.StorePath) + //} + err := os.RemoveAll(e.StorePath) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("RepairTranslationStores() os.RemoveAll(e.StorePath='%v')", e.StorePath)) + } + store, err := boltdb.OpenTranslateStore(e.StorePath, e.Index, e.Field, e.PartitionID, pilosa.DefaultPartitionN) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("RepairTranslationStores() create empty boldtdb: boltdb.OpenTranslateStore e.StorePath='%v'", e.StorePath)) + } + err = store.Close() + if err != nil { + return errors.Wrap(err, fmt.Sprintf("RepairTranslationStores() closing empty boltdb at path '%v'", e.StorePath)) + } + continue + } + // INVAR: e is a replica for this paritionID. + // Copy from primary if checksums are different. + if e.Checksum != primaryChecksum { + from := group.elem[prim].StorePath + dest := e.StorePath + if verbose { + fmt.Printf("# e.Checksum '%v' != primaryChecksum '%v': copying from primary translation store '%v' -> '%v'\n", e.Checksum, primaryChecksum, from, dest) + } + err := cp(from, dest) + if err != nil { + return fmt.Errorf("error: could not copy from primary '%v' to replica translation store '%v': '%v' ... try to keep going...\n", from, dest, err) + } } } } @@ -555,7 +575,7 @@ func (cfg *FsckConfig) readOneDir(dir string) (idx2frag map[string]*pilosa.Index quiet := cfg.Quiet if !quiet { - fmt.Printf("# opening dir '%v'... this may take a few minutes... fixcol=%v\n\n", dir, cfg.FixCol) + fmt.Printf("# opening dir '%v'... this may take a few minutes...\n\n", dir) } jmphasher := &pilosa.Jmphasher{} @@ -819,7 +839,7 @@ func (cfg *FsckConfig) analyzeThisIndex( } nDir := len(nodes2fragsum) - keyCount, idCount := cfg.getKeyIDCounts(ats) + keyCount, idCount := cfg.getKeyIDCounts(index, ats) fixNeeded = changedFiles > 0 || ats.RepairNeeded var actionTaken string @@ -835,7 +855,7 @@ func (cfg *FsckConfig) analyzeThisIndex( } else { if fixNeeded { wouldBe = "sync actions that would be taken under -fix:" - actionTaken = "*REPAIRS NEEDED BUT WERE NOT APPLIED* ; pilosa-fsck -fix and -fixcol were omitted." + actionTaken = "*REPAIRS NEEDED BUT WERE NOT APPLIED* ; pilosa-fsck -fix was omitted." } else { wouldBe = "" actionTaken = "NO REPAIR NEEDED." @@ -860,7 +880,6 @@ func (cfg *FsckConfig) analyzeThisIndex( # pilosa-fsck final report # # run with -fix: %v -# -fixcol: %v # # index examined: '%v' # @@ -878,7 +897,7 @@ func (cfg *FsckConfig) analyzeThisIndex( # %v # ======================================================== `, - cfg.Fix, cfg.FixCol, index, nDir, cfg.ReplicaN, humanize.Comma(totalBytes), humanize.Comma(totalFiles), humanize.Comma(int64(nDir*pilosa.DefaultPartitionN)), humanize.Comma(int64(keyCount)), humanize.Comma(int64(idCount)), actionTaken, fragUpdate) + cfg.Fix, index, nDir, cfg.ReplicaN, humanize.Comma(totalBytes), humanize.Comma(totalFiles), humanize.Comma(int64(nDir*pilosa.DefaultPartitionN)), humanize.Comma(int64(keyCount)), humanize.Comma(int64(idCount)), actionTaken, fragUpdate) return } @@ -915,10 +934,12 @@ func cp(fromPath, toPath string) (err error) { return os.Rename(tmpTo, toPath) } -func (cfg *FsckConfig) getKeyIDCounts(ats *pilosa.AllTranslatorSummary) (keyCount, idCount int) { +func (cfg *FsckConfig) getKeyIDCounts(index string, ats *pilosa.AllTranslatorSummary) (keyCount, idCount int) { for _, sum := range ats.Sums { - keyCount += sum.KeyCount - idCount += sum.IDCount + if sum.Index == index { + keyCount += sum.KeyCount + idCount += sum.IDCount + } } return } diff --git a/cmd/pilosa-fsck/fsck_test.go b/cmd/pilosa-fsck/fsck_test.go index b8002f78f..e6169c4f5 100644 --- a/cmd/pilosa-fsck/fsck_test.go +++ b/cmd/pilosa-fsck/fsck_test.go @@ -88,91 +88,109 @@ func Test_Repair(t *testing.T) { ctx := context.Background() - index := "rick" - fieldName := "f" + index := []string{"rick", "morty"} + fieldName := []string{"f", "flying_car"} + idx := make([]*pilosa.Index, len(index)) + field := make([]*pilosa.Field, len(index)) + var err error - idx, err := nodes[0].API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true, TrackExistence: true}) - if err != nil { - t.Fatalf("creating index: %v", err) - } - if idx.CreatedAt() == 0 { - t.Fatal("index createdAt is empty") - } + for i := range index { - field, err := nodes[0].API.CreateField(ctx, index, fieldName, pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100)) - if err != nil { - t.Fatalf("creating field: %v", err) - } - if field.CreatedAt() == 0 { - t.Fatal("field createdAt is empty") + idx[i], err = nodes[0].API.CreateIndex(ctx, index[i], pilosa.IndexOptions{Keys: true, TrackExistence: true}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + if idx[i].CreatedAt() == 0 { + t.Fatal("index createdAt is empty") + } + + field[i], err = nodes[0].API.CreateField(ctx, index[i], fieldName[i], pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100)) + if err != nil { + t.Fatalf("creating field: %v", err) + } + if field[i].CreatedAt() == 0 { + t.Fatal("field createdAt is empty") + } } rowID := uint64(1) timestamp := int64(0) - // Generate some keyed records. - rowIDs := []uint64{} - timestamps := []int64{} - N := 10 - for i := 1; i <= N; i++ { - rowIDs = append(rowIDs, rowID) - timestamps = append(timestamps, timestamp) - } + for i := range index { - // Keys are sharded so ordering is not guaranteed. - colKeys := []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"} - - colKeys = colKeys[:N] - - // Import data with keys to the coordinator (node0) and verify that it gets - // translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher) - req := &pilosa.ImportRequest{ - Index: index, - IndexCreatedAt: idx.CreatedAt(), - Field: fieldName, - FieldCreatedAt: field.CreatedAt(), - - // even though this says Shard: 0, that won't matter. The column keys - // get hashed and that decides the actual shard. - Shard: 0, - RowIDs: rowIDs, - ColumnKeys: colKeys, - Timestamps: timestamps, - } - - qcx := nodes[0].API.Txf().NewQcx() - - if err := nodes[0].API.Import(ctx, qcx, req); err != nil { - t.Fatal(err) - } - panicOn(qcx.Finish()) - - pql := fmt.Sprintf("Row(%s=%d)", fieldName, rowID) - - // Query node0. - if res, err := nodes[0].API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil { - t.Fatal(err) - } else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) { - t.Fatalf("expected colKeys='%#v'; observed column keys: %#v", colKeys, keys) - } - - // Query node1. - if err := test.RetryUntil(5*time.Second, func() error { - if res, err := nodes[1].API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil { - return err - } else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) { - return fmt.Errorf("unexpected column keys: %#v", keys) + // Generate some keyed records. + rowIDs := []uint64{} + timestamps := []int64{} + N := 10 + for j := 1; j <= N; j++ { + rowIDs = append(rowIDs, rowID) + timestamps = append(timestamps, timestamp) } - return nil - }); err != nil { - t.Fatal(err) - } + var colKeys []string + switch i { + case 0: + // Keys are sharded so ordering is not guaranteed. + colKeys = []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"} + colKeys = colKeys[:N] + case 1: + colKeys = []string{"col11", "col12"} + N = len(colKeys) + rowIDs = rowIDs[:N] + timestamps = timestamps[:N] + } + + // Import data with keys to the coordinator (node0) and verify that it gets + // translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher) + req := &pilosa.ImportRequest{ + Index: index[i], + IndexCreatedAt: idx[i].CreatedAt(), + Field: fieldName[i], + FieldCreatedAt: field[i].CreatedAt(), + + // even though this says Shard: 0, that won't matter. The column keys + // get hashed and that decides the actual shard. + Shard: 0, + RowIDs: rowIDs, + ColumnKeys: colKeys, + Timestamps: timestamps, + } + + qcx := nodes[0].API.Txf().NewQcx() + + if err := nodes[0].API.Import(ctx, qcx, req); err != nil { + t.Fatal(err) + } + panicOn(qcx.Finish()) + //qcx.Reset() + + pql := fmt.Sprintf("Row(%s=%d)", fieldName[i], rowID) + + // Query node0. + if res, err := nodes[0].API.Query(ctx, &pilosa.QueryRequest{Index: index[i], Query: pql}); err != nil { + t.Fatal(err) + } else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) { + t.Fatalf("expected colKeys='%#v'; observed column keys: %#v", colKeys, keys) + } + + // Query node1. + if err := test.RetryUntil(5*time.Second, func() error { + if res, err := nodes[1].API.Query(ctx, &pilosa.QueryRequest{Index: index[i], Query: pql}); err != nil { + return err + } else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) { + return fmt.Errorf("unexpected column keys: %#v", keys) + } + return nil + }); err != nil { + t.Fatal(err) + } + } // end of setup. // partitionID in use: 6, 31, 57, 133, 185, 235 targetPartition := 31 // which partitionID we mess with. targetNode := nodes[0] // this is the first replica. + targetIndex := index[0] // 0 first replica // 1 second replica // 2 -- not a replica @@ -191,8 +209,8 @@ func Test_Repair(t *testing.T) { // for this test, mess up a replica that is not the primary. h := targetNode.API.Holder() - idx = h.Index(index) - store := idx.TranslateStore(targetPartition) + idx[0] = h.Index(index[0]) + store := idx[0].TranslateStore(targetPartition) fwd, rev := getFwdRev(store, targetPartition) //vv("targetPartition=%v, store.PartitionID=%v, before corruption, fwd='%#v', rev='%#v'", targetPartition, store.PartitionID, fwd, rev) @@ -218,6 +236,28 @@ func Test_Repair(t *testing.T) { //fwd3, rev3 := getFwdRev(store, targetPartition) //vv("after corruption, fwd='%#v', rev='%#v'", fwd3, rev3) + targetIndex1 := "morty" + targetPartition1 := 226 // for "col11" + // # fsck_test.go:248 2020-10-06T20:24:33.755576-05:00 on k=47, idx[1]: targetPartition=47, store.PartitionID=0x4abe160, before corruption, fwd1='map[string]uint64{"col12":0xcf00001}', rev1='map[uint64]string{0xcf00001:"col12"}' + //# fsck_test.go:248 2020-10-06T20:24:35.608568-05:00 on k=226, idx[1]: targetPartition=226, store.PartitionID=0x4abe160, before corruption, fwd1='map[string]uint64{"col11":0xcc00001}', rev1='map[uint64]string{0xcc00001:"col11"}' + idx[1] = h.Index(index[1]) + store1 := idx[1].TranslateStore(targetPartition1) + fwd1, rev1 := getFwdRev(store1, targetPartition1) + //vv("on k=%v, idx[1]: targetPartition=%v, store.PartitionID=%v, before corruption, fwd1='%#v', rev1='%#v'", k, targetPartition1, store.PartitionID, fwd1, rev1) + + presz1 := len(rev1) + delete(rev1, fwd1["col11"]) + postsz1 := len(rev1) + + if postsz1 == presz1 { + panic("did not delete any key!") + } + bolt1 := store1.(*boltdb.TranslateStore) + if err := bolt1.SetFwdRevMaps(nil, fwd1, rev1); err != nil { + t.Fatal(err) + } + + // done corrupting. for _, nd := range nodes { nd.Command.Close() } @@ -231,7 +271,7 @@ func Test_Repair(t *testing.T) { // first we check that the corruption can be detected // by our test with the checksums. - chk, err := check(dirs, cfg, targetPartition) + chk, err := check(dirs, cfg, targetIndex, targetPartition) _ = chk //vv("pre-fix, chk='%v'; err='%v'", chk, err) @@ -239,6 +279,14 @@ func Test_Repair(t *testing.T) { panic("expected to see checksums not match! but no corruption detected.") } + chk1, err := check(dirs, cfg, targetIndex1, targetPartition1) + _ = chk1 + //vv("pre-fix, chk1='%v'; err='%v'", chk1, err) + + if err == nil { + panic("expected to see checksums not match! but no corruption detected.") + } + // b) running in reporting mode only should report that a fix is needed. fixNeeded, err := cfg.Run() panicOn(err) @@ -249,6 +297,7 @@ func Test_Repair(t *testing.T) { // c) run the fix. cfg.Fix = true cfg.FixCol = true + fixNeeded, err = cfg.Run() panicOn(err) if !fixNeeded { @@ -260,18 +309,22 @@ func Test_Repair(t *testing.T) { //chksums = getChecksums(dirs, cfg, targetPartition) //vv("after repair chksums = '%#v'", chksums) - chk, err = check(dirs, cfg, targetPartition) + chk, err = check(dirs, cfg, targetIndex, targetPartition) _ = chk //vv("chk = '%v' after repair; err='%v'", chk, err) panicOn(err) + chk1, err = check(dirs, cfg, targetIndex1, targetPartition1) + _ = chk1 + //vv("chk = '%v' after repair; err='%v'", chk, err) + panicOn(err) + // e) run again, should see no fix needed. fixNeeded, err = cfg.Run() panicOn(err) if fixNeeded { panic("should see no fix needed after the prior repair") } - } func getFwdRev(store pilosa.TranslateStore, partitionID int) (fwd map[string]uint64, rev map[uint64]string) { @@ -288,10 +341,13 @@ func getFwdRev(store pilosa.TranslateStore, partitionID int) (fwd map[string]uin return } -func check(dirs []string, cfg *FsckConfig, targetPartition int) (chksum string, err error) { +func check(dirs []string, cfg *FsckConfig, targetIndex string, targetPartition int) (chksum string, err error) { + //vv("top of check, dirs = '%#v', targetIndex='%v', targetPartition='%v'", dirs, targetIndex, targetPartition) + //defer vv("returning from check()") firstChecksum := "" firstDir := "" + firstStorePath := "" quiet := cfg.Quiet defer func() { cfg.Quiet = quiet @@ -301,19 +357,42 @@ func check(dirs []string, cfg *FsckConfig, targetPartition int) (chksum string, dir := dirs[i] _, _, ats, err := cfg.readOneDir(dir) panicOn(err) + indexes := indexesFromAts(ats) + //vv("indexes = '%#v'", indexes) - for _, s := range ats.Sums { - if s.PartitionID != targetPartition { + for _, index := range indexes { + + if index != targetIndex { continue } - if s.IsPrimary || s.IsReplica { - chksum := s.Checksum - if firstChecksum == "" { - firstChecksum = chksum - firstDir = dir - } else { - if chksum != firstChecksum { - return chksum, fmt.Errorf("bolt chksum on node %v '%v' disagrees with '%v' on '%v'", dir, chksum, firstChecksum, firstDir) + for _, s := range ats.Sums { + //vv(" s= '%#v'", s) + if s.Index != index { + //vv("skipping s.Index '%v' != index '%v'", s.Index, index) + continue + } + if s.PartitionID != targetPartition { + continue + } + //vv("accepting s.PartitionID(%v) == targetPartition(%v); s.Index '%v'; "+ + //"index '%v'; s.IsPrimary=%v, s.IsReplica=%v, s='%#v'; s.Checksum='%v', firstChecksum='%v'", + //s.PartitionID, targetPartition, s.Index, index, + //s.IsPrimary, s.IsReplica, s, s.Checksum, firstChecksum) + + if s.IsPrimary || s.IsReplica { + chksum := s.Checksum + if firstChecksum == "" { + + firstChecksum = chksum + firstDir = dir + firstStorePath = s.StorePath + + } else { + //vv("targetIndex = '%v'; firstChecksum='%v', chksum='%v'", targetIndex, firstChecksum, chksum) + + if chksum != firstChecksum { + return chksum, fmt.Errorf("bolt chksum on node %v '%v' disagrees with '%v' on '%v'; index='%v'; s.StorePath = '%v'; firstStorePath='%v'", dir, chksum, firstChecksum, firstDir, index, s.StorePath, firstStorePath) + } } } } From 937a1afade122885cec82bcf367a785dbaf8d4fe Mon Sep 17 00:00:00 2001 From: Jason Aten Date: Wed, 7 Oct 2020 14:58:48 -0500 Subject: [PATCH 44/53] pilosa/cmd/random-query: generate random queries from existing schema - fixes molecula/molecula#136 - flag -n 0 does continuous queries until process is killed. --- cmd/random-query/main.go | 377 ++++++++++++++++++++++++++++++++++ cmd/random-query/main_test.go | 165 +++++++++++++++ cmd/random-query/vprint.go | 177 ++++++++++++++++ license.exceptions | 2 +- 4 files changed, 720 insertions(+), 1 deletion(-) create mode 100644 cmd/random-query/main.go create mode 100644 cmd/random-query/main_test.go create mode 100644 cmd/random-query/vprint.go diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go new file mode 100644 index 000000000..10fdfa6d5 --- /dev/null +++ b/cmd/random-query/main.go @@ -0,0 +1,377 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "flag" + "fmt" + "math/rand" + nethttp "net/http" + "os" + "strings" + "time" + + "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/http" +) + +// RandomQueryConfig +type RandomQueryConfig struct { + + // user facing flags + HostPort string // -hostport + TreeDepth int // -d + QueryCount int // -n + Verbose bool // -v + + IndexMap map[string]*Features + + API *pilosa.API + Info []*pilosa.IndexInfo + + BitmapFunc []string + + Rnd *rand.Rand +} + +type API interface { + + // InternalClient + Schema(ctx context.Context) ([]*pilosa.IndexInfo, error) + Query(ctx context.Context, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) + + // API for contrast; just a little different: + //Schema(ctx context.Context) []*IndexInfo + //Query(ctx context.Context, req *pilosa.QueryRequest) (pilosa.QueryResponse, error) +} + +// have to wrap because the ugly little differences between InternalClient and API +type wrapper struct { + api *pilosa.API +} + +func (w *wrapper) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error) { + return w.api.Schema(ctx), nil +} + +func (w *wrapper) Query(ctx context.Context, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) { + r, err := w.api.Query(ctx, queryRequest) + return &r, err +} + +func wrapApiToInternalClient(api *pilosa.API) *wrapper { + return &wrapper{api: api} +} + +// call DefineFlags before myflags.Parse() +func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) { + fs.StringVar(&cfg.HostPort, "hostport", "localhost:10101", "host:port of pilosa to run random queries on.") + fs.IntVar(&cfg.TreeDepth, "d", 4, "depth of random queries to generate.") + fs.IntVar(&cfg.QueryCount, "n", 100, "number of random queries to generate. Set to 0 for inifinite queries.") + fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated") +} + +// call c.ValidateConfig() after myflags.Parse() +func (c *RandomQueryConfig) ValidateConfig() error { + if c.TreeDepth < 1 { + return fmt.Errorf("-d depth must be 1 or greater; saw %v", c.TreeDepth) + } + if c.QueryCount < 0 { + return fmt.Errorf("-n count must be 0 or greater; saw %v", c.QueryCount) + } + return nil +} + +var ProgramName = "random-query" + +func main() { + + myflags := flag.NewFlagSet(ProgramName, flag.ExitOnError) + cfg := NewRandomQueryConfig() + cfg.DefineFlags(myflags) + + err := myflags.Parse(os.Args[1:]) + if err != nil { + fmt.Fprintf(os.Stderr, "\n%v\n", err.Error()) + os.Exit(1) + } + err = cfg.ValidateConfig() + if err != nil { + fmt.Fprintf(os.Stderr, "%s error: %s\n", ProgramName, err) + os.Exit(1) + } + + err = cfg.Run() + + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } +} + +func (cfg *RandomQueryConfig) Run() (err error) { + remoteClient := nethttp.DefaultClient + cli, err := http.NewInternalClient(cfg.HostPort, remoteClient) + if err != nil { + return err + } + ctx := context.Background() + totalQ := 0 + loops := 0 + t0 := time.Now() + +NewSetup: + err = cfg.Setup(cli) + if err != nil { + return err + } + + if len(cfg.IndexMap) == 0 { + return fmt.Errorf("no rows to query") + } + + var indexes []string + for index := range cfg.IndexMap { + indexes = append(indexes, index) + } + + for j := 0; ; j++ { + if cfg.QueryCount > 0 { + if j >= cfg.QueryCount { + break + } + } else { + // else keep doing queries forever... + if loops > 0 && loops%500 == 0 { + // ...but account for any new data arrived by getting + // the schema and rows again every so often. + loops++ + goto NewSetup + } + } + if totalQ%100 == 0 { + dur := time.Since(t0) + if dur > 0 { + qps := 1e9 * float64(totalQ) / float64(dur) + AlwaysPrintf("totalQueries run: %v elapsed: %v qps: %0.02f", totalQ, dur, qps) + } + } + + index := indexes[rand.Intn(len(indexes))] + + pql, err := cfg.GenQuery(index) + panicOn(err) + + if cfg.Verbose { + fmt.Printf("pql = '%v'\n", pql) + } + + // Query node0. + res, err := cli.Query(ctx, index, &pilosa.QueryRequest{Index: index, Query: pql}) + if err != nil { + AlwaysPrintf("QUERY FAILED! queries before this=%v; err = '%v', pql='%v'", loops, err, pql) + return err + } + if cfg.Verbose { + fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0]) + } + totalQ++ + loops++ + } + + return nil +} + +type Features struct { + Slc []IndexFieldRow +} + +func NewRandomQueryConfig() *RandomQueryConfig { + return &RandomQueryConfig{ + IndexMap: make(map[string]*Features), + } +} + +type IndexFieldRow struct { + Index string + Field string + RowID uint64 + RowKey string + IsRowKey bool +} + +// Run a RandomQuery takes a list of RowIDFeatures and ColumnKeyObjects +// and spits back a PQL query +// +func (cfg *RandomQueryConfig) Setup(api API) (err error) { + + ctx := context.Background() + cfg.Info, err = api.Schema(ctx) + if err != nil { + return err + } + for i, ii := range cfg.Info { + _ = i + for k, fld := range ii.Fields { + _ = k + if fld.Options.Type == "set" { + pql := fmt.Sprintf("Rows(%v)", fld.Name) + + res, err := api.Query(ctx, ii.Name, &pilosa.QueryRequest{Index: ii.Name, Query: pql}) + panicOn(err) + if cfg.Verbose { + fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0]) + } + // if the option is set to use RowKeys, then must get the Keys instead of the Rows from the RowIdentifiers. + // e.g. + // success on pql = 'Rows(aba)'; res='&pilosa.RowIdentifiers{Rows:[]uint64(nil), Keys:[]string{"aba1", "aba2"} + // success on pql = 'Rows(f)'; res='pilosa.RowIdentifiers{Rows:[]uint64{0x1}, Keys:[]string(nil), field:"f"}' + + switch x := res.Results[0].(type) { + case *pilosa.RowIdentifiers: + // internalClient gets this + cfg.AddResponse(ii.Name, fld.Name, x) + case pilosa.RowIdentifiers: + // test gets this + cfg.AddResponse(ii.Name, fld.Name, &x) + } + } + } + } + + cfg.BitmapFunc = []string{"Union", "Intersect", "Xor", "Not", "Difference"} + seed := int64(42) + cfg.Rnd = rand.New(rand.NewSource(seed)) + + return nil +} + +func (cfg *RandomQueryConfig) AddResponse(index, field string, x *pilosa.RowIdentifiers) { + for _, rowID := range x.Rows { + cfg.AddFeature(index, field, rowID, "", false) + } + for _, rowKey := range x.Keys { + cfg.AddFeature(index, field, 0, rowKey, true) + } +} + +func (cfg *RandomQueryConfig) GenQuery(index string) (pql string, err error) { + + tree := cfg.GenTree(index, cfg.TreeDepth) + + if cfg.Verbose { + fmt.Printf("%v\n", tree.StringIndent(0)) + } + pql = tree.ToPQL() + + // avoid using too much bandwidth, just count the final bitmap. + pql = fmt.Sprintf("Count(%v)", pql) + return +} + +type Tree struct { + Chd []*Tree + + S string +} + +func (tr *Tree) StringIndent(ind int) (s string) { + spc := strings.Repeat(" ", ind) + spc1 := strings.Repeat(" ", ind+1) + var chds []string + leaf := true + if len(tr.Chd) == 0 { + // leaf + } else { + leaf = false + for _, chd := range tr.Chd { + chds = append(chds, chd.StringIndent(ind+1)) + } + } + if leaf { + s += fmt.Sprintf("%v %v\n", spc1, tr.S) + } else { + for i, c := range chds { + if i == 0 { + s += fmt.Sprintf("%v %v\n%v", spc, tr.S, c) + } else { + s += fmt.Sprintf("%v", c) + } + } + } + return +} + +func (cfg *RandomQueryConfig) GenTree(index string, depth int) (tr *Tree) { + if depth == 0 { + slc := cfg.IndexMap[index].Slc + //vv("depth is 0, slc = '%#v'", slc) + r := cfg.Rnd.Intn(len(slc)) + fea := slc[r] + if fea.IsRowKey { + return &Tree{S: fmt.Sprintf("Row(%v='%v')", fea.Field, fea.RowKey)} + } + return &Tree{S: fmt.Sprintf("Row(%v=%v)", fea.Field, fea.RowID)} + } + + r := cfg.Rnd.Intn(len(cfg.BitmapFunc)) + f := cfg.BitmapFunc[r] + tr = &Tree{S: f} + numChild := 2 + switch f { + case "Union", "Intersect", "Xor": + numChild = cfg.Rnd.Intn(8) + 2 + case "Not": + numChild = 1 + case "Difference": + numChild = 2 + } + for i := 0; i < numChild; i++ { + tr.Chd = append(tr.Chd, cfg.GenTree(index, depth-1)) + } + return +} + +func (tr *Tree) ToPQL() (s string) { + + if len(tr.Chd) == 0 { + // leaf + return tr.S + } + + var chds []string + for _, c := range tr.Chd { + chds = append(chds, c.ToPQL()) + } + all := strings.Join(chds, ", ") + return fmt.Sprintf("%v(%v)", tr.S, all) +} + +func (cfg *RandomQueryConfig) AddFeature(index, field string, rowID uint64, rowKey string, isRowKey bool) { + + f, ok := cfg.IndexMap[index] + if !ok { + f = &Features{} + cfg.IndexMap[index] = f + } + f.Slc = append(f.Slc, IndexFieldRow{ + Index: index, + Field: field, + RowID: rowID, + RowKey: rowKey, + IsRowKey: isRowKey, + }) +} diff --git a/cmd/random-query/main_test.go b/cmd/random-query/main_test.go new file mode 100644 index 000000000..646846899 --- /dev/null +++ b/cmd/random-query/main_test.go @@ -0,0 +1,165 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "math/rand" + "strconv" + "testing" + + "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/boltdb" + "github.com/pilosa/pilosa/v2/http" + "github.com/pilosa/pilosa/v2/server" + "github.com/pilosa/pilosa/v2/test" +) + +func Test_RandomQuery(t *testing.T) { + + cfg := NewRandomQueryConfig() + + nNodes := 1 + nReplicas := 1 + + name := t.Name() + var nodeid []string + for i := 0; i < nNodes; i++ { + // work around a bug in the test.MustRunCluster that corrupts + // the .topology file if we only join name with one "_" underscore. + nodeid = append(nodeid, name+"__"+strconv.Itoa(i)) + } + + c := test.MustRunCluster(t, nNodes, + []server.CommandOption{ + server.OptCommandServerOptions( + pilosa.OptServerNodeID(nodeid[0]), + pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), + pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerReplicaN(nReplicas), + )}, + ) + defer c.Close() + + var nodes []*test.Command + var dirs []string + for i := 0; i < nNodes; i++ { + nd := c.GetNode(i) + nodes = append(nodes, nd) + dirs = append(dirs, nd.Server.Holder().Path()) + } + _ = dirs + + ctx := context.Background() + + indexes := []string{"rick", "morty"} + fieldName := []string{"f", "flying_car"} + idx := make([]*pilosa.Index, len(indexes)) + field := make([]*pilosa.Field, len(indexes)) + + var err error + + for i := range indexes { + + idx[i], err = nodes[0].API.CreateIndex(ctx, indexes[i], pilosa.IndexOptions{Keys: true, TrackExistence: true}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + if idx[i].CreatedAt() == 0 { + t.Fatal("index createdAt is empty") + } + + field[i], err = nodes[0].API.CreateField(ctx, indexes[i], fieldName[i], pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100)) + if err != nil { + t.Fatalf("creating field: %v", err) + } + if field[i].CreatedAt() == 0 { + t.Fatal("field createdAt is empty") + } + } + + timestamp := int64(0) + + for i := range indexes { + + // Generate some keyed records. + rowIDs := []uint64{} + timestamps := []int64{} + N := 10 + for j := 1; j <= N; j++ { + rowIDs = append(rowIDs, uint64(j)) + timestamps = append(timestamps, timestamp) + } + + var colKeys []string + switch i { + case 0: + // Keys are sharded so ordering is not guaranteed. + colKeys = []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"} + colKeys = colKeys[:N] + case 1: + colKeys = []string{"col11", "col12"} + N = len(colKeys) + rowIDs = rowIDs[:N] + timestamps = timestamps[:N] + } + + // Import data with keys to the coordinator (node0) and verify that it gets + // translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher) + req := &pilosa.ImportRequest{ + Index: indexes[i], + IndexCreatedAt: idx[i].CreatedAt(), + Field: fieldName[i], + FieldCreatedAt: field[i].CreatedAt(), + + // even though this says Shard: 0, that won't matter. The column keys + // get hashed and that decides the actual shard. + Shard: 0, + RowIDs: rowIDs, + ColumnKeys: colKeys, + Timestamps: timestamps, + } + //vv("rowIDs = '%#v'", rowIDs) + //vv("colKeys = '%#v'", colKeys) + + qcx := nodes[0].API.Txf().NewQcx() + + if err := nodes[0].API.Import(ctx, qcx, req); err != nil { + t.Fatal(err) + } + panicOn(qcx.Finish()) + //qcx.Reset() + } + // end of setup. + + panicOn(cfg.Setup(wrapApiToInternalClient(nodes[0].API))) + + for j := 0; j < 4; j++ { + index := indexes[rand.Intn(len(indexes))] + + pql, err := cfg.GenQuery(index) + panicOn(err) + + //vv("pql = '%v'", pql) + + // Query node0. + res, err := nodes[0].API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}) + if err != nil { + t.Fatal(err) + } + _ = res + //vv("success on pql = '%v'; res='%v'", pql, res.Results[0]) + } +} diff --git a/cmd/random-query/vprint.go b/cmd/random-query/vprint.go new file mode 100644 index 000000000..83b1681f7 --- /dev/null +++ b/cmd/random-query/vprint.go @@ -0,0 +1,177 @@ +// home: https://github.com/glycerine/vprint +// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved. +// License: MIT +// +// MIT License +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package main + +import ( + "fmt" + "io" + "os" + "path" + "runtime" + "runtime/debug" + "sync" + "time" +) + +const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00" +const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00" + +// for tons of debug output +var VerboseVerbose bool = false + +// convience functions for . import +var pp = PP +var vv = VV + +var panicOn = PanicOn + +func init() { + // keeper linter happy + _ = pp + _ = vv +} + +func PanicOn(err error) { + if err != nil { + panic(err) + } +} + +func PP(format string, a ...interface{}) { + if VerboseVerbose { + TSPrintf(format, a...) + } +} + +func VV(format string, a ...interface{}) { + TSPrintf(format, a...) +} + +func AlwaysPrintf(format string, a ...interface{}) { + TSPrintf(format, a...) +} + +var tsPrintfMut sync.Mutex + +// time-stamped printf +func TSPrintf(format string, a ...interface{}) { + tsPrintfMut.Lock() + Printf("# %s %s ", FileLine(3), ts()) + Printf(format+"\n", a...) + tsPrintfMut.Unlock() +} + +// get timestamp for logging purposes +func ts() string { + return time.Now().Format(RFC3339UsecTz0) +} + +// so we can multi write easily, use our own printf +var OurStdout io.Writer = os.Stdout + +// Printf formats according to a format specifier and writes to standard output. +// It returns the number of bytes written and any write error encountered. +func Printf(format string, a ...interface{}) (n int, err error) { + return fmt.Fprintf(OurStdout, format, a...) +} + +func FileLine(depth int) string { + _, fileName, fileLine, ok := runtime.Caller(depth) + var s string + if ok { + s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine) + } else { + s = "" + } + return s +} + +func stack() string { + return string(debug.Stack()) +} + +func FileExists(name string) bool { + fi, err := os.Stat(name) + if err != nil { + return false + } + if fi.IsDir() { + return false + } + return true +} + +func DirExists(name string) bool { + fi, err := os.Stat(name) + if err != nil { + return false + } + if fi.IsDir() { + return true + } + return false +} + +func FileSize(name string) int64 { + fi, err := os.Stat(name) + if err != nil { + return 0 + } + return fi.Size() +} + +// Caller returns the name of the calling function. +func Caller(upStack int) string { + // elide ourself and runtime.Callers + target := upStack + 2 + + pc := make([]uintptr, target+2) + n := runtime.Callers(0, pc) + + f := runtime.Frame{Function: "unknown"} + if n > 0 { + frames := runtime.CallersFrames(pc[:n]) + for i := 0; i <= target; i++ { + contender, more := frames.Next() + if i == target { + f = contender + } + if !more { + break + } + } + } + return f.Function +} + +// happy linter: +var _ = DirExists +var _ = FileExists +var _ = Caller +var _ = stack +var _ = RFC3339MsecTz0 +var _ = RFC3339UsecTz0 +var _ = AlwaysPrintf +var _ = FileSize diff --git a/license.exceptions b/license.exceptions index 2c8c4162f..46230d311 100644 --- a/license.exceptions +++ b/license.exceptions @@ -22,4 +22,4 @@ ./proto/vdsm/vdsm.proto ./proto/vdsm/vdsm.pb.go ./cmd/pilosa-fsck/vprint.go - +./cmd/random-query/vprint.go From a58c345232bbe5ad7c80e34c03a8e4f0ed940b9f Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 9 Oct 2020 12:05:03 -0500 Subject: [PATCH 45/53] Use copy of vdsm QueryPQLRequest for better compatibility --- proto/vdsm/vdsm.pb.go | 139 +++++++++++++++++++++++++++++------------- proto/vdsm/vdsm.proto | 8 ++- server/grpc.go | 5 +- 3 files changed, 105 insertions(+), 47 deletions(-) diff --git a/proto/vdsm/vdsm.pb.go b/proto/vdsm/vdsm.pb.go index 99f04cda2..8ed0afcd8 100644 --- a/proto/vdsm/vdsm.pb.go +++ b/proto/vdsm/vdsm.pb.go @@ -562,6 +562,54 @@ func (m *DeleteVDSResponse) XXX_DiscardUnknown() { var xxx_messageInfo_DeleteVDSResponse proto.InternalMessageInfo +// copied from pilosa.proto to use "vds" instead of "index" +type QueryPQLRequest struct { + Vds string `protobuf:"bytes,1,opt,name=vds,proto3" json:"vds,omitempty"` + Pql string `protobuf:"bytes,2,opt,name=pql,proto3" json:"pql,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *QueryPQLRequest) Reset() { *m = QueryPQLRequest{} } +func (m *QueryPQLRequest) String() string { return proto.CompactTextString(m) } +func (*QueryPQLRequest) ProtoMessage() {} +func (*QueryPQLRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{10} +} + +func (m *QueryPQLRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_QueryPQLRequest.Unmarshal(m, b) +} +func (m *QueryPQLRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_QueryPQLRequest.Marshal(b, m, deterministic) +} +func (m *QueryPQLRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_QueryPQLRequest.Merge(m, src) +} +func (m *QueryPQLRequest) XXX_Size() int { + return xxx_messageInfo_QueryPQLRequest.Size(m) +} +func (m *QueryPQLRequest) XXX_DiscardUnknown() { + xxx_messageInfo_QueryPQLRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_QueryPQLRequest proto.InternalMessageInfo + +func (m *QueryPQLRequest) GetVds() string { + if m != nil { + return m.Vds + } + return "" +} + +func (m *QueryPQLRequest) GetPql() string { + if m != nil { + return m.Pql + } + return "" +} + func init() { proto.RegisterType((*DataSource)(nil), "vdsm.DataSource") proto.RegisterMapType((map[string]string)(nil), "vdsm.DataSource.ConfigEntry") @@ -574,50 +622,53 @@ func init() { proto.RegisterType((*PostVDSResponse)(nil), "vdsm.PostVDSResponse") proto.RegisterType((*DeleteVDSRequest)(nil), "vdsm.DeleteVDSRequest") proto.RegisterType((*DeleteVDSResponse)(nil), "vdsm.DeleteVDSResponse") + proto.RegisterType((*QueryPQLRequest)(nil), "vdsm.QueryPQLRequest") } func init() { proto.RegisterFile("vdsm/vdsm.proto", fileDescriptor_3849c3e549495f50) } var fileDescriptor_3849c3e549495f50 = []byte{ - // 604 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x54, 0xcd, 0x6e, 0xd3, 0x4c, - 0x14, 0x8d, 0xe3, 0x34, 0x6d, 0x6e, 0xbe, 0xb6, 0xfe, 0xa6, 0x69, 0x6b, 0x85, 0x1f, 0x05, 0x2f, - 0x50, 0x36, 0xa4, 0x51, 0x0a, 0x52, 0xa8, 0x10, 0x12, 0x6d, 0x80, 0x56, 0xe2, 0x27, 0x75, 0x20, - 0x0b, 0x36, 0x68, 0x62, 0x4f, 0xcb, 0x08, 0xc7, 0x36, 0x9e, 0x71, 0x20, 0xaf, 0xc3, 0x2b, 0xf0, - 0x60, 0xbc, 0x02, 0xf2, 0xfc, 0x38, 0x4e, 0x5a, 0x04, 0xa8, 0x9b, 0xf6, 0xfa, 0xcc, 0x3d, 0xf7, - 0x9c, 0x73, 0x27, 0x1a, 0xd8, 0x9e, 0xf9, 0x6c, 0x7a, 0x90, 0xfd, 0xe9, 0xc4, 0x49, 0xc4, 0x23, - 0x54, 0xc9, 0xea, 0xe6, 0x7f, 0x31, 0x0d, 0x22, 0x86, 0x25, 0xe6, 0xfc, 0x34, 0x00, 0x06, 0x98, - 0xe3, 0x51, 0x94, 0x26, 0x1e, 0x41, 0x5b, 0x50, 0xa6, 0xbe, 0x6d, 0xb4, 0x8c, 0xb6, 0xe9, 0x96, - 0xa9, 0x8f, 0x10, 0x54, 0x42, 0x3c, 0x25, 0x76, 0xb9, 0x65, 0xb4, 0x6b, 0xae, 0xa8, 0x51, 0x0b, - 0xea, 0x3e, 0x61, 0x5e, 0x42, 0x63, 0x4e, 0xa3, 0xd0, 0x36, 0xc5, 0x51, 0x11, 0xca, 0x58, 0x7c, - 0x1e, 0x13, 0xbb, 0x22, 0x59, 0x59, 0x8d, 0x1e, 0x42, 0xd5, 0x8b, 0xc2, 0x0b, 0x7a, 0x69, 0xaf, - 0xb5, 0xcc, 0x76, 0xbd, 0x77, 0xbb, 0x23, 0x9c, 0x2d, 0xb4, 0x3b, 0x27, 0xe2, 0xf8, 0x79, 0xc8, - 0x93, 0xb9, 0xab, 0x7a, 0xd1, 0x1e, 0x54, 0x19, 0xc7, 0x3c, 0x65, 0x76, 0x55, 0xcc, 0x52, 0x5f, - 0xcd, 0xc7, 0x50, 0x2f, 0xb4, 0x23, 0x0b, 0xcc, 0xcf, 0x64, 0x2e, 0x7c, 0xd7, 0xdc, 0xac, 0x44, - 0x0d, 0x58, 0x9b, 0xe1, 0x20, 0xd5, 0xce, 0xe5, 0xc7, 0x51, 0xb9, 0x6f, 0x38, 0xdf, 0x0d, 0x30, - 0xc7, 0x83, 0x51, 0x21, 0x6a, 0xed, 0x06, 0x51, 0xef, 0x81, 0xda, 0xe7, 0x47, 0x1a, 0xfa, 0xe4, - 0x9b, 0x8a, 0x5c, 0x97, 0xd8, 0x59, 0x06, 0xa1, 0x1e, 0xd4, 0x7d, 0xcc, 0x31, 0x13, 0x29, 0x99, - 0x8a, 0x6f, 0xad, 0xc6, 0x77, 0x8b, 0x4d, 0x8e, 0x05, 0x5b, 0x2f, 0x09, 0x1f, 0x0f, 0x46, 0xcc, - 0x25, 0x5f, 0x52, 0xc2, 0xb8, 0xd3, 0x85, 0xed, 0x1c, 0x61, 0x71, 0x14, 0x32, 0x82, 0xee, 0x40, - 0x76, 0xa3, 0xcc, 0x36, 0xc4, 0xc4, 0x9a, 0x9c, 0x38, 0x1e, 0x8c, 0x5c, 0x01, 0x3b, 0x27, 0xb0, - 0x29, 0x19, 0x6a, 0x04, 0xb2, 0x16, 0x89, 0x4f, 0x4b, 0x22, 0x73, 0xa3, 0x98, 0xf9, 0xb4, 0x24, - 0x53, 0x1f, 0x03, 0x6c, 0x50, 0xff, 0x6d, 0xf2, 0x06, 0x4f, 0x89, 0xf3, 0x40, 0x1b, 0xc9, 0x55, - 0x6f, 0x81, 0x39, 0xf3, 0x99, 0x18, 0xb3, 0x24, 0x9a, 0xa1, 0x4e, 0x17, 0xb6, 0x86, 0x11, 0x2b, - 0x8a, 0xde, 0x05, 0xf0, 0xc9, 0x05, 0x0d, 0xa9, 0xd8, 0xa0, 0x5c, 0x77, 0x01, 0x71, 0x0e, 0x61, - 0x3b, 0x67, 0x28, 0x85, 0xd5, 0x9b, 0xb1, 0xc0, 0x4c, 0x13, 0xaa, 0x2e, 0x26, 0x2b, 0x9d, 0x17, - 0x60, 0x0d, 0x48, 0x40, 0x38, 0xb9, 0x61, 0xba, 0x1d, 0xf8, 0xbf, 0x30, 0x47, 0xca, 0xf7, 0x7e, - 0x54, 0x60, 0xe3, 0x75, 0x14, 0x10, 0x2f, 0x0d, 0x30, 0xea, 0xc3, 0xba, 0x5a, 0x3b, 0x6a, 0xc8, - 0xac, 0xcb, 0xf7, 0xd2, 0xdc, 0x5d, 0x41, 0xe5, 0x10, 0xa7, 0x84, 0x1e, 0x41, 0x55, 0x82, 0x68, - 0xa7, 0xd8, 0xa2, 0x79, 0x8d, 0x65, 0x30, 0xa7, 0xf5, 0x61, 0x5d, 0xed, 0x43, 0x0b, 0x2e, 0x2f, - 0x54, 0x0b, 0xae, 0x2c, 0xcd, 0x29, 0xa1, 0xa7, 0x50, 0xcb, 0xc3, 0xa0, 0x3d, 0xf5, 0xfb, 0x5a, - 0xd9, 0x52, 0x73, 0xff, 0x0a, 0x9e, 0xf3, 0x9f, 0xc0, 0xc6, 0x79, 0x4a, 0x92, 0xf9, 0xe8, 0xfc, - 0x15, 0xda, 0xef, 0xa8, 0x57, 0x42, 0x23, 0x9a, 0xbf, 0xa3, 0x0f, 0xdc, 0xe8, 0xeb, 0x82, 0xdb, - 0x35, 0xd0, 0x33, 0xd8, 0xd4, 0xbd, 0xef, 0x43, 0x9c, 0xcc, 0x7f, 0x3f, 0x62, 0x57, 0x1f, 0xbc, - 0xc3, 0x93, 0x80, 0x5c, 0x63, 0x60, 0x78, 0xc5, 0xc0, 0xf0, 0x1f, 0x0c, 0x0c, 0xaf, 0x37, 0x30, - 0xfc, 0x0b, 0x03, 0x47, 0xb0, 0x7e, 0x16, 0xb2, 0x98, 0x78, 0x1c, 0xed, 0xe9, 0x1e, 0x05, 0xfc, - 0x49, 0xfe, 0xb8, 0xfd, 0xe1, 0xfe, 0x25, 0xe5, 0x9f, 0xd2, 0x49, 0xc7, 0x8b, 0xa6, 0x07, 0xb2, - 0x49, 0xff, 0x9b, 0xf5, 0x0e, 0xc4, 0x6b, 0x2b, 0x1e, 0xe3, 0x49, 0x55, 0xd4, 0x87, 0xbf, 0x02, - 0x00, 0x00, 0xff, 0xff, 0x39, 0x92, 0xf5, 0x8e, 0xa0, 0x05, 0x00, 0x00, + // 629 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x54, 0xdb, 0x6e, 0xd3, 0x40, + 0x10, 0x8d, 0xe3, 0x34, 0x6d, 0x26, 0xb4, 0x09, 0xdb, 0xb4, 0xb5, 0xc2, 0x45, 0xc1, 0x0f, 0x28, + 0x2f, 0xa4, 0x55, 0x4a, 0xa5, 0x52, 0x21, 0x24, 0xda, 0x00, 0xad, 0x54, 0x20, 0x71, 0xa0, 0x0f, + 0xbc, 0xa0, 0xad, 0xbd, 0x2d, 0x2b, 0x1c, 0xdb, 0xf1, 0xae, 0x03, 0xf9, 0x1d, 0xfe, 0x80, 0x2f, + 0xe3, 0x17, 0x90, 0xf7, 0xe2, 0x38, 0x06, 0x54, 0xa4, 0xbe, 0x24, 0xb3, 0x67, 0x67, 0xe6, 0xcc, + 0x39, 0x63, 0x2d, 0x34, 0x66, 0x1e, 0x9b, 0xec, 0xa6, 0x3f, 0xbd, 0x28, 0x0e, 0x79, 0x88, 0x2a, + 0x69, 0xdc, 0xbe, 0x13, 0x51, 0x3f, 0x64, 0x58, 0x62, 0xf6, 0x2f, 0x03, 0x60, 0x80, 0x39, 0x1e, + 0x87, 0x49, 0xec, 0x12, 0xb4, 0x01, 0x65, 0xea, 0x59, 0x46, 0xc7, 0xe8, 0x9a, 0x4e, 0x99, 0x7a, + 0x08, 0x41, 0x25, 0xc0, 0x13, 0x62, 0x95, 0x3b, 0x46, 0xb7, 0xe6, 0x88, 0x18, 0x75, 0xa0, 0xee, + 0x11, 0xe6, 0xc6, 0x34, 0xe2, 0x34, 0x0c, 0x2c, 0x53, 0x5c, 0xe5, 0xa1, 0xb4, 0x8a, 0xcf, 0x23, + 0x62, 0x55, 0x64, 0x55, 0x1a, 0xa3, 0xa7, 0x50, 0x75, 0xc3, 0xe0, 0x8a, 0x5e, 0x5b, 0x2b, 0x1d, + 0xb3, 0x5b, 0xef, 0xdf, 0xef, 0x89, 0xc9, 0x16, 0xdc, 0xbd, 0x13, 0x71, 0xfd, 0x2a, 0xe0, 0xf1, + 0xdc, 0x51, 0xb9, 0x68, 0x1b, 0xaa, 0x8c, 0x63, 0x9e, 0x30, 0xab, 0x2a, 0x7a, 0xa9, 0x53, 0xfb, + 0x19, 0xd4, 0x73, 0xe9, 0xa8, 0x09, 0xe6, 0x57, 0x32, 0x17, 0x73, 0xd7, 0x9c, 0x34, 0x44, 0x2d, + 0x58, 0x99, 0x61, 0x3f, 0xd1, 0x93, 0xcb, 0xc3, 0x51, 0xf9, 0xd0, 0xb0, 0x7f, 0x18, 0x60, 0x5e, + 0x0c, 0xc6, 0x39, 0xa9, 0xb5, 0x5b, 0x48, 0x7d, 0x04, 0xca, 0xcf, 0xcf, 0x34, 0xf0, 0xc8, 0x77, + 0x25, 0xb9, 0x2e, 0xb1, 0xb3, 0x14, 0x42, 0x7d, 0xa8, 0x7b, 0x98, 0x63, 0x26, 0x54, 0x32, 0x25, + 0xbf, 0x59, 0x94, 0xef, 0xe4, 0x93, 0xec, 0x26, 0x6c, 0xbc, 0x21, 0xfc, 0x62, 0x30, 0x66, 0x0e, + 0x99, 0x26, 0x84, 0x71, 0x7b, 0x0f, 0x1a, 0x19, 0xc2, 0xa2, 0x30, 0x60, 0x04, 0x3d, 0x80, 0x74, + 0xa3, 0xcc, 0x32, 0x44, 0xc7, 0x9a, 0xec, 0x78, 0x31, 0x18, 0x3b, 0x02, 0xb6, 0x4f, 0x60, 0x5d, + 0x56, 0xa8, 0x16, 0xa8, 0xb9, 0x50, 0x7c, 0x5a, 0x12, 0x9a, 0x5b, 0x79, 0xcd, 0xa7, 0x25, 0xa9, + 0xfa, 0x18, 0x60, 0x8d, 0x7a, 0xef, 0xe3, 0x77, 0x78, 0x42, 0xec, 0x27, 0x7a, 0x90, 0x8c, 0xf5, + 0x1e, 0x98, 0x33, 0x8f, 0x89, 0x36, 0x4b, 0xa4, 0x29, 0x6a, 0xef, 0xc1, 0xc6, 0x30, 0x64, 0x79, + 0xd2, 0x87, 0x00, 0x1e, 0xb9, 0xa2, 0x01, 0x15, 0x0e, 0x4a, 0xbb, 0x73, 0x88, 0xbd, 0x0f, 0x8d, + 0xac, 0x42, 0x31, 0x14, 0x37, 0xd3, 0x04, 0x33, 0x89, 0xa9, 0x5a, 0x4c, 0x1a, 0xda, 0xaf, 0xa1, + 0x39, 0x20, 0x3e, 0xe1, 0xe4, 0x96, 0xea, 0x36, 0xe1, 0x6e, 0xae, 0x8f, 0xa4, 0xb7, 0x0f, 0xa0, + 0x31, 0x4a, 0x48, 0x3c, 0x1f, 0x8e, 0xce, 0x17, 0xbd, 0x33, 0xcd, 0x35, 0x21, 0x34, 0x45, 0xa2, + 0xa9, 0xaf, 0x67, 0x8a, 0xa6, 0x7e, 0xff, 0x67, 0x05, 0xd6, 0xde, 0x86, 0x3e, 0x71, 0x13, 0x1f, + 0xa3, 0x43, 0x58, 0x55, 0xdb, 0x42, 0x2d, 0x69, 0xd1, 0xf2, 0x3a, 0xdb, 0x5b, 0x05, 0x54, 0x71, + 0x97, 0xd0, 0x01, 0x54, 0x25, 0x88, 0x36, 0xf3, 0x29, 0xba, 0xae, 0xb5, 0x0c, 0x66, 0x65, 0x87, + 0xb0, 0xaa, 0x6c, 0xd4, 0x84, 0xcb, 0x7b, 0xd0, 0x84, 0x05, 0xaf, 0xed, 0x12, 0x7a, 0x01, 0xb5, + 0xcc, 0x03, 0xb4, 0xad, 0x3e, 0xcb, 0x82, 0xb9, 0xed, 0x9d, 0x3f, 0xf0, 0xac, 0xfe, 0x39, 0xac, + 0x09, 0xbb, 0xc6, 0xa3, 0x73, 0xb4, 0xd3, 0x53, 0x8f, 0x8b, 0x46, 0x74, 0xfd, 0xa6, 0xbe, 0x70, + 0xc2, 0x6f, 0x8b, 0xda, 0x3d, 0x03, 0xbd, 0x84, 0x75, 0x9d, 0xfb, 0x31, 0xc0, 0xf1, 0xfc, 0xdf, + 0x2d, 0xb6, 0xf4, 0xc5, 0x07, 0x7c, 0xe9, 0x93, 0xdc, 0x00, 0x47, 0x6a, 0x80, 0xe1, 0xe8, 0x1c, + 0x29, 0x95, 0x85, 0xfd, 0xdd, 0x4c, 0x3f, 0xfc, 0x3b, 0xfd, 0xf0, 0xbf, 0xe8, 0x57, 0xcf, 0x02, + 0x16, 0x11, 0x97, 0xa3, 0x6d, 0x9d, 0xa3, 0x80, 0x9b, 0xe8, 0x8f, 0xbb, 0x9f, 0x1e, 0x5f, 0x53, + 0xfe, 0x25, 0xb9, 0xec, 0xb9, 0xe1, 0x64, 0x57, 0x26, 0xe9, 0xbf, 0x59, 0x7f, 0x57, 0x3c, 0xd1, + 0xe2, 0x05, 0xbf, 0xac, 0x8a, 0x78, 0xff, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0xb0, 0x14, 0xc9, + 0x0e, 0xd5, 0x05, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -638,7 +689,7 @@ type MoleculaClient interface { DeleteVDS(ctx context.Context, in *DeleteVDSRequest, opts ...grpc.CallOption) (*DeleteVDSResponse, error) QuerySQL(ctx context.Context, in *proto1.QuerySQLRequest, opts ...grpc.CallOption) (Molecula_QuerySQLClient, error) QuerySQLUnary(ctx context.Context, in *proto1.QuerySQLRequest, opts ...grpc.CallOption) (*proto1.TableResponse, error) - QueryPQL(ctx context.Context, in *proto1.QueryPQLRequest, opts ...grpc.CallOption) (Molecula_QueryPQLClient, error) + QueryPQL(ctx context.Context, in *QueryPQLRequest, opts ...grpc.CallOption) (Molecula_QueryPQLClient, error) QueryPQLUnary(ctx context.Context, in *proto1.QueryPQLRequest, opts ...grpc.CallOption) (*proto1.TableResponse, error) Inspect(ctx context.Context, in *proto1.InspectRequest, opts ...grpc.CallOption) (Molecula_InspectClient, error) } @@ -728,7 +779,7 @@ func (c *moleculaClient) QuerySQLUnary(ctx context.Context, in *proto1.QuerySQLR return out, nil } -func (c *moleculaClient) QueryPQL(ctx context.Context, in *proto1.QueryPQLRequest, opts ...grpc.CallOption) (Molecula_QueryPQLClient, error) { +func (c *moleculaClient) QueryPQL(ctx context.Context, in *QueryPQLRequest, opts ...grpc.CallOption) (Molecula_QueryPQLClient, error) { stream, err := c.cc.NewStream(ctx, &_Molecula_serviceDesc.Streams[1], "/vdsm.Molecula/QueryPQL", opts...) if err != nil { return nil, err @@ -809,7 +860,7 @@ type MoleculaServer interface { DeleteVDS(context.Context, *DeleteVDSRequest) (*DeleteVDSResponse, error) QuerySQL(*proto1.QuerySQLRequest, Molecula_QuerySQLServer) error QuerySQLUnary(context.Context, *proto1.QuerySQLRequest) (*proto1.TableResponse, error) - QueryPQL(*proto1.QueryPQLRequest, Molecula_QueryPQLServer) error + QueryPQL(*QueryPQLRequest, Molecula_QueryPQLServer) error QueryPQLUnary(context.Context, *proto1.QueryPQLRequest) (*proto1.TableResponse, error) Inspect(*proto1.InspectRequest, Molecula_InspectServer) error } @@ -836,7 +887,7 @@ func (*UnimplementedMoleculaServer) QuerySQL(req *proto1.QuerySQLRequest, srv Mo func (*UnimplementedMoleculaServer) QuerySQLUnary(ctx context.Context, req *proto1.QuerySQLRequest) (*proto1.TableResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method QuerySQLUnary not implemented") } -func (*UnimplementedMoleculaServer) QueryPQL(req *proto1.QueryPQLRequest, srv Molecula_QueryPQLServer) error { +func (*UnimplementedMoleculaServer) QueryPQL(req *QueryPQLRequest, srv Molecula_QueryPQLServer) error { return status.Errorf(codes.Unimplemented, "method QueryPQL not implemented") } func (*UnimplementedMoleculaServer) QueryPQLUnary(ctx context.Context, req *proto1.QueryPQLRequest) (*proto1.TableResponse, error) { @@ -962,7 +1013,7 @@ func _Molecula_QuerySQLUnary_Handler(srv interface{}, ctx context.Context, dec f } func _Molecula_QueryPQL_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(proto1.QueryPQLRequest) + m := new(QueryPQLRequest) if err := stream.RecvMsg(m); err != nil { return err } diff --git a/proto/vdsm/vdsm.proto b/proto/vdsm/vdsm.proto index b72622ac9..645cf8e21 100644 --- a/proto/vdsm/vdsm.proto +++ b/proto/vdsm/vdsm.proto @@ -60,6 +60,12 @@ message DeleteVDSRequest { message DeleteVDSResponse { } +// copied from pilosa.proto to use "vds" instead of "index" +message QueryPQLRequest { + string vds = 1; + string pql = 2; +} + service Molecula { rpc GetVDSs(GetVDSsRequest) returns (GetVDSsResponse) {}; rpc GetVDS(GetVDSRequest) returns (GetVDSResponse) {}; @@ -67,7 +73,7 @@ service Molecula { rpc DeleteVDS(DeleteVDSRequest) returns (DeleteVDSResponse) {}; rpc QuerySQL(pilosa.QuerySQLRequest) returns (stream pilosa.RowResponse) {}; rpc QuerySQLUnary(pilosa.QuerySQLRequest) returns (pilosa.TableResponse) {}; - rpc QueryPQL(pilosa.QueryPQLRequest) returns (stream pilosa.RowResponse) {}; + rpc QueryPQL(QueryPQLRequest) returns (stream pilosa.RowResponse) {}; rpc QueryPQLUnary(pilosa.QueryPQLRequest) returns (pilosa.TableResponse) {}; rpc Inspect(pilosa.InspectRequest) returns (stream pilosa.RowResponse) {}; } diff --git a/server/grpc.go b/server/grpc.go index 47dcf6a07..85d8afc84 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -333,8 +333,9 @@ func (h *VDSMGRPCHandler) QuerySQLUnary(ctx context.Context, req *pb.QuerySQLReq return h.grpcHandler.QuerySQLUnary(ctx, req) } -func (h *VDSMGRPCHandler) QueryPQL(req *pb.QueryPQLRequest, srv vdsm_pb.Molecula_QueryPQLServer) error { - return h.grpcHandler.QueryPQL(req, srv) +func (h *VDSMGRPCHandler) QueryPQL(req *vdsm_pb.QueryPQLRequest, srv vdsm_pb.Molecula_QueryPQLServer) error { + preq := &pb.QueryPQLRequest{Index: req.Vds, Pql: req.Pql} + return h.grpcHandler.QueryPQL(preq, srv) } func (h *VDSMGRPCHandler) QueryPQLUnary(ctx context.Context, req *pb.QueryPQLRequest) (*pb.TableResponse, error) { From 3039aa44af88973c67e6f4b3c2e7775dab0dd328 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 9 Oct 2020 13:21:46 -0500 Subject: [PATCH 46/53] Use copy of vdsm InspectRequest for better compatibility --- proto/vdsm/vdsm.pb.go | 178 +++++++++++++++++++++++++++++++----------- proto/vdsm/vdsm.proto | 12 ++- server/grpc.go | 5 +- 3 files changed, 146 insertions(+), 49 deletions(-) diff --git a/proto/vdsm/vdsm.pb.go b/proto/vdsm/vdsm.pb.go index 8ed0afcd8..f2285f560 100644 --- a/proto/vdsm/vdsm.pb.go +++ b/proto/vdsm/vdsm.pb.go @@ -610,6 +610,86 @@ func (m *QueryPQLRequest) GetPql() string { return "" } +// copied from pilosa.proto to use "vds" instead of "index" and "records" instead of "columns" +type InspectRequest struct { + Vds string `protobuf:"bytes,1,opt,name=vds,proto3" json:"vds,omitempty"` + Records *proto1.IdsOrKeys `protobuf:"bytes,2,opt,name=records,proto3" json:"records,omitempty"` + FilterFields []string `protobuf:"bytes,3,rep,name=filterFields,proto3" json:"filterFields,omitempty"` + Limit uint64 `protobuf:"varint,4,opt,name=limit,proto3" json:"limit,omitempty"` + Offset uint64 `protobuf:"varint,5,opt,name=offset,proto3" json:"offset,omitempty"` + Query string `protobuf:"bytes,6,opt,name=query,proto3" json:"query,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *InspectRequest) Reset() { *m = InspectRequest{} } +func (m *InspectRequest) String() string { return proto.CompactTextString(m) } +func (*InspectRequest) ProtoMessage() {} +func (*InspectRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{11} +} + +func (m *InspectRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_InspectRequest.Unmarshal(m, b) +} +func (m *InspectRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_InspectRequest.Marshal(b, m, deterministic) +} +func (m *InspectRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_InspectRequest.Merge(m, src) +} +func (m *InspectRequest) XXX_Size() int { + return xxx_messageInfo_InspectRequest.Size(m) +} +func (m *InspectRequest) XXX_DiscardUnknown() { + xxx_messageInfo_InspectRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_InspectRequest proto.InternalMessageInfo + +func (m *InspectRequest) GetVds() string { + if m != nil { + return m.Vds + } + return "" +} + +func (m *InspectRequest) GetRecords() *proto1.IdsOrKeys { + if m != nil { + return m.Records + } + return nil +} + +func (m *InspectRequest) GetFilterFields() []string { + if m != nil { + return m.FilterFields + } + return nil +} + +func (m *InspectRequest) GetLimit() uint64 { + if m != nil { + return m.Limit + } + return 0 +} + +func (m *InspectRequest) GetOffset() uint64 { + if m != nil { + return m.Offset + } + return 0 +} + +func (m *InspectRequest) GetQuery() string { + if m != nil { + return m.Query + } + return "" +} + func init() { proto.RegisterType((*DataSource)(nil), "vdsm.DataSource") proto.RegisterMapType((map[string]string)(nil), "vdsm.DataSource.ConfigEntry") @@ -623,52 +703,58 @@ func init() { proto.RegisterType((*DeleteVDSRequest)(nil), "vdsm.DeleteVDSRequest") proto.RegisterType((*DeleteVDSResponse)(nil), "vdsm.DeleteVDSResponse") proto.RegisterType((*QueryPQLRequest)(nil), "vdsm.QueryPQLRequest") + proto.RegisterType((*InspectRequest)(nil), "vdsm.InspectRequest") } func init() { proto.RegisterFile("vdsm/vdsm.proto", fileDescriptor_3849c3e549495f50) } var fileDescriptor_3849c3e549495f50 = []byte{ - // 629 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x54, 0xdb, 0x6e, 0xd3, 0x40, - 0x10, 0x8d, 0xe3, 0x34, 0x6d, 0x26, 0xb4, 0x09, 0xdb, 0xb4, 0xb5, 0xc2, 0x45, 0xc1, 0x0f, 0x28, - 0x2f, 0xa4, 0x55, 0x4a, 0xa5, 0x52, 0x21, 0x24, 0xda, 0x00, 0xad, 0x54, 0x20, 0x71, 0xa0, 0x0f, - 0xbc, 0xa0, 0xad, 0xbd, 0x2d, 0x2b, 0x1c, 0xdb, 0xf1, 0xae, 0x03, 0xf9, 0x1d, 0xfe, 0x80, 0x2f, - 0xe3, 0x17, 0x90, 0xf7, 0xe2, 0x38, 0x06, 0x54, 0xa4, 0xbe, 0x24, 0xb3, 0x67, 0x67, 0xe6, 0xcc, - 0x39, 0x63, 0x2d, 0x34, 0x66, 0x1e, 0x9b, 0xec, 0xa6, 0x3f, 0xbd, 0x28, 0x0e, 0x79, 0x88, 0x2a, - 0x69, 0xdc, 0xbe, 0x13, 0x51, 0x3f, 0x64, 0x58, 0x62, 0xf6, 0x2f, 0x03, 0x60, 0x80, 0x39, 0x1e, - 0x87, 0x49, 0xec, 0x12, 0xb4, 0x01, 0x65, 0xea, 0x59, 0x46, 0xc7, 0xe8, 0x9a, 0x4e, 0x99, 0x7a, - 0x08, 0x41, 0x25, 0xc0, 0x13, 0x62, 0x95, 0x3b, 0x46, 0xb7, 0xe6, 0x88, 0x18, 0x75, 0xa0, 0xee, - 0x11, 0xe6, 0xc6, 0x34, 0xe2, 0x34, 0x0c, 0x2c, 0x53, 0x5c, 0xe5, 0xa1, 0xb4, 0x8a, 0xcf, 0x23, - 0x62, 0x55, 0x64, 0x55, 0x1a, 0xa3, 0xa7, 0x50, 0x75, 0xc3, 0xe0, 0x8a, 0x5e, 0x5b, 0x2b, 0x1d, - 0xb3, 0x5b, 0xef, 0xdf, 0xef, 0x89, 0xc9, 0x16, 0xdc, 0xbd, 0x13, 0x71, 0xfd, 0x2a, 0xe0, 0xf1, - 0xdc, 0x51, 0xb9, 0x68, 0x1b, 0xaa, 0x8c, 0x63, 0x9e, 0x30, 0xab, 0x2a, 0x7a, 0xa9, 0x53, 0xfb, - 0x19, 0xd4, 0x73, 0xe9, 0xa8, 0x09, 0xe6, 0x57, 0x32, 0x17, 0x73, 0xd7, 0x9c, 0x34, 0x44, 0x2d, - 0x58, 0x99, 0x61, 0x3f, 0xd1, 0x93, 0xcb, 0xc3, 0x51, 0xf9, 0xd0, 0xb0, 0x7f, 0x18, 0x60, 0x5e, - 0x0c, 0xc6, 0x39, 0xa9, 0xb5, 0x5b, 0x48, 0x7d, 0x04, 0xca, 0xcf, 0xcf, 0x34, 0xf0, 0xc8, 0x77, - 0x25, 0xb9, 0x2e, 0xb1, 0xb3, 0x14, 0x42, 0x7d, 0xa8, 0x7b, 0x98, 0x63, 0x26, 0x54, 0x32, 0x25, - 0xbf, 0x59, 0x94, 0xef, 0xe4, 0x93, 0xec, 0x26, 0x6c, 0xbc, 0x21, 0xfc, 0x62, 0x30, 0x66, 0x0e, - 0x99, 0x26, 0x84, 0x71, 0x7b, 0x0f, 0x1a, 0x19, 0xc2, 0xa2, 0x30, 0x60, 0x04, 0x3d, 0x80, 0x74, - 0xa3, 0xcc, 0x32, 0x44, 0xc7, 0x9a, 0xec, 0x78, 0x31, 0x18, 0x3b, 0x02, 0xb6, 0x4f, 0x60, 0x5d, - 0x56, 0xa8, 0x16, 0xa8, 0xb9, 0x50, 0x7c, 0x5a, 0x12, 0x9a, 0x5b, 0x79, 0xcd, 0xa7, 0x25, 0xa9, - 0xfa, 0x18, 0x60, 0x8d, 0x7a, 0xef, 0xe3, 0x77, 0x78, 0x42, 0xec, 0x27, 0x7a, 0x90, 0x8c, 0xf5, - 0x1e, 0x98, 0x33, 0x8f, 0x89, 0x36, 0x4b, 0xa4, 0x29, 0x6a, 0xef, 0xc1, 0xc6, 0x30, 0x64, 0x79, - 0xd2, 0x87, 0x00, 0x1e, 0xb9, 0xa2, 0x01, 0x15, 0x0e, 0x4a, 0xbb, 0x73, 0x88, 0xbd, 0x0f, 0x8d, - 0xac, 0x42, 0x31, 0x14, 0x37, 0xd3, 0x04, 0x33, 0x89, 0xa9, 0x5a, 0x4c, 0x1a, 0xda, 0xaf, 0xa1, - 0x39, 0x20, 0x3e, 0xe1, 0xe4, 0x96, 0xea, 0x36, 0xe1, 0x6e, 0xae, 0x8f, 0xa4, 0xb7, 0x0f, 0xa0, - 0x31, 0x4a, 0x48, 0x3c, 0x1f, 0x8e, 0xce, 0x17, 0xbd, 0x33, 0xcd, 0x35, 0x21, 0x34, 0x45, 0xa2, - 0xa9, 0xaf, 0x67, 0x8a, 0xa6, 0x7e, 0xff, 0x67, 0x05, 0xd6, 0xde, 0x86, 0x3e, 0x71, 0x13, 0x1f, - 0xa3, 0x43, 0x58, 0x55, 0xdb, 0x42, 0x2d, 0x69, 0xd1, 0xf2, 0x3a, 0xdb, 0x5b, 0x05, 0x54, 0x71, - 0x97, 0xd0, 0x01, 0x54, 0x25, 0x88, 0x36, 0xf3, 0x29, 0xba, 0xae, 0xb5, 0x0c, 0x66, 0x65, 0x87, - 0xb0, 0xaa, 0x6c, 0xd4, 0x84, 0xcb, 0x7b, 0xd0, 0x84, 0x05, 0xaf, 0xed, 0x12, 0x7a, 0x01, 0xb5, - 0xcc, 0x03, 0xb4, 0xad, 0x3e, 0xcb, 0x82, 0xb9, 0xed, 0x9d, 0x3f, 0xf0, 0xac, 0xfe, 0x39, 0xac, - 0x09, 0xbb, 0xc6, 0xa3, 0x73, 0xb4, 0xd3, 0x53, 0x8f, 0x8b, 0x46, 0x74, 0xfd, 0xa6, 0xbe, 0x70, - 0xc2, 0x6f, 0x8b, 0xda, 0x3d, 0x03, 0xbd, 0x84, 0x75, 0x9d, 0xfb, 0x31, 0xc0, 0xf1, 0xfc, 0xdf, - 0x2d, 0xb6, 0xf4, 0xc5, 0x07, 0x7c, 0xe9, 0x93, 0xdc, 0x00, 0x47, 0x6a, 0x80, 0xe1, 0xe8, 0x1c, - 0x29, 0x95, 0x85, 0xfd, 0xdd, 0x4c, 0x3f, 0xfc, 0x3b, 0xfd, 0xf0, 0xbf, 0xe8, 0x57, 0xcf, 0x02, - 0x16, 0x11, 0x97, 0xa3, 0x6d, 0x9d, 0xa3, 0x80, 0x9b, 0xe8, 0x8f, 0xbb, 0x9f, 0x1e, 0x5f, 0x53, - 0xfe, 0x25, 0xb9, 0xec, 0xb9, 0xe1, 0x64, 0x57, 0x26, 0xe9, 0xbf, 0x59, 0x7f, 0x57, 0x3c, 0xd1, - 0xe2, 0x05, 0xbf, 0xac, 0x8a, 0x78, 0xff, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0xb0, 0x14, 0xc9, - 0x0e, 0xd5, 0x05, 0x00, 0x00, + // 718 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x55, 0x5d, 0x6f, 0x12, 0x4d, + 0x14, 0x66, 0x59, 0x0a, 0xe5, 0xd0, 0x02, 0x9d, 0xd2, 0x76, 0xc3, 0xfb, 0x11, 0xde, 0xbd, 0x78, + 0x43, 0x62, 0xa4, 0x0d, 0xb5, 0x09, 0x36, 0xc6, 0xc4, 0x16, 0x6b, 0x1b, 0xab, 0x85, 0x45, 0x7b, + 0xe1, 0x8d, 0xd9, 0xb2, 0x43, 0x9d, 0xb8, 0xec, 0x2e, 0x3b, 0xb3, 0x28, 0x7f, 0xc7, 0x3f, 0xe0, + 0xa5, 0x3f, 0xcb, 0xbf, 0x60, 0xe6, 0x0b, 0x16, 0xb4, 0xd1, 0xa4, 0x37, 0x30, 0xf3, 0xcc, 0x39, + 0xe7, 0x39, 0xcf, 0x9c, 0x67, 0xb2, 0x50, 0x99, 0x7a, 0x74, 0xbc, 0xcf, 0x7f, 0x5a, 0x51, 0x1c, + 0xb2, 0x10, 0xe5, 0xf8, 0xba, 0xbe, 0x11, 0x11, 0x3f, 0xa4, 0xae, 0xc4, 0xec, 0xef, 0x06, 0x40, + 0xd7, 0x65, 0xee, 0x20, 0x4c, 0xe2, 0x21, 0x46, 0x65, 0xc8, 0x12, 0xcf, 0x32, 0x1a, 0x46, 0xd3, + 0x74, 0xb2, 0xc4, 0x43, 0x08, 0x72, 0x81, 0x3b, 0xc6, 0x56, 0xb6, 0x61, 0x34, 0x8b, 0x8e, 0x58, + 0xa3, 0x06, 0x94, 0x3c, 0x4c, 0x87, 0x31, 0x89, 0x18, 0x09, 0x03, 0xcb, 0x14, 0x47, 0x69, 0x88, + 0x67, 0xb1, 0x59, 0x84, 0xad, 0x9c, 0xcc, 0xe2, 0x6b, 0xf4, 0x08, 0xf2, 0xc3, 0x30, 0x18, 0x91, + 0x5b, 0x6b, 0xad, 0x61, 0x36, 0x4b, 0xed, 0xbf, 0x5b, 0xa2, 0xb3, 0x05, 0x77, 0xeb, 0x54, 0x1c, + 0x3f, 0x0f, 0x58, 0x3c, 0x73, 0x54, 0x2c, 0xda, 0x85, 0x3c, 0x65, 0x2e, 0x4b, 0xa8, 0x95, 0x17, + 0xb5, 0xd4, 0xae, 0xfe, 0x18, 0x4a, 0xa9, 0x70, 0x54, 0x05, 0xf3, 0x23, 0x9e, 0x89, 0xbe, 0x8b, + 0x0e, 0x5f, 0xa2, 0x1a, 0xac, 0x4d, 0x5d, 0x3f, 0xd1, 0x9d, 0xcb, 0xcd, 0x71, 0xb6, 0x63, 0xd8, + 0x5f, 0x0c, 0x30, 0xaf, 0xbb, 0x83, 0x94, 0xd4, 0xe2, 0x3d, 0xa4, 0xfe, 0x07, 0xea, 0x3e, 0xdf, + 0x93, 0xc0, 0xc3, 0x9f, 0x95, 0xe4, 0x92, 0xc4, 0x2e, 0x38, 0x84, 0xda, 0x50, 0xf2, 0x5c, 0xe6, + 0x52, 0xa1, 0x92, 0x2a, 0xf9, 0xd5, 0x55, 0xf9, 0x4e, 0x3a, 0xc8, 0xae, 0x42, 0xf9, 0x05, 0x66, + 0xd7, 0xdd, 0x01, 0x75, 0xf0, 0x24, 0xc1, 0x94, 0xd9, 0x07, 0x50, 0x99, 0x23, 0x34, 0x0a, 0x03, + 0x8a, 0xd1, 0x3f, 0xc0, 0x27, 0x4a, 0x2d, 0x43, 0x54, 0x2c, 0xca, 0x8a, 0xd7, 0xdd, 0x81, 0x23, + 0x60, 0xfb, 0x14, 0x36, 0x65, 0x86, 0x2a, 0x81, 0xaa, 0x0b, 0xc5, 0xe7, 0x19, 0xa1, 0xb9, 0x96, + 0xd6, 0x7c, 0x9e, 0x91, 0xaa, 0x4f, 0x00, 0xd6, 0x89, 0x77, 0x15, 0xbf, 0x76, 0xc7, 0xd8, 0x7e, + 0xa8, 0x1b, 0x99, 0xb3, 0xfe, 0x05, 0xe6, 0xd4, 0xa3, 0xa2, 0xcc, 0x12, 0x29, 0x47, 0xed, 0x03, + 0x28, 0xf7, 0x42, 0x9a, 0x26, 0xfd, 0x17, 0xc0, 0xc3, 0x23, 0x12, 0x10, 0x71, 0x83, 0xf2, 0xba, + 0x53, 0x88, 0x7d, 0x08, 0x95, 0x79, 0x86, 0x62, 0x58, 0x9d, 0x4c, 0x15, 0xcc, 0x24, 0x26, 0x6a, + 0x30, 0x7c, 0x69, 0x9f, 0x41, 0xb5, 0x8b, 0x7d, 0xcc, 0xf0, 0x3d, 0xd5, 0x6d, 0xc3, 0x56, 0xaa, + 0x8e, 0xa4, 0xb7, 0x8f, 0xa0, 0xd2, 0x4f, 0x70, 0x3c, 0xeb, 0xf5, 0x2f, 0x17, 0xb5, 0xe7, 0x9a, + 0x8b, 0x42, 0x28, 0x47, 0xa2, 0x89, 0xaf, 0x7b, 0x8a, 0x26, 0xbe, 0xfd, 0xcd, 0x80, 0xf2, 0x45, + 0x40, 0x23, 0x3c, 0x64, 0x77, 0xa7, 0x3d, 0x80, 0x42, 0x8c, 0x87, 0x61, 0xec, 0x51, 0x91, 0x5a, + 0x6a, 0x6f, 0xb5, 0xd4, 0x73, 0xbc, 0xf0, 0xe8, 0x55, 0xfc, 0x12, 0xcf, 0xa8, 0xa3, 0x23, 0x90, + 0x0d, 0x1b, 0x23, 0xe2, 0x33, 0x1c, 0x9f, 0x11, 0xec, 0x7b, 0xd4, 0x32, 0x1b, 0x66, 0xb3, 0xe8, + 0x2c, 0x61, 0xdc, 0xe7, 0x3e, 0x19, 0x13, 0x26, 0x8c, 0x97, 0x73, 0xe4, 0x86, 0x3f, 0x9b, 0x70, + 0x34, 0xa2, 0x98, 0x59, 0x6b, 0x02, 0x56, 0x3b, 0x1e, 0x3d, 0xe1, 0xd2, 0xd4, 0x6b, 0x92, 0x9b, + 0xf6, 0xd7, 0x1c, 0xac, 0xbf, 0x0a, 0x7d, 0x3c, 0x4c, 0x7c, 0x17, 0x75, 0xa0, 0xa0, 0x7c, 0x86, + 0x6a, 0x72, 0xb8, 0xcb, 0x46, 0xac, 0xef, 0xac, 0xa0, 0xea, 0xd6, 0x32, 0xe8, 0x08, 0xf2, 0x12, + 0x44, 0xdb, 0xe9, 0x10, 0x9d, 0x57, 0x5b, 0x06, 0xe7, 0x69, 0x1d, 0x28, 0x28, 0x03, 0x68, 0xc2, + 0x65, 0x07, 0x69, 0xc2, 0x15, 0x97, 0xd8, 0x19, 0xf4, 0x14, 0x8a, 0xf3, 0xe9, 0xa1, 0x5d, 0xf5, + 0xa0, 0x56, 0x6c, 0x51, 0xdf, 0xfb, 0x09, 0x9f, 0xe7, 0x3f, 0x81, 0x75, 0x31, 0xe8, 0x41, 0xff, + 0x12, 0xed, 0xe9, 0x39, 0x68, 0x44, 0xe7, 0x6f, 0xeb, 0x03, 0x27, 0xfc, 0xb4, 0xc8, 0x3d, 0x30, + 0xd0, 0x33, 0xd8, 0xd4, 0xb1, 0x6f, 0x03, 0x37, 0x9e, 0xdd, 0x5d, 0x62, 0x47, 0x1f, 0xbc, 0x71, + 0x6f, 0x7c, 0x9c, 0x6a, 0xe0, 0x58, 0x35, 0xd0, 0xeb, 0x5f, 0x22, 0xa5, 0x72, 0xc5, 0x79, 0xbf, + 0xa7, 0xef, 0xfd, 0x9a, 0xbe, 0xf7, 0x07, 0xf4, 0x1d, 0x28, 0x28, 0xc3, 0xea, 0x9b, 0x5f, 0xf6, + 0xef, 0x9d, 0xe4, 0x27, 0xcd, 0x77, 0xff, 0xdf, 0x12, 0xf6, 0x21, 0xb9, 0x69, 0x0d, 0xc3, 0xf1, + 0xbe, 0x0c, 0xd2, 0x7f, 0xd3, 0xf6, 0xbe, 0xf8, 0xb4, 0x88, 0x2f, 0xcf, 0x4d, 0x5e, 0xac, 0x0f, + 0x7f, 0x04, 0x00, 0x00, 0xff, 0xff, 0xff, 0x97, 0x0c, 0xf4, 0x8d, 0x06, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -691,7 +777,7 @@ type MoleculaClient interface { QuerySQLUnary(ctx context.Context, in *proto1.QuerySQLRequest, opts ...grpc.CallOption) (*proto1.TableResponse, error) QueryPQL(ctx context.Context, in *QueryPQLRequest, opts ...grpc.CallOption) (Molecula_QueryPQLClient, error) QueryPQLUnary(ctx context.Context, in *proto1.QueryPQLRequest, opts ...grpc.CallOption) (*proto1.TableResponse, error) - Inspect(ctx context.Context, in *proto1.InspectRequest, opts ...grpc.CallOption) (Molecula_InspectClient, error) + Inspect(ctx context.Context, in *InspectRequest, opts ...grpc.CallOption) (Molecula_InspectClient, error) } type moleculaClient struct { @@ -820,7 +906,7 @@ func (c *moleculaClient) QueryPQLUnary(ctx context.Context, in *proto1.QueryPQLR return out, nil } -func (c *moleculaClient) Inspect(ctx context.Context, in *proto1.InspectRequest, opts ...grpc.CallOption) (Molecula_InspectClient, error) { +func (c *moleculaClient) Inspect(ctx context.Context, in *InspectRequest, opts ...grpc.CallOption) (Molecula_InspectClient, error) { stream, err := c.cc.NewStream(ctx, &_Molecula_serviceDesc.Streams[2], "/vdsm.Molecula/Inspect", opts...) if err != nil { return nil, err @@ -862,7 +948,7 @@ type MoleculaServer interface { QuerySQLUnary(context.Context, *proto1.QuerySQLRequest) (*proto1.TableResponse, error) QueryPQL(*QueryPQLRequest, Molecula_QueryPQLServer) error QueryPQLUnary(context.Context, *proto1.QueryPQLRequest) (*proto1.TableResponse, error) - Inspect(*proto1.InspectRequest, Molecula_InspectServer) error + Inspect(*InspectRequest, Molecula_InspectServer) error } // UnimplementedMoleculaServer can be embedded to have forward compatible implementations. @@ -893,7 +979,7 @@ func (*UnimplementedMoleculaServer) QueryPQL(req *QueryPQLRequest, srv Molecula_ func (*UnimplementedMoleculaServer) QueryPQLUnary(ctx context.Context, req *proto1.QueryPQLRequest) (*proto1.TableResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method QueryPQLUnary not implemented") } -func (*UnimplementedMoleculaServer) Inspect(req *proto1.InspectRequest, srv Molecula_InspectServer) error { +func (*UnimplementedMoleculaServer) Inspect(req *InspectRequest, srv Molecula_InspectServer) error { return status.Errorf(codes.Unimplemented, "method Inspect not implemented") } @@ -1052,7 +1138,7 @@ func _Molecula_QueryPQLUnary_Handler(srv interface{}, ctx context.Context, dec f } func _Molecula_Inspect_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(proto1.InspectRequest) + m := new(InspectRequest) if err := stream.RecvMsg(m); err != nil { return err } diff --git a/proto/vdsm/vdsm.proto b/proto/vdsm/vdsm.proto index 645cf8e21..0755fd8ee 100644 --- a/proto/vdsm/vdsm.proto +++ b/proto/vdsm/vdsm.proto @@ -66,6 +66,16 @@ message QueryPQLRequest { string pql = 2; } +// copied from pilosa.proto to use "vds" instead of "index" and "records" instead of "columns" +message InspectRequest { + string vds = 1; + pilosa.IdsOrKeys records = 2; + repeated string filterFields = 3; + uint64 limit = 4; + uint64 offset = 5; + string query = 6; +} + service Molecula { rpc GetVDSs(GetVDSsRequest) returns (GetVDSsResponse) {}; rpc GetVDS(GetVDSRequest) returns (GetVDSResponse) {}; @@ -75,5 +85,5 @@ service Molecula { rpc QuerySQLUnary(pilosa.QuerySQLRequest) returns (pilosa.TableResponse) {}; rpc QueryPQL(QueryPQLRequest) returns (stream pilosa.RowResponse) {}; rpc QueryPQLUnary(pilosa.QueryPQLRequest) returns (pilosa.TableResponse) {}; - rpc Inspect(pilosa.InspectRequest) returns (stream pilosa.RowResponse) {}; + rpc Inspect(InspectRequest) returns (stream pilosa.RowResponse) {}; } diff --git a/server/grpc.go b/server/grpc.go index 85d8afc84..649d29b86 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -342,8 +342,9 @@ func (h *VDSMGRPCHandler) QueryPQLUnary(ctx context.Context, req *pb.QueryPQLReq return h.grpcHandler.QueryPQLUnary(ctx, req) } -func (h *VDSMGRPCHandler) Inspect(req *pb.InspectRequest, srv vdsm_pb.Molecula_InspectServer) error { - return h.grpcHandler.Inspect(req, srv) +func (h *VDSMGRPCHandler) Inspect(req *vdsm_pb.InspectRequest, srv vdsm_pb.Molecula_InspectServer) error { + preq := &pb.InspectRequest{Index: req.Vds, Columns: req.Records, FilterFields: req.FilterFields, Limit: req.Limit, Offset: req.Offset, Query: req.Query} + return h.grpcHandler.Inspect(preq, srv) } // ResultUint64 is a wrapper around a uint64 result type From 6c10917ceb6d8ff238622f6bd48678e58435fa62 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Fri, 9 Oct 2020 13:34:20 -0500 Subject: [PATCH 47/53] Use vdsm.QueryPQLRequest for vdsm.QueryPQLUnary --- proto/vdsm/vdsm.pb.go | 104 +++++++++++++++++++++--------------------- proto/vdsm/vdsm.proto | 2 +- server/grpc.go | 5 +- 3 files changed, 56 insertions(+), 55 deletions(-) diff --git a/proto/vdsm/vdsm.pb.go b/proto/vdsm/vdsm.pb.go index f2285f560..083d258e8 100644 --- a/proto/vdsm/vdsm.pb.go +++ b/proto/vdsm/vdsm.pb.go @@ -709,52 +709,52 @@ func init() { func init() { proto.RegisterFile("vdsm/vdsm.proto", fileDescriptor_3849c3e549495f50) } var fileDescriptor_3849c3e549495f50 = []byte{ - // 718 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x55, 0x5d, 0x6f, 0x12, 0x4d, - 0x14, 0x66, 0x59, 0x0a, 0xe5, 0xd0, 0x02, 0x9d, 0xd2, 0x76, 0xc3, 0xfb, 0x11, 0xde, 0xbd, 0x78, - 0x43, 0x62, 0xa4, 0x0d, 0xb5, 0x09, 0x36, 0xc6, 0xc4, 0x16, 0x6b, 0x1b, 0xab, 0x85, 0x45, 0x7b, - 0xe1, 0x8d, 0xd9, 0xb2, 0x43, 0x9d, 0xb8, 0xec, 0x2e, 0x3b, 0xb3, 0x28, 0x7f, 0xc7, 0x3f, 0xe0, - 0xa5, 0x3f, 0xcb, 0xbf, 0x60, 0xe6, 0x0b, 0x16, 0xb4, 0xd1, 0xa4, 0x37, 0x30, 0xf3, 0xcc, 0x39, - 0xe7, 0x39, 0xcf, 0x9c, 0x67, 0xb2, 0x50, 0x99, 0x7a, 0x74, 0xbc, 0xcf, 0x7f, 0x5a, 0x51, 0x1c, - 0xb2, 0x10, 0xe5, 0xf8, 0xba, 0xbe, 0x11, 0x11, 0x3f, 0xa4, 0xae, 0xc4, 0xec, 0xef, 0x06, 0x40, - 0xd7, 0x65, 0xee, 0x20, 0x4c, 0xe2, 0x21, 0x46, 0x65, 0xc8, 0x12, 0xcf, 0x32, 0x1a, 0x46, 0xd3, - 0x74, 0xb2, 0xc4, 0x43, 0x08, 0x72, 0x81, 0x3b, 0xc6, 0x56, 0xb6, 0x61, 0x34, 0x8b, 0x8e, 0x58, - 0xa3, 0x06, 0x94, 0x3c, 0x4c, 0x87, 0x31, 0x89, 0x18, 0x09, 0x03, 0xcb, 0x14, 0x47, 0x69, 0x88, - 0x67, 0xb1, 0x59, 0x84, 0xad, 0x9c, 0xcc, 0xe2, 0x6b, 0xf4, 0x08, 0xf2, 0xc3, 0x30, 0x18, 0x91, - 0x5b, 0x6b, 0xad, 0x61, 0x36, 0x4b, 0xed, 0xbf, 0x5b, 0xa2, 0xb3, 0x05, 0x77, 0xeb, 0x54, 0x1c, - 0x3f, 0x0f, 0x58, 0x3c, 0x73, 0x54, 0x2c, 0xda, 0x85, 0x3c, 0x65, 0x2e, 0x4b, 0xa8, 0x95, 0x17, - 0xb5, 0xd4, 0xae, 0xfe, 0x18, 0x4a, 0xa9, 0x70, 0x54, 0x05, 0xf3, 0x23, 0x9e, 0x89, 0xbe, 0x8b, - 0x0e, 0x5f, 0xa2, 0x1a, 0xac, 0x4d, 0x5d, 0x3f, 0xd1, 0x9d, 0xcb, 0xcd, 0x71, 0xb6, 0x63, 0xd8, - 0x5f, 0x0c, 0x30, 0xaf, 0xbb, 0x83, 0x94, 0xd4, 0xe2, 0x3d, 0xa4, 0xfe, 0x07, 0xea, 0x3e, 0xdf, - 0x93, 0xc0, 0xc3, 0x9f, 0x95, 0xe4, 0x92, 0xc4, 0x2e, 0x38, 0x84, 0xda, 0x50, 0xf2, 0x5c, 0xe6, - 0x52, 0xa1, 0x92, 0x2a, 0xf9, 0xd5, 0x55, 0xf9, 0x4e, 0x3a, 0xc8, 0xae, 0x42, 0xf9, 0x05, 0x66, - 0xd7, 0xdd, 0x01, 0x75, 0xf0, 0x24, 0xc1, 0x94, 0xd9, 0x07, 0x50, 0x99, 0x23, 0x34, 0x0a, 0x03, - 0x8a, 0xd1, 0x3f, 0xc0, 0x27, 0x4a, 0x2d, 0x43, 0x54, 0x2c, 0xca, 0x8a, 0xd7, 0xdd, 0x81, 0x23, - 0x60, 0xfb, 0x14, 0x36, 0x65, 0x86, 0x2a, 0x81, 0xaa, 0x0b, 0xc5, 0xe7, 0x19, 0xa1, 0xb9, 0x96, - 0xd6, 0x7c, 0x9e, 0x91, 0xaa, 0x4f, 0x00, 0xd6, 0x89, 0x77, 0x15, 0xbf, 0x76, 0xc7, 0xd8, 0x7e, - 0xa8, 0x1b, 0x99, 0xb3, 0xfe, 0x05, 0xe6, 0xd4, 0xa3, 0xa2, 0xcc, 0x12, 0x29, 0x47, 0xed, 0x03, - 0x28, 0xf7, 0x42, 0x9a, 0x26, 0xfd, 0x17, 0xc0, 0xc3, 0x23, 0x12, 0x10, 0x71, 0x83, 0xf2, 0xba, - 0x53, 0x88, 0x7d, 0x08, 0x95, 0x79, 0x86, 0x62, 0x58, 0x9d, 0x4c, 0x15, 0xcc, 0x24, 0x26, 0x6a, - 0x30, 0x7c, 0x69, 0x9f, 0x41, 0xb5, 0x8b, 0x7d, 0xcc, 0xf0, 0x3d, 0xd5, 0x6d, 0xc3, 0x56, 0xaa, - 0x8e, 0xa4, 0xb7, 0x8f, 0xa0, 0xd2, 0x4f, 0x70, 0x3c, 0xeb, 0xf5, 0x2f, 0x17, 0xb5, 0xe7, 0x9a, - 0x8b, 0x42, 0x28, 0x47, 0xa2, 0x89, 0xaf, 0x7b, 0x8a, 0x26, 0xbe, 0xfd, 0xcd, 0x80, 0xf2, 0x45, - 0x40, 0x23, 0x3c, 0x64, 0x77, 0xa7, 0x3d, 0x80, 0x42, 0x8c, 0x87, 0x61, 0xec, 0x51, 0x91, 0x5a, - 0x6a, 0x6f, 0xb5, 0xd4, 0x73, 0xbc, 0xf0, 0xe8, 0x55, 0xfc, 0x12, 0xcf, 0xa8, 0xa3, 0x23, 0x90, - 0x0d, 0x1b, 0x23, 0xe2, 0x33, 0x1c, 0x9f, 0x11, 0xec, 0x7b, 0xd4, 0x32, 0x1b, 0x66, 0xb3, 0xe8, - 0x2c, 0x61, 0xdc, 0xe7, 0x3e, 0x19, 0x13, 0x26, 0x8c, 0x97, 0x73, 0xe4, 0x86, 0x3f, 0x9b, 0x70, - 0x34, 0xa2, 0x98, 0x59, 0x6b, 0x02, 0x56, 0x3b, 0x1e, 0x3d, 0xe1, 0xd2, 0xd4, 0x6b, 0x92, 0x9b, - 0xf6, 0xd7, 0x1c, 0xac, 0xbf, 0x0a, 0x7d, 0x3c, 0x4c, 0x7c, 0x17, 0x75, 0xa0, 0xa0, 0x7c, 0x86, - 0x6a, 0x72, 0xb8, 0xcb, 0x46, 0xac, 0xef, 0xac, 0xa0, 0xea, 0xd6, 0x32, 0xe8, 0x08, 0xf2, 0x12, - 0x44, 0xdb, 0xe9, 0x10, 0x9d, 0x57, 0x5b, 0x06, 0xe7, 0x69, 0x1d, 0x28, 0x28, 0x03, 0x68, 0xc2, - 0x65, 0x07, 0x69, 0xc2, 0x15, 0x97, 0xd8, 0x19, 0xf4, 0x14, 0x8a, 0xf3, 0xe9, 0xa1, 0x5d, 0xf5, - 0xa0, 0x56, 0x6c, 0x51, 0xdf, 0xfb, 0x09, 0x9f, 0xe7, 0x3f, 0x81, 0x75, 0x31, 0xe8, 0x41, 0xff, - 0x12, 0xed, 0xe9, 0x39, 0x68, 0x44, 0xe7, 0x6f, 0xeb, 0x03, 0x27, 0xfc, 0xb4, 0xc8, 0x3d, 0x30, - 0xd0, 0x33, 0xd8, 0xd4, 0xb1, 0x6f, 0x03, 0x37, 0x9e, 0xdd, 0x5d, 0x62, 0x47, 0x1f, 0xbc, 0x71, - 0x6f, 0x7c, 0x9c, 0x6a, 0xe0, 0x58, 0x35, 0xd0, 0xeb, 0x5f, 0x22, 0xa5, 0x72, 0xc5, 0x79, 0xbf, - 0xa7, 0xef, 0xfd, 0x9a, 0xbe, 0xf7, 0x07, 0xf4, 0x1d, 0x28, 0x28, 0xc3, 0xea, 0x9b, 0x5f, 0xf6, - 0xef, 0x9d, 0xe4, 0x27, 0xcd, 0x77, 0xff, 0xdf, 0x12, 0xf6, 0x21, 0xb9, 0x69, 0x0d, 0xc3, 0xf1, - 0xbe, 0x0c, 0xd2, 0x7f, 0xd3, 0xf6, 0xbe, 0xf8, 0xb4, 0x88, 0x2f, 0xcf, 0x4d, 0x5e, 0xac, 0x0f, - 0x7f, 0x04, 0x00, 0x00, 0xff, 0xff, 0xff, 0x97, 0x0c, 0xf4, 0x8d, 0x06, 0x00, 0x00, + // 719 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x55, 0xcd, 0x6e, 0xda, 0x4c, + 0x14, 0xc5, 0x98, 0x40, 0xb8, 0x24, 0x40, 0x26, 0x24, 0xb1, 0xf8, 0x7e, 0xc4, 0xe7, 0xc5, 0x27, + 0xa4, 0xaa, 0x24, 0x22, 0x8d, 0x44, 0xa3, 0xaa, 0x55, 0x13, 0x9a, 0x26, 0x6a, 0xda, 0x80, 0x69, + 0xb3, 0xe8, 0xa6, 0x72, 0xf0, 0x90, 0x8e, 0x6a, 0x6c, 0xe3, 0x19, 0xd3, 0xf2, 0x3a, 0xdd, 0x77, + 0xdd, 0xc7, 0xea, 0x2b, 0x54, 0xf3, 0x07, 0x86, 0x8a, 0x55, 0x36, 0x30, 0x73, 0xe6, 0xde, 0x7b, + 0xee, 0x99, 0x7b, 0x46, 0x86, 0xca, 0xd4, 0xa3, 0xe3, 0x43, 0xfe, 0xd3, 0x8a, 0xe2, 0x90, 0x85, + 0x28, 0xc7, 0xd7, 0xf5, 0xad, 0x88, 0xf8, 0x21, 0x75, 0x25, 0x66, 0xff, 0x32, 0x00, 0xba, 0x2e, + 0x73, 0x07, 0x61, 0x12, 0x0f, 0x31, 0x2a, 0x43, 0x96, 0x78, 0x96, 0xd1, 0x30, 0x9a, 0xa6, 0x93, + 0x25, 0x1e, 0x42, 0x90, 0x0b, 0xdc, 0x31, 0xb6, 0xb2, 0x0d, 0xa3, 0x59, 0x74, 0xc4, 0x1a, 0x35, + 0xa0, 0xe4, 0x61, 0x3a, 0x8c, 0x49, 0xc4, 0x48, 0x18, 0x58, 0xa6, 0x38, 0x4a, 0x43, 0x3c, 0x8b, + 0xcd, 0x22, 0x6c, 0xe5, 0x64, 0x16, 0x5f, 0xa3, 0x27, 0x90, 0x1f, 0x86, 0xc1, 0x88, 0xdc, 0x5b, + 0x1b, 0x0d, 0xb3, 0x59, 0x6a, 0xff, 0xdd, 0x12, 0x9d, 0x2d, 0xb8, 0x5b, 0xe7, 0xe2, 0xf8, 0x55, + 0xc0, 0xe2, 0x99, 0xa3, 0x62, 0xd1, 0x3e, 0xe4, 0x29, 0x73, 0x59, 0x42, 0xad, 0xbc, 0xa8, 0xa5, + 0x76, 0xf5, 0xa7, 0x50, 0x4a, 0x85, 0xa3, 0x2a, 0x98, 0x5f, 0xf0, 0x4c, 0xf4, 0x5d, 0x74, 0xf8, + 0x12, 0xd5, 0x60, 0x63, 0xea, 0xfa, 0x89, 0xee, 0x5c, 0x6e, 0x4e, 0xb3, 0x1d, 0xc3, 0xfe, 0x6e, + 0x80, 0x79, 0xdb, 0x1d, 0xa4, 0xa4, 0x16, 0x1f, 0x20, 0xf5, 0x3f, 0x50, 0xf7, 0xf9, 0x89, 0x04, + 0x1e, 0xfe, 0xa6, 0x24, 0x97, 0x24, 0x76, 0xc5, 0x21, 0xd4, 0x86, 0x92, 0xe7, 0x32, 0x97, 0x0a, + 0x95, 0x54, 0xc9, 0xaf, 0xae, 0xca, 0x77, 0xd2, 0x41, 0x76, 0x15, 0xca, 0xaf, 0x31, 0xbb, 0xed, + 0x0e, 0xa8, 0x83, 0x27, 0x09, 0xa6, 0xcc, 0x3e, 0x82, 0xca, 0x1c, 0xa1, 0x51, 0x18, 0x50, 0x8c, + 0xfe, 0x01, 0x3e, 0x51, 0x6a, 0x19, 0xa2, 0x62, 0x51, 0x56, 0xbc, 0xed, 0x0e, 0x1c, 0x01, 0xdb, + 0xe7, 0xb0, 0x2d, 0x33, 0x54, 0x09, 0x54, 0x5d, 0x28, 0xbe, 0xcc, 0x08, 0xcd, 0xb5, 0xb4, 0xe6, + 0xcb, 0x8c, 0x54, 0x7d, 0x06, 0xb0, 0x49, 0xbc, 0x9b, 0xf8, 0x9d, 0x3b, 0xc6, 0xf6, 0x63, 0xdd, + 0xc8, 0x9c, 0xf5, 0x2f, 0x30, 0xa7, 0x1e, 0x15, 0x65, 0x96, 0x48, 0x39, 0x6a, 0x1f, 0x41, 0xb9, + 0x17, 0xd2, 0x34, 0xe9, 0xbf, 0x00, 0x1e, 0x1e, 0x91, 0x80, 0x88, 0x1b, 0x94, 0xd7, 0x9d, 0x42, + 0xec, 0x63, 0xa8, 0xcc, 0x33, 0x14, 0xc3, 0xea, 0x64, 0xaa, 0x60, 0x26, 0x31, 0x51, 0x83, 0xe1, + 0x4b, 0xfb, 0x02, 0xaa, 0x5d, 0xec, 0x63, 0x86, 0x1f, 0xa8, 0x6e, 0x17, 0x76, 0x52, 0x75, 0x24, + 0xbd, 0x7d, 0x02, 0x95, 0x7e, 0x82, 0xe3, 0x59, 0xaf, 0x7f, 0xbd, 0xa8, 0x3d, 0xd7, 0x5c, 0x14, + 0x42, 0x39, 0x12, 0x4d, 0x7c, 0xdd, 0x53, 0x34, 0xf1, 0xed, 0x9f, 0x06, 0x94, 0xaf, 0x02, 0x1a, + 0xe1, 0x21, 0x5b, 0x9f, 0xf6, 0x08, 0x0a, 0x31, 0x1e, 0x86, 0xb1, 0x47, 0x45, 0x6a, 0xa9, 0xbd, + 0xd3, 0x52, 0xcf, 0xf1, 0xca, 0xa3, 0x37, 0xf1, 0x1b, 0x3c, 0xa3, 0x8e, 0x8e, 0x40, 0x36, 0x6c, + 0x8d, 0x88, 0xcf, 0x70, 0x7c, 0x41, 0xb0, 0xef, 0x51, 0xcb, 0x6c, 0x98, 0xcd, 0xa2, 0xb3, 0x84, + 0x71, 0x9f, 0xfb, 0x64, 0x4c, 0x98, 0x30, 0x5e, 0xce, 0x91, 0x1b, 0xfe, 0x6c, 0xc2, 0xd1, 0x88, + 0x62, 0x66, 0x6d, 0x08, 0x58, 0xed, 0x78, 0xf4, 0x84, 0x4b, 0x53, 0xaf, 0x49, 0x6e, 0xda, 0x3f, + 0x72, 0xb0, 0xf9, 0x36, 0xf4, 0xf1, 0x30, 0xf1, 0x5d, 0xd4, 0x81, 0x82, 0xf2, 0x19, 0xaa, 0xc9, + 0xe1, 0x2e, 0x1b, 0xb1, 0xbe, 0xb7, 0x82, 0xaa, 0x5b, 0xcb, 0xa0, 0x13, 0xc8, 0x4b, 0x10, 0xed, + 0xa6, 0x43, 0x74, 0x5e, 0x6d, 0x19, 0x9c, 0xa7, 0x75, 0xa0, 0xa0, 0x0c, 0xa0, 0x09, 0x97, 0x1d, + 0xa4, 0x09, 0x57, 0x5c, 0x62, 0x67, 0xd0, 0x73, 0x28, 0xce, 0xa7, 0x87, 0xf6, 0xd5, 0x83, 0x5a, + 0xb1, 0x45, 0xfd, 0xe0, 0x0f, 0x7c, 0x9e, 0xff, 0x0c, 0x36, 0xc5, 0xa0, 0x07, 0xfd, 0x6b, 0x74, + 0xa0, 0xe7, 0xa0, 0x11, 0x9d, 0xbf, 0xab, 0x0f, 0x9c, 0xf0, 0xeb, 0x22, 0xf7, 0xc8, 0x40, 0x2f, + 0x61, 0x5b, 0xc7, 0x7e, 0x08, 0xdc, 0x78, 0xb6, 0xbe, 0xc4, 0x9e, 0x3e, 0x78, 0xef, 0xde, 0xf9, + 0x38, 0xd5, 0xc0, 0xa9, 0x6a, 0xa0, 0xd7, 0xbf, 0x46, 0x4a, 0xe5, 0x8a, 0xf3, 0xd6, 0xd3, 0xbf, + 0x50, 0xf4, 0x3d, 0x4d, 0xbf, 0xa6, 0xc0, 0x5a, 0xf2, 0x0e, 0x14, 0x94, 0x5d, 0xf5, 0xbd, 0x2f, + 0xbb, 0x77, 0x2d, 0xf5, 0x59, 0xf3, 0xe3, 0xff, 0xf7, 0x84, 0x7d, 0x4e, 0xee, 0x5a, 0xc3, 0x70, + 0x7c, 0x28, 0x83, 0xf4, 0xdf, 0xb4, 0x7d, 0x28, 0x3e, 0x2c, 0xe2, 0xbb, 0x73, 0x97, 0x17, 0xeb, + 0xe3, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xc7, 0x63, 0x8c, 0x6a, 0x8b, 0x06, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -776,7 +776,7 @@ type MoleculaClient interface { QuerySQL(ctx context.Context, in *proto1.QuerySQLRequest, opts ...grpc.CallOption) (Molecula_QuerySQLClient, error) QuerySQLUnary(ctx context.Context, in *proto1.QuerySQLRequest, opts ...grpc.CallOption) (*proto1.TableResponse, error) QueryPQL(ctx context.Context, in *QueryPQLRequest, opts ...grpc.CallOption) (Molecula_QueryPQLClient, error) - QueryPQLUnary(ctx context.Context, in *proto1.QueryPQLRequest, opts ...grpc.CallOption) (*proto1.TableResponse, error) + QueryPQLUnary(ctx context.Context, in *QueryPQLRequest, opts ...grpc.CallOption) (*proto1.TableResponse, error) Inspect(ctx context.Context, in *InspectRequest, opts ...grpc.CallOption) (Molecula_InspectClient, error) } @@ -897,7 +897,7 @@ func (x *moleculaQueryPQLClient) Recv() (*proto1.RowResponse, error) { return m, nil } -func (c *moleculaClient) QueryPQLUnary(ctx context.Context, in *proto1.QueryPQLRequest, opts ...grpc.CallOption) (*proto1.TableResponse, error) { +func (c *moleculaClient) QueryPQLUnary(ctx context.Context, in *QueryPQLRequest, opts ...grpc.CallOption) (*proto1.TableResponse, error) { out := new(proto1.TableResponse) err := c.cc.Invoke(ctx, "/vdsm.Molecula/QueryPQLUnary", in, out, opts...) if err != nil { @@ -947,7 +947,7 @@ type MoleculaServer interface { QuerySQL(*proto1.QuerySQLRequest, Molecula_QuerySQLServer) error QuerySQLUnary(context.Context, *proto1.QuerySQLRequest) (*proto1.TableResponse, error) QueryPQL(*QueryPQLRequest, Molecula_QueryPQLServer) error - QueryPQLUnary(context.Context, *proto1.QueryPQLRequest) (*proto1.TableResponse, error) + QueryPQLUnary(context.Context, *QueryPQLRequest) (*proto1.TableResponse, error) Inspect(*InspectRequest, Molecula_InspectServer) error } @@ -976,7 +976,7 @@ func (*UnimplementedMoleculaServer) QuerySQLUnary(ctx context.Context, req *prot func (*UnimplementedMoleculaServer) QueryPQL(req *QueryPQLRequest, srv Molecula_QueryPQLServer) error { return status.Errorf(codes.Unimplemented, "method QueryPQL not implemented") } -func (*UnimplementedMoleculaServer) QueryPQLUnary(ctx context.Context, req *proto1.QueryPQLRequest) (*proto1.TableResponse, error) { +func (*UnimplementedMoleculaServer) QueryPQLUnary(ctx context.Context, req *QueryPQLRequest) (*proto1.TableResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method QueryPQLUnary not implemented") } func (*UnimplementedMoleculaServer) Inspect(req *InspectRequest, srv Molecula_InspectServer) error { @@ -1120,7 +1120,7 @@ func (x *moleculaQueryPQLServer) Send(m *proto1.RowResponse) error { } func _Molecula_QueryPQLUnary_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(proto1.QueryPQLRequest) + in := new(QueryPQLRequest) if err := dec(in); err != nil { return nil, err } @@ -1132,7 +1132,7 @@ func _Molecula_QueryPQLUnary_Handler(srv interface{}, ctx context.Context, dec f FullMethod: "/vdsm.Molecula/QueryPQLUnary", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(MoleculaServer).QueryPQLUnary(ctx, req.(*proto1.QueryPQLRequest)) + return srv.(MoleculaServer).QueryPQLUnary(ctx, req.(*QueryPQLRequest)) } return interceptor(ctx, in, info, handler) } diff --git a/proto/vdsm/vdsm.proto b/proto/vdsm/vdsm.proto index 0755fd8ee..8222a2eb8 100644 --- a/proto/vdsm/vdsm.proto +++ b/proto/vdsm/vdsm.proto @@ -84,6 +84,6 @@ service Molecula { rpc QuerySQL(pilosa.QuerySQLRequest) returns (stream pilosa.RowResponse) {}; rpc QuerySQLUnary(pilosa.QuerySQLRequest) returns (pilosa.TableResponse) {}; rpc QueryPQL(QueryPQLRequest) returns (stream pilosa.RowResponse) {}; - rpc QueryPQLUnary(pilosa.QueryPQLRequest) returns (pilosa.TableResponse) {}; + rpc QueryPQLUnary(QueryPQLRequest) returns (pilosa.TableResponse) {}; rpc Inspect(InspectRequest) returns (stream pilosa.RowResponse) {}; } diff --git a/server/grpc.go b/server/grpc.go index 649d29b86..b25b48e84 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -338,8 +338,9 @@ func (h *VDSMGRPCHandler) QueryPQL(req *vdsm_pb.QueryPQLRequest, srv vdsm_pb.Mol return h.grpcHandler.QueryPQL(preq, srv) } -func (h *VDSMGRPCHandler) QueryPQLUnary(ctx context.Context, req *pb.QueryPQLRequest) (*pb.TableResponse, error) { - return h.grpcHandler.QueryPQLUnary(ctx, req) +func (h *VDSMGRPCHandler) QueryPQLUnary(ctx context.Context, req *vdsm_pb.QueryPQLRequest) (*pb.TableResponse, error) { + preq := &pb.QueryPQLRequest{Index: req.Vds, Pql: req.Pql} + return h.grpcHandler.QueryPQLUnary(ctx, preq) } func (h *VDSMGRPCHandler) Inspect(req *vdsm_pb.InspectRequest, srv vdsm_pb.Molecula_InspectServer) error { From 8da15b18d421a687dadeb786d0eea45333ff109f Mon Sep 17 00:00:00 2001 From: Jason Aten Date: Fri, 9 Oct 2020 11:14:41 -0500 Subject: [PATCH 48/53] pilosa-fsck: parallelize Index.ComputeTranslatorSummary - add -index option --- cmd/pilosa-fsck/fsck.go | 50 ++++++++++++--- index.go | 134 +++++++++++++++++++++++----------------- 2 files changed, 118 insertions(+), 66 deletions(-) diff --git a/cmd/pilosa-fsck/fsck.go b/cmd/pilosa-fsck/fsck.go index 2bb6b892e..240276268 100644 --- a/cmd/pilosa-fsck/fsck.go +++ b/cmd/pilosa-fsck/fsck.go @@ -52,15 +52,15 @@ type FsckConfig struct { Fix bool // -fix FixCol bool // -fixcol - Colkeydump bool // -col + Colkeydump bool // -col + JustThisIndex string // -index // -col column key dump only options: - Dir string - Index string - PartitionID int - ShowHeader bool - ShowKey bool - ShowID bool + // Dir string + // PartitionID int + // ShowHeader bool + // ShowKey bool + // ShowID bool // not flags, just the Args() left after all other flags. Should be the list // of pilosa (holder) directories for the cluster. @@ -87,6 +87,8 @@ func (cfg *FsckConfig) DefineFlags(fs *flag.FlagSet) { fs.StringVar(&cfg.PilosaConfigPath, "config", "", "(required: -replicas or -config, with -config preferred) path to the pilosa.conf for the cluster (e.g. /etc/pilosa.conf)") + fs.StringVar(&cfg.JustThisIndex, "index", "", "(optional) restrict to just this index. Otherwise we default to all indexes.") + fs.Usage = func() { fmt.Fprintf(os.Stderr, "pilosa-fsck version: %v\n\n", pilosa.VersionInfo()) fmt.Fprintf(os.Stderr, `Use: pilosa-fsck -replicas R {-fix} {-q} /backup/1/.pilosa /backup/2/.pilosa ... /backup/N/.pilosa @@ -99,6 +101,9 @@ func (cfg *FsckConfig) DefineFlags(fs *flag.FlagSet) { the number of replicas maintained in the cluster. Must be the same as the [cluster] 'replicas = R' entry shared across all the pilosa.conf files on each node. + -index index_name + (optional) restrict to just this index. Otherwise we default to all indexes. + -q be very quiet during analysis and repair @@ -323,9 +328,9 @@ func main() { func (cfg *FsckConfig) Run() (fixNeeded bool, err error) { - if cfg.Colkeydump { - cfg.dumpcols() - } + // if cfg.Colkeydump { + // cfg.dumpcols() + //} perNodeIndexMaps, clusterNodes, ats, err := cfg.read() if err != nil { @@ -398,6 +403,10 @@ func (cfg *FsckConfig) RepairTranslationStores(ats *pilosa.AllTranslatorSummary) for _, index := range indexes { + if !cfg.DoingIndex(index) { + continue + } + m := make(map[int]*group) for _, sum := range ats.Sums { @@ -474,6 +483,7 @@ func (cfg *FsckConfig) RepairTranslationStores(ats *pilosa.AllTranslatorSummary) return nil } +/* func (cfg *FsckConfig) dumpcols() { verbose := cfg.Verbose @@ -551,6 +561,7 @@ func (cfg *FsckConfig) dumpcols() { } } } +*/ func (cfg *FsckConfig) read() (perNodeIndexMaps []map[string]*pilosa.IndexFragmentSummary, clusterNodes []string, final *pilosa.AllTranslatorSummary, err error) { @@ -615,6 +626,13 @@ func (cfg *FsckConfig) readOneDir(dir string) (idx2frag map[string]*pilosa.Index const checkKeys = true atsNode = pilosa.NewAllTranslatorSummary() for _, idx := range holder.Indexes() { + + if !cfg.DoingIndex(idx.Name()) { + continue + } + + //vv("calling idx.ComputeTranslatorSummary(verbose, checkKeys=%v, cfg.FixCol='%v')", checkKeys, cfg.FixCol) + asum, err := idx.ComputeTranslatorSummary(verbose, checkKeys, cfg.FixCol, topo, nodeID) if err != nil { log.Fatal(err) @@ -666,6 +684,18 @@ func (cfg *FsckConfig) readOneDir(dir string) (idx2frag map[string]*pilosa.Index return } +func (cfg *FsckConfig) DoingIndex(index string) bool { + if cfg.JustThisIndex == "" { + // scan all indexes + return true + } + if index == cfg.JustThisIndex { + // scan just this one + return true + } + return false +} + // from cluster.go:1924 func loadTopology(holderDir string, hasher pilosa.Hasher, partitionN, replicaN int) (*pilosa.Topology, error) { diff --git a/index.go b/index.go index 56adbd91b..b5c652c1c 100644 --- a/index.go +++ b/index.go @@ -792,84 +792,106 @@ func (i *Index) ComputeTranslatorSummary(verbose, checkKeys, applyKeyRepairs boo defer i.mu.RUnlock() ats = &AllTranslatorSummary{} + var atsMu sync.Mutex if verbose { fmt.Printf("\n# index: %v\n# =================\n", i.name) } + + var g errgroup.Group + for _, fld := range i.fields { - sum, err := fld.translateStore.ComputeTranslatorSummaryRows() - if err != nil { - return ats, err - } - sum.Field = fld.name - sum.Index = i.Name() - sum.Checksum = hash.Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, fld.name, i.Name()))) - sum.IsColKey = false - if verbose { - fmt.Printf("# row blake3-%v keyN: %5v idN: %5v field: '%v'\n", sum.Checksum, sum.KeyCount, sum.IDCount, fld.name) - } - ats.Sums = append(ats.Sums, sum) + fld := fld + g.Go(func() error { + //vv("g.Go() on fld '%v'", fld.name) + sum, err := fld.translateStore.ComputeTranslatorSummaryRows() + if err != nil { + return err + } + sum.Field = fld.name + sum.Index = i.Name() + sum.Checksum = hash.Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, fld.name, i.Name()))) + sum.IsColKey = false + if verbose { + fmt.Printf("# row blake3-%v keyN: %5v idN: %5v field: '%v'\n", sum.Checksum, sum.KeyCount, sum.IDCount, fld.name) + } + atsMu.Lock() + ats.Sums = append(ats.Sums, sum) + atsMu.Unlock() + return nil + }) } if verbose { fmt.Printf("# ====================\n") } for partitionID, store := range i.translateStores { - if checkKeys { - prim := topo.PrimaryNodeIndex(partitionID) - primID := topo.nodeIDs[prim] + partitionID := partitionID + store := store + g.Go(func() error { + //vv("g.Go() running on store.Path = '%v'", store.GetStorePath()) + if checkKeys { + prim := topo.PrimaryNodeIndex(partitionID) + primID := topo.nodeIDs[prim] - // note: we fix irrespective of nodeID == primID now, so that we - // get a fine grain report of what maps were off. + // note: we fix irrespective of nodeID == primID now, so that we + // get a fine grain report of what maps were off. - if verbose { - // This is pilosa-fsck output, not regular log. - fmt.Printf("# doing analysis of keys on nodeID '%v', and primID '%v'\n", nodeID, primID) + if verbose { + // This is pilosa-fsck output, not regular log. + fmt.Printf("# doing analysis of keys on nodeID '%v', and primID '%v'\n", nodeID, primID) + } + changed, err := store.RepairKeys(topo, verbose, applyKeyRepairs) + if err != nil { + return errors.Wrap(err, "ComputeTranslatorSummary() call to store.Repair()") + } + if changed { + atsMu.Lock() + ats.RepairNeeded = true + atsMu.Unlock() + } } - changed, err := store.RepairKeys(topo, verbose, applyKeyRepairs) + + // key repair has to be above, because we compute the checksum below. + + sum, err := store.ComputeTranslatorSummaryCols(partitionID, topo) if err != nil { - return nil, errors.Wrap(err, "ComputeTranslatorSummary() call to store.Repair()") + return err } - if changed { - ats.RepairNeeded = true + if sum == nil { + // probably one of the Noop stores from the tests. + return nil } - } + sum.IsColKey = true + sum.PartitionID = partitionID + sum.Index = i.Name() + sum.StorePath = store.GetStorePath() + sum.NodeID = nodeID + sum.IsPrimary = topo.IsPrimary(nodeID, partitionID) - // key repair has to be above, because we compute the checksum below. - - sum, err := store.ComputeTranslatorSummaryCols(partitionID, topo) - if err != nil { - return ats, err - } - if sum == nil { - // probably one of the Noop stores from the tests. - continue - } - sum.IsColKey = true - sum.PartitionID = partitionID - sum.Index = i.Name() - sum.StorePath = store.GetStorePath() - sum.NodeID = nodeID - sum.IsPrimary = topo.IsPrimary(nodeID, partitionID) - - replicas := topo.GetNonPrimaryReplicas(partitionID) - for _, replica := range replicas { - if nodeID == replica { - sum.IsReplica = true - break + replicas := topo.GetNonPrimaryReplicas(partitionID) + for _, replica := range replicas { + if nodeID == replica { + sum.IsReplica = true + break + } } - } - sum.Checksum = hash.Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, partitionID, i.Name()))) - if verbose { - // This is not regular index logging. This is output of the pilosa-fsck tool. - // So it must be printing straight to stdout. - fmt.Printf("# col blake3-%v keyN: %10v idN: %10v paritionID: %03v primary: %03v\n", sum.Checksum, sum.KeyCount, sum.IDCount, partitionID, sum.PrimaryNodeIndex) - } + sum.Checksum = hash.Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, partitionID, i.Name()))) + if verbose { + // This is not regular index logging. This is output of the pilosa-fsck tool. + // So it must be printing straight to stdout. + fmt.Printf("# col blake3-%v keyN: %10v idN: %10v paritionID: %03v primary: %03v\n", sum.Checksum, sum.KeyCount, sum.IDCount, partitionID, sum.PrimaryNodeIndex) + } + atsMu.Lock() + ats.Sums = append(ats.Sums, sum) + atsMu.Unlock() - ats.Sums = append(ats.Sums, sum) + return nil + }) } - return + err = g.Wait() + return ats, err } // returned by WriteFragmentChecksums From bfe570ef9d98f778f759073dbdda19f7a742b48f Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Fri, 9 Oct 2020 12:20:19 -0600 Subject: [PATCH 49/53] Fix deadlock in importWorker() This commit wraps the work for a view in a function so that the defer on the finisher executes after each view is processed instead of at the end. --- api.go | 55 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/api.go b/api.go index 9f4190535..35d81db7f 100644 --- a/api.go +++ b/api.go @@ -378,37 +378,42 @@ func importWorker(importWork chan importJob) { } } - tx, finisher := j.qcx.GetTx(Txo{Write: writable, Index: j.field.idx, Shard: j.shard}) - defer finisher(&err0) + if err := func() error { + tx, finisher := j.qcx.GetTx(Txo{Write: writable, Index: j.field.idx, Shard: j.shard}) + defer finisher(&err0) - var doClear bool - switch doAction { - case RequestActionOverwrite: - err := j.field.importRoaringOverwrite(j.ctx, tx, viewData, j.shard, viewName, j.req.Block) - if err != nil { - return errors.Wrap(err, "importing roaring as overwrite") - } - case RequestActionClear: - doClear = true - fallthrough - case RequestActionSet: - fileMagic := uint32(binary.LittleEndian.Uint16(viewData[0:2])) - if fileMagic == roaring.MagicNumber { // if pilosa roaring format - err := j.field.importRoaring(j.ctx, tx, viewData, j.shard, viewName, doClear) + var doClear bool + switch doAction { + case RequestActionOverwrite: + err := j.field.importRoaringOverwrite(j.ctx, tx, viewData, j.shard, viewName, j.req.Block) if err != nil { - return errors.Wrap(err, "importing pilosa roaring") + return errors.Wrap(err, "importing roaring as overwrite") } - } else { - // must make a copy of data to operate on locally on standard roaring format. - // field.importRoaring changes the standard roaring run format to pilosa roaring - data := make([]byte, len(viewData)) - copy(data, viewData) - err := j.field.importRoaring(j.ctx, tx, data, j.shard, viewName, doClear) + case RequestActionClear: + doClear = true + fallthrough + case RequestActionSet: + fileMagic := uint32(binary.LittleEndian.Uint16(viewData[0:2])) + if fileMagic == roaring.MagicNumber { // if pilosa roaring format + err := j.field.importRoaring(j.ctx, tx, viewData, j.shard, viewName, doClear) + if err != nil { + return errors.Wrap(err, "importing pilosa roaring") + } + } else { + // must make a copy of data to operate on locally on standard roaring format. + // field.importRoaring changes the standard roaring run format to pilosa roaring + data := make([]byte, len(viewData)) + copy(data, viewData) + err := j.field.importRoaring(j.ctx, tx, data, j.shard, viewName, doClear) - if err != nil { - return errors.Wrap(err, "importing standard roaring") + if err != nil { + return errors.Wrap(err, "importing standard roaring") + } } } + return nil + }(); err != nil { + return err } } return nil From 9b83b71c8f627c4f5abfa1fa5976fb466ef53fdb Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Mon, 12 Oct 2020 11:28:18 -0600 Subject: [PATCH 50/53] Fix error check reference in importWorker() --- api.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api.go b/api.go index 35d81db7f..dfbd33e01 100644 --- a/api.go +++ b/api.go @@ -378,9 +378,9 @@ func importWorker(importWork chan importJob) { } } - if err := func() error { + if err := func() (err1 error) { tx, finisher := j.qcx.GetTx(Txo{Write: writable, Index: j.field.idx, Shard: j.shard}) - defer finisher(&err0) + defer finisher(&err1) var doClear bool switch doAction { From 676208ea73f8b976a0648e8eed4bab111d1f2b14 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Mon, 12 Oct 2020 12:32:58 -0600 Subject: [PATCH 51/53] Add test to reproduce deadlock --- http/client_test.go | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/http/client_test.go b/http/client_test.go index cfe51bde4..b1b154647 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -588,6 +588,42 @@ func TestClient_ImportRoaring(t *testing.T) { } } +// Ensure client can bulk import data with multiple views and not deadlock. +func TestClient_ImportRoaring_MultiView(t *testing.T) { + cluster := test.MustNewCluster(t, 2) + for _, c := range cluster.Nodes { + c.Config.Cluster.ReplicaN = 2 + } + err := cluster.Start() + if err != nil { + t.Fatalf("starting cluster: %v", err) + } + defer cluster.Close() + + _, err = cluster.GetNode(0).API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + _, err = cluster.GetNode(0).API.CreateField(context.Background(), "i", "f", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 100)) + if err != nil { + t.Fatalf("creating field: %v", err) + } + _, err = cluster.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: "Set(0, f=1)"}) + if err != nil { + t.Fatalf("querying: %v", err) + } + + // Send import request. + host := cluster.GetNode(0).URL() + c := MustNewClient(host, http.GetHTTPClient(nil)) + req := &pilosa.ImportRoaringRequest{Views: map[string][]byte{}} + req.Views["a"], _ = hex.DecodeString("3B3001000100000900010000000100010009000100") + req.Views["b"], _ = hex.DecodeString("3B3001000100000900010000000100010009000100") + if err := c.ImportRoaring(context.Background(), &cluster.GetNode(0).API.Node().URI, "i", "f", 0, false, req); err != nil { + t.Fatal(err) + } +} + // Ensure client can bulk import data. func TestClient_ImportKeys(t *testing.T) { t.Run("SingleNode", func(t *testing.T) { From b3e6cdc0d8f4a334380ec93c2ff65626c9fd44f0 Mon Sep 17 00:00:00 2001 From: "Jason E. Aten" Date: Mon, 12 Oct 2020 17:44:40 -0500 Subject: [PATCH 52/53] pilosa/dbshard: allow deleteIndex and then re-use of index - fixes #843 --- dbshard.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dbshard.go b/dbshard.go index c2e0b3dbe..27bc4c6f9 100644 --- a/dbshard.go +++ b/dbshard.go @@ -320,6 +320,9 @@ func (per *DBPerShard) DeleteIndex(index string) (err error) { } } } + // allow the index to be created again anew. + delete(per.dbh.Index, index) + return } @@ -436,7 +439,7 @@ func (per *DBPerShard) GetDBShard(index string, shard uint64, idx *Index) (dbs * if len(per.types) == 1 && per.types[0] == roaringTxn { // roaring txn are nil/fake anyway. Don't freak out. } else { - panic(fmt.Sprintf("cannot retain closed dbs across holder ReOpen dbs='%p'", dbs)) + panic(fmt.Sprintf("cannot retain closed dbs across holder ReOpen dbs='%p'; per.types[0]='%v'; len(per.types)=%v", dbs, per.types[0], len(per.types))) } } if !ok { From 50a8db854dc78e488d475f2d2b7c4c174909348a Mon Sep 17 00:00:00 2001 From: "Jason E. Aten" Date: Tue, 13 Oct 2020 17:50:16 -0500 Subject: [PATCH 53/53] pilosa-fsck: the -readers flag controls parallelism - add path info to the panic if we find a corrupt boltdb translation store. --- boltdb/translate.go | 9 +++ cmd/pilosa-chk/chk.go | 2 +- cmd/pilosa-fsck/fsck.go | 12 +++- cmd/pilosa-fsck/fsck_test.go | 5 +- go.mod | 1 + index.go | 103 ++++++++++++++++++++++++++++------- 6 files changed, 109 insertions(+), 23 deletions(-) diff --git a/boltdb/translate.go b/boltdb/translate.go index f5d2681ea..b5a0e5651 100644 --- a/boltdb/translate.go +++ b/boltdb/translate.go @@ -116,6 +116,15 @@ func NewTranslateStore(index, field string, partitionID, partitionN int) *Transl // Open opens the translate file. func (s *TranslateStore) Open() (err error) { + + // add the path to the problem database if we panic handling it. + defer func() { + r := recover() + if r != nil { + panic(fmt.Sprintf("pilosa/boltdb/TranslateStore.Open(s.Path='%v') panic with '%v'", s.Path, r)) + } + }() + if err := os.MkdirAll(filepath.Dir(s.Path), 0777); err != nil { return errors.Wrapf(err, "mkdir %s", filepath.Dir(s.Path)) } else if s.db, err = bolt.Open(s.Path, 0666, &bolt.Options{Timeout: 1 * time.Second}); err != nil { diff --git a/cmd/pilosa-chk/chk.go b/cmd/pilosa-chk/chk.go index 7e315700c..353b33b3b 100644 --- a/cmd/pilosa-chk/chk.go +++ b/cmd/pilosa-chk/chk.go @@ -79,7 +79,7 @@ func main() { const checkKeys = false const applyKeyRepairs = false for _, idx := range holder.Indexes() { - asum, err := idx.ComputeTranslatorSummary(verbose, checkKeys, applyKeyRepairs, nil, "fake-nodeID") + asum, err := idx.ComputeTranslatorSummary(verbose, checkKeys, applyKeyRepairs, nil, "fake-nodeID", 10) if err != nil { log.Fatal(err) } diff --git a/cmd/pilosa-fsck/fsck.go b/cmd/pilosa-fsck/fsck.go index 240276268..34ed071f8 100644 --- a/cmd/pilosa-fsck/fsck.go +++ b/cmd/pilosa-fsck/fsck.go @@ -73,6 +73,8 @@ type FsckConfig struct { ReplicaN int // -replicas PilosaConfigPath string // -config + ParallelReaders int // -readers + topo *pilosa.Topology } @@ -85,6 +87,8 @@ func (cfg *FsckConfig) DefineFlags(fs *flag.FlagSet) { fs.IntVar(&cfg.ReplicaN, "replicas", 0, "(required) manually entered replicaN; the number of replicas maintained in the cluster. Must be the same as the [cluster] 'replicas = R' entry in the pilosa.conf file for the cluster.") + fs.IntVar(&cfg.ParallelReaders, "readers", 10, "how many parallel readers to use to scan at once. 0 means do everything possible in parallel. 1 means serialize everything through a single reader. Can be adjusted to control memory consumption.") + fs.StringVar(&cfg.PilosaConfigPath, "config", "", "(required: -replicas or -config, with -config preferred) path to the pilosa.conf for the cluster (e.g. /etc/pilosa.conf)") fs.StringVar(&cfg.JustThisIndex, "index", "", "(optional) restrict to just this index. Otherwise we default to all indexes.") @@ -104,6 +108,12 @@ func (cfg *FsckConfig) DefineFlags(fs *flag.FlagSet) { -index index_name (optional) restrict to just this index. Otherwise we default to all indexes. + -readers PR + how many parallel readers to use to scan at once. PR==0 means do everything + possible in parallel. PR==1 means serialize everything through a single reader. + Adjust PR to control memory consumption if needed. As a practical limit, setting + PR > 10000 will have no effect. (default is 10). + -q be very quiet during analysis and repair @@ -633,7 +643,7 @@ func (cfg *FsckConfig) readOneDir(dir string) (idx2frag map[string]*pilosa.Index //vv("calling idx.ComputeTranslatorSummary(verbose, checkKeys=%v, cfg.FixCol='%v')", checkKeys, cfg.FixCol) - asum, err := idx.ComputeTranslatorSummary(verbose, checkKeys, cfg.FixCol, topo, nodeID) + asum, err := idx.ComputeTranslatorSummary(verbose, checkKeys, cfg.FixCol, topo, nodeID, cfg.ParallelReaders) if err != nil { log.Fatal(err) } diff --git a/cmd/pilosa-fsck/fsck_test.go b/cmd/pilosa-fsck/fsck_test.go index e6169c4f5..41e0d7a8c 100644 --- a/cmd/pilosa-fsck/fsck_test.go +++ b/cmd/pilosa-fsck/fsck_test.go @@ -201,8 +201,9 @@ func Test_Repair(t *testing.T) { FixCol: false, Quiet: true, //Verbose: true, - ReplicaN: nReplicas, - Dirs: dirs, + ReplicaN: nReplicas, + Dirs: dirs, + ParallelReaders: 5, } panicOn(cfg.ValidateConfig()) diff --git a/go.mod b/go.mod index 0425a1b03..6f3b9fa55 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361 github.com/dustin/go-humanize v1.0.0 + github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 github.com/glycerine/lmdb-go v1.9.32 github.com/go-ole/go-ole v1.2.4 // indirect github.com/gogo/protobuf v1.2.1 diff --git a/index.go b/index.go index b5c652c1c..44200b191 100644 --- a/index.go +++ b/index.go @@ -26,6 +26,7 @@ import ( "sync" "time" + "github.com/glycerine/idem" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/v2/hash" "github.com/pilosa/pilosa/v2/internal" @@ -787,30 +788,74 @@ func (ats *AllTranslatorSummary) Sort() { } // sums is only guaranteed to be sorted by (index, PartitionID, field) iff err returns nil -func (i *Index) ComputeTranslatorSummary(verbose, checkKeys, applyKeyRepairs bool, topo *Topology, nodeID string) (ats *AllTranslatorSummary, err error) { - i.mu.RLock() - defer i.mu.RUnlock() +func (idx *Index) ComputeTranslatorSummary(verbose, checkKeys, applyKeyRepairs bool, topo *Topology, nodeID string, parallelReaders int) (ats *AllTranslatorSummary, err error) { + idx.mu.RLock() + defer idx.mu.RUnlock() ats = &AllTranslatorSummary{} var atsMu sync.Mutex if verbose { - fmt.Printf("\n# index: %v\n# =================\n", i.name) + fmt.Printf("\n# index: %v\n# =================\n", idx.name) } - var g errgroup.Group + jobQ := make(chan func() error, 10000) + var errmu sync.Mutex - for _, fld := range i.fields { + if parallelReaders < 1 { + // turn it up to 11 + parallelReaders = 10000 + } + + halters := make([]*idem.Halter, parallelReaders) + for j := 0; j < parallelReaders; j++ { + h := idem.NewHalter() + halters[j] = h + } + for _, h := range halters { + go func(h *idem.Halter) { + defer h.MarkDone() + for { + select { + case <-h.ReqStop.Chan: + return + case f, ok := <-jobQ: + if !ok || f == nil { + // channel closed, finish up + return + } + + err1 := f() + if err1 != nil { + errmu.Lock() + if err == nil { + err = err1 + } + errmu.Unlock() + // an error occurred, tell everyone to stop + for _, h2 := range halters { + h2.RequestStop() + } + return + } + } + } + }(h) + } + +floop: + for _, fld := range idx.fields { fld := fld - g.Go(func() error { - //vv("g.Go() on fld '%v'", fld.name) + + fun := func() error { + //vv("ComputeTranslatorSummary() on fld '%v'", fld.name) sum, err := fld.translateStore.ComputeTranslatorSummaryRows() if err != nil { return err } sum.Field = fld.name - sum.Index = i.Name() - sum.Checksum = hash.Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, fld.name, i.Name()))) + sum.Index = idx.Name() + sum.Checksum = hash.Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, fld.name, idx.Name()))) sum.IsColKey = false if verbose { fmt.Printf("# row blake3-%v keyN: %5v idN: %5v field: '%v'\n", sum.Checksum, sum.KeyCount, sum.IDCount, fld.name) @@ -819,17 +864,26 @@ func (i *Index) ComputeTranslatorSummary(verbose, checkKeys, applyKeyRepairs boo ats.Sums = append(ats.Sums, sum) atsMu.Unlock() return nil - }) - } + } + + select { + case <-halters[0].ReqStop.Chan: + break floop + case jobQ <- fun: + } + } // end floop + if verbose { fmt.Printf("# ====================\n") } - for partitionID, store := range i.translateStores { +tloop: + for partitionID, store := range idx.translateStores { partitionID := partitionID store := store - g.Go(func() error { - //vv("g.Go() running on store.Path = '%v'", store.GetStorePath()) + + fun2 := func() error { + //vv("ComputeTranslatorSummary() running on store.Path = '%v'", store.GetStorePath()) if checkKeys { prim := topo.PrimaryNodeIndex(partitionID) primID := topo.nodeIDs[prim] @@ -864,7 +918,7 @@ func (i *Index) ComputeTranslatorSummary(verbose, checkKeys, applyKeyRepairs boo } sum.IsColKey = true sum.PartitionID = partitionID - sum.Index = i.Name() + sum.Index = idx.Name() sum.StorePath = store.GetStorePath() sum.NodeID = nodeID sum.IsPrimary = topo.IsPrimary(nodeID, partitionID) @@ -877,7 +931,7 @@ func (i *Index) ComputeTranslatorSummary(verbose, checkKeys, applyKeyRepairs boo } } - sum.Checksum = hash.Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, partitionID, i.Name()))) + sum.Checksum = hash.Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, partitionID, idx.Name()))) if verbose { // This is not regular index logging. This is output of the pilosa-fsck tool. // So it must be printing straight to stdout. @@ -888,9 +942,20 @@ func (i *Index) ComputeTranslatorSummary(verbose, checkKeys, applyKeyRepairs boo atsMu.Unlock() return nil - }) + } + select { + case <-halters[0].ReqStop.Chan: + break tloop + case jobQ <- fun2: + } + } // end tloop + + close(jobQ) // tell the workers no more jobs. + + // wait for everyone to finish + for _, h := range halters { + <-h.Done.Chan } - err = g.Wait() return ats, err }