From 9c05155db4ace6e922cc6f02a659fafc7b8d8871 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 21 Nov 2018 16:35:50 +0300 Subject: [PATCH 001/125] Added /internal/translate/keys endpoint --- api.go | 30 ++ encoding/proto/proto.go | 44 +++ handler.go | 10 + http/handler.go | 28 +- internal/public.pb.go | 645 +++++++++++++++++++++++++++++++++++----- internal/public.proto | 10 + server/handler_test.go | 88 +++++- 7 files changed, 781 insertions(+), 74 deletions(-) diff --git a/api.go b/api.go index 7a95ce76e..20a7e7bb3 100644 --- a/api.go +++ b/api.go @@ -1071,6 +1071,36 @@ func (api *API) Info() serverInfo { } } +func (api *API) TranslateKeys(body io.Reader) ([]byte, error) { + reqBytes, err := ioutil.ReadAll(body) + if err != nil { + return nil, NewBadRequestError(errors.Wrap(err, "read body error")) + } + var req TranslateKeysRequest + if err := api.Serializer.Unmarshal(reqBytes, &req); err != nil { + return nil, NewBadRequestError(errors.Wrap(err, "unmarshal body error")) + } + var ids []uint64 + if req.Field == "" { + ids, err = api.holder.translateFile.TranslateColumnsToUint64(req.Index, req.Keys) + } else { + ids, err = api.holder.translateFile.TranslateRowsToUint64(req.Index, req.Field, req.Keys) + } + if err != nil { + return nil, err + } + + resp := TranslateKeysResponse{ + IDs: ids, + } + // Encode response. + buf, err := api.Serializer.Marshal(&resp) + if err != nil { + return nil, errors.Wrap(err, "translate keys response encoding error") + } + return buf, nil +} + type serverInfo struct { ShardWidth uint64 `json:"shardWidth"` } diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 9826c7ef9..6d8a8e409 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -241,6 +241,22 @@ func (Serializer) Unmarshal(buf []byte, m pilosa.Message) error { } decodeBlockDataResponse(msg, mt) return nil + case *pilosa.TranslateKeysRequest: + msg := &internal.TranslateKeysRequest{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling TranslateKeysRequest") + } + decodeTranslateKeysRequest(msg, mt) + return nil + case *pilosa.TranslateKeysResponse: + msg := &internal.TranslateKeysResponse{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling TranslateKeysResponse") + } + decodeTranslateKeysResponse(msg, mt) + return nil default: panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m)) } @@ -298,6 +314,10 @@ func encodeToProto(m pilosa.Message) proto.Message { return encodeBlockDataRequest(mt) case *pilosa.BlockDataResponse: return encodeBlockDataResponse(mt) + case *pilosa.TranslateKeysRequest: + return encodeTranslateKeysRequest(mt) + case *pilosa.TranslateKeysResponse: + return encodeTranslateKeysResponse(mt) } return nil } @@ -669,6 +689,20 @@ func encodeRecalculateCaches(*pilosa.RecalculateCaches) *internal.RecalculateCac return &internal.RecalculateCaches{} } +func encodeTranslateKeysResponse(response *pilosa.TranslateKeysResponse) *internal.TranslateKeysResponse { + return &internal.TranslateKeysResponse{ + IDs: response.IDs, + } +} + +func encodeTranslateKeysRequest(request *pilosa.TranslateKeysRequest) *internal.TranslateKeysRequest { + return &internal.TranslateKeysRequest{ + Index: request.Index, + Field: request.Field, + Keys: request.Keys, + } +} + func decodeResizeInstruction(ri *internal.ResizeInstruction, m *pilosa.ResizeInstruction) { m.JobID = ri.JobID m.Node = &pilosa.Node{} @@ -964,6 +998,16 @@ func decodeQueryResults(pb []*internal.QueryResult, m []interface{}) { } } +func decodeTranslateKeysRequest(pb *internal.TranslateKeysRequest, m *pilosa.TranslateKeysRequest) { + m.Index = pb.Index + m.Field = pb.Field + m.Keys = pb.Keys +} + +func decodeTranslateKeysResponse(pb *internal.TranslateKeysResponse, m *pilosa.TranslateKeysResponse) { + m.IDs = pb.IDs +} + // QueryResult types. const ( queryResultTypeNil uint32 = iota diff --git a/handler.go b/handler.go index 9fc3af368..a8fe5528a 100644 --- a/handler.go +++ b/handler.go @@ -112,3 +112,13 @@ type BlockDataResponse struct { RowIDs []uint64 ColumnIDs []uint64 } + +type TranslateKeysRequest struct { + Index string + Field string + Keys []string +} + +type TranslateKeysResponse struct { + IDs []uint64 +} diff --git a/http/handler.go b/http/handler.go index 1ecbd2fb1..1d52d95ad 100644 --- a/http/handler.go +++ b/http/handler.go @@ -24,9 +24,8 @@ import ( "io/ioutil" "net" "net/http" - "net/url" - // Imported for its side-effect of registering pprof endpoints with the server. _ "net/http/pprof" + "net/url" // Imported for its side-effect of registering pprof endpoints with the server. "reflect" "runtime/debug" "strconv" @@ -37,7 +36,6 @@ import ( "github.com/gorilla/mux" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/logger" - "github.com/pkg/errors" ) @@ -199,6 +197,7 @@ func (h *Handler) populateValidators() { h.validators["GetNodes"] = queryValidationSpecRequired() h.validators["GetShardMax"] = queryValidationSpecRequired() h.validators["GetTranslateData"] = queryValidationSpecRequired("offset") + h.validators["PostTranslateKeys"] = queryValidationSpecRequired() } func (h *Handler) queryArgValidator(next http.Handler) http.Handler { @@ -260,6 +259,7 @@ func newRouter(handler *Handler) *mux.Router { router.HandleFunc("/internal/nodes", handler.handleGetNodes).Methods("GET").Name("GetNodes") router.HandleFunc("/internal/shards/max", handler.handleGetShardsMax).Methods("GET").Name("GetShardsMax") // TODO: deprecate, but it's being used by the client router.HandleFunc("/internal/translate/data", handler.handleGetTranslateData).Methods("GET").Name("GetTranslateData") + router.HandleFunc("/internal/translate/keys", handler.handlePostTranslateKeys).Methods("POST").Name("PostTranslateKeys") router.Use(handler.queryArgValidator) return router @@ -1549,3 +1549,25 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request h.logger.Printf("writing import-roaring response: %v", err) } } + +func (h *Handler) handlePostTranslateKeys(w http.ResponseWriter, r *http.Request) { + // Verify that request is only communicating over protobufs. + if r.Header.Get("Content-Type") != "application/x-protobuf" { + http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) + return + } else if r.Header.Get("Accept") != "application/x-protobuf" { + http.Error(w, "Not acceptable", http.StatusNotAcceptable) + return + } + + buf, err := h.api.TranslateKeys(r.Body) + if err != nil { + http.Error(w, fmt.Sprintf("translate keys: %v", err), http.StatusInternalServerError) + } + + // Write response. + _, err = w.Write(buf) + if err != nil { + h.logger.Printf("writing translate keys response: %v", err) + } +} diff --git a/internal/public.pb.go b/internal/public.pb.go index 8d78db985..167247854 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -35,7 +35,7 @@ func (m *Row) Reset() { *m = Row{} } func (m *Row) String() string { return proto.CompactTextString(m) } func (*Row) ProtoMessage() {} func (*Row) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{0} + return fileDescriptor_public_4fd446cb72375657, []int{0} } func (m *Row) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -97,7 +97,7 @@ func (m *RowIdentifiers) Reset() { *m = RowIdentifiers{} } func (m *RowIdentifiers) String() string { return proto.CompactTextString(m) } func (*RowIdentifiers) ProtoMessage() {} func (*RowIdentifiers) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{1} + return fileDescriptor_public_4fd446cb72375657, []int{1} } func (m *RowIdentifiers) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -153,7 +153,7 @@ func (m *Pair) Reset() { *m = Pair{} } func (m *Pair) String() string { return proto.CompactTextString(m) } func (*Pair) ProtoMessage() {} func (*Pair) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{2} + return fileDescriptor_public_4fd446cb72375657, []int{2} } func (m *Pair) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -215,7 +215,7 @@ func (m *FieldRow) Reset() { *m = FieldRow{} } func (m *FieldRow) String() string { return proto.CompactTextString(m) } func (*FieldRow) ProtoMessage() {} func (*FieldRow) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{3} + return fileDescriptor_public_4fd446cb72375657, []int{3} } func (m *FieldRow) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -270,7 +270,7 @@ func (m *GroupCount) Reset() { *m = GroupCount{} } func (m *GroupCount) String() string { return proto.CompactTextString(m) } func (*GroupCount) ProtoMessage() {} func (*GroupCount) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{4} + return fileDescriptor_public_4fd446cb72375657, []int{4} } func (m *GroupCount) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -325,7 +325,7 @@ func (m *ValCount) Reset() { *m = ValCount{} } func (m *ValCount) String() string { return proto.CompactTextString(m) } func (*ValCount) ProtoMessage() {} func (*ValCount) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{5} + return fileDescriptor_public_4fd446cb72375657, []int{5} } func (m *ValCount) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -381,7 +381,7 @@ func (m *Bit) Reset() { *m = Bit{} } func (m *Bit) String() string { return proto.CompactTextString(m) } func (*Bit) ProtoMessage() {} func (*Bit) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{6} + return fileDescriptor_public_4fd446cb72375657, []int{6} } func (m *Bit) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -444,7 +444,7 @@ func (m *ColumnAttrSet) Reset() { *m = ColumnAttrSet{} } func (m *ColumnAttrSet) String() string { return proto.CompactTextString(m) } func (*ColumnAttrSet) ProtoMessage() {} func (*ColumnAttrSet) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{7} + return fileDescriptor_public_4fd446cb72375657, []int{7} } func (m *ColumnAttrSet) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -510,7 +510,7 @@ func (m *Attr) Reset() { *m = Attr{} } func (m *Attr) String() string { return proto.CompactTextString(m) } func (*Attr) ProtoMessage() {} func (*Attr) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{8} + return fileDescriptor_public_4fd446cb72375657, []int{8} } func (m *Attr) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -592,7 +592,7 @@ func (m *AttrMap) Reset() { *m = AttrMap{} } func (m *AttrMap) String() string { return proto.CompactTextString(m) } func (*AttrMap) ProtoMessage() {} func (*AttrMap) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{9} + return fileDescriptor_public_4fd446cb72375657, []int{9} } func (m *AttrMap) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -644,7 +644,7 @@ func (m *QueryRequest) Reset() { *m = QueryRequest{} } func (m *QueryRequest) String() string { return proto.CompactTextString(m) } func (*QueryRequest) ProtoMessage() {} func (*QueryRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{10} + return fileDescriptor_public_4fd446cb72375657, []int{10} } func (m *QueryRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -728,7 +728,7 @@ func (m *QueryResponse) Reset() { *m = QueryResponse{} } func (m *QueryResponse) String() string { return proto.CompactTextString(m) } func (*QueryResponse) ProtoMessage() {} func (*QueryResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{11} + return fileDescriptor_public_4fd446cb72375657, []int{11} } func (m *QueryResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -797,7 +797,7 @@ func (m *QueryResult) Reset() { *m = QueryResult{} } func (m *QueryResult) String() string { return proto.CompactTextString(m) } func (*QueryResult) ProtoMessage() {} func (*QueryResult) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{12} + return fileDescriptor_public_4fd446cb72375657, []int{12} } func (m *QueryResult) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -907,7 +907,7 @@ func (m *ImportRequest) Reset() { *m = ImportRequest{} } func (m *ImportRequest) String() string { return proto.CompactTextString(m) } func (*ImportRequest) ProtoMessage() {} func (*ImportRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{13} + return fileDescriptor_public_4fd446cb72375657, []int{13} } func (m *ImportRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1008,7 +1008,7 @@ func (m *ImportValueRequest) Reset() { *m = ImportValueRequest{} } func (m *ImportValueRequest) String() string { return proto.CompactTextString(m) } func (*ImportValueRequest) ProtoMessage() {} func (*ImportValueRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{14} + return fileDescriptor_public_4fd446cb72375657, []int{14} } func (m *ImportValueRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1079,6 +1079,116 @@ func (m *ImportValueRequest) GetValues() []int64 { return nil } +type TranslateKeysRequest struct { + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Field string `protobuf:"bytes,2,opt,name=Field,proto3" json:"Field,omitempty"` + Keys []string `protobuf:"bytes,3,rep,name=Keys" json:"Keys,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *TranslateKeysRequest) Reset() { *m = TranslateKeysRequest{} } +func (m *TranslateKeysRequest) String() string { return proto.CompactTextString(m) } +func (*TranslateKeysRequest) ProtoMessage() {} +func (*TranslateKeysRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_public_4fd446cb72375657, []int{15} +} +func (m *TranslateKeysRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *TranslateKeysRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_TranslateKeysRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (dst *TranslateKeysRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_TranslateKeysRequest.Merge(dst, src) +} +func (m *TranslateKeysRequest) XXX_Size() int { + return m.Size() +} +func (m *TranslateKeysRequest) XXX_DiscardUnknown() { + xxx_messageInfo_TranslateKeysRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_TranslateKeysRequest proto.InternalMessageInfo + +func (m *TranslateKeysRequest) GetIndex() string { + if m != nil { + return m.Index + } + return "" +} + +func (m *TranslateKeysRequest) GetField() string { + if m != nil { + return m.Field + } + return "" +} + +func (m *TranslateKeysRequest) GetKeys() []string { + if m != nil { + return m.Keys + } + return nil +} + +type TranslateKeysResponse struct { + IDs []uint64 `protobuf:"varint,3,rep,packed,name=IDs" json:"IDs,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *TranslateKeysResponse) Reset() { *m = TranslateKeysResponse{} } +func (m *TranslateKeysResponse) String() string { return proto.CompactTextString(m) } +func (*TranslateKeysResponse) ProtoMessage() {} +func (*TranslateKeysResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_public_4fd446cb72375657, []int{16} +} +func (m *TranslateKeysResponse) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *TranslateKeysResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_TranslateKeysResponse.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (dst *TranslateKeysResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_TranslateKeysResponse.Merge(dst, src) +} +func (m *TranslateKeysResponse) XXX_Size() int { + return m.Size() +} +func (m *TranslateKeysResponse) XXX_DiscardUnknown() { + xxx_messageInfo_TranslateKeysResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_TranslateKeysResponse proto.InternalMessageInfo + +func (m *TranslateKeysResponse) GetIDs() []uint64 { + if m != nil { + return m.IDs + } + return nil +} + func init() { proto.RegisterType((*Row)(nil), "internal.Row") proto.RegisterType((*RowIdentifiers)(nil), "internal.RowIdentifiers") @@ -1095,6 +1205,8 @@ func init() { proto.RegisterType((*QueryResult)(nil), "internal.QueryResult") proto.RegisterType((*ImportRequest)(nil), "internal.ImportRequest") proto.RegisterType((*ImportValueRequest)(nil), "internal.ImportValueRequest") + proto.RegisterType((*TranslateKeysRequest)(nil), "internal.TranslateKeysRequest") + proto.RegisterType((*TranslateKeysResponse)(nil), "internal.TranslateKeysResponse") } func (m *Row) Marshal() (dAtA []byte, err error) { size := m.Size() @@ -1979,6 +2091,92 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *TranslateKeysRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *TranslateKeysRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Index) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Index))) + i += copy(dAtA[i:], m.Index) + } + if len(m.Field) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Field))) + i += copy(dAtA[i:], m.Field) + } + if len(m.Keys) > 0 { + for _, s := range m.Keys { + dAtA[i] = 0x1a + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *TranslateKeysResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *TranslateKeysResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.IDs) > 0 { + dAtA23 := make([]byte, len(m.IDs)*10) + var j22 int + for _, num := range m.IDs { + for num >= 1<<7 { + dAtA23[j22] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j22++ + } + dAtA23[j22] = uint8(num) + j22++ + } + dAtA[i] = 0x1a + i++ + i = encodeVarintPublic(dAtA, i, uint64(j22)) + i += copy(dAtA[i:], dAtA23[:j22]) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + func encodeVarintPublic(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -2434,6 +2632,51 @@ func (m *ImportValueRequest) Size() (n int) { return n } +func (m *TranslateKeysRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Index) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } + l = len(m.Field) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } + if len(m.Keys) > 0 { + for _, s := range m.Keys { + l = len(s) + n += 1 + l + sovPublic(uint64(l)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *TranslateKeysResponse) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.IDs) > 0 { + l = 0 + for _, e := range m.IDs { + l += sovPublic(uint64(e)) + } + n += 1 + sovPublic(uint64(l)) + l + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func sovPublic(x uint64) (n int) { for { n++ @@ -5115,6 +5358,268 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } return nil } +func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: TranslateKeysRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: TranslateKeysRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Index = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Field = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Keys", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Keys = append(m.Keys, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPublic(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPublic + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *TranslateKeysResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: TranslateKeysResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: TranslateKeysResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 3: + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.IDs = append(m.IDs, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + var elementCount int + var count int + for _, integer := range dAtA { + if integer < 128 { + count++ + } + } + elementCount = count + if elementCount != 0 && len(m.IDs) == 0 { + m.IDs = make([]uint64, 0, elementCount) + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.IDs = append(m.IDs, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field IDs", wireType) + } + default: + iNdEx = preIndex + skippy, err := skipPublic(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPublic + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipPublic(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 @@ -5220,59 +5725,61 @@ var ( ErrIntOverflowPublic = fmt.Errorf("proto: integer overflow") ) -func init() { proto.RegisterFile("public.proto", fileDescriptor_public_fc5da89825239896) } +func init() { proto.RegisterFile("public.proto", fileDescriptor_public_4fd446cb72375657) } -var fileDescriptor_public_fc5da89825239896 = []byte{ - // 804 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6e, 0xdb, 0x46, - 0x14, 0xed, 0x88, 0x94, 0x44, 0x5d, 0x59, 0xaa, 0x31, 0x70, 0x5d, 0xa2, 0x30, 0x54, 0x82, 0x28, - 0x0a, 0xae, 0x64, 0x40, 0x05, 0x8c, 0xae, 0xfa, 0xf0, 0xab, 0x10, 0xdc, 0x1a, 0xcd, 0xd8, 0x71, - 0x90, 0x25, 0x6d, 0x4d, 0x6c, 0x02, 0x14, 0x87, 0xe1, 0x03, 0xb2, 0xbe, 0x23, 0x9b, 0x7c, 0x42, - 0x16, 0xf9, 0x10, 0x2f, 0x83, 0x7c, 0x41, 0xe2, 0xfc, 0x48, 0x30, 0x77, 0x38, 0x1a, 0x8a, 0x0e, - 0x8c, 0x2c, 0xb2, 0x9b, 0x73, 0x5f, 0xbc, 0xe7, 0xbe, 0x08, 0x1b, 0x69, 0x79, 0x19, 0x47, 0x57, - 0xe3, 0x34, 0x13, 0x85, 0xa0, 0x4e, 0x94, 0x14, 0x3c, 0x4b, 0xc2, 0xd8, 0x7f, 0x0e, 0x16, 0x13, - 0x0b, 0xea, 0x42, 0xf7, 0x40, 0xc4, 0xe5, 0x3c, 0xc9, 0x5d, 0xe2, 0x59, 0x81, 0xcd, 0x34, 0xa4, - 0xbf, 0x40, 0xfb, 0xef, 0xa2, 0xc8, 0x72, 0xb7, 0xe5, 0x59, 0x41, 0x7f, 0x32, 0x1c, 0x6b, 0xd7, - 0xb1, 0x14, 0x33, 0xa5, 0xa4, 0x14, 0xec, 0x13, 0xbe, 0xcc, 0x5d, 0xcb, 0xb3, 0x82, 0x1e, 0xc3, - 0xb7, 0xff, 0x3b, 0x0c, 0x99, 0x58, 0x4c, 0x67, 0x3c, 0x29, 0xa2, 0x17, 0x11, 0x57, 0x56, 0x4c, - 0x2c, 0xf4, 0x27, 0xf0, 0xbd, 0xf2, 0x6c, 0xd5, 0x3c, 0xff, 0x00, 0xfb, 0xff, 0x30, 0xca, 0xe8, - 0x10, 0x5a, 0xd3, 0x43, 0x97, 0x78, 0x24, 0xb0, 0x59, 0x6b, 0x7a, 0x48, 0xb7, 0xa0, 0x7d, 0x20, - 0xca, 0xa4, 0x70, 0x5b, 0x28, 0x52, 0x80, 0x6e, 0x82, 0x75, 0xc2, 0x97, 0xae, 0xe5, 0x91, 0xa0, - 0xc7, 0xe4, 0xd3, 0xdf, 0x03, 0xe7, 0x38, 0xe2, 0xf1, 0x4c, 0x32, 0xdb, 0x82, 0x36, 0xbe, 0x31, - 0x4c, 0x8f, 0x29, 0x20, 0xa5, 0x32, 0xb7, 0x43, 0x1d, 0x09, 0x81, 0xff, 0x2f, 0xc0, 0x3f, 0x99, - 0x28, 0x53, 0x15, 0x37, 0x80, 0x36, 0x22, 0x4c, 0xb7, 0x3f, 0xa1, 0x86, 0xb9, 0x0e, 0xce, 0x94, - 0xc1, 0x97, 0xf3, 0xf2, 0x27, 0xe0, 0x5c, 0x84, 0xf1, 0x2a, 0xc7, 0x8b, 0x30, 0xc6, 0x1c, 0x2c, - 0x26, 0x9f, 0xeb, 0x3e, 0x96, 0xf6, 0x79, 0x0a, 0xd6, 0x7e, 0x54, 0x98, 0xf4, 0x48, 0x2d, 0x3d, - 0xfa, 0x13, 0x38, 0xaa, 0x2b, 0xab, 0xbc, 0x57, 0x98, 0xee, 0x40, 0xef, 0x3c, 0x9a, 0xf3, 0xbc, - 0x08, 0xe7, 0x29, 0x96, 0xc2, 0x62, 0x46, 0xe0, 0x3f, 0x83, 0x81, 0xb2, 0x94, 0xdd, 0x3a, 0xe3, - 0xc5, 0x83, 0xca, 0x7e, 0x5d, 0x97, 0x1f, 0x56, 0xfa, 0x0d, 0x01, 0x5b, 0xea, 0xb4, 0x8a, 0xac, - 0x54, 0xb2, 0xb1, 0xe7, 0xcb, 0x94, 0x57, 0x99, 0xe2, 0x9b, 0x7a, 0xd0, 0x3f, 0x2b, 0xb2, 0x28, - 0xb9, 0xbe, 0x08, 0xe3, 0x92, 0x57, 0x81, 0xea, 0x22, 0xc9, 0x71, 0x9a, 0x14, 0x4a, 0x6d, 0x23, - 0x8d, 0x15, 0x96, 0x1c, 0xf7, 0x85, 0x88, 0x95, 0xb2, 0xed, 0x91, 0xc0, 0x61, 0x46, 0x40, 0x47, - 0x00, 0xc7, 0xb1, 0x08, 0x2b, 0xdf, 0x8e, 0x47, 0x02, 0xc2, 0x6a, 0x12, 0x7f, 0x17, 0xba, 0x32, - 0xd3, 0xff, 0xc2, 0xd4, 0xb0, 0x25, 0x8f, 0xb0, 0xf5, 0xef, 0x08, 0x6c, 0x3c, 0x29, 0x79, 0xb6, - 0x64, 0xfc, 0x65, 0xc9, 0x73, 0xec, 0x0a, 0x62, 0x3d, 0x4a, 0x08, 0xe8, 0x36, 0x74, 0xce, 0x6e, - 0xc2, 0x6c, 0xa6, 0x6a, 0x67, 0xb3, 0x0a, 0x49, 0xae, 0xa6, 0xe6, 0x39, 0x72, 0x75, 0x58, 0x5d, - 0x24, 0x3d, 0x19, 0x9f, 0x8b, 0x42, 0x93, 0xa9, 0x10, 0x0d, 0xe0, 0xfb, 0xa3, 0xdb, 0xab, 0xb8, - 0x9c, 0x71, 0x26, 0x16, 0xca, 0xbb, 0x83, 0x06, 0x4d, 0x31, 0xfd, 0x15, 0x86, 0x95, 0x48, 0x6f, - 0x6f, 0x17, 0x0d, 0x1b, 0x52, 0xff, 0x15, 0x81, 0x41, 0x45, 0x25, 0x4f, 0x45, 0x92, 0x73, 0xd9, - 0xaf, 0xa3, 0x2c, 0xd3, 0xfd, 0x3a, 0xca, 0x32, 0xba, 0x0b, 0x5d, 0xc6, 0xf3, 0x32, 0x2e, 0xf4, - 0x10, 0xfc, 0x60, 0xca, 0xa2, 0x7d, 0xcb, 0xb8, 0x60, 0xda, 0x8a, 0xfe, 0x09, 0xc3, 0xb5, 0xa1, - 0x52, 0xdb, 0xdf, 0x9f, 0xfc, 0x68, 0xfc, 0xd6, 0xf4, 0xac, 0x61, 0xee, 0xbf, 0x6f, 0x41, 0xbf, - 0x16, 0x99, 0xfe, 0x8c, 0xb7, 0x08, 0x73, 0xea, 0x4f, 0x06, 0x26, 0x8a, 0xdc, 0x34, 0xbc, 0x52, - 0x1b, 0x40, 0x4e, 0xab, 0x79, 0x22, 0xa7, 0xb2, 0x8b, 0xf2, 0x4a, 0xe8, 0xcf, 0xd6, 0xba, 0x28, - 0xc5, 0x4c, 0x29, 0xf1, 0xb2, 0xdd, 0x84, 0xc9, 0x35, 0x9f, 0xe1, 0x3c, 0x39, 0x4c, 0x43, 0x3a, - 0x36, 0xfb, 0x89, 0x0d, 0x58, 0x5b, 0x71, 0xad, 0x61, 0x66, 0x87, 0xf5, 0x40, 0xcb, 0x5e, 0x0c, - 0xaa, 0x81, 0x96, 0x2d, 0x94, 0xbb, 0x29, 0x0b, 0x8f, 0xcd, 0x57, 0x88, 0xee, 0x41, 0xdf, 0x5c, - 0x92, 0xdc, 0x75, 0x30, 0xc3, 0x2d, 0x13, 0xde, 0x28, 0x59, 0xdd, 0x90, 0xfe, 0xd5, 0xbc, 0x99, - 0x6e, 0x0f, 0x33, 0x73, 0xd7, 0xaa, 0x51, 0xd3, 0xb3, 0x86, 0xbd, 0xff, 0x91, 0xc0, 0x60, 0x3a, - 0x4f, 0x45, 0x56, 0xd4, 0xc6, 0x76, 0x9a, 0xcc, 0xf8, 0xad, 0x1e, 0x5b, 0x04, 0xe6, 0x2e, 0xb6, - 0x1a, 0x77, 0x11, 0xc7, 0x17, 0xc7, 0xd5, 0x66, 0x0a, 0xd4, 0x58, 0xda, 0x6b, 0x2c, 0x77, 0xa0, - 0xa7, 0x0f, 0x50, 0xee, 0xb6, 0x51, 0x65, 0x04, 0x72, 0x21, 0x57, 0x17, 0x48, 0x4e, 0xb0, 0x15, - 0x58, 0xac, 0x26, 0x91, 0x9d, 0x61, 0x62, 0x81, 0xc7, 0xbf, 0x8b, 0xc7, 0x5f, 0x43, 0xe9, 0xa9, - 0xc2, 0xa0, 0xd2, 0x41, 0x65, 0x4d, 0xe2, 0xbf, 0x25, 0x40, 0x15, 0x47, 0x5c, 0xed, 0x6f, 0x47, - 0xf4, 0x71, 0x42, 0xdb, 0xd0, 0xc1, 0xef, 0x69, 0x32, 0x15, 0x6a, 0xa4, 0xdb, 0x6d, 0xa6, 0xbb, - 0xbf, 0x79, 0x77, 0x3f, 0x22, 0xef, 0xee, 0x47, 0xe4, 0xc3, 0xfd, 0x88, 0xbc, 0xfe, 0x34, 0xfa, - 0xee, 0xb2, 0x83, 0xbf, 0xe1, 0xdf, 0x3e, 0x07, 0x00, 0x00, 0xff, 0xff, 0xa6, 0x62, 0xa8, 0x25, - 0x96, 0x07, 0x00, 0x00, +var fileDescriptor_public_4fd446cb72375657 = []byte{ + // 835 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x6e, 0xe3, 0x44, + 0x14, 0x66, 0x62, 0x27, 0x71, 0x4e, 0x9a, 0xb0, 0x1a, 0x65, 0x17, 0x0b, 0xad, 0x82, 0x65, 0x21, + 0x64, 0x6e, 0xb2, 0x52, 0x90, 0x56, 0x5c, 0xf1, 0xd3, 0x6d, 0x8b, 0xa2, 0x42, 0x05, 0xd3, 0x12, + 0xc4, 0xa5, 0xdb, 0x0c, 0xad, 0x25, 0xc7, 0x63, 0xec, 0xb1, 0xd2, 0x3c, 0x07, 0x37, 0x3c, 0x02, + 0x17, 0x3c, 0x48, 0x2f, 0x11, 0x4f, 0x00, 0xe5, 0x45, 0xd0, 0x9c, 0xf1, 0x64, 0x1c, 0xb7, 0xaa, + 0x10, 0xda, 0xbb, 0xf9, 0xce, 0x9f, 0xcf, 0x77, 0xfe, 0x12, 0x38, 0xc8, 0xab, 0xcb, 0x34, 0xb9, + 0x9a, 0xe5, 0x85, 0x90, 0x82, 0x7a, 0x49, 0x26, 0x79, 0x91, 0xc5, 0x69, 0xf8, 0x23, 0x38, 0x4c, + 0x6c, 0xa8, 0x0f, 0xfd, 0x37, 0x22, 0xad, 0xd6, 0x59, 0xe9, 0x93, 0xc0, 0x89, 0x5c, 0x66, 0x20, + 0xfd, 0x10, 0xba, 0x5f, 0x4a, 0x59, 0x94, 0x7e, 0x27, 0x70, 0xa2, 0xe1, 0x7c, 0x3c, 0x33, 0xae, + 0x33, 0x25, 0x66, 0x5a, 0x49, 0x29, 0xb8, 0xa7, 0x7c, 0x5b, 0xfa, 0x4e, 0xe0, 0x44, 0x03, 0x86, + 0xef, 0xf0, 0x53, 0x18, 0x33, 0xb1, 0x59, 0xac, 0x78, 0x26, 0x93, 0x9f, 0x12, 0xae, 0xad, 0x98, + 0xd8, 0x98, 0x4f, 0xe0, 0x7b, 0xe7, 0xd9, 0x69, 0x78, 0x7e, 0x06, 0xee, 0xb7, 0x71, 0x52, 0xd0, + 0x31, 0x74, 0x16, 0x47, 0x3e, 0x09, 0x48, 0xe4, 0xb2, 0xce, 0xe2, 0x88, 0x4e, 0xa0, 0xfb, 0x46, + 0x54, 0x99, 0xf4, 0x3b, 0x28, 0xd2, 0x80, 0x3e, 0x03, 0xe7, 0x94, 0x6f, 0x7d, 0x27, 0x20, 0xd1, + 0x80, 0xa9, 0x67, 0xf8, 0x1a, 0xbc, 0x93, 0x84, 0xa7, 0x2b, 0xc5, 0x6c, 0x02, 0x5d, 0x7c, 0x63, + 0x98, 0x01, 0xd3, 0x40, 0x49, 0x55, 0x6e, 0x47, 0x26, 0x12, 0x82, 0xf0, 0x6b, 0x80, 0xaf, 0x0a, + 0x51, 0xe5, 0x3a, 0x6e, 0x04, 0x5d, 0x44, 0x98, 0xee, 0x70, 0x4e, 0x2d, 0x73, 0x13, 0x9c, 0x69, + 0x83, 0xc7, 0xf3, 0x0a, 0xe7, 0xe0, 0x2d, 0xe3, 0x74, 0x97, 0xe3, 0x32, 0x4e, 0x31, 0x07, 0x87, + 0xa9, 0xe7, 0xbe, 0x8f, 0x63, 0x7c, 0xbe, 0x07, 0xe7, 0x30, 0x91, 0x36, 0x3d, 0xd2, 0x48, 0x8f, + 0xbe, 0x0f, 0x9e, 0xee, 0xca, 0x2e, 0xef, 0x1d, 0xa6, 0x2f, 0x61, 0x70, 0x91, 0xac, 0x79, 0x29, + 0xe3, 0x75, 0x8e, 0xa5, 0x70, 0x98, 0x15, 0x84, 0x3f, 0xc0, 0x48, 0x5b, 0xaa, 0x6e, 0x9d, 0x73, + 0xf9, 0xa0, 0xb2, 0xff, 0xad, 0xcb, 0x0f, 0x2b, 0xfd, 0x1b, 0x01, 0x57, 0xe9, 0x8c, 0x8a, 0xec, + 0x54, 0xaa, 0xb1, 0x17, 0xdb, 0x9c, 0xd7, 0x99, 0xe2, 0x9b, 0x06, 0x30, 0x3c, 0x97, 0x45, 0x92, + 0x5d, 0x2f, 0xe3, 0xb4, 0xe2, 0x75, 0xa0, 0xa6, 0x48, 0x71, 0x5c, 0x64, 0x52, 0xab, 0x5d, 0xa4, + 0xb1, 0xc3, 0x8a, 0xe3, 0xa1, 0x10, 0xa9, 0x56, 0x76, 0x03, 0x12, 0x79, 0xcc, 0x0a, 0xe8, 0x14, + 0xe0, 0x24, 0x15, 0x71, 0xed, 0xdb, 0x0b, 0x48, 0x44, 0x58, 0x43, 0x12, 0xbe, 0x82, 0xbe, 0xca, + 0xf4, 0x9b, 0x38, 0xb7, 0x6c, 0xc9, 0x13, 0x6c, 0xc3, 0x3b, 0x02, 0x07, 0xdf, 0x55, 0xbc, 0xd8, + 0x32, 0xfe, 0x73, 0xc5, 0x4b, 0xec, 0x0a, 0x62, 0x33, 0x4a, 0x08, 0xe8, 0x0b, 0xe8, 0x9d, 0xdf, + 0xc4, 0xc5, 0x4a, 0xd7, 0xce, 0x65, 0x35, 0x52, 0x5c, 0x6d, 0xcd, 0x4b, 0xe4, 0xea, 0xb1, 0xa6, + 0x48, 0x79, 0x32, 0xbe, 0x16, 0xd2, 0x90, 0xa9, 0x11, 0x8d, 0xe0, 0xdd, 0xe3, 0xdb, 0xab, 0xb4, + 0x5a, 0x71, 0x26, 0x36, 0xda, 0xbb, 0x87, 0x06, 0x6d, 0x31, 0xfd, 0x08, 0xc6, 0xb5, 0xc8, 0x6c, + 0x6f, 0x1f, 0x0d, 0x5b, 0xd2, 0xf0, 0x17, 0x02, 0xa3, 0x9a, 0x4a, 0x99, 0x8b, 0xac, 0xe4, 0xaa, + 0x5f, 0xc7, 0x45, 0x61, 0xfa, 0x75, 0x5c, 0x14, 0xf4, 0x15, 0xf4, 0x19, 0x2f, 0xab, 0x54, 0x9a, + 0x21, 0x78, 0x6e, 0xcb, 0x62, 0x7c, 0xab, 0x54, 0x32, 0x63, 0x45, 0x3f, 0x87, 0xf1, 0xde, 0x50, + 0xe9, 0xed, 0x1f, 0xce, 0xdf, 0xb3, 0x7e, 0x7b, 0x7a, 0xd6, 0x32, 0x0f, 0xff, 0xec, 0xc0, 0xb0, + 0x11, 0x99, 0x7e, 0x80, 0xb7, 0x08, 0x73, 0x1a, 0xce, 0x47, 0x36, 0x8a, 0xda, 0x34, 0xbc, 0x52, + 0x07, 0x40, 0xce, 0xea, 0x79, 0x22, 0x67, 0xaa, 0x8b, 0xea, 0x4a, 0x98, 0xcf, 0x36, 0xba, 0xa8, + 0xc4, 0x4c, 0x2b, 0xf1, 0xb2, 0xdd, 0xc4, 0xd9, 0x35, 0x5f, 0xe1, 0x3c, 0x79, 0xcc, 0x40, 0x3a, + 0xb3, 0xfb, 0x89, 0x0d, 0xd8, 0x5b, 0x71, 0xa3, 0x61, 0x76, 0x87, 0xcd, 0x40, 0xab, 0x5e, 0x8c, + 0xea, 0x81, 0x56, 0x2d, 0x54, 0xbb, 0xa9, 0x0a, 0x8f, 0xcd, 0xd7, 0x88, 0xbe, 0x86, 0xa1, 0xbd, + 0x24, 0xa5, 0xef, 0x61, 0x86, 0x13, 0x1b, 0xde, 0x2a, 0x59, 0xd3, 0x90, 0x7e, 0xd1, 0xbe, 0x99, + 0xfe, 0x00, 0x33, 0xf3, 0xf7, 0xaa, 0xd1, 0xd0, 0xb3, 0x96, 0x7d, 0xf8, 0x37, 0x81, 0xd1, 0x62, + 0x9d, 0x8b, 0x42, 0x36, 0xc6, 0x76, 0x91, 0xad, 0xf8, 0xad, 0x19, 0x5b, 0x04, 0xf6, 0x2e, 0x76, + 0x5a, 0x77, 0x11, 0xc7, 0x17, 0xc7, 0xd5, 0x65, 0x1a, 0x34, 0x58, 0xba, 0x7b, 0x2c, 0x5f, 0xc2, + 0xc0, 0x1c, 0xa0, 0xd2, 0xef, 0xa2, 0xca, 0x0a, 0xd4, 0x42, 0xee, 0x2e, 0x90, 0x9a, 0x60, 0x27, + 0x72, 0x58, 0x43, 0xa2, 0x3a, 0xc3, 0xc4, 0x06, 0x8f, 0x7f, 0x1f, 0x8f, 0xbf, 0x81, 0xca, 0x53, + 0x87, 0x41, 0xa5, 0x87, 0xca, 0x86, 0x24, 0xfc, 0x9d, 0x00, 0xd5, 0x1c, 0x71, 0xb5, 0xdf, 0x1e, + 0xd1, 0xa7, 0x09, 0xbd, 0x80, 0x1e, 0x7e, 0xcf, 0x90, 0xa9, 0x51, 0x2b, 0xdd, 0xfe, 0x83, 0x74, + 0x97, 0x30, 0xb9, 0x28, 0xe2, 0xac, 0x4c, 0x63, 0xc9, 0x95, 0xe0, 0xff, 0xe4, 0xfb, 0xd8, 0x0f, + 0xec, 0xc7, 0xf0, 0xbc, 0x15, 0xd7, 0x2e, 0xb7, 0x22, 0xe0, 0x20, 0x01, 0xf5, 0x3c, 0x7c, 0x76, + 0x77, 0x3f, 0x25, 0x7f, 0xdc, 0x4f, 0xc9, 0x5f, 0xf7, 0x53, 0xf2, 0xeb, 0x3f, 0xd3, 0x77, 0x2e, + 0x7b, 0xf8, 0x4f, 0xe0, 0x93, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0x58, 0xdf, 0x75, 0xf7, 0x19, + 0x08, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index 229fcfadd..a972340b1 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -105,3 +105,13 @@ message ImportValueRequest { repeated string ColumnKeys = 7; repeated int64 Values = 6; } + +message TranslateKeysRequest { + string Index = 1; + string Field = 2; + repeated string Keys = 3; +} + +message TranslateKeysResponse { + repeated uint64 IDs = 3; +} \ No newline at end of file diff --git a/server/handler_test.go b/server/handler_test.go index 9e76d77b9..4fa6aaf93 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -22,14 +22,13 @@ import ( "fmt" "io" "io/ioutil" + gohttp "net/http" "net/http/httptest" "reflect" "strings" "testing" "time" - gohttp "net/http" - "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/server" @@ -689,6 +688,91 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected body: %q", w.Body.String()) } }) + + t.Run("translate keys", func(t *testing.T) { + // create index + w := httptest.NewRecorder() + r := test.MustNewHTTPRequest("POST", "/index/i1-tr", strings.NewReader(`{"options":{"keys":true}}`)) + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if w.Body.String() != `{"success":true}`+"\n" { + t.Fatalf("unexpected body: %q", w.Body.String()) + } + + // create field + w = httptest.NewRecorder() + r = test.MustNewHTTPRequest("POST", "/index/i1-tr/field/f1", strings.NewReader(`{"options":{"keys":true}}`)) + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if w.Body.String() != `{"success":true}`+"\n" { + t.Fatalf("unexpected body: %q", w.Body.String()) + } + + // set some bits + w = httptest.NewRecorder() + r = test.MustNewHTTPRequest("POST", "/index/i1-tr/query", strings.NewReader(`Set("col1", f1="row1")Set("col2", f1="row1")`)) + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + + // Generate request body for translate column keys request + reqBody, err := cmd.API.Serializer.Marshal(&pilosa.TranslateKeysRequest{ + Index: "i1-tr", + Keys: []string{"col1", "col2"}, + }) + if err != nil { + t.Fatal(err) + } + // Generate protobuf request. + w = httptest.NewRecorder() + r = test.MustNewHTTPRequest("POST", "/internal/translate/keys", bytes.NewReader(reqBody)) + r.Header.Set("Content-Type", "application/x-protobuf") + r.Header.Set("Accept", "application/x-protobuf") + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + target := []uint64{1, 2} + resp := pilosa.TranslateKeysResponse{} + err = cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(target, resp.IDs) { + t.Fatalf("%v != %v", target, resp.IDs) + } + + // Generate request body for translate row keys request + reqBody, err = cmd.API.Serializer.Marshal(&pilosa.TranslateKeysRequest{ + Index: "i1-tr", + Field: "f1", + Keys: []string{"row1"}, + }) + if err != nil { + t.Fatal(err) + } + // Generate protobuf request. + w = httptest.NewRecorder() + r = test.MustNewHTTPRequest("POST", "/internal/translate/keys", bytes.NewReader(reqBody)) + r.Header.Set("Content-Type", "application/x-protobuf") + r.Header.Set("Accept", "application/x-protobuf") + h.ServeHTTP(w, r) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + target = []uint64{1} + resp = pilosa.TranslateKeysResponse{} + err = cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(target, resp.IDs) { + t.Fatalf("%v != %v", target, resp.IDs) + } + }) } func TestClusterTranslator(t *testing.T) { From d543689868cee1c73e6d097020f5d5d620b259d0 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Mon, 26 Nov 2018 17:52:53 +0300 Subject: [PATCH 002/125] updated translate keys test to include new keys --- server/handler_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/handler_test.go b/server/handler_test.go index 9034418d5..f6652632d 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -737,7 +737,7 @@ func TestHandler_Endpoints(t *testing.T) { // set some bits w = httptest.NewRecorder() - r = test.MustNewHTTPRequest("POST", "/index/i1-tr/query", strings.NewReader(`Set("col1", f1="row1")Set("col2", f1="row1")`)) + r = test.MustNewHTTPRequest("POST", "/index/i1-tr/query", strings.NewReader(`Set("col1", f1="row1")`)) h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) @@ -746,7 +746,7 @@ func TestHandler_Endpoints(t *testing.T) { // Generate request body for translate column keys request reqBody, err := cmd.API.Serializer.Marshal(&pilosa.TranslateKeysRequest{ Index: "i1-tr", - Keys: []string{"col1", "col2"}, + Keys: []string{"col1", "col2", "col3"}, }) if err != nil { t.Fatal(err) @@ -760,7 +760,7 @@ func TestHandler_Endpoints(t *testing.T) { if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } - target := []uint64{1, 2} + target := []uint64{1, 2, 3} resp := pilosa.TranslateKeysResponse{} err = cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp) if err != nil { @@ -774,7 +774,7 @@ func TestHandler_Endpoints(t *testing.T) { reqBody, err = cmd.API.Serializer.Marshal(&pilosa.TranslateKeysRequest{ Index: "i1-tr", Field: "f1", - Keys: []string{"row1"}, + Keys: []string{"row1", "row2"}, }) if err != nil { t.Fatal(err) @@ -788,7 +788,7 @@ func TestHandler_Endpoints(t *testing.T) { if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } - target = []uint64{1} + target = []uint64{1, 2} resp = pilosa.TranslateKeysResponse{} err = cmd.API.Serializer.Unmarshal(w.Body.Bytes(), &resp) if err != nil { From f0c6394c6102320ae6f9e6f4001b04f4750c2fa9 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 27 Nov 2018 11:00:55 -0600 Subject: [PATCH 003/125] Use syscall.Dup3 on ARM64 as Dup2 is unsupported --- server/server.go | 24 ------------------------ server/setup_logger.go | 35 +++++++++++++++++++++++++++++++++++ server/setup_logger_arm64.go | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 24 deletions(-) create mode 100644 server/setup_logger.go create mode 100644 server/setup_logger_arm64.go diff --git a/server/server.go b/server/server.go index c6b69a7f5..c80f8268e 100644 --- a/server/server.go +++ b/server/server.go @@ -174,30 +174,6 @@ func (m *Command) Wait() error { } } -// setupLogger sets up the logger based on the configuration. -func (m *Command) setupLogger() error { - if m.Config.LogPath == "" { - m.logOutput = m.Stderr - } else { - f, err := os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) - if err != nil { - return errors.Wrap(err, "opening file") - } - m.logOutput = f - err = syscall.Dup2(int(f.Fd()), int(os.Stderr.Fd())) - if err != nil { - return errors.Wrap(err, "dup2ing stderr onto logfile") - } - } - - if m.Config.Verbose { - m.logger = logger.NewVerboseLogger(m.logOutput) - } else { - m.logger = logger.NewStandardLogger(m.logOutput) - } - return nil -} - // SetupServer uses the cluster configuration to set up this server. func (m *Command) SetupServer() error { err := m.setupLogger() diff --git a/server/setup_logger.go b/server/setup_logger.go new file mode 100644 index 000000000..06d61eced --- /dev/null +++ b/server/setup_logger.go @@ -0,0 +1,35 @@ +// +build !arm64 + +package server + +import ( + "os" + "syscall" + + "github.com/pilosa/pilosa/logger" + "github.com/pkg/errors" +) + +// setupLogger sets up the logger based on the configuration. +func (m *Command) setupLogger() error { + if m.Config.LogPath == "" { + m.logOutput = m.Stderr + } else { + f, err := os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) + if err != nil { + return errors.Wrap(err, "opening file") + } + m.logOutput = f + err = syscall.Dup2(int(f.Fd()), int(os.Stderr.Fd())) + if err != nil { + return errors.Wrap(err, "dup2ing stderr onto logfile") + } + } + + if m.Config.Verbose { + m.logger = logger.NewVerboseLogger(m.logOutput) + } else { + m.logger = logger.NewStandardLogger(m.logOutput) + } + return nil +} diff --git a/server/setup_logger_arm64.go b/server/setup_logger_arm64.go new file mode 100644 index 000000000..dd6718741 --- /dev/null +++ b/server/setup_logger_arm64.go @@ -0,0 +1,33 @@ +package server + +import ( + "os" + "syscall" + + "github.com/pilosa/pilosa/logger" + "github.com/pkg/errors" +) + +// setupLogger sets up the logger based on the configuration. +func (m *Command) setupLogger() error { + if m.Config.LogPath == "" { + m.logOutput = m.Stderr + } else { + f, err := os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) + if err != nil { + return errors.Wrap(err, "opening file") + } + m.logOutput = f + err = syscall.Dup3(int(f.Fd()), int(os.Stderr.Fd()), 0) + if err != nil { + return errors.Wrap(err, "dup2ing stderr onto logfile") + } + } + + if m.Config.Verbose { + m.logger = logger.NewVerboseLogger(m.logOutput) + } else { + m.logger = logger.NewStandardLogger(m.logOutput) + } + return nil +} From 7330daa2227cc56a69a85483f5ccbe63b4fe806c Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 27 Nov 2018 11:09:46 -0600 Subject: [PATCH 004/125] Add ARM build to CI --- .circleci/config.yml | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2ba2868e6..08b615b3b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ fast-checkout: &fast-checkout attach_workspace: at: . jobs: - build: + setup: <<: *defaults steps: - checkout @@ -30,6 +30,14 @@ jobs: - run: gometalinter --install - run: go get github.com/remyoudompheng/go-misc/deadcode - run: make gometalinter + test-build-arm: + <<: *defaults + steps: + - *fast-checkout + - run: make build GOOS=linux GOARCH=arm GOARM=5 + - run: make build GOOS=linux GOARCH=arm GOARM=6 + - run: make build GOOS=linux GOARCH=arm GOARM=7 + - run: make build GOOS=linux GOARCH=arm64 test-golang-1.11: &base-test <<: *defaults steps: @@ -100,25 +108,28 @@ workflows: version: 2 test: jobs: - - build + - setup - linter: requires: - - build + - setup + - test-build-arm: + requires: + - setup - test-golang-1.11: requires: - - build + - setup - test-golang-1.11-race: requires: - - build + - setup - test-golang-1.11-386: requires: - - build + - setup - test-golang-1.10: requires: - - build + - setup - cluster-tests: requires: - - build + - setup - prerelease: requires: - linter From ef6db5cc1a53a0bd2a79d8016fe926548c3bff35 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 4 Dec 2018 14:03:14 -0600 Subject: [PATCH 005/125] propogate updates to node details (not just additions and deletions) to all nodes in cluster, not just coordinator --- cluster.go | 9 ++++++++- server/server_test.go | 9 +++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/cluster.go b/cluster.go index 451ce8ba1..155c4a4aa 100644 --- a/cluster.go +++ b/cluster.go @@ -70,7 +70,7 @@ type Node struct { } func (n Node) String() string { - return fmt.Sprintf("Node:%s:%s:%s", n.URI, n.State, n.ID[:6]) + return fmt.Sprintf("Node:%s:%s:%s", n.URI, n.State, n.ID) } // Nodes represents a list of nodes. @@ -364,6 +364,7 @@ func (c *cluster) addNode(node *Node) error { if !c.Topology.addID(node.ID) { return nil } + c.Topology.nodeStates[node.ID] = node.State // save topology return c.saveTopology() @@ -588,6 +589,12 @@ func (c *cluster) nodePositionByID(nodeID string) int { func (c *cluster) addNodeBasicSorted(node *Node) bool { n := c.unprotectedNodeByID(node.ID) if n != nil { + if n.State != node.State || n.IsCoordinator != node.IsCoordinator || n.URI != node.URI { + n.State = node.State + n.IsCoordinator = node.IsCoordinator + n.URI = node.URI + return true + } return false } diff --git a/server/server_test.go b/server/server_test.go index 3aa11b8a2..627e83862 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -678,6 +678,15 @@ func TestClusterQueriesAfterRestart(t *testing.T) { defer cluster.Close() cmd1 := cluster[1] + for _, com := range cluster { + nodes := com.API.Hosts(context.Background()) + for _, n := range nodes { + if n.State != "READY" { + t.Fatalf("unexpected node state after upping cluster: %v", nodes) + } + } + } + cmd1.MustCreateIndex(t, "testidx", pilosa.IndexOptions{}) cmd1.MustCreateField(t, "testidx", "testfield", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 10)) From 727659644b87fcbcc64df5a50547d4f4edec5bd1 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Wed, 21 Nov 2018 14:42:01 -0700 Subject: [PATCH 006/125] Add GroupBy filter. --- executor.go | 38 +- executor_test.go | 10 + pql/ast.go | 156 ++-- pql/pql.peg | 1 + pql/pql.peg.go | 1766 +++++++++++++++++++++++--------------------- pql/pqlpeg_test.go | 17 + 6 files changed, 1062 insertions(+), 926 deletions(-) diff --git a/executor.go b/executor.go index 0458aea0d..d8a24f8dd 100644 --- a/executor.go +++ b/executor.go @@ -884,6 +884,10 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call } else if hasLimit { limit = int(lim) } + filter, _, err := c.CallArg("filter") + if err != nil { + return nil, err + } // perform necessary Rows queries (any that have limit or columns args) - // TODO, call async? would only help if multiple Rows queries had a column @@ -892,7 +896,7 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call childRows := make([]RowIDs, len(c.Children)) for i, child := range c.Children { if child.Name != "Rows" { - return nil, errors.Errorf("'%s' is not a valid child query for GroupBy, must be 'Rows'", c.Name) + return nil, errors.Errorf("'%s' is not a valid child query for GroupBy, must be 'Rows'", child.Name) } _, hasLimit, err := child.UintArg("limit") if err != nil { @@ -915,7 +919,7 @@ func (e *executor) executeGroupBy(ctx context.Context, index string, c *pql.Call // Execute calls in bulk on each remote node and merge. mapFn := func(shard uint64) (interface{}, error) { - return e.executeGroupByShard(ctx, index, c, shard, childRows) + return e.executeGroupByShard(ctx, index, c, filter, shard, childRows) } // Merge returned results at coordinating node. reduceFn := func(prev, v interface{}) interface{} { @@ -1028,8 +1032,15 @@ func (g GroupCount) Compare(o GroupCount) int { return 0 } -func (e *executor) executeGroupByShard(_ context.Context, index string, c *pql.Call, shard uint64, childRows []RowIDs) ([]GroupCount, error) { - iter, err := newGroupByIterator(childRows, c.Children, index, shard, e.Holder) +func (e *executor) executeGroupByShard(ctx context.Context, index string, c *pql.Call, filter *pql.Call, shard uint64, childRows []RowIDs) (_ []GroupCount, err error) { + var filterRow *Row + if filter != nil { + if filterRow, err = e.executeBitmapCallShard(ctx, index, filter, shard); err != nil { + return nil, errors.Wrapf(err, "executing group by filter for shard %d", shard) + } + } + + iter, err := newGroupByIterator(childRows, c.Children, filterRow, index, shard, e.Holder) if err != nil { return nil, errors.Wrapf(err, "getting group by iterator for shard %d", shard) } @@ -2687,16 +2698,20 @@ type groupByIterator struct { // fields and then sets the row ids. fields []FieldRow done bool + + // Optional filter row to intersect against first level of values. + filter *Row } // newGroupByIterator initializes a new groupByIterator. -func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, index string, shard uint64, holder *Holder) (*groupByIterator, error) { +func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, filter *Row, index string, shard uint64, holder *Holder) (*groupByIterator, error) { gbi := &groupByIterator{ rowIters: make([]*rowIterator, len(children)), rows: make([]struct { row *Row id uint64 }, len(children)), + filter: filter, fields: make([]FieldRow, len(children)), } @@ -2756,6 +2771,11 @@ func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, index string, sha } } + // Apply filter to first level, if available. + if gbi.filter != nil && len(gbi.rows) > 0 { + gbi.rows[0].row = gbi.rows[0].row.Intersect(gbi.filter) + } + for i := 1; i < len(gbi.rows)-1; i++ { gbi.rows[i].row = gbi.rows[i].row.Intersect(gbi.rows[i-1].row) } @@ -2774,10 +2794,12 @@ func (gbi *groupByIterator) nextAtIdx(i int) { if wrapped && i != 0 { gbi.nextAtIdx(i - 1) } - if i != 0 && i != len(gbi.rows)-1 { - gbi.rows[i].row = nr.Intersect(gbi.rows[i-1].row) - } else { + if i == 0 && gbi.filter != nil { + gbi.rows[i].row = nr.Intersect(gbi.filter) + } else if i == 0 || i == len(gbi.rows)-1 { gbi.rows[i].row = nr + } else { + gbi.rows[i].row = nr.Intersect(gbi.rows[i-1].row) } gbi.rows[i].id = rowID } diff --git a/executor_test.go b/executor_test.go index 87b72689c..3c54fbefb 100644 --- a/executor_test.go +++ b/executor_test.go @@ -2845,6 +2845,16 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { checkGroupBy(t, expected, results) }) + t.Run("Filter", func(t *testing.T) { + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 100}}, Count: 3}, + {Group: []pilosa.FieldRow{{Field: "general", RowID: 10}, {Field: "sub", RowID: 110}}, Count: 1}, + } + + results := c.Query(t, "i", `GroupBy(Rows(field=general), Rows(field=sub), filter=Row(general=10))`).Results[0].([]pilosa.GroupCount) + checkGroupBy(t, expected, results) + }) + t.Run("check field offset no limit", func(t *testing.T) { expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "general", RowID: 11}}, Count: 2}, diff --git a/pql/ast.go b/pql/ast.go index 0e2ff6aef..32255b6a8 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -27,28 +27,34 @@ import ( type Query struct { Calls []*Call - lastField string - lastCond Token - inList bool - callStack []*Call - + callStack []*callStackElem conditional []string } func (q *Query) startCall(name string) { newCall := &Call{Name: name} - q.callStack = append(q.callStack, newCall) + q.callStack = append(q.callStack, &callStackElem{call: newCall}) if len(q.callStack) == 1 { q.Calls = append(q.Calls, newCall) - } else { - calls := q.callStack[len(q.callStack)-2].Children - q.callStack[len(q.callStack)-2].Children = append(calls, newCall) + } else if prevElem := q.callStack[len(q.callStack)-2]; prevElem.lastField == "" { + prevElem.call.Children = append(prevElem.call.Children, newCall) } } -func (q *Query) endCall() { +// endCall removes the last element from the call stack and returns the call. +func (q *Query) endCall() *Call { + elem := q.callStack[len(q.callStack)-1] + q.callStack[len(q.callStack)-1] = nil q.callStack = q.callStack[:len(q.callStack)-1] + return elem.call +} + +func (q *Query) lastCallStackElem() *callStackElem { + if len(q.callStack) == 0 { + return nil + } + return q.callStack[len(q.callStack)-1] } func (q *Query) addPosNum(key, value string) { @@ -63,9 +69,9 @@ func (q *Query) addPosStr(key, value string) { func (q *Query) startConditional() { q.conditional = make([]string, 0) - call := q.callStack[len(q.callStack)-1] - if call.Args == nil { - call.Args = make(map[string]interface{}) + elem := q.lastCallStackElem() + if elem.call.Args == nil { + elem.call.Args = make(map[string]interface{}) } } @@ -89,47 +95,48 @@ func (q *Query) endConditional() { high++ } - call := q.callStack[len(q.callStack)-1] - call.Args[field] = &Condition{Op: BETWEEN, Value: []interface{}{low, high}} + elem := q.lastCallStackElem() + elem.call.Args[field] = &Condition{Op: BETWEEN, Value: []interface{}{low, high}} q.conditional = nil } func (q *Query) addField(field string) { - if q.lastField != "" { - panic(fmt.Sprintf("addField called with '%s' while field is not empty, it's: %s", field, q.lastField)) + elem := q.lastCallStackElem() + if elem == nil || elem.lastField != "" { + panic(fmt.Sprintf("addField called with '%s' while field is not empty, it's: %s", field, elem.lastField)) } - q.lastField = field - call := q.callStack[len(q.callStack)-1] - if call.Args == nil { - call.Args = make(map[string]interface{}) + elem.lastField = field + if elem.call.Args == nil { + elem.call.Args = make(map[string]interface{}) } } func (q *Query) addVal(val interface{}) { - if q.lastField == "" { + elem := q.lastCallStackElem() + if elem == nil || elem.lastField == "" { panic(fmt.Sprintf("addVal called with '%s' when lastField is empty", val)) } - call := q.callStack[len(q.callStack)-1] - if q.inList { - list := call.Args[q.lastField].([]interface{}) - call.Args[q.lastField] = append(list, val) + if elem.inList { + list := elem.call.Args[elem.lastField].([]interface{}) + elem.call.Args[elem.lastField] = append(list, val) return } - if q.lastCond != ILLEGAL { - call.Args[q.lastField] = &Condition{ - Op: q.lastCond, + if elem.lastCond != ILLEGAL { + elem.call.Args[elem.lastField] = &Condition{ + Op: elem.lastCond, Value: val, } } else { - call.Args[q.lastField] = val + elem.call.Args[elem.lastField] = val } - q.lastField = "" - q.lastCond = ILLEGAL + elem.lastField = "" + elem.lastCond = ILLEGAL } func (q *Query) addNumVal(val string) { - if q.lastField == "" { + elem := q.lastCallStackElem() + if elem == nil || elem.lastField == "" { panic(fmt.Sprintf("addIntVal called with '%s' when lastField is empty", val)) } var ival interface{} @@ -142,70 +149,70 @@ func (q *Query) addNumVal(val string) { if err != nil { panic(err) } - call := q.callStack[len(q.callStack)-1] - if q.inList { - if q.lastCond != ILLEGAL { - list := call.Args[q.lastField].(*Condition).Value.([]interface{}) - call.Args[q.lastField] = &Condition{ - Op: q.lastCond, + if elem.inList { + if elem.lastCond != ILLEGAL { + list := elem.call.Args[elem.lastField].(*Condition).Value.([]interface{}) + elem.call.Args[elem.lastField] = &Condition{ + Op: elem.lastCond, Value: append(list, ival), } } else { - list := call.Args[q.lastField].([]interface{}) - call.Args[q.lastField] = append(list, ival) + list := elem.call.Args[elem.lastField].([]interface{}) + elem.call.Args[elem.lastField] = append(list, ival) } return - } else if q.lastCond != ILLEGAL { - call.Args[q.lastField] = &Condition{ - Op: q.lastCond, + } else if elem.lastCond != ILLEGAL { + elem.call.Args[elem.lastField] = &Condition{ + Op: elem.lastCond, Value: ival, } } else { - call.Args[q.lastField] = ival + elem.call.Args[elem.lastField] = ival } - q.lastField = "" - q.lastCond = ILLEGAL + elem.lastField = "" + elem.lastCond = ILLEGAL } func (q *Query) startList() { - call := q.callStack[len(q.callStack)-1] - if q.lastCond != ILLEGAL { - call.Args[q.lastField] = &Condition{ - Op: q.lastCond, + elem := q.lastCallStackElem() + if elem.lastCond != ILLEGAL { + elem.call.Args[elem.lastField] = &Condition{ + Op: elem.lastCond, Value: make([]interface{}, 0), } } else { - call.Args[q.lastField] = make([]interface{}, 0) + elem.call.Args[elem.lastField] = make([]interface{}, 0) } - q.inList = true + elem.inList = true } func (q *Query) endList() { - q.inList = false - q.lastField = "" - q.lastCond = ILLEGAL + elem := q.lastCallStackElem() + elem.inList = false + elem.lastField = "" + elem.lastCond = ILLEGAL } func (q *Query) addGT() { - q.lastCond = GT + q.lastCallStackElem().lastCond = GT } func (q *Query) addLT() { - q.lastCond = LT + q.lastCallStackElem().lastCond = LT } func (q *Query) addGTE() { - q.lastCond = GTE + q.lastCallStackElem().lastCond = GTE } func (q *Query) addLTE() { - q.lastCond = LTE + q.lastCallStackElem().lastCond = LTE } func (q *Query) addEQ() { - q.lastCond = EQ + q.lastCallStackElem().lastCond = EQ } func (q *Query) addNEQ() { - q.lastCond = NEQ + q.lastCallStackElem().lastCond = NEQ } func (q *Query) addBTWN() { - q.lastCond = BETWEEN + q.lastCallStackElem().lastCond = BETWEEN } // WriteCallN returns the number of mutating calls. @@ -229,6 +236,13 @@ func (q *Query) String() string { return strings.Join(a, "\n") } +type callStackElem struct { + call *Call + lastField string + lastCond Token + inList bool +} + // Call represents a function call in the AST. type Call struct { Name string @@ -329,6 +343,22 @@ func (c *Call) UintSliceArg(key string) ([]uint64, bool, error) { } } +// CallArg is for reading the value at key from call.Args as a Call. If the +// key is not in Call.Args, the value of the returned value will be nil, and +// the error will be nil. An error is returned if the value is not a Call. +func (c *Call) CallArg(key string) (*Call, bool, error) { + val, ok := c.Args[key] + if !ok { + return nil, false, nil + } + switch tval := val.(type) { + case *Call: + return tval, true, nil + default: + return nil, true, fmt.Errorf("could not convert %v of type %T to Call in Call.CallArg", tval, tval) + } +} + // keys returns a list of argument keys in sorted order. func (c *Call) keys() []string { a := make([]string, 0, len(c.Args)) diff --git a/pql/pql.peg b/pql/pql.peg index 15d33ac65..13e900c33 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -44,6 +44,7 @@ item <- ( 'null' &(comma / sp close) { p.addVal(nil) } / 'false' &(comma / sp close) { p.addVal(false) } / < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end]) } / < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end]) } + / < IDENT > { p.startCall(buffer[begin:end]) } open allargs comma? close { p.addVal(p.endCall()) } / < ([[A-Z]] / [0-9] / '-' / '_' / ':')+ > { p.addVal(buffer[begin:end]) } / < '"' doublequotedstring '"' > { s, _ := strconv.Unquote(buffer[begin:end]); p.addVal(s) } / '\'' < singlequotedstring > '\'' { p.addVal(buffer[begin:end]) } diff --git a/pql/pql.peg.go b/pql/pql.peg.go index 5c3fb9ee9..edb6b6a70 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -101,6 +101,8 @@ const ( ruleAction48 ruleAction49 ruleAction50 + ruleAction51 + ruleAction52 ) var rul3s = [...]string{ @@ -190,6 +192,8 @@ var rul3s = [...]string{ "Action48", "Action49", "Action50", + "Action51", + "Action52", } type token32 struct { @@ -306,7 +310,7 @@ type PQL struct { Buffer string buffer []rune - rules [86]func() bool + rules [88]func() bool parse func(rule ...int) error reset func() Pretty bool @@ -477,29 +481,33 @@ func (p *PQL) Execute() { case ruleAction38: p.addNumVal(buffer[begin:end]) case ruleAction39: - p.addVal(buffer[begin:end]) + p.startCall(buffer[begin:end]) case ruleAction40: - s, _ := strconv.Unquote(buffer[begin:end]) - p.addVal(s) + p.addVal(p.endCall()) case ruleAction41: p.addVal(buffer[begin:end]) case ruleAction42: - p.addField(buffer[begin:end]) + s, _ := strconv.Unquote(buffer[begin:end]) + p.addVal(s) case ruleAction43: - p.addPosStr("_field", buffer[begin:end]) + p.addVal(buffer[begin:end]) case ruleAction44: - p.addPosNum("_col", buffer[begin:end]) + p.addField(buffer[begin:end]) case ruleAction45: - p.addPosStr("_col", buffer[begin:end]) + p.addPosStr("_field", buffer[begin:end]) case ruleAction46: - p.addPosStr("_col", buffer[begin:end]) + p.addPosNum("_col", buffer[begin:end]) case ruleAction47: - p.addPosNum("_row", buffer[begin:end]) + p.addPosStr("_col", buffer[begin:end]) case ruleAction48: - p.addPosStr("_row", buffer[begin:end]) + p.addPosStr("_col", buffer[begin:end]) case ruleAction49: - p.addPosStr("_row", buffer[begin:end]) + p.addPosNum("_row", buffer[begin:end]) case ruleAction50: + p.addPosStr("_row", buffer[begin:end]) + case ruleAction51: + p.addPosStr("_row", buffer[begin:end]) + case ruleAction52: p.addPosStr("_timestamp", buffer[begin:end]) } @@ -661,7 +669,7 @@ func (p *PQL) Init() { add(rulePegText, position13) } { - add(ruleAction50, position) + add(ruleAction52, position) } add(ruletimestamp, position12) } @@ -747,7 +755,7 @@ func (p *PQL) Init() { add(rulePegText, position21) } { - add(ruleAction47, position) + add(ruleAction49, position) } goto l19 l20: @@ -768,7 +776,7 @@ func (p *PQL) Init() { } position++ { - add(ruleAction48, position) + add(ruleAction50, position) } goto l19 l23: @@ -789,7 +797,7 @@ func (p *PQL) Init() { } position++ { - add(ruleAction49, position) + add(ruleAction51, position) } } l19: @@ -1210,53 +1218,8 @@ func (p *PQL) Init() { position, tokenIndex = position7, tokenIndex7 { position63 := position - { - position64 := position - { - position65, tokenIndex65 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l66 - } - position++ - goto l65 - l66: - position, tokenIndex = position65, tokenIndex65 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l5 - } - position++ - } - l65: - l67: - { - position68, tokenIndex68 := position, tokenIndex - { - position69, tokenIndex69 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l70 - } - position++ - goto l69 - l70: - position, tokenIndex = position69, tokenIndex69 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l71 - } - position++ - goto l69 - l71: - position, tokenIndex = position69, tokenIndex69 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l68 - } - position++ - } - l69: - goto l67 - l68: - position, tokenIndex = position68, tokenIndex68 - } - add(ruleIDENT, position64) + if !_rules[ruleIDENT]() { + goto l5 } add(rulePegText, position63) } @@ -1270,15 +1233,15 @@ func (p *PQL) Init() { goto l5 } { - position73, tokenIndex73 := position, tokenIndex + position65, tokenIndex65 := position, tokenIndex if !_rules[rulecomma]() { - goto l73 + goto l65 } - goto l74 - l73: - position, tokenIndex = position73, tokenIndex73 + goto l66 + l65: + position, tokenIndex = position65, tokenIndex65 } - l74: + l66: if !_rules[ruleclose]() { goto l5 } @@ -1296,232 +1259,232 @@ func (p *PQL) Init() { }, /* 2 allargs <- <((Call (comma Call)* (comma args)?) / args / sp)> */ func() bool { - position76, tokenIndex76 := position, tokenIndex + position68, tokenIndex68 := position, tokenIndex { - position77 := position + position69 := position { - position78, tokenIndex78 := position, tokenIndex + position70, tokenIndex70 := position, tokenIndex if !_rules[ruleCall]() { - goto l79 + goto l71 } - l80: + l72: { - position81, tokenIndex81 := position, tokenIndex + position73, tokenIndex73 := position, tokenIndex if !_rules[rulecomma]() { - goto l81 + goto l73 } if !_rules[ruleCall]() { - goto l81 + goto l73 } - goto l80 - l81: - position, tokenIndex = position81, tokenIndex81 + goto l72 + l73: + position, tokenIndex = position73, tokenIndex73 } { - position82, tokenIndex82 := position, tokenIndex + position74, tokenIndex74 := position, tokenIndex if !_rules[rulecomma]() { - goto l82 + goto l74 } if !_rules[ruleargs]() { - goto l82 + goto l74 } - goto l83 - l82: - position, tokenIndex = position82, tokenIndex82 + goto l75 + l74: + position, tokenIndex = position74, tokenIndex74 } - l83: - goto l78 - l79: - position, tokenIndex = position78, tokenIndex78 + l75: + goto l70 + l71: + position, tokenIndex = position70, tokenIndex70 if !_rules[ruleargs]() { - goto l84 - } - goto l78 - l84: - position, tokenIndex = position78, tokenIndex78 - if !_rules[rulesp]() { goto l76 } + goto l70 + l76: + position, tokenIndex = position70, tokenIndex70 + if !_rules[rulesp]() { + goto l68 + } } - l78: - add(ruleallargs, position77) + l70: + add(ruleallargs, position69) } return true - l76: - position, tokenIndex = position76, tokenIndex76 + l68: + position, tokenIndex = position68, tokenIndex68 return false }, /* 3 args <- <(arg (comma args)? sp)> */ func() bool { - position85, tokenIndex85 := position, tokenIndex + position77, tokenIndex77 := position, tokenIndex { - position86 := position + position78 := position if !_rules[rulearg]() { - goto l85 + goto l77 } { - position87, tokenIndex87 := position, tokenIndex + position79, tokenIndex79 := position, tokenIndex if !_rules[rulecomma]() { - goto l87 + goto l79 } if !_rules[ruleargs]() { - goto l87 + goto l79 } - goto l88 - l87: - position, tokenIndex = position87, tokenIndex87 + goto l80 + l79: + position, tokenIndex = position79, tokenIndex79 } - l88: + l80: if !_rules[rulesp]() { - goto l85 + goto l77 } - add(ruleargs, position86) + add(ruleargs, position78) } return true - l85: - position, tokenIndex = position85, tokenIndex85 + l77: + position, tokenIndex = position77, tokenIndex77 return false }, /* 4 arg <- <((field sp '=' sp value) / (field sp COND sp value))> */ func() bool { - position89, tokenIndex89 := position, tokenIndex + position81, tokenIndex81 := position, tokenIndex { - position90 := position + position82 := position { - position91, tokenIndex91 := position, tokenIndex + position83, tokenIndex83 := position, tokenIndex if !_rules[rulefield]() { - goto l92 + goto l84 } if !_rules[rulesp]() { - goto l92 + goto l84 } if buffer[position] != rune('=') { - goto l92 + goto l84 } position++ if !_rules[rulesp]() { - goto l92 + goto l84 } if !_rules[rulevalue]() { - goto l92 + goto l84 } - goto l91 - l92: - position, tokenIndex = position91, tokenIndex91 + goto l83 + l84: + position, tokenIndex = position83, tokenIndex83 if !_rules[rulefield]() { - goto l89 + goto l81 } if !_rules[rulesp]() { - goto l89 + goto l81 } { - position93 := position + position85 := position { - position94, tokenIndex94 := position, tokenIndex + position86, tokenIndex86 := position, tokenIndex if buffer[position] != rune('>') { - goto l95 + goto l87 } position++ if buffer[position] != rune('<') { - goto l95 + goto l87 } position++ { add(ruleAction18, position) } - goto l94 - l95: - position, tokenIndex = position94, tokenIndex94 + goto l86 + l87: + position, tokenIndex = position86, tokenIndex86 if buffer[position] != rune('<') { - goto l97 + goto l89 } position++ if buffer[position] != rune('=') { - goto l97 + goto l89 } position++ { add(ruleAction19, position) } - goto l94 - l97: - position, tokenIndex = position94, tokenIndex94 + goto l86 + l89: + position, tokenIndex = position86, tokenIndex86 if buffer[position] != rune('>') { - goto l99 + goto l91 } position++ if buffer[position] != rune('=') { - goto l99 + goto l91 } position++ { add(ruleAction20, position) } - goto l94 - l99: - position, tokenIndex = position94, tokenIndex94 + goto l86 + l91: + position, tokenIndex = position86, tokenIndex86 if buffer[position] != rune('=') { - goto l101 + goto l93 } position++ if buffer[position] != rune('=') { - goto l101 + goto l93 } position++ { add(ruleAction21, position) } - goto l94 - l101: - position, tokenIndex = position94, tokenIndex94 + goto l86 + l93: + position, tokenIndex = position86, tokenIndex86 if buffer[position] != rune('!') { - goto l103 + goto l95 } position++ if buffer[position] != rune('=') { - goto l103 + goto l95 } position++ { add(ruleAction22, position) } - goto l94 - l103: - position, tokenIndex = position94, tokenIndex94 + goto l86 + l95: + position, tokenIndex = position86, tokenIndex86 if buffer[position] != rune('<') { - goto l105 + goto l97 } position++ { add(ruleAction23, position) } - goto l94 - l105: - position, tokenIndex = position94, tokenIndex94 + goto l86 + l97: + position, tokenIndex = position86, tokenIndex86 if buffer[position] != rune('>') { - goto l89 + goto l81 } position++ { add(ruleAction24, position) } } - l94: - add(ruleCOND, position93) + l86: + add(ruleCOND, position85) } if !_rules[rulesp]() { - goto l89 + goto l81 } if !_rules[rulevalue]() { - goto l89 + goto l81 } } - l91: - add(rulearg, position90) + l83: + add(rulearg, position82) } return true - l89: - position, tokenIndex = position89, tokenIndex89 + l81: + position, tokenIndex = position81, tokenIndex81 return false }, /* 5 COND <- <(('>' '<' Action18) / ('<' '=' Action19) / ('>' '=' Action20) / ('=' '=' Action21) / ('!' '=' Action22) / ('<' Action23) / ('>' Action24))> */ @@ -1530,102 +1493,102 @@ func (p *PQL) Init() { nil, /* 7 condint <- <(<(('-'? [1-9] [0-9]*) / '0')> sp Action27)> */ func() bool { - position110, tokenIndex110 := position, tokenIndex + position102, tokenIndex102 := position, tokenIndex { - position111 := position + position103 := position { - position112 := position + position104 := position { - position113, tokenIndex113 := position, tokenIndex + position105, tokenIndex105 := position, tokenIndex { - position115, tokenIndex115 := position, tokenIndex + position107, tokenIndex107 := position, tokenIndex if buffer[position] != rune('-') { - goto l115 + goto l107 } position++ - goto l116 - l115: - position, tokenIndex = position115, tokenIndex115 + goto l108 + l107: + position, tokenIndex = position107, tokenIndex107 } - l116: + l108: if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l114 + goto l106 } position++ - l117: + l109: { - position118, tokenIndex118 := position, tokenIndex + position110, tokenIndex110 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l118 + goto l110 } position++ - goto l117 - l118: - position, tokenIndex = position118, tokenIndex118 + goto l109 + l110: + position, tokenIndex = position110, tokenIndex110 } - goto l113 - l114: - position, tokenIndex = position113, tokenIndex113 + goto l105 + l106: + position, tokenIndex = position105, tokenIndex105 if buffer[position] != rune('0') { - goto l110 + goto l102 } position++ } - l113: - add(rulePegText, position112) + l105: + add(rulePegText, position104) } if !_rules[rulesp]() { - goto l110 + goto l102 } { add(ruleAction27, position) } - add(rulecondint, position111) + add(rulecondint, position103) } return true - l110: - position, tokenIndex = position110, tokenIndex110 + l102: + position, tokenIndex = position102, tokenIndex102 return false }, /* 8 condLT <- <(<(('<' '=') / '<')> sp Action28)> */ func() bool { - position120, tokenIndex120 := position, tokenIndex + position112, tokenIndex112 := position, tokenIndex { - position121 := position + position113 := position { - position122 := position + position114 := position { - position123, tokenIndex123 := position, tokenIndex + position115, tokenIndex115 := position, tokenIndex if buffer[position] != rune('<') { - goto l124 + goto l116 } position++ if buffer[position] != rune('=') { - goto l124 + goto l116 } position++ - goto l123 - l124: - position, tokenIndex = position123, tokenIndex123 + goto l115 + l116: + position, tokenIndex = position115, tokenIndex115 if buffer[position] != rune('<') { - goto l120 + goto l112 } position++ } - l123: - add(rulePegText, position122) + l115: + add(rulePegText, position114) } if !_rules[rulesp]() { - goto l120 + goto l112 } { add(ruleAction28, position) } - add(rulecondLT, position121) + add(rulecondLT, position113) } return true - l120: - position, tokenIndex = position120, tokenIndex120 + l112: + position, tokenIndex = position112, tokenIndex112 return false }, /* 9 condfield <- <( sp Action29)> */ @@ -1634,1067 +1597,1102 @@ func (p *PQL) Init() { nil, /* 11 value <- <(item / (lbrack Action32 list rbrack Action33))> */ func() bool { - position128, tokenIndex128 := position, tokenIndex + position120, tokenIndex120 := position, tokenIndex { - position129 := position + position121 := position { - position130, tokenIndex130 := position, tokenIndex + position122, tokenIndex122 := position, tokenIndex if !_rules[ruleitem]() { - goto l131 + goto l123 } - goto l130 - l131: - position, tokenIndex = position130, tokenIndex130 + goto l122 + l123: + position, tokenIndex = position122, tokenIndex122 { - position132 := position + position124 := position if buffer[position] != rune('[') { - goto l128 + goto l120 } position++ if !_rules[rulesp]() { - goto l128 + goto l120 } - add(rulelbrack, position132) + add(rulelbrack, position124) } { add(ruleAction32, position) } if !_rules[rulelist]() { - goto l128 + goto l120 } { - position134 := position + position126 := position if !_rules[rulesp]() { - goto l128 + goto l120 } if buffer[position] != rune(']') { - goto l128 + goto l120 } position++ if !_rules[rulesp]() { - goto l128 + goto l120 } - add(rulerbrack, position134) + add(rulerbrack, position126) } { add(ruleAction33, position) } } - l130: - add(rulevalue, position129) + l122: + add(rulevalue, position121) + } + return true + l120: + position, tokenIndex = position120, tokenIndex120 + return false + }, + /* 12 list <- <(item (comma list)?)> */ + func() bool { + position128, tokenIndex128 := position, tokenIndex + { + position129 := position + if !_rules[ruleitem]() { + goto l128 + } + { + position130, tokenIndex130 := position, tokenIndex + if !_rules[rulecomma]() { + goto l130 + } + if !_rules[rulelist]() { + goto l130 + } + goto l131 + l130: + position, tokenIndex = position130, tokenIndex130 + } + l131: + add(rulelist, position129) } return true l128: position, tokenIndex = position128, tokenIndex128 return false }, - /* 12 list <- <(item (comma list)?)> */ + /* 13 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action34) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action35) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action36) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action37) / (<('-'? '.' [0-9]+)> Action38) / ( Action39 open allargs comma? close Action40) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action41) / (<('"' doublequotedstring '"')> Action42) / ('\'' '\'' Action43))> */ func() bool { - position136, tokenIndex136 := position, tokenIndex + position132, tokenIndex132 := position, tokenIndex { - position137 := position - if !_rules[ruleitem]() { - goto l136 - } + position133 := position { - position138, tokenIndex138 := position, tokenIndex - if !_rules[rulecomma]() { - goto l138 - } - if !_rules[rulelist]() { - goto l138 - } - goto l139 - l138: - position, tokenIndex = position138, tokenIndex138 - } - l139: - add(rulelist, position137) - } - return true - l136: - position, tokenIndex = position136, tokenIndex136 - return false - }, - /* 13 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action34) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action35) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action36) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action37) / (<('-'? '.' [0-9]+)> Action38) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action39) / (<('"' doublequotedstring '"')> Action40) / ('\'' '\'' Action41))> */ - func() bool { - position140, tokenIndex140 := position, tokenIndex - { - position141 := position - { - position142, tokenIndex142 := position, tokenIndex + position134, tokenIndex134 := position, tokenIndex if buffer[position] != rune('n') { - goto l143 + goto l135 } position++ if buffer[position] != rune('u') { - goto l143 + goto l135 } position++ if buffer[position] != rune('l') { - goto l143 + goto l135 } position++ if buffer[position] != rune('l') { - goto l143 + goto l135 } position++ { - position144, tokenIndex144 := position, tokenIndex + position136, tokenIndex136 := position, tokenIndex { - position145, tokenIndex145 := position, tokenIndex + position137, tokenIndex137 := position, tokenIndex if !_rules[rulecomma]() { - goto l146 + goto l138 } - goto l145 - l146: - position, tokenIndex = position145, tokenIndex145 + goto l137 + l138: + position, tokenIndex = position137, tokenIndex137 if !_rules[rulesp]() { - goto l143 + goto l135 } if !_rules[ruleclose]() { - goto l143 + goto l135 } } - l145: - position, tokenIndex = position144, tokenIndex144 + l137: + position, tokenIndex = position136, tokenIndex136 } { add(ruleAction34, position) } - goto l142 - l143: - position, tokenIndex = position142, tokenIndex142 + goto l134 + l135: + position, tokenIndex = position134, tokenIndex134 if buffer[position] != rune('t') { - goto l148 + goto l140 } position++ if buffer[position] != rune('r') { - goto l148 + goto l140 } position++ if buffer[position] != rune('u') { - goto l148 + goto l140 } position++ if buffer[position] != rune('e') { - goto l148 + goto l140 } position++ { - position149, tokenIndex149 := position, tokenIndex + position141, tokenIndex141 := position, tokenIndex { - position150, tokenIndex150 := position, tokenIndex + position142, tokenIndex142 := position, tokenIndex if !_rules[rulecomma]() { - goto l151 + goto l143 } - goto l150 - l151: - position, tokenIndex = position150, tokenIndex150 + goto l142 + l143: + position, tokenIndex = position142, tokenIndex142 if !_rules[rulesp]() { - goto l148 + goto l140 } if !_rules[ruleclose]() { - goto l148 + goto l140 } } - l150: - position, tokenIndex = position149, tokenIndex149 + l142: + position, tokenIndex = position141, tokenIndex141 } { add(ruleAction35, position) } - goto l142 - l148: - position, tokenIndex = position142, tokenIndex142 + goto l134 + l140: + position, tokenIndex = position134, tokenIndex134 if buffer[position] != rune('f') { - goto l153 + goto l145 } position++ if buffer[position] != rune('a') { - goto l153 + goto l145 } position++ if buffer[position] != rune('l') { - goto l153 + goto l145 } position++ if buffer[position] != rune('s') { - goto l153 + goto l145 } position++ if buffer[position] != rune('e') { - goto l153 + goto l145 } position++ { - position154, tokenIndex154 := position, tokenIndex + position146, tokenIndex146 := position, tokenIndex { - position155, tokenIndex155 := position, tokenIndex + position147, tokenIndex147 := position, tokenIndex if !_rules[rulecomma]() { - goto l156 + goto l148 } - goto l155 - l156: - position, tokenIndex = position155, tokenIndex155 + goto l147 + l148: + position, tokenIndex = position147, tokenIndex147 if !_rules[rulesp]() { - goto l153 + goto l145 } if !_rules[ruleclose]() { - goto l153 + goto l145 } } - l155: - position, tokenIndex = position154, tokenIndex154 + l147: + position, tokenIndex = position146, tokenIndex146 } { add(ruleAction36, position) } - goto l142 - l153: - position, tokenIndex = position142, tokenIndex142 + goto l134 + l145: + position, tokenIndex = position134, tokenIndex134 { - position159 := position + position151 := position { - position160, tokenIndex160 := position, tokenIndex + position152, tokenIndex152 := position, tokenIndex if buffer[position] != rune('-') { - goto l160 + goto l152 } position++ - goto l161 - l160: - position, tokenIndex = position160, tokenIndex160 + goto l153 + l152: + position, tokenIndex = position152, tokenIndex152 } - l161: + l153: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l158 + goto l150 } position++ - l162: + l154: { - position163, tokenIndex163 := position, tokenIndex + position155, tokenIndex155 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l163 + goto l155 } position++ - goto l162 - l163: - position, tokenIndex = position163, tokenIndex163 + goto l154 + l155: + position, tokenIndex = position155, tokenIndex155 } { - position164, tokenIndex164 := position, tokenIndex + position156, tokenIndex156 := position, tokenIndex if buffer[position] != rune('.') { - goto l164 + goto l156 } position++ - l166: + l158: { - position167, tokenIndex167 := position, tokenIndex + position159, tokenIndex159 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l167 + goto l159 } position++ - goto l166 - l167: - position, tokenIndex = position167, tokenIndex167 + goto l158 + l159: + position, tokenIndex = position159, tokenIndex159 } - goto l165 - l164: - position, tokenIndex = position164, tokenIndex164 + goto l157 + l156: + position, tokenIndex = position156, tokenIndex156 } - l165: - add(rulePegText, position159) + l157: + add(rulePegText, position151) } { add(ruleAction37, position) } - goto l142 - l158: - position, tokenIndex = position142, tokenIndex142 + goto l134 + l150: + position, tokenIndex = position134, tokenIndex134 { - position170 := position + position162 := position { - position171, tokenIndex171 := position, tokenIndex + position163, tokenIndex163 := position, tokenIndex if buffer[position] != rune('-') { - goto l171 + goto l163 } position++ - goto l172 - l171: - position, tokenIndex = position171, tokenIndex171 + goto l164 + l163: + position, tokenIndex = position163, tokenIndex163 } - l172: + l164: if buffer[position] != rune('.') { - goto l169 + goto l161 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l169 + goto l161 } position++ - l173: + l165: { - position174, tokenIndex174 := position, tokenIndex + position166, tokenIndex166 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l174 + goto l166 } position++ - goto l173 - l174: - position, tokenIndex = position174, tokenIndex174 + goto l165 + l166: + position, tokenIndex = position166, tokenIndex166 } - add(rulePegText, position170) + add(rulePegText, position162) } { add(ruleAction38, position) } - goto l142 - l169: - position, tokenIndex = position142, tokenIndex142 + goto l134 + l161: + position, tokenIndex = position134, tokenIndex134 { - position177 := position - { - position180, tokenIndex180 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l181 - } - position++ - goto l180 - l181: - position, tokenIndex = position180, tokenIndex180 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l182 - } - position++ - goto l180 - l182: - position, tokenIndex = position180, tokenIndex180 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l183 - } - position++ - goto l180 - l183: - position, tokenIndex = position180, tokenIndex180 - if buffer[position] != rune('-') { - goto l184 - } - position++ - goto l180 - l184: - position, tokenIndex = position180, tokenIndex180 - if buffer[position] != rune('_') { - goto l185 - } - position++ - goto l180 - l185: - position, tokenIndex = position180, tokenIndex180 - if buffer[position] != rune(':') { - goto l176 - } - position++ + position169 := position + if !_rules[ruleIDENT]() { + goto l168 } - l180: - l178: - { - position179, tokenIndex179 := position, tokenIndex - { - position186, tokenIndex186 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l187 - } - position++ - goto l186 - l187: - position, tokenIndex = position186, tokenIndex186 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l188 - } - position++ - goto l186 - l188: - position, tokenIndex = position186, tokenIndex186 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l189 - } - position++ - goto l186 - l189: - position, tokenIndex = position186, tokenIndex186 - if buffer[position] != rune('-') { - goto l190 - } - position++ - goto l186 - l190: - position, tokenIndex = position186, tokenIndex186 - if buffer[position] != rune('_') { - goto l191 - } - position++ - goto l186 - l191: - position, tokenIndex = position186, tokenIndex186 - if buffer[position] != rune(':') { - goto l179 - } - position++ - } - l186: - goto l178 - l179: - position, tokenIndex = position179, tokenIndex179 - } - add(rulePegText, position177) + add(rulePegText, position169) } { add(ruleAction39, position) } - goto l142 - l176: - position, tokenIndex = position142, tokenIndex142 + if !_rules[ruleopen]() { + goto l168 + } + if !_rules[ruleallargs]() { + goto l168 + } { - position194 := position - if buffer[position] != rune('"') { - goto l193 + position171, tokenIndex171 := position, tokenIndex + if !_rules[rulecomma]() { + goto l171 } - position++ - if !_rules[ruledoublequotedstring]() { - goto l193 - } - if buffer[position] != rune('"') { - goto l193 - } - position++ - add(rulePegText, position194) + goto l172 + l171: + position, tokenIndex = position171, tokenIndex171 + } + l172: + if !_rules[ruleclose]() { + goto l168 } { add(ruleAction40, position) } - goto l142 - l193: - position, tokenIndex = position142, tokenIndex142 - if buffer[position] != rune('\'') { - goto l140 - } - position++ + goto l134 + l168: + position, tokenIndex = position134, tokenIndex134 { - position196 := position - if !_rules[rulesinglequotedstring]() { - goto l140 + position175 := position + { + position178, tokenIndex178 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l179 + } + position++ + goto l178 + l179: + position, tokenIndex = position178, tokenIndex178 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l180 + } + position++ + goto l178 + l180: + position, tokenIndex = position178, tokenIndex178 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l181 + } + position++ + goto l178 + l181: + position, tokenIndex = position178, tokenIndex178 + if buffer[position] != rune('-') { + goto l182 + } + position++ + goto l178 + l182: + position, tokenIndex = position178, tokenIndex178 + if buffer[position] != rune('_') { + goto l183 + } + position++ + goto l178 + l183: + position, tokenIndex = position178, tokenIndex178 + if buffer[position] != rune(':') { + goto l174 + } + position++ } - add(rulePegText, position196) + l178: + l176: + { + position177, tokenIndex177 := position, tokenIndex + { + position184, tokenIndex184 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l185 + } + position++ + goto l184 + l185: + position, tokenIndex = position184, tokenIndex184 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l186 + } + position++ + goto l184 + l186: + position, tokenIndex = position184, tokenIndex184 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l187 + } + position++ + goto l184 + l187: + position, tokenIndex = position184, tokenIndex184 + if buffer[position] != rune('-') { + goto l188 + } + position++ + goto l184 + l188: + position, tokenIndex = position184, tokenIndex184 + if buffer[position] != rune('_') { + goto l189 + } + position++ + goto l184 + l189: + position, tokenIndex = position184, tokenIndex184 + if buffer[position] != rune(':') { + goto l177 + } + position++ + } + l184: + goto l176 + l177: + position, tokenIndex = position177, tokenIndex177 + } + add(rulePegText, position175) } - if buffer[position] != rune('\'') { - goto l140 - } - position++ { add(ruleAction41, position) } + goto l134 + l174: + position, tokenIndex = position134, tokenIndex134 + { + position192 := position + if buffer[position] != rune('"') { + goto l191 + } + position++ + if !_rules[ruledoublequotedstring]() { + goto l191 + } + if buffer[position] != rune('"') { + goto l191 + } + position++ + add(rulePegText, position192) + } + { + add(ruleAction42, position) + } + goto l134 + l191: + position, tokenIndex = position134, tokenIndex134 + if buffer[position] != rune('\'') { + goto l132 + } + position++ + { + position194 := position + if !_rules[rulesinglequotedstring]() { + goto l132 + } + add(rulePegText, position194) + } + if buffer[position] != rune('\'') { + goto l132 + } + position++ + { + add(ruleAction43, position) + } } - l142: - add(ruleitem, position141) + l134: + add(ruleitem, position133) } return true - l140: - position, tokenIndex = position140, tokenIndex140 + l132: + position, tokenIndex = position132, tokenIndex132 return false }, /* 14 doublequotedstring <- <(('\\' '"') / ('\\' '\\') / (!'"' .))*> */ func() bool { { - position199 := position - l200: + position197 := position + l198: { - position201, tokenIndex201 := position, tokenIndex + position199, tokenIndex199 := position, tokenIndex { - position202, tokenIndex202 := position, tokenIndex + position200, tokenIndex200 := position, tokenIndex if buffer[position] != rune('\\') { - goto l203 + goto l201 } position++ if buffer[position] != rune('"') { - goto l203 + goto l201 } position++ - goto l202 - l203: - position, tokenIndex = position202, tokenIndex202 + goto l200 + l201: + position, tokenIndex = position200, tokenIndex200 if buffer[position] != rune('\\') { - goto l204 + goto l202 } position++ if buffer[position] != rune('\\') { - goto l204 + goto l202 } position++ - goto l202 - l204: - position, tokenIndex = position202, tokenIndex202 + goto l200 + l202: + position, tokenIndex = position200, tokenIndex200 { - position205, tokenIndex205 := position, tokenIndex + position203, tokenIndex203 := position, tokenIndex if buffer[position] != rune('"') { - goto l205 + goto l203 } position++ - goto l201 - l205: - position, tokenIndex = position205, tokenIndex205 + goto l199 + l203: + position, tokenIndex = position203, tokenIndex203 } if !matchDot() { - goto l201 + goto l199 } } - l202: - goto l200 - l201: - position, tokenIndex = position201, tokenIndex201 + l200: + goto l198 + l199: + position, tokenIndex = position199, tokenIndex199 } - add(ruledoublequotedstring, position199) + add(ruledoublequotedstring, position197) } return true }, /* 15 singlequotedstring <- <(('\\' '\'') / ('\\' '\\') / (!'\'' .))*> */ func() bool { { - position207 := position - l208: + position205 := position + l206: { - position209, tokenIndex209 := position, tokenIndex + position207, tokenIndex207 := position, tokenIndex { - position210, tokenIndex210 := position, tokenIndex + position208, tokenIndex208 := position, tokenIndex if buffer[position] != rune('\\') { - goto l211 + goto l209 } position++ if buffer[position] != rune('\'') { - goto l211 + goto l209 } position++ - goto l210 - l211: - position, tokenIndex = position210, tokenIndex210 + goto l208 + l209: + position, tokenIndex = position208, tokenIndex208 if buffer[position] != rune('\\') { - goto l212 + goto l210 } position++ if buffer[position] != rune('\\') { - goto l212 + goto l210 } position++ - goto l210 - l212: - position, tokenIndex = position210, tokenIndex210 + goto l208 + l210: + position, tokenIndex = position208, tokenIndex208 { - position213, tokenIndex213 := position, tokenIndex + position211, tokenIndex211 := position, tokenIndex if buffer[position] != rune('\'') { - goto l213 + goto l211 } position++ - goto l209 - l213: - position, tokenIndex = position213, tokenIndex213 + goto l207 + l211: + position, tokenIndex = position211, tokenIndex211 } if !matchDot() { - goto l209 + goto l207 } } - l210: - goto l208 - l209: - position, tokenIndex = position209, tokenIndex209 + l208: + goto l206 + l207: + position, tokenIndex = position207, tokenIndex207 } - add(rulesinglequotedstring, position207) + add(rulesinglequotedstring, position205) } return true }, /* 16 fieldExpr <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '_' / '-')*)> */ func() bool { - position214, tokenIndex214 := position, tokenIndex + position212, tokenIndex212 := position, tokenIndex { - position215 := position + position213 := position { - position216, tokenIndex216 := position, tokenIndex + position214, tokenIndex214 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l217 + goto l215 } position++ - goto l216 - l217: - position, tokenIndex = position216, tokenIndex216 + goto l214 + l215: + position, tokenIndex = position214, tokenIndex214 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l214 + goto l212 } position++ } + l214: l216: - l218: { - position219, tokenIndex219 := position, tokenIndex + position217, tokenIndex217 := position, tokenIndex { - position220, tokenIndex220 := position, tokenIndex + position218, tokenIndex218 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l221 - } - position++ - goto l220 - l221: - position, tokenIndex = position220, tokenIndex220 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l222 - } - position++ - goto l220 - l222: - position, tokenIndex = position220, tokenIndex220 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l223 - } - position++ - goto l220 - l223: - position, tokenIndex = position220, tokenIndex220 - if buffer[position] != rune('_') { - goto l224 - } - position++ - goto l220 - l224: - position, tokenIndex = position220, tokenIndex220 - if buffer[position] != rune('-') { goto l219 } position++ + goto l218 + l219: + position, tokenIndex = position218, tokenIndex218 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l220 + } + position++ + goto l218 + l220: + position, tokenIndex = position218, tokenIndex218 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l221 + } + position++ + goto l218 + l221: + position, tokenIndex = position218, tokenIndex218 + if buffer[position] != rune('_') { + goto l222 + } + position++ + goto l218 + l222: + position, tokenIndex = position218, tokenIndex218 + if buffer[position] != rune('-') { + goto l217 + } + position++ } - l220: - goto l218 - l219: - position, tokenIndex = position219, tokenIndex219 + l218: + goto l216 + l217: + position, tokenIndex = position217, tokenIndex217 } - add(rulefieldExpr, position215) + add(rulefieldExpr, position213) } return true - l214: - position, tokenIndex = position214, tokenIndex214 + l212: + position, tokenIndex = position212, tokenIndex212 return false }, - /* 17 field <- <(<(fieldExpr / reserved)> Action42)> */ + /* 17 field <- <(<(fieldExpr / reserved)> Action44)> */ func() bool { - position225, tokenIndex225 := position, tokenIndex + position223, tokenIndex223 := position, tokenIndex { - position226 := position + position224 := position { - position227 := position + position225 := position { - position228, tokenIndex228 := position, tokenIndex + position226, tokenIndex226 := position, tokenIndex if !_rules[rulefieldExpr]() { - goto l229 + goto l227 } - goto l228 - l229: - position, tokenIndex = position228, tokenIndex228 + goto l226 + l227: + position, tokenIndex = position226, tokenIndex226 { - position230 := position + position228 := position { - position231, tokenIndex231 := position, tokenIndex + position229, tokenIndex229 := position, tokenIndex if buffer[position] != rune('_') { - goto l232 + goto l230 } position++ if buffer[position] != rune('r') { - goto l232 + goto l230 } position++ if buffer[position] != rune('o') { - goto l232 + goto l230 } position++ if buffer[position] != rune('w') { - goto l232 + goto l230 } position++ - goto l231 - l232: - position, tokenIndex = position231, tokenIndex231 + goto l229 + l230: + position, tokenIndex = position229, tokenIndex229 if buffer[position] != rune('_') { - goto l233 + goto l231 } position++ if buffer[position] != rune('c') { - goto l233 + goto l231 } position++ if buffer[position] != rune('o') { - goto l233 + goto l231 } position++ if buffer[position] != rune('l') { - goto l233 + goto l231 } position++ - goto l231 - l233: - position, tokenIndex = position231, tokenIndex231 + goto l229 + l231: + position, tokenIndex = position229, tokenIndex229 if buffer[position] != rune('_') { - goto l234 + goto l232 } position++ if buffer[position] != rune('s') { - goto l234 + goto l232 } position++ if buffer[position] != rune('t') { - goto l234 + goto l232 } position++ if buffer[position] != rune('a') { - goto l234 + goto l232 } position++ if buffer[position] != rune('r') { - goto l234 + goto l232 } position++ if buffer[position] != rune('t') { - goto l234 + goto l232 } position++ - goto l231 - l234: - position, tokenIndex = position231, tokenIndex231 + goto l229 + l232: + position, tokenIndex = position229, tokenIndex229 if buffer[position] != rune('_') { - goto l235 + goto l233 } position++ if buffer[position] != rune('e') { - goto l235 + goto l233 } position++ if buffer[position] != rune('n') { - goto l235 + goto l233 } position++ if buffer[position] != rune('d') { - goto l235 + goto l233 } position++ - goto l231 - l235: - position, tokenIndex = position231, tokenIndex231 + goto l229 + l233: + position, tokenIndex = position229, tokenIndex229 if buffer[position] != rune('_') { - goto l236 + goto l234 } position++ if buffer[position] != rune('t') { - goto l236 + goto l234 } position++ if buffer[position] != rune('i') { - goto l236 + goto l234 } position++ if buffer[position] != rune('m') { - goto l236 + goto l234 } position++ if buffer[position] != rune('e') { - goto l236 + goto l234 } position++ if buffer[position] != rune('s') { - goto l236 + goto l234 } position++ if buffer[position] != rune('t') { - goto l236 + goto l234 } position++ if buffer[position] != rune('a') { - goto l236 + goto l234 } position++ if buffer[position] != rune('m') { - goto l236 + goto l234 } position++ if buffer[position] != rune('p') { - goto l236 + goto l234 } position++ - goto l231 - l236: - position, tokenIndex = position231, tokenIndex231 + goto l229 + l234: + position, tokenIndex = position229, tokenIndex229 if buffer[position] != rune('_') { - goto l225 + goto l223 } position++ if buffer[position] != rune('f') { - goto l225 + goto l223 } position++ if buffer[position] != rune('i') { - goto l225 + goto l223 } position++ if buffer[position] != rune('e') { - goto l225 + goto l223 } position++ if buffer[position] != rune('l') { - goto l225 + goto l223 } position++ if buffer[position] != rune('d') { - goto l225 + goto l223 } position++ } - l231: - add(rulereserved, position230) + l229: + add(rulereserved, position228) } } - l228: - add(rulePegText, position227) + l226: + add(rulePegText, position225) } { - add(ruleAction42, position) + add(ruleAction44, position) } - add(rulefield, position226) + add(rulefield, position224) } return true - l225: - position, tokenIndex = position225, tokenIndex225 + l223: + position, tokenIndex = position223, tokenIndex223 return false }, /* 18 reserved <- <(('_' 'r' 'o' 'w') / ('_' 'c' 'o' 'l') / ('_' 's' 't' 'a' 'r' 't') / ('_' 'e' 'n' 'd') / ('_' 't' 'i' 'm' 'e' 's' 't' 'a' 'm' 'p') / ('_' 'f' 'i' 'e' 'l' 'd'))> */ nil, - /* 19 posfield <- <( Action43)> */ + /* 19 posfield <- <( Action45)> */ func() bool { - position239, tokenIndex239 := position, tokenIndex + position237, tokenIndex237 := position, tokenIndex { - position240 := position + position238 := position { - position241 := position + position239 := position if !_rules[rulefieldExpr]() { - goto l239 + goto l237 } - add(rulePegText, position241) + add(rulePegText, position239) } { - add(ruleAction43, position) + add(ruleAction45, position) } - add(ruleposfield, position240) + add(ruleposfield, position238) } return true - l239: - position, tokenIndex = position239, tokenIndex239 + l237: + position, tokenIndex = position237, tokenIndex237 return false }, /* 20 uint <- <(([1-9] [0-9]*) / '0')> */ func() bool { - position243, tokenIndex243 := position, tokenIndex + position241, tokenIndex241 := position, tokenIndex { - position244 := position + position242 := position { - position245, tokenIndex245 := position, tokenIndex + position243, tokenIndex243 := position, tokenIndex if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l246 + goto l244 } position++ - l247: + l245: { - position248, tokenIndex248 := position, tokenIndex + position246, tokenIndex246 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l248 + goto l246 } position++ - goto l247 - l248: - position, tokenIndex = position248, tokenIndex248 + goto l245 + l246: + position, tokenIndex = position246, tokenIndex246 } - goto l245 - l246: - position, tokenIndex = position245, tokenIndex245 + goto l243 + l244: + position, tokenIndex = position243, tokenIndex243 if buffer[position] != rune('0') { - goto l243 + goto l241 } position++ } - l245: - add(ruleuint, position244) + l243: + add(ruleuint, position242) } return true - l243: - position, tokenIndex = position243, tokenIndex243 + l241: + position, tokenIndex = position241, tokenIndex241 return false }, - /* 21 col <- <(( Action44) / ('\'' '\'' Action45) / ('"' '"' Action46))> */ + /* 21 col <- <(( Action46) / ('\'' '\'' Action47) / ('"' '"' Action48))> */ func() bool { - position249, tokenIndex249 := position, tokenIndex + position247, tokenIndex247 := position, tokenIndex { - position250 := position + position248 := position { - position251, tokenIndex251 := position, tokenIndex + position249, tokenIndex249 := position, tokenIndex { - position253 := position + position251 := position if !_rules[ruleuint]() { - goto l252 + goto l250 } - add(rulePegText, position253) + add(rulePegText, position251) } { - add(ruleAction44, position) + add(ruleAction46, position) } - goto l251 - l252: - position, tokenIndex = position251, tokenIndex251 + goto l249 + l250: + position, tokenIndex = position249, tokenIndex249 if buffer[position] != rune('\'') { - goto l255 + goto l253 + } + position++ + { + position254 := position + if !_rules[rulesinglequotedstring]() { + goto l253 + } + add(rulePegText, position254) + } + if buffer[position] != rune('\'') { + goto l253 + } + position++ + { + add(ruleAction47, position) + } + goto l249 + l253: + position, tokenIndex = position249, tokenIndex249 + if buffer[position] != rune('"') { + goto l247 } position++ { position256 := position - if !_rules[rulesinglequotedstring]() { - goto l255 + if !_rules[ruledoublequotedstring]() { + goto l247 } add(rulePegText, position256) } - if buffer[position] != rune('\'') { - goto l255 - } - position++ - { - add(ruleAction45, position) - } - goto l251 - l255: - position, tokenIndex = position251, tokenIndex251 if buffer[position] != rune('"') { - goto l249 + goto l247 } position++ { - position258 := position - if !_rules[ruledoublequotedstring]() { - goto l249 - } - add(rulePegText, position258) - } - if buffer[position] != rune('"') { - goto l249 - } - position++ - { - add(ruleAction46, position) + add(ruleAction48, position) } } - l251: - add(rulecol, position250) + l249: + add(rulecol, position248) } return true - l249: - position, tokenIndex = position249, tokenIndex249 + l247: + position, tokenIndex = position247, tokenIndex247 return false }, - /* 22 row <- <(( Action47) / ('\'' '\'' Action48) / ('"' '"' Action49))> */ + /* 22 row <- <(( Action49) / ('\'' '\'' Action50) / ('"' '"' Action51))> */ nil, /* 23 open <- <('(' sp)> */ + func() bool { + position259, tokenIndex259 := position, tokenIndex + { + position260 := position + if buffer[position] != rune('(') { + goto l259 + } + position++ + if !_rules[rulesp]() { + goto l259 + } + add(ruleopen, position260) + } + return true + l259: + position, tokenIndex = position259, tokenIndex259 + return false + }, + /* 24 close <- <(')' sp)> */ func() bool { position261, tokenIndex261 := position, tokenIndex { position262 := position - if buffer[position] != rune('(') { + if buffer[position] != rune(')') { goto l261 } position++ if !_rules[rulesp]() { goto l261 } - add(ruleopen, position262) + add(ruleclose, position262) } return true l261: position, tokenIndex = position261, tokenIndex261 return false }, - /* 24 close <- <(')' sp)> */ - func() bool { - position263, tokenIndex263 := position, tokenIndex - { - position264 := position - if buffer[position] != rune(')') { - goto l263 - } - position++ - if !_rules[rulesp]() { - goto l263 - } - add(ruleclose, position264) - } - return true - l263: - position, tokenIndex = position263, tokenIndex263 - return false - }, /* 25 sp <- <(' ' / '\t' / '\n')*> */ func() bool { { - position266 := position - l267: + position264 := position + l265: { - position268, tokenIndex268 := position, tokenIndex + position266, tokenIndex266 := position, tokenIndex { - position269, tokenIndex269 := position, tokenIndex + position267, tokenIndex267 := position, tokenIndex if buffer[position] != rune(' ') { - goto l270 - } - position++ - goto l269 - l270: - position, tokenIndex = position269, tokenIndex269 - if buffer[position] != rune('\t') { - goto l271 - } - position++ - goto l269 - l271: - position, tokenIndex = position269, tokenIndex269 - if buffer[position] != rune('\n') { goto l268 } position++ + goto l267 + l268: + position, tokenIndex = position267, tokenIndex267 + if buffer[position] != rune('\t') { + goto l269 + } + position++ + goto l267 + l269: + position, tokenIndex = position267, tokenIndex267 + if buffer[position] != rune('\n') { + goto l266 + } + position++ } - l269: - goto l267 - l268: - position, tokenIndex = position268, tokenIndex268 + l267: + goto l265 + l266: + position, tokenIndex = position266, tokenIndex266 } - add(rulesp, position266) + add(rulesp, position264) } return true }, /* 26 comma <- <(sp ',' sp)> */ func() bool { - position272, tokenIndex272 := position, tokenIndex + position270, tokenIndex270 := position, tokenIndex { - position273 := position + position271 := position if !_rules[rulesp]() { - goto l272 + goto l270 } if buffer[position] != rune(',') { - goto l272 + goto l270 } position++ if !_rules[rulesp]() { - goto l272 + goto l270 } - add(rulecomma, position273) + add(rulecomma, position271) } return true - l272: - position, tokenIndex = position272, tokenIndex272 + l270: + position, tokenIndex = position270, tokenIndex270 return false }, /* 27 lbrack <- <('[' sp)> */ @@ -2702,142 +2700,196 @@ func (p *PQL) Init() { /* 28 rbrack <- <(sp ']' sp)> */ nil, /* 29 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ - nil, - /* 30 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ func() bool { - position277, tokenIndex277 := position, tokenIndex + position274, tokenIndex274 := position, tokenIndex { - position278 := position - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 - } - position++ - if buffer[position] != rune('-') { - goto l277 - } - position++ + position275 := position { - position279, tokenIndex279 := position, tokenIndex - if buffer[position] != rune('0') { - goto l280 - } - position++ - goto l279 - l280: - position, tokenIndex = position279, tokenIndex279 - if buffer[position] != rune('1') { + position276, tokenIndex276 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { goto l277 } position++ + goto l276 + l277: + position, tokenIndex = position276, tokenIndex276 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l274 + } + position++ } - l279: + l276: + l278: + { + position279, tokenIndex279 := position, tokenIndex + { + position280, tokenIndex280 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l281 + } + position++ + goto l280 + l281: + position, tokenIndex = position280, tokenIndex280 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l282 + } + position++ + goto l280 + l282: + position, tokenIndex = position280, tokenIndex280 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l279 + } + position++ + } + l280: + goto l278 + l279: + position, tokenIndex = position279, tokenIndex279 + } + add(ruleIDENT, position275) + } + return true + l274: + position, tokenIndex = position274, tokenIndex274 + return false + }, + /* 30 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ + func() bool { + position283, tokenIndex283 := position, tokenIndex + { + position284 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 + goto l283 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l283 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l283 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l283 } position++ if buffer[position] != rune('-') { - goto l277 + goto l283 + } + position++ + { + position285, tokenIndex285 := position, tokenIndex + if buffer[position] != rune('0') { + goto l286 + } + position++ + goto l285 + l286: + position, tokenIndex = position285, tokenIndex285 + if buffer[position] != rune('1') { + goto l283 + } + position++ + } + l285: + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l283 + } + position++ + if buffer[position] != rune('-') { + goto l283 } position++ if c := buffer[position]; c < rune('0') || c > rune('3') { - goto l277 + goto l283 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 + goto l283 } position++ if buffer[position] != rune('T') { - goto l277 + goto l283 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 + goto l283 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 + goto l283 } position++ if buffer[position] != rune(':') { - goto l277 + goto l283 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 + goto l283 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l277 + goto l283 } position++ - add(ruletimestampbasicfmt, position278) + add(ruletimestampbasicfmt, position284) } return true - l277: - position, tokenIndex = position277, tokenIndex277 + l283: + position, tokenIndex = position283, tokenIndex283 return false }, /* 31 timestampfmt <- <(('"' timestampbasicfmt '"') / ('\'' timestampbasicfmt '\'') / timestampbasicfmt)> */ func() bool { - position281, tokenIndex281 := position, tokenIndex + position287, tokenIndex287 := position, tokenIndex { - position282 := position + position288 := position { - position283, tokenIndex283 := position, tokenIndex + position289, tokenIndex289 := position, tokenIndex if buffer[position] != rune('"') { - goto l284 + goto l290 } position++ if !_rules[ruletimestampbasicfmt]() { - goto l284 + goto l290 } if buffer[position] != rune('"') { - goto l284 + goto l290 } position++ - goto l283 - l284: - position, tokenIndex = position283, tokenIndex283 + goto l289 + l290: + position, tokenIndex = position289, tokenIndex289 if buffer[position] != rune('\'') { - goto l285 + goto l291 } position++ if !_rules[ruletimestampbasicfmt]() { - goto l285 + goto l291 } if buffer[position] != rune('\'') { - goto l285 + goto l291 } position++ - goto l283 - l285: - position, tokenIndex = position283, tokenIndex283 + goto l289 + l291: + position, tokenIndex = position289, tokenIndex289 if !_rules[ruletimestampbasicfmt]() { - goto l281 + goto l287 } } - l283: - add(ruletimestampfmt, position282) + l289: + add(ruletimestampfmt, position288) } return true - l281: - position, tokenIndex = position281, tokenIndex281 + l287: + position, tokenIndex = position287, tokenIndex287 return false }, - /* 32 timestamp <- <( Action50)> */ + /* 32 timestamp <- <( Action52)> */ nil, /* 34 Action0 <- <{p.startCall("Set")}> */ nil, @@ -2918,29 +2970,33 @@ func (p *PQL) Init() { nil, /* 73 Action38 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 74 Action39 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 74 Action39 <- <{ p.startCall(buffer[begin:end]) }> */ nil, - /* 75 Action40 <- <{ s, _ := strconv.Unquote(buffer[begin:end]); p.addVal(s) }> */ + /* 75 Action40 <- <{ p.addVal(p.endCall()) }> */ nil, /* 76 Action41 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 77 Action42 <- <{ p.addField(buffer[begin:end]) }> */ + /* 77 Action42 <- <{ s, _ := strconv.Unquote(buffer[begin:end]); p.addVal(s) }> */ nil, - /* 78 Action43 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ + /* 78 Action43 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 79 Action44 <- <{p.addPosNum("_col", buffer[begin:end])}> */ + /* 79 Action44 <- <{ p.addField(buffer[begin:end]) }> */ nil, - /* 80 Action45 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + /* 80 Action45 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ nil, - /* 81 Action46 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + /* 81 Action46 <- <{p.addPosNum("_col", buffer[begin:end])}> */ nil, - /* 82 Action47 <- <{p.addPosNum("_row", buffer[begin:end])}> */ + /* 82 Action47 <- <{p.addPosStr("_col", buffer[begin:end])}> */ nil, - /* 83 Action48 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + /* 83 Action48 <- <{p.addPosStr("_col", buffer[begin:end])}> */ nil, - /* 84 Action49 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + /* 84 Action49 <- <{p.addPosNum("_row", buffer[begin:end])}> */ nil, - /* 85 Action50 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ + /* 85 Action50 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + nil, + /* 86 Action51 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + nil, + /* 87 Action52 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ nil, } p.rules = _rules diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index 2025120e9..f79780c8c 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -612,6 +612,23 @@ func TestPQLDeepEquality(t *testing.T) { }, }, }}, + { + name: "GroupBy", + call: "GroupBy(Rows(), filter=Row(a=1))", + exp: &Call{ + Name: "GroupBy", + Args: map[string]interface{}{ + "filter": &Call{ + Name: "Row", + Args: map[string]interface{}{ + "a": int64(1), + }, + }, + }, + Children: []*Call{ + {Name: "Rows"}, + }, + }}, } for i, test := range tests { From 82a9ef2059386483d364f7cdbbd2a040ef6a0e7a Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 21 Nov 2018 16:35:50 +0300 Subject: [PATCH 007/125] Added /internal/translate/keys endpoint --- http/handler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/http/handler.go b/http/handler.go index 330acae0d..68719fdfe 100644 --- a/http/handler.go +++ b/http/handler.go @@ -24,8 +24,8 @@ import ( "io/ioutil" "net" "net/http" - _ "net/http/pprof" - "net/url" // Imported for its side-effect of registering pprof endpoints with the server. + _ "net/http/pprof" // Imported for its side-effect of registering pprof endpoints with the server. + "net/url" "reflect" "runtime/debug" "strconv" From b77c8a630b741ee2222ddd71bdca1f4018da7587 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Wed, 28 Nov 2018 14:46:15 -0500 Subject: [PATCH 008/125] Add in place union --- roaring/roaring.go | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 9c4274df9..8cc856bfc 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -396,9 +396,24 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { } // Union returns the bitwise union of b and other. -func (b *Bitmap) Union(other *Bitmap) *Bitmap { +func (b *Bitmap) Union(others ...*Bitmap) *Bitmap { output := NewBitmap() + output.UnionInPlace(others...) + return output +} +// UnionInPlace returns the bitwise union of b and other, modifying +// b in place. +func (b *Bitmap) UnionInPlace(others ...*Bitmap) { + for _, other := range others { + b.unionIntoTarget(other, b) + } +} + +// unionIntoTarget stores the union of b and other into target. b and other will +// be left unchanged, but target will be modified in place. Used to share +// the union logic between the copy-on-write and in-place functions. +func (b *Bitmap) unionIntoTarget(other *Bitmap, target *Bitmap) { iiter, _ := b.Containers.Iterator(0) jiter, _ := other.Containers.Iterator(0) i, j := iiter.Next(), jiter.Next() @@ -406,21 +421,20 @@ func (b *Bitmap) Union(other *Bitmap) *Bitmap { kj, cj := jiter.Value() for i || j { if i && (!j || ki < kj) { - output.Containers.Put(ki, ci.Clone()) + target.Containers.Put(ki, ci.Clone()) i = iiter.Next() ki, ci = iiter.Value() } else if j && (!i || ki > kj) { - output.Containers.Put(kj, cj.Clone()) + target.Containers.Put(kj, cj.Clone()) j = jiter.Next() kj, cj = jiter.Value() } else { // ki == kj - output.Containers.Put(ki, union(ci, cj)) + target.Containers.Put(ki, union(ci, cj)) i, j = iiter.Next(), jiter.Next() ki, ci = iiter.Value() kj, cj = jiter.Value() } } - return output } // Difference returns the difference of b and other. From d3606e274dd84363859011dcf43dbb043556dacb Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Wed, 28 Nov 2018 15:07:56 -0500 Subject: [PATCH 009/125] fix bug --- roaring/roaring.go | 1 + 1 file changed, 1 insertion(+) diff --git a/roaring/roaring.go b/roaring/roaring.go index 8cc856bfc..99f7a06fa 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -398,6 +398,7 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { // Union returns the bitwise union of b and other. func (b *Bitmap) Union(others ...*Bitmap) *Bitmap { output := NewBitmap() + output.UnionInPlace(b) output.UnionInPlace(others...) return output } From d2da91fdde0638b49db541879e3da9355deda750 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Wed, 28 Nov 2018 15:20:36 -0500 Subject: [PATCH 010/125] add test --- roaring/roaring_test.go | 83 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index f5c016ea3..f554d1df3 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -396,7 +396,44 @@ func TestBitmap_Union1(t *testing.T) { if n := result.Count(); n != 75007 { t.Fatalf("unexpected n: %d", n) } +} +func TestBitmap_UnionInPlace1(t *testing.T) { + var ( + bm0 = roaring.NewFileBitmap(0, 2683177) + bm1 = roaring.NewFileBitmap() + result = roaring.NewBitmap() + ) + for i := uint64(628); i < 2683301; i++ { + bm1.Add(i) + } + bm1.Add(4000000) + + result.UnionInPlace(bm0, bm1) + if n := result.Count(); n != 2682675 { + t.Fatalf("unexpected n: %d", n) + } + + bm := testBM() + result = roaring.NewBitmap() + result.UnionInPlace(bm, bm0) + if n := result.Count(); n != 75009 { + t.Fatalf("unexpected n: %d", n) + } + + result = roaring.NewBitmap() + result.UnionInPlace(bm, bm) + if n := result.Count(); n != 75007 { + t.Fatalf("unexpected n: %d", n) + } + + // Make sure the bitmaps weren't mutated. + if n := bm0.Count(); n != 2 { + t.Fatalf("unexpected n: %d", n) + } + if n := bm1.Count(); n != 2682674 { + t.Fatalf("unexpected n: %d", n) + } } func TestBitmap_Intersection_Empty(t *testing.T) { @@ -544,11 +581,35 @@ func TestBitmap_Union(t *testing.T) { bm0 := roaring.NewFileBitmap(0, 1000001, 1000002, 1000003) bm1 := roaring.NewFileBitmap(0, 50000, 1000001, 1000002) result := bm0.Union(bm1) + if n := result.Count(); n != 5 { t.Fatalf("unexpected n: %d", n) } } +func TestBitmap_UnionInPlace(t *testing.T) { + var ( + bm0 = roaring.NewFileBitmap(0, 1000001, 1000002, 1000003) + bm1 = roaring.NewFileBitmap(0, 50000, 1000001, 1000002) + result = roaring.NewBitmap() + ) + result.UnionInPlace(bm0, bm1) + + // Make sure the union worked. + if n := result.Count(); n != 5 { + t.Fatalf("unexpected n: %d", n) + } + + // Make sure the other bitmaps weren't mutated. + if n := bm0.Count(); n != 4 { + t.Fatalf("unexpected n: %d", n) + } + if n := bm1.Count(); n != 4 { + t.Fatalf("unexpected n: %d", n) + } + +} + func TestBitmap_Xor(t *testing.T) { bm0 := testBM() bm1 := roaring.NewFileBitmap(0, 1, 2, 3) @@ -1350,3 +1411,25 @@ func BenchmarkSliceDescending(b *testing.B) { } } } + +func BenchmarkUnion(b *testing.B) { + // a1, a2, b, r1, r2 *roaring.Bitmap + data := getBenchData(b) + for n := 0; n < b.N; n++ { + data.a1. + Union(data.a2). + Union(data.b). + Union(data.r1). + Union(data.r2) + } +} + +// func BenchmarkUnionBulk(b *testing.B) { +// // a1, a2, b, r1, r2 *roaring.Bitmap +// data := getBenchData(b) +// yolo := roaring.NewBitmap() +// for n := 0; n < b.N; n++ { +// yolo. +// BulkUnion(data.a1, data.a2, data.b, data.r1, data.r2) +// } +// } From 2e1f60ac426f2fac9388b2dbf61107b2676ed669 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Wed, 28 Nov 2018 18:40:32 -0500 Subject: [PATCH 011/125] horrible wip --- roaring/roaring.go | 252 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 232 insertions(+), 20 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 99f7a06fa..bc75b5119 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -411,31 +411,176 @@ func (b *Bitmap) UnionInPlace(others ...*Bitmap) { } } +type wrapperIter struct { + iter ContainerIterator + hasNext bool + handled bool +} + // unionIntoTarget stores the union of b and other into target. b and other will // be left unchanged, but target will be modified in place. Used to share // the union logic between the copy-on-write and in-place functions. -func (b *Bitmap) unionIntoTarget(other *Bitmap, target *Bitmap) { - iiter, _ := b.Containers.Iterator(0) - jiter, _ := other.Containers.Iterator(0) - i, j := iiter.Next(), jiter.Next() - ki, ci := iiter.Value() - kj, cj := jiter.Value() - for i || j { - if i && (!j || ki < kj) { - target.Containers.Put(ki, ci.Clone()) - i = iiter.Next() - ki, ci = iiter.Value() - } else if j && (!i || ki > kj) { - target.Containers.Put(kj, cj.Clone()) - j = jiter.Next() - kj, cj = jiter.Value() - } else { // ki == kj - target.Containers.Put(ki, union(ci, cj)) - i, j = iiter.Next(), jiter.Next() - ki, ci = iiter.Value() - kj, cj = jiter.Value() +func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { + otherIters := make([]wrapperIter, 0, len(others)) + for _, other := range others { + otherIter, _ := other.Containers.Iterator(0) + otherIters = append(otherIters, wrapperIter{ + iter: otherIter, + }) + } + + // Loop until we've exhausted every iter. + for { + hasNext := false + for i, otherIter := range otherIters { + next := otherIter.iter.Next() + otherIters[i].hasNext = next + otherIters[i].handled = false + if next { + hasNext = true + } + } + + if !hasNext { + // None of the iters had any more values, we're done. + break + } + + // Loop until every iters current value has been handled. + for { + for i, iIter := range otherIters { + if !iIter.hasNext || iIter.handled { + continue + } + + // Can store key-level statistics here + iKey, iContainer := iIter.iter.Value() + n := iContainer.n + needsUnion := false + hasMaxRange := iContainer.n == maxContainerVal+1 + for _, jIter := range otherIters[i:] { + if hasMaxRange { + continue + } + + // Calculate key-level statistics here + jKey, jContainer := jIter.iter.Value() + + if iKey == jKey { + needsUnion = true + n += jContainer.n + if !hasMaxRange { + hasMaxRange = jContainer.n == maxContainerVal+1 + } + } + } + + if !needsUnion { + // TODO: Don't clone if sealed + target.Containers.Put(iKey, iContainer.Clone()) + otherIters[i].handled = true + continue + } + + // Need to union + if hasMaxRange { + // Use the max range + container := &Container{ + runs: []interval16{{start: 0, last: maxContainerVal}}, + containerType: containerRun, + n: maxContainerVal + 1, + } + target.Containers.Put(iKey, container) + } else { + // TODO: Implement this + // if n < ArrayMaxSize { + // // Use an array + // // container := &Container{ + // // containerType: containerArray, + // // array: make([]uint16, 0, n), + // // } + // } + // else { + // Use a bitmap + buf := make([]uint64, bitmapN) + ob := buf[:bitmapN] + output := &Container{ + bitmap: ob, + n: n, + containerType: containerBitmap, + } + + for _, jIter := range otherIters { + jKey, jContainer := jIter.iter.Value() + + if iKey == jKey { + } + } + } + } } } + // iiter, _ := b.Containers.Iterator(0) + // i := iiter.Next() + // for i { + // for _, otherIter := range otherIters { + // jiter := otherIter + // j := jiter.Next() + // if !j { + // continue + // } + + // ki, ci := iiter.Value() + // kj, cj := jiter.Value() + + // if i && (!j || ki < kj) { + // // TODO: Don't clone if sealed + // target.Containers.Put(ki, ci.Clone()) + // i = iiter.Next() + // ki, ci = iiter.Value() + // } else if j && (!i || ki > kj) { + // // TODO: Don't clone if sealed + // target.Containers.Put(kj, cj.Clone()) + // j = jiter.Next() + // kj, cj = jiter.Value() + // } else { // ki == kj + // // TODO: unionIntoTarget? + // target.Containers.Put(ki, union(ci, cj)) + // i, j = iiter.Next(), jiter.Next() + // ki, ci = iiter.Value() + // kj, cj = jiter.Value() + // } + + // } + // } + // } + + // for _, otherIter := range otherIters { + // jiter := otherIter + + // i, j := iiter.Next(), jiter.Next() + // ki, ci := iiter.Value() + // kj, cj := jiter.Value() + // for i || j { + // if i && (!j || ki < kj) { + // // TODO: Don't clone if sealed + // target.Containers.Put(ki, ci.Clone()) + // i = iiter.Next() + // ki, ci = iiter.Value() + // } else if j && (!i || ki > kj) { + // // TODO: Don't clone if sealed + // target.Containers.Put(kj, cj.Clone()) + // j = jiter.Next() + // kj, cj = jiter.Value() + // } else { // ki == kj + // // TODO: unionIntoTarget? + // target.Containers.Put(ki, union(ci, cj)) + // i, j = iiter.Next(), jiter.Next() + // ki, ci = iiter.Value() + // kj, cj = jiter.Value() + // } + // } + // } } // Difference returns the difference of b and other. @@ -2289,6 +2434,38 @@ func unionArrayArray(a, b *Container) *Container { return output } +// func unionArrayArrayInPlace(a, b *Container) *Container { +// statsHit("union/ArrayArray") +// output := a +// na, nb := len(a.array), len(b.array) +// for i, j := 0, 0; ; { +// if i >= na && j >= nb { +// break +// } else if i < na && j >= nb { +// output.add(a.array[i]) +// i++ +// continue +// } else if i >= na && j < nb { +// output.add(b.array[j]) +// j++ +// continue +// } + +// va, vb := a.array[i], b.array[j] +// if va < vb { +// output.add(va) +// i++ +// } else if va > vb { +// output.add(vb) +// j++ +// } else { +// output.add(va) +// i, j = i+1, j+1 +// } +// } +// return output +// } + // unionArrayRun optimistically assumes that the result will be a run container, // and converts to a bitmap or array container afterwards if necessary. func unionArrayRun(a, b *Container) *Container { @@ -2399,6 +2576,13 @@ func unionBitmapRun(a, b *Container) *Container { return output } +func unionBitmapRunInPlace(a, b *Container) { + statsHit("union/BitmapRun") + for j := 0; j < len(b.runs); j++ { + a.bitmapSetRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1) + } +} + const maxBitmap = 0xFFFFFFFFFFFFFFFF // sets all bits in [i, j) (c must be a bitmap container) @@ -2518,6 +2702,15 @@ func unionArrayBitmap(a, b *Container) *Container { return output } +func unionBitmapArrayInPlace(a, b *Container) { + for _, v := range b.array { + if !a.bitmapContains(v) { + a.bitmap[v/64] |= (1 << uint64(v%64)) + a.n++ + } + } +} + func unionBitmapBitmap(a, b *Container) *Container { // local variables added to prevent BCE checks in loop // see https://go101.org/article/bounds-check-elimination.html @@ -2544,6 +2737,25 @@ func unionBitmapBitmap(a, b *Container) *Container { return output } +func unionBitmapBitmapInPlace(a, b *Container) { + // local variables added to prevent BCE checks in loop + // see https://go101.org/article/bounds-check-elimination.html + + var ( + ab = a.bitmap[:bitmapN] + bb = b.bitmap[:bitmapN] + + n int32 + ) + + for i := 0; i < bitmapN; i++ { + ab[i] = ab[i] | bb[i] + n += int32(popcount(ab[i])) + } + + a.n = n +} + func difference(a, b *Container) *Container { if a.isArray() { if b.isArray() { From af3cb91a40c6b217afc8084cd164f2fff4c03d5d Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Wed, 28 Nov 2018 18:42:38 -0500 Subject: [PATCH 012/125] first --- roaring/roaring.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index bc75b5119..b8b5734c5 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -504,7 +504,7 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { // Use a bitmap buf := make([]uint64, bitmapN) ob := buf[:bitmapN] - output := &Container{ + container := &Container{ bitmap: ob, n: n, containerType: containerBitmap, @@ -514,6 +514,13 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { jKey, jContainer := jIter.iter.Value() if iKey == jKey { + if jContainer.isArray() { + unionBitmapArrayInPlace(container, jContainer) + } else if jContainer.isRun() { + unionBitmapRunInPlace(container, jContainer) + } else { + unionBitmapBitmapInPlace(container, jContainer) + } } } } From 30946a037267664570c267f8abe72308e19725b1 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Wed, 28 Nov 2018 20:15:31 -0500 Subject: [PATCH 013/125] working --- roaring/roaring.go | 280 ++++++++++++++++++---------------------- roaring/roaring_test.go | 4 + 2 files changed, 133 insertions(+), 151 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index b8b5734c5..7f88cb651 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -406,9 +406,10 @@ func (b *Bitmap) Union(others ...*Bitmap) *Bitmap { // UnionInPlace returns the bitwise union of b and other, modifying // b in place. func (b *Bitmap) UnionInPlace(others ...*Bitmap) { - for _, other := range others { - b.unionIntoTarget(other, b) - } + b.unionIntoTarget(b, others...) + // for _, other := range others { + // b.unionIntoTarget(other, b) + // } } type wrapperIter struct { @@ -421,17 +422,135 @@ type wrapperIter struct { // be left unchanged, but target will be modified in place. Used to share // the union logic between the copy-on-write and in-place functions. func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { - otherIters := make([]wrapperIter, 0, len(others)) - for _, other := range others { - otherIter, _ := other.Containers.Iterator(0) + fmt.Println("huh: ", len(others)) + otherIters := make([]wrapperIter, 0, len(others)+1) + bIter, _ := b.Containers.Iterator(0) + next := bIter.Next() + if next { + fmt.Println("Appending self") otherIters = append(otherIters, wrapperIter{ - iter: otherIter, + iter: bIter, + hasNext: true, + handled: false, }) } + for _, other := range others { + otherIter, _ := other.Containers.Iterator(0) + next := otherIter.Next() + if next { + otherIters = append(otherIters, wrapperIter{ + iter: otherIter, + hasNext: true, + handled: false, + }) + } + } + // Loop until we've exhausted every iter. - for { - hasNext := false + hasNext := true + fmt.Println(len(otherIters)) + for hasNext { + // Loop until every iters current value has been handled. + // for { + for i, iIter := range otherIters { + fmt.Println("copter") + if !iIter.hasNext || iIter.handled { + continue + } + + fmt.Println("here?") + // Can store key-level statistics here + iKey, iContainer := iIter.iter.Value() + fmt.Println("iKey: ", iKey) + fmt.Println("iContainer: ", iContainer) + n := iContainer.n + needsUnion := false + hasMaxRange := iContainer.n == maxContainerVal+1 + for _, jIter := range otherIters[i:] { + if hasMaxRange { + continue + } + + // Calculate key-level statistics here + jKey, jContainer := jIter.iter.Value() + fmt.Println("jContainer.n: ", jContainer.n) + + if iKey == jKey { + needsUnion = true + n += jContainer.n + if !hasMaxRange { + hasMaxRange = jContainer.n == maxContainerVal+1 + } + } + } + + if !needsUnion { + fmt.Println("Cloning") + // TODO: Don't clone if sealed + target.Containers.Put(iKey, iContainer.Clone()) + otherIters[i].handled = true + continue + } + + // Need to union + fmt.Println("ikey: ", iKey) + if hasMaxRange { + panic("maxRange") + fmt.Println("maxRange") + // Use the max range + container := &Container{ + runs: []interval16{{start: 0, last: maxContainerVal}}, + containerType: containerRun, + n: maxContainerVal + 1, + } + target.Containers.Put(iKey, container) + } else { + // TODO: Implement this + // if n < ArrayMaxSize { + // // Use an array + // // container := &Container{ + // // containerType: containerArray, + // // array: make([]uint16, 0, n), + // // } + // } + // else { + // Use a bitmap + fmt.Println("bitmap") + buf := make([]uint64, bitmapN) + ob := buf[:bitmapN] + container := &Container{ + bitmap: ob, + n: 0, + containerType: containerBitmap, + } + + for _, jIter := range otherIters { + jKey, jContainer := jIter.iter.Value() + + if iKey == jKey { + if jContainer.isArray() { + // panic("unionBitmapArrayInPlace") + fmt.Println("array into bitmap") + unionBitmapArrayInPlace(container, jContainer) + fmt.Println("After union array: ", container.n) + } else if jContainer.isRun() { + // panic("unionBitmapRunInPlace") + unionBitmapRunInPlace(container, jContainer) + } else { + fmt.Println("bitmap into bitmap") + // panic("unionBitmapBitmapInPlace") + unionBitmapBitmapInPlace(container, jContainer) + fmt.Println("After union bitmap: ", container.n) + } + } + } + target.Containers.Put(iKey, container) + } + } + // } + + hasNext = false for i, otherIter := range otherIters { next := otherIter.iter.Next() otherIters[i].hasNext = next @@ -445,149 +564,7 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { // None of the iters had any more values, we're done. break } - - // Loop until every iters current value has been handled. - for { - for i, iIter := range otherIters { - if !iIter.hasNext || iIter.handled { - continue - } - - // Can store key-level statistics here - iKey, iContainer := iIter.iter.Value() - n := iContainer.n - needsUnion := false - hasMaxRange := iContainer.n == maxContainerVal+1 - for _, jIter := range otherIters[i:] { - if hasMaxRange { - continue - } - - // Calculate key-level statistics here - jKey, jContainer := jIter.iter.Value() - - if iKey == jKey { - needsUnion = true - n += jContainer.n - if !hasMaxRange { - hasMaxRange = jContainer.n == maxContainerVal+1 - } - } - } - - if !needsUnion { - // TODO: Don't clone if sealed - target.Containers.Put(iKey, iContainer.Clone()) - otherIters[i].handled = true - continue - } - - // Need to union - if hasMaxRange { - // Use the max range - container := &Container{ - runs: []interval16{{start: 0, last: maxContainerVal}}, - containerType: containerRun, - n: maxContainerVal + 1, - } - target.Containers.Put(iKey, container) - } else { - // TODO: Implement this - // if n < ArrayMaxSize { - // // Use an array - // // container := &Container{ - // // containerType: containerArray, - // // array: make([]uint16, 0, n), - // // } - // } - // else { - // Use a bitmap - buf := make([]uint64, bitmapN) - ob := buf[:bitmapN] - container := &Container{ - bitmap: ob, - n: n, - containerType: containerBitmap, - } - - for _, jIter := range otherIters { - jKey, jContainer := jIter.iter.Value() - - if iKey == jKey { - if jContainer.isArray() { - unionBitmapArrayInPlace(container, jContainer) - } else if jContainer.isRun() { - unionBitmapRunInPlace(container, jContainer) - } else { - unionBitmapBitmapInPlace(container, jContainer) - } - } - } - } - } - } } - // iiter, _ := b.Containers.Iterator(0) - // i := iiter.Next() - // for i { - // for _, otherIter := range otherIters { - // jiter := otherIter - // j := jiter.Next() - // if !j { - // continue - // } - - // ki, ci := iiter.Value() - // kj, cj := jiter.Value() - - // if i && (!j || ki < kj) { - // // TODO: Don't clone if sealed - // target.Containers.Put(ki, ci.Clone()) - // i = iiter.Next() - // ki, ci = iiter.Value() - // } else if j && (!i || ki > kj) { - // // TODO: Don't clone if sealed - // target.Containers.Put(kj, cj.Clone()) - // j = jiter.Next() - // kj, cj = jiter.Value() - // } else { // ki == kj - // // TODO: unionIntoTarget? - // target.Containers.Put(ki, union(ci, cj)) - // i, j = iiter.Next(), jiter.Next() - // ki, ci = iiter.Value() - // kj, cj = jiter.Value() - // } - - // } - // } - // } - - // for _, otherIter := range otherIters { - // jiter := otherIter - - // i, j := iiter.Next(), jiter.Next() - // ki, ci := iiter.Value() - // kj, cj := jiter.Value() - // for i || j { - // if i && (!j || ki < kj) { - // // TODO: Don't clone if sealed - // target.Containers.Put(ki, ci.Clone()) - // i = iiter.Next() - // ki, ci = iiter.Value() - // } else if j && (!i || ki > kj) { - // // TODO: Don't clone if sealed - // target.Containers.Put(kj, cj.Clone()) - // j = jiter.Next() - // kj, cj = jiter.Value() - // } else { // ki == kj - // // TODO: unionIntoTarget? - // target.Containers.Put(ki, union(ci, cj)) - // i, j = iiter.Next(), jiter.Next() - // ki, ci = iiter.Value() - // kj, cj = jiter.Value() - // } - // } - // } } // Difference returns the difference of b and other. @@ -2714,6 +2691,7 @@ func unionBitmapArrayInPlace(a, b *Container) { if !a.bitmapContains(v) { a.bitmap[v/64] |= (1 << uint64(v%64)) a.n++ + fmt.Println("added: ", v) } } } diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index f554d1df3..18d4858ae 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -411,10 +411,14 @@ func TestBitmap_UnionInPlace1(t *testing.T) { result.UnionInPlace(bm0, bm1) if n := result.Count(); n != 2682675 { + // for _, val := range result.Slice() { + // fmt.Println(val) + // } t.Fatalf("unexpected n: %d", n) } bm := testBM() + fmt.Println("bm.Count(): ", bm.Count()) result = roaring.NewBitmap() result.UnionInPlace(bm, bm0) if n := result.Count(); n != 75009 { From e8e4369f763c6c18404829884086d9d10931a707 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Wed, 28 Nov 2018 20:18:18 -0500 Subject: [PATCH 014/125] all passing --- roaring/roaring.go | 22 ---------------------- roaring/roaring_test.go | 4 ---- 2 files changed, 26 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 7f88cb651..0e288d25e 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -422,12 +422,10 @@ type wrapperIter struct { // be left unchanged, but target will be modified in place. Used to share // the union logic between the copy-on-write and in-place functions. func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { - fmt.Println("huh: ", len(others)) otherIters := make([]wrapperIter, 0, len(others)+1) bIter, _ := b.Containers.Iterator(0) next := bIter.Next() if next { - fmt.Println("Appending self") otherIters = append(otherIters, wrapperIter{ iter: bIter, hasNext: true, @@ -449,21 +447,16 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { // Loop until we've exhausted every iter. hasNext := true - fmt.Println(len(otherIters)) for hasNext { // Loop until every iters current value has been handled. // for { for i, iIter := range otherIters { - fmt.Println("copter") if !iIter.hasNext || iIter.handled { continue } - fmt.Println("here?") // Can store key-level statistics here iKey, iContainer := iIter.iter.Value() - fmt.Println("iKey: ", iKey) - fmt.Println("iContainer: ", iContainer) n := iContainer.n needsUnion := false hasMaxRange := iContainer.n == maxContainerVal+1 @@ -474,7 +467,6 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { // Calculate key-level statistics here jKey, jContainer := jIter.iter.Value() - fmt.Println("jContainer.n: ", jContainer.n) if iKey == jKey { needsUnion = true @@ -486,7 +478,6 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { } if !needsUnion { - fmt.Println("Cloning") // TODO: Don't clone if sealed target.Containers.Put(iKey, iContainer.Clone()) otherIters[i].handled = true @@ -494,10 +485,7 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { } // Need to union - fmt.Println("ikey: ", iKey) if hasMaxRange { - panic("maxRange") - fmt.Println("maxRange") // Use the max range container := &Container{ runs: []interval16{{start: 0, last: maxContainerVal}}, @@ -516,7 +504,6 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { // } // else { // Use a bitmap - fmt.Println("bitmap") buf := make([]uint64, bitmapN) ob := buf[:bitmapN] container := &Container{ @@ -530,25 +517,17 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { if iKey == jKey { if jContainer.isArray() { - // panic("unionBitmapArrayInPlace") - fmt.Println("array into bitmap") unionBitmapArrayInPlace(container, jContainer) - fmt.Println("After union array: ", container.n) } else if jContainer.isRun() { - // panic("unionBitmapRunInPlace") unionBitmapRunInPlace(container, jContainer) } else { - fmt.Println("bitmap into bitmap") - // panic("unionBitmapBitmapInPlace") unionBitmapBitmapInPlace(container, jContainer) - fmt.Println("After union bitmap: ", container.n) } } } target.Containers.Put(iKey, container) } } - // } hasNext = false for i, otherIter := range otherIters { @@ -2691,7 +2670,6 @@ func unionBitmapArrayInPlace(a, b *Container) { if !a.bitmapContains(v) { a.bitmap[v/64] |= (1 << uint64(v%64)) a.n++ - fmt.Println("added: ", v) } } } diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index 18d4858ae..f554d1df3 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -411,14 +411,10 @@ func TestBitmap_UnionInPlace1(t *testing.T) { result.UnionInPlace(bm0, bm1) if n := result.Count(); n != 2682675 { - // for _, val := range result.Slice() { - // fmt.Println(val) - // } t.Fatalf("unexpected n: %d", n) } bm := testBM() - fmt.Println("bm.Count(): ", bm.Count()) result = roaring.NewBitmap() result.UnionInPlace(bm, bm0) if n := result.Count(); n != 75009 { From 339a78d88d2e518faef9794723f0296680b13ae2 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Wed, 28 Nov 2018 21:34:44 -0500 Subject: [PATCH 015/125] wokring --- roaring/roaring.go | 18 +++++++++--------- roaring/roaring_test.go | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 0e288d25e..7685f7a94 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -407,9 +407,6 @@ func (b *Bitmap) Union(others ...*Bitmap) *Bitmap { // b in place. func (b *Bitmap) UnionInPlace(others ...*Bitmap) { b.unionIntoTarget(b, others...) - // for _, other := range others { - // b.unionIntoTarget(other, b) - // } } type wrapperIter struct { @@ -504,12 +501,15 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { // } // else { // Use a bitmap - buf := make([]uint64, bitmapN) - ob := buf[:bitmapN] - container := &Container{ - bitmap: ob, - n: 0, - containerType: containerBitmap, + container := target.Containers.Get(iKey) + if container == nil { + buf := make([]uint64, bitmapN) + ob := buf[:bitmapN] + container = &Container{ + bitmap: ob, + n: 0, + containerType: containerBitmap, + } } for _, jIter := range otherIters { diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index f554d1df3..8e389c586 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -1424,12 +1424,12 @@ func BenchmarkUnion(b *testing.B) { } } -// func BenchmarkUnionBulk(b *testing.B) { -// // a1, a2, b, r1, r2 *roaring.Bitmap -// data := getBenchData(b) -// yolo := roaring.NewBitmap() -// for n := 0; n < b.N; n++ { -// yolo. -// BulkUnion(data.a1, data.a2, data.b, data.r1, data.r2) -// } -// } +func BenchmarkUnionBulk(b *testing.B) { + // a1, a2, b, r1, r2 *roaring.Bitmap + data := getBenchData(b) + yolo := roaring.NewBitmap() + for n := 0; n < b.N; n++ { + yolo. + UnionInPlace(data.a1, data.a2, data.b, data.r1, data.r2) + } +} From b47292aa26842c4c5d44ec0d52dc56a9f1ec35a8 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Wed, 28 Nov 2018 21:43:28 -0500 Subject: [PATCH 016/125] switch to bitmap repairs for inplace algo --- roaring/roaring.go | 45 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 7685f7a94..3085b22bd 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -544,6 +544,19 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { break } } + + // Repair bitmaps after the fact + iter, _ := target.Containers.Iterator(0) + for iter.Next() { + _, container := iter.Value() + if container.isBitmap() { + n := int32(0) + for i := 0; i < bitmapN; i++ { + n += int32(popcount(container.bitmap[i])) + } + container.n = n + } + } } // Difference returns the difference of b and other. @@ -2539,10 +2552,12 @@ func unionBitmapRun(a, b *Container) *Container { return output } +// unions the run b into the bitmap a, mutating a in place. The n value of +// a will need to be repaired after the fact. func unionBitmapRunInPlace(a, b *Container) { statsHit("union/BitmapRun") for j := 0; j < len(b.runs); j++ { - a.bitmapSetRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1) + a.bitmapSetRangeIgnoreN(uint64(b.runs[j].start), uint64(b.runs[j].last)+1) } } @@ -2571,6 +2586,25 @@ func (c *Container) bitmapSetRange(i, j uint64) { } } +// sets all bits in [i, j) (c must be a bitmap container) without updating +// the value of n, meaning it will need to be repaired after the fact. +func (c *Container) bitmapSetRangeIgnoreN(i, j uint64) { + x := i >> 6 + y := (j - 1) >> 6 + var X uint64 = maxBitmap << (i % 64) + var Y uint64 = maxBitmap >> (63 - ((j - 1) % 64)) + + if x == y { + c.bitmap[x] |= (X & Y) + } else { + c.bitmap[x] |= X + for i := x + 1; i < y; i++ { + c.bitmap[i] = maxBitmap + } + c.bitmap[y] |= Y + } +} + // xor's all bits in [i, j) with all true (c must be a bitmap container). func (c *Container) bitmapXorRange(i, j uint64) { x := i >> 6 @@ -2665,6 +2699,8 @@ func unionArrayBitmap(a, b *Container) *Container { return output } +// unions array b into bitmap a, mutating a in place. The n value +// of a will need to be repaired after the fact. func unionBitmapArrayInPlace(a, b *Container) { for _, v := range b.array { if !a.bitmapContains(v) { @@ -2700,6 +2736,8 @@ func unionBitmapBitmap(a, b *Container) *Container { return output } +// unions bitmap b into bitmap a, mutating a in place. The n value of +// a will need to be repaired after the fact. func unionBitmapBitmapInPlace(a, b *Container) { // local variables added to prevent BCE checks in loop // see https://go101.org/article/bounds-check-elimination.html @@ -2707,16 +2745,11 @@ func unionBitmapBitmapInPlace(a, b *Container) { var ( ab = a.bitmap[:bitmapN] bb = b.bitmap[:bitmapN] - - n int32 ) for i := 0; i < bitmapN; i++ { ab[i] = ab[i] | bb[i] - n += int32(popcount(ab[i])) } - - a.n = n } func difference(a, b *Container) *Container { From eb5ad7bf49a494e25d9f3aca38d4e0ce6e969749 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Wed, 28 Nov 2018 21:44:37 -0500 Subject: [PATCH 017/125] dont keep n in sync with bitmaprun in place --- roaring/roaring.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 3085b22bd..98285d543 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2703,10 +2703,7 @@ func unionArrayBitmap(a, b *Container) *Container { // of a will need to be repaired after the fact. func unionBitmapArrayInPlace(a, b *Container) { for _, v := range b.array { - if !a.bitmapContains(v) { - a.bitmap[v/64] |= (1 << uint64(v%64)) - a.n++ - } + a.bitmap[v/64] |= (1 << uint64(v%64)) } } From 3eb060c4c681497b85c7ca38749ba71c6bc80076 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 00:15:57 -0500 Subject: [PATCH 018/125] fix bug --- roaring/roaring.go | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 98285d543..f152a5656 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -27,6 +27,8 @@ import ( "github.com/pkg/errors" ) +func statshit() {} + const ( // magicNumber is an identifier, in bytes 0-1 of the file. magicNumber = uint32(12348) @@ -419,6 +421,15 @@ type wrapperIter struct { // be left unchanged, but target will be modified in place. Used to share // the union logic between the copy-on-write and in-place functions. func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { + numArrayIntoBitmap := 0 + numRunIntoBitmap := 0 + numBitmapIntoBitmap := 0 + // defer func() { + // fmt.Println("numArrayIntoBitmap: ", numArrayIntoBitmap) + // fmt.Println("numRunIntoBitmap: ", numRunIntoBitmap) + // fmt.Println("numBitmapIntoBitmap: ", numBitmapIntoBitmap) + // }() + otherIters := make([]wrapperIter, 0, len(others)+1) bIter, _ := b.Containers.Iterator(0) next := bIter.Next() @@ -490,6 +501,12 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { n: maxContainerVal + 1, } target.Containers.Put(iKey, container) + for j, jIter := range otherIters { + jKey, _ := jIter.iter.Value() + if iKey == jKey { + otherIters[j].handled = true + } + } } else { // TODO: Implement this // if n < ArrayMaxSize { @@ -512,17 +529,21 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { } } - for _, jIter := range otherIters { + for j, jIter := range otherIters { jKey, jContainer := jIter.iter.Value() if iKey == jKey { if jContainer.isArray() { + numArrayIntoBitmap++ unionBitmapArrayInPlace(container, jContainer) } else if jContainer.isRun() { + numRunIntoBitmap++ unionBitmapRunInPlace(container, jContainer) } else { + numBitmapIntoBitmap++ unionBitmapBitmapInPlace(container, jContainer) } + otherIters[j].handled = true } } target.Containers.Put(iKey, container) @@ -2703,7 +2724,9 @@ func unionArrayBitmap(a, b *Container) *Container { // of a will need to be repaired after the fact. func unionBitmapArrayInPlace(a, b *Container) { for _, v := range b.array { - a.bitmap[v/64] |= (1 << uint64(v%64)) + // a.bitmap[v>>6] |= (1 << uint64(v%64)) + i := v >> 6 + a.bitmap[i] = a.bitmap[i] | (uint64(1) << (v % 64)) } } From 50f119df4e21779495057ef43ef2b9451c2a8387 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 10:02:24 -0500 Subject: [PATCH 019/125] Allocate bitmap if needed (existing wrong type --- roaring/roaring.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index f152a5656..9b40399f0 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -519,7 +519,7 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { // else { // Use a bitmap container := target.Containers.Get(iKey) - if container == nil { + if container == nil || container.containerType != containerBitmap { buf := make([]uint64, bitmapN) ob := buf[:bitmapN] container = &Container{ From d78e2fc87c896d32927a873d3ce5a80dd6be90f7 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 11:31:54 -0500 Subject: [PATCH 020/125] Move repair logic to helpers --- roaring/containers.go | 8 ++++++++ roaring/roaring.go | 23 ++++++++++++----------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/roaring/containers.go b/roaring/containers.go index 3fe0814cc..9862b894c 100644 --- a/roaring/containers.go +++ b/roaring/containers.go @@ -156,6 +156,14 @@ func (sc *sliceContainers) Iterator(key uint64) (citer ContainerIterator, found return &sliceIterator{e: sc, i: i}, found } +func (sc *sliceContainers) Repair() { + for _, c := range sc.containers { + if c.isBitmap() { + c.bitmapRepair() + } + } +} + type sliceIterator struct { e *sliceContainers i int diff --git a/roaring/roaring.go b/roaring/roaring.go index 9b40399f0..60476d89f 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -100,6 +100,9 @@ type Containers interface { Count() uint64 + // Repair will n values after in-place operations. + Repair() + //Reset will clear the containers collection to allow for recycling during snapshot Reset() } @@ -567,17 +570,7 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { } // Repair bitmaps after the fact - iter, _ := target.Containers.Iterator(0) - for iter.Next() { - _, container := iter.Value() - if container.isBitmap() { - n := int32(0) - for i := 0; i < bitmapN; i++ { - n += int32(popcount(container.bitmap[i])) - } - container.n = n - } - } + target.Containers.Repair() } // Difference returns the difference of b and other. @@ -1981,6 +1974,14 @@ func (c *Container) check() error { return a } +func (c *Container) bitmapRepair() { + n := int32(0) + for i := 0; i < bitmapN; i++ { + n += int32(popcount(c.bitmap[i])) + } + c.n = n +} + // containerInfo represents a point-in-time snapshot of container stats. type containerInfo struct { Key uint64 // container key From fccc7070060720cb8174ea635a95a80d1a41bede Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 11:32:44 -0500 Subject: [PATCH 021/125] remove debug code --- roaring/roaring.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 60476d89f..b2b6fdf39 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -424,15 +424,6 @@ type wrapperIter struct { // be left unchanged, but target will be modified in place. Used to share // the union logic between the copy-on-write and in-place functions. func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { - numArrayIntoBitmap := 0 - numRunIntoBitmap := 0 - numBitmapIntoBitmap := 0 - // defer func() { - // fmt.Println("numArrayIntoBitmap: ", numArrayIntoBitmap) - // fmt.Println("numRunIntoBitmap: ", numRunIntoBitmap) - // fmt.Println("numBitmapIntoBitmap: ", numBitmapIntoBitmap) - // }() - otherIters := make([]wrapperIter, 0, len(others)+1) bIter, _ := b.Containers.Iterator(0) next := bIter.Next() @@ -537,13 +528,10 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { if iKey == jKey { if jContainer.isArray() { - numArrayIntoBitmap++ unionBitmapArrayInPlace(container, jContainer) } else if jContainer.isRun() { - numRunIntoBitmap++ unionBitmapRunInPlace(container, jContainer) } else { - numBitmapIntoBitmap++ unionBitmapBitmapInPlace(container, jContainer) } otherIters[j].handled = true From 13dbe22b18163d1e4d3759b4c06b3f80b8c24234 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 13:53:10 -0500 Subject: [PATCH 022/125] Move next logic into helper --- roaring/roaring.go | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index b2b6fdf39..bbafbae69 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -420,11 +420,28 @@ type wrapperIter struct { handled bool } +type wrappedIters []wrapperIter + +func (w wrappedIters) next() bool { + hasNext := false + + for i, wrapped := range w { + next := wrapped.iter.Next() + w[i].hasNext = next + w[i].handled = false + if next { + hasNext = true + } + } + + return hasNext +} + // unionIntoTarget stores the union of b and other into target. b and other will // be left unchanged, but target will be modified in place. Used to share // the union logic between the copy-on-write and in-place functions. func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { - otherIters := make([]wrapperIter, 0, len(others)+1) + otherIters := make(wrappedIters, 0, len(others)+1) bIter, _ := b.Containers.Iterator(0) next := bIter.Next() if next { @@ -541,15 +558,7 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { } } - hasNext = false - for i, otherIter := range otherIters { - next := otherIter.iter.Next() - otherIters[i].hasNext = next - otherIters[i].handled = false - if next { - hasNext = true - } - } + hasNext = otherIters.next() if !hasNext { // None of the iters had any more values, we're done. From 5b50ecfd8e62b299f0829d6d38cfe41280068bc2 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 13:53:38 -0500 Subject: [PATCH 023/125] delete unused code --- roaring/roaring.go | 1 - 1 file changed, 1 deletion(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index bbafbae69..77419c2db 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -468,7 +468,6 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { hasNext := true for hasNext { // Loop until every iters current value has been handled. - // for { for i, iIter := range otherIters { if !iIter.hasNext || iIter.handled { continue From 8e3da346c3bf6e5365d592a06f268627040c97b7 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 14:16:31 -0500 Subject: [PATCH 024/125] Add more comments and add helper method for bulk marking handled --- roaring/roaring.go | 64 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 49 insertions(+), 15 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 77419c2db..ec0cec4d9 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -437,6 +437,15 @@ func (w wrappedIters) next() bool { return hasNext } +func (w wrappedIters) markItersWithCurrentKeyAsHandled(key uint64) { + for i, wrapped := range w { + currKey, _ := wrapped.iter.Value() + if currKey == key { + w[i].handled = true + } + } +} + // unionIntoTarget stores the union of b and other into target. b and other will // be left unchanged, but target will be modified in place. Used to share // the union logic between the copy-on-write and in-place functions. @@ -470,16 +479,40 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { // Loop until every iters current value has been handled. for i, iIter := range otherIters { if !iIter.hasNext || iIter.handled { + // Either we've exhausted this iter (it has no more containers), or + // we've already handled the current container by unioning it with + // one of the containers we encountered earlier. continue } - // Can store key-level statistics here iKey, iContainer := iIter.iter.Value() - n := iContainer.n - needsUnion := false - hasMaxRange := iContainer.n == maxContainerVal+1 + + // Summary statistics about all the containers in the other bitmaps + // that share the same key so we can make smarter union strategy + // decisions later. + var ( + // Estimated cardinality of the union of all containers with the same + // key as iKey across all bitmaps. This calculation is very rough as + // we just sum the cardinality of the container across the different + // bitmaps which could result in very inflated values, but it allows + // us to avoid allocating expensive bitmaps when unioning many low + // density containers. + n = iContainer.n + // Whether iContainer is the only container across all the bitmaps + // with the key iKey. If true, we can skip all the unioning logic + // and just clone the container into target. + isOnlyContainerWithKey = true + // Whether any of the containers are storing every possible value that + // they can. If so, we can short-circuit all the unioning logic and use + // a RLE container with a single value in it. This is an optimization to + // avoid using an expensive bitmap container for bitmaps that have some + // extremely dense containers. + hasMaxRange = iContainer.n == maxContainerVal+1 + ) for _, jIter := range otherIters[i:] { if hasMaxRange { + // If we already know that we're going to use a max range RLE container, + // then there is no reason to continue calculating statistics. continue } @@ -487,7 +520,7 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { jKey, jContainer := jIter.iter.Value() if iKey == jKey { - needsUnion = true + isOnlyContainerWithKey = false n += jContainer.n if !hasMaxRange { hasMaxRange = jContainer.n == maxContainerVal+1 @@ -495,28 +528,29 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { } } - if !needsUnion { - // TODO: Don't clone if sealed + if isOnlyContainerWithKey { + // TODO(rartoul): We can avoid these clones if we can determine + // if the container is coming from an immutable bitmap and we + // know that we can mark the target bitmap as immutable as well. target.Containers.Put(iKey, iContainer.Clone()) otherIters[i].handled = true continue } - // Need to union + // There was more than one container across the bitmaps with key iKey + // so we need to calculate a union. if hasMaxRange { - // Use the max range + // One (or more) of the containers represented the maximum possible + // range that a container can store, so instead of calculating a + // union we can generate an RLE container that represents the entire + // range. container := &Container{ runs: []interval16{{start: 0, last: maxContainerVal}}, containerType: containerRun, n: maxContainerVal + 1, } target.Containers.Put(iKey, container) - for j, jIter := range otherIters { - jKey, _ := jIter.iter.Value() - if iKey == jKey { - otherIters[j].handled = true - } - } + otherIters[i:].markItersWithCurrentKeyAsHandled(iKey) } else { // TODO: Implement this // if n < ArrayMaxSize { From c02479a5f9063d49b444a0f9dc600416629c72d0 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 14:23:11 -0500 Subject: [PATCH 025/125] more comments and cleanup --- roaring/roaring.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index ec0cec4d9..457973e8f 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -552,17 +552,18 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { target.Containers.Put(iKey, container) otherIters[i:].markItersWithCurrentKeyAsHandled(iKey) } else { - // TODO: Implement this - // if n < ArrayMaxSize { - // // Use an array - // // container := &Container{ - // // containerType: containerArray, - // // array: make([]uint16, 0, n), - // // } - // } - // else { - // Use a bitmap + // Use a bitmap container for the target bitmap, and union everything + // into it. + // + // TODO(rartoul): Add another conditional case for n < ArrayMaxSize + // (or some fraction of that value) to avoid allocating expensive + // bitmaps when unioning many low-density array containers, but this + // will require writing a union in place algorithm for an array container + // that accepts multiple different containers to union into it for + // efficiency. container := target.Containers.Get(iKey) + // If target already has a bitmap container for iKey then we can reuse that, + // otherwise we have to allocate a new one. if container == nil || container.containerType != containerBitmap { buf := make([]uint64, bitmapN) ob := buf[:bitmapN] @@ -573,6 +574,10 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { } } + // Once we've acquire a bitmap container (either by reusing the existing one + // or allocating a new one) then the last step is to iterate through all the + // other containers to see which ones have the same key, and union all of them + // into the target bitmap container. for j, jIter := range otherIters { jKey, jContainer := jIter.iter.Value() @@ -592,11 +597,6 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { } hasNext = otherIters.next() - - if !hasNext { - // None of the iters had any more values, we're done. - break - } } // Repair bitmaps after the fact From defcb40f8c177e4eeedd76e6a779e838e55be659 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 14:25:39 -0500 Subject: [PATCH 026/125] Add comment --- roaring/roaring.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 457973e8f..e0fa9948c 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -599,7 +599,12 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { hasNext = otherIters.next() } - // Repair bitmaps after the fact + // Performing the popcount() operation with every union is wasteful because + // its likely the value will be invalidated by the next union operation. As + // a result, when we're performing all our in-place unions, we don't repair + // the value of n (container cardinality), and then at the very end we perform + // a "Repair" to recalculate all the container values. That way we never popcount() + // an entire bitmap container more than once per bulk union operation. target.Containers.Repair() } From 7931bc2c3703ef22adfb601fa0e21820699e84f2 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 14:26:57 -0500 Subject: [PATCH 027/125] Add comment --- roaring/roaring.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roaring/roaring.go b/roaring/roaring.go index e0fa9948c..8d612b5e5 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -592,6 +592,10 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { otherIters[j].handled = true } } + + // Now that we've calculated a container is that a union of all the containers + // with the same key across all the bitmaps, we store it in the list of containers + // for the target. target.Containers.Put(iKey, container) } } From 06e7dbdc757f797c569d7a82c479858ca6814fd6 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 14:30:16 -0500 Subject: [PATCH 028/125] simplify and remove dead code --- roaring/roaring.go | 36 +----------------------------------- 1 file changed, 1 insertion(+), 35 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 8d612b5e5..269f9ec57 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2471,38 +2471,6 @@ func unionArrayArray(a, b *Container) *Container { return output } -// func unionArrayArrayInPlace(a, b *Container) *Container { -// statsHit("union/ArrayArray") -// output := a -// na, nb := len(a.array), len(b.array) -// for i, j := 0, 0; ; { -// if i >= na && j >= nb { -// break -// } else if i < na && j >= nb { -// output.add(a.array[i]) -// i++ -// continue -// } else if i >= na && j < nb { -// output.add(b.array[j]) -// j++ -// continue -// } - -// va, vb := a.array[i], b.array[j] -// if va < vb { -// output.add(va) -// i++ -// } else if va > vb { -// output.add(vb) -// j++ -// } else { -// output.add(va) -// i, j = i+1, j+1 -// } -// } -// return output -// } - // unionArrayRun optimistically assumes that the result will be a run container, // and converts to a bitmap or array container afterwards if necessary. func unionArrayRun(a, b *Container) *Container { @@ -2764,9 +2732,7 @@ func unionArrayBitmap(a, b *Container) *Container { // of a will need to be repaired after the fact. func unionBitmapArrayInPlace(a, b *Container) { for _, v := range b.array { - // a.bitmap[v>>6] |= (1 << uint64(v%64)) - i := v >> 6 - a.bitmap[i] = a.bitmap[i] | (uint64(1) << (v % 64)) + a.bitmap[v>>6] |= (uint64(1) << (v % 64)) } } From 4cdf2adbe509a669d4edee7ae1abe89ddca9534f Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 15:33:26 -0500 Subject: [PATCH 029/125] fix comment --- roaring/roaring.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 269f9ec57..aef03a82c 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -605,8 +605,8 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { // Performing the popcount() operation with every union is wasteful because // its likely the value will be invalidated by the next union operation. As - // a result, when we're performing all our in-place unions, we don't repair - // the value of n (container cardinality), and then at the very end we perform + // a result, when we're performing all our in-place unions we allow the value of + // n (container cardinality) to fall out of sync, and then at the very end we perform // a "Repair" to recalculate all the container values. That way we never popcount() // an entire bitmap container more than once per bulk union operation. target.Containers.Repair() From 49df3fcd305fbd4bcb15476a079ee5015e9e3f7c Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 15:34:29 -0500 Subject: [PATCH 030/125] Dont shadow statshit --- roaring/roaring.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index aef03a82c..bed3672e5 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -27,8 +27,6 @@ import ( "github.com/pkg/errors" ) -func statshit() {} - const ( // magicNumber is an identifier, in bytes 0-1 of the file. magicNumber = uint32(12348) From 70338be7ca7ecf8ba9d77f680e7483c097d49254 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 16:25:12 -0500 Subject: [PATCH 031/125] Add crazy comment --- roaring/roaring.go | 163 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 130 insertions(+), 33 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index bed3672e5..8efd6e0cf 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -406,47 +406,112 @@ func (b *Bitmap) Union(others ...*Bitmap) *Bitmap { return output } -// UnionInPlace returns the bitwise union of b and other, modifying +// UnionInPlace returns the bitwise union of b and others, modifying // b in place. func (b *Bitmap) UnionInPlace(others ...*Bitmap) { b.unionIntoTarget(b, others...) } -type wrapperIter struct { - iter ContainerIterator - hasNext bool - handled bool -} - -type wrappedIters []wrapperIter - -func (w wrappedIters) next() bool { - hasNext := false - - for i, wrapped := range w { - next := wrapped.iter.Next() - w[i].hasNext = next - w[i].handled = false - if next { - hasNext = true - } - } - - return hasNext -} - -func (w wrappedIters) markItersWithCurrentKeyAsHandled(key uint64) { - for i, wrapped := range w { - currKey, _ := wrapped.iter.Value() - if currKey == key { - w[i].handled = true - } - } -} - // unionIntoTarget stores the union of b and other into target. b and other will // be left unchanged, but target will be modified in place. Used to share // the union logic between the copy-on-write and in-place functions. +// +// This function performs an n-way union of n bitmaps. It performs this in an +// optimized manner looping through all the bitmaps and performing unions one +// container at a time. As a result, instead of generating many intermediary +// containers for each union operation for a given container key, only one +// new container needs to be allocated (or re-used) regardless of how many bitmaps +// participate in the union. This significantly reduces allocations. In addition, +// because we perform the unions one container at a time accross all the bitmaps, we +// can calculate summary statistics that allow us to make more efficient decisions +// up front. For example, imagine trying to combine a union accross the following three +// bitsets: +// +// 1. Bitmap A: Single array container at key 0 with 400 values in it. +// 2. Bitmap B: Single array container at key 0 with 500 values in it. +// 3. Bitmap C: Single array container at key 0 with 3500 values in it. +// +// Naive approach: +// +// 1. Perform union of bitmap A and B, container by container +// a. 400 + 500 < ArrayMaxSize so likely we will choose to allocate a new array +// container and then perform a unionArrayArray operation to merge the two +// arrays into the new array container. +// 2. Perform a union of the bitmap generated in the step above with bitmap C. +// 900 + 3500 > ArrayMaxSize so we will need to upgrade to a bitset container which +// we will have to allocate, and then we will have to perform two unions into the +// new bitmap container: one for the array container generated in the previous step, +// and one for the bitset container in bitmap C. +// +// Approach taken by this function: +// +// 1. Detect that bitmaps A, B, and C all have containers for key 0. +// 2. Estimate the resulting cardinality of the union of all their containers to be +// 400 + 500 + 3500 > ArrayMaxSize and decide upfront to use a bitset for the target +// container. +// 3. Union the containers from bitmaps A, B, and C into the new bitset container directly +// using fast bitwise operations. +// +// In the naive approach, we had to allocate two containers, whereas in the optimized approach +// we only had to allocate one container, and we also had to perform less union operations. This +// example is simplistic, but the impact in terms of CPU cycles and memory allocations achieved +// by using the optimized alogorithm when working with a large number of large bitmaps is huge. +// +// An additional optimization that this function makes is that it recognizes that even when +// CPU support is present, performing the popcount() operation isn't free. Imagine a scenario +// where 10 bitset containers are being unioned together one after the next. If every +// bitset<->bitset union operation needs to keep the containers cardinality up to date, then +// the algorithm will waste a lot of time performing intermediary popcount() operations that +// will immediately be invalidated by the next union operation. As a result, we allow the cardinality +// of containers to degrade when we perform the in-place union operations, and then when the algorithm +// completes we "repair" all the containers by performing the popcount() operation one time. This means +// that we only ever have to do O(1) popcount operations per container instead of O(n) where n is the +// number of containers with the same key that are being unioned together. +// +// The algorithm works by iterating through all of the containers in all of the bitmaps concurrently. +// At every "tick" of the outermost loop, we increment our pointer into the bitmaps list of containers +// by 1 (if we haven't reached the end of the containers for that bitmap.) +// +// We then loop through all of the "current" values of the current container for all of the bitmaps +// and for each container with a specific key that we encounter, we scan forward to see if any of the +// other bitmaps have a container for the same key. If so, we calculate some summary statistics and +// then use that information to make a decision about how to union all of the containers with the same +// key together, perform the union, and then move on to the next batch of containers that share the same +// key. +// +// We repeat this process until every single bitmaps current container has been "handled". Then we start the +// outer loop over again and the process repeats until we've iterated through every container in every bitmap +// and unioned everything into a single target bitmap. +// +// The diagram below shows the iteration state of four different maps as the algorithm progresses. The diagrams should be +// interpreted from left -> right, top -> bottom. The ^ symbol represents the bitmaps current container iteration position, +// and the - symbol represents a container that is at the current iteration position, but has been marked as "handled". +// +// ---------------------------- | ---------------------------- | ---------------------------- +// Bitmap 1 |___X____________X__________| | |___X____________X__________| | |___X____________X__________| +// ^ | _ | +// ---------------------------- | ---------------------------- | ---------------------------- +// Bitmap 2 |_______X________X______X___| | |_______X_______________X___| | |_______X_______________X___| +// ^ | ^ | +// ---------------------------- | ---------------------------- | ---------------------------- +// Bitmap 3 |_______X___________________| | |_______X___________________| | |_______X___________________| +// ^ | ^ | +// ---------------------------- | ---------------------------- | ---------------------------- +// Bitmap 4 |___X_______________________| | |___X_______________________| | |___X_______________________| +// ^ | _ | +// ------------------------------------------------------------------------------------------------------------------------ +// ---------------------------- | ---------------------------- | ---------------------------- +// Bitmap 1 |___X____________X__________| | |___X____________X__________| | |___X____________X__________| +// _ | ^ | _ +// ---------------------------- | ---------------------------- | ---------------------------- +// Bitmap 2 |_______X_______________X___| | |_______X_______________X___| | |_______X_______________X___| +// _ | ^ | ^ +// ---------------------------- | ---------------------------- | ---------------------------- +// Bitmap 3 |_______X___________________| | |_______X___________________| | |_______X___________________| +// _ | | +// ---------------------------- | ---------------------------- | ---------------------------- +// Bitmap 4 |___X_______________________| | |___X_______________________| | |___X_______________________| +// _ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { otherIters := make(wrappedIters, 0, len(others)+1) bIter, _ := b.Containers.Iterator(0) @@ -3884,3 +3949,35 @@ func readWithRuns(b *Bitmap, data []byte, pos int, keyN uint32) { } } } + +type wrapperIter struct { + iter ContainerIterator + hasNext bool + handled bool +} + +type wrappedIters []wrapperIter + +func (w wrappedIters) next() bool { + hasNext := false + + for i, wrapped := range w { + next := wrapped.iter.Next() + w[i].hasNext = next + w[i].handled = false + if next { + hasNext = true + } + } + + return hasNext +} + +func (w wrappedIters) markItersWithCurrentKeyAsHandled(key uint64) { + for i, wrapped := range w { + currKey, _ := wrapped.iter.Value() + if currKey == key { + w[i].handled = true + } + } +} From 25eae0204d0728aa5d613db18f3bf007e7162ce6 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 16:26:44 -0500 Subject: [PATCH 032/125] Update benchmarks --- roaring/roaring_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index 8e389c586..72153538f 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -1413,7 +1413,6 @@ func BenchmarkSliceDescending(b *testing.B) { } func BenchmarkUnion(b *testing.B) { - // a1, a2, b, r1, r2 *roaring.Bitmap data := getBenchData(b) for n := 0; n < b.N; n++ { data.a1. @@ -1425,11 +1424,10 @@ func BenchmarkUnion(b *testing.B) { } func BenchmarkUnionBulk(b *testing.B) { - // a1, a2, b, r1, r2 *roaring.Bitmap data := getBenchData(b) - yolo := roaring.NewBitmap() + bm := roaring.NewBitmap() for n := 0; n < b.N; n++ { - yolo. + bm. UnionInPlace(data.a1, data.a2, data.b, data.r1, data.r2) } } From feb19c62b99e0df19bb6411daaca8ba657e3fb0e Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 16:28:33 -0500 Subject: [PATCH 033/125] Change repair functions to specify they are bitmap only --- roaring/containers.go | 2 +- roaring/roaring.go | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/roaring/containers.go b/roaring/containers.go index 9862b894c..102ffd3bd 100644 --- a/roaring/containers.go +++ b/roaring/containers.go @@ -156,7 +156,7 @@ func (sc *sliceContainers) Iterator(key uint64) (citer ContainerIterator, found return &sliceIterator{e: sc, i: i}, found } -func (sc *sliceContainers) Repair() { +func (sc *sliceContainers) RepairBitmaps() { for _, c := range sc.containers { if c.isBitmap() { c.bitmapRepair() diff --git a/roaring/roaring.go b/roaring/roaring.go index 8efd6e0cf..c242bf94f 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -98,8 +98,9 @@ type Containers interface { Count() uint64 - // Repair will n values after in-place operations. - Repair() + // RepairBitmaps will repair cardinality(n) values on bitmap containers after + // in-place operations. + RepairBitmaps() //Reset will clear the containers collection to allow for recycling during snapshot Reset() @@ -672,7 +673,7 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { // n (container cardinality) to fall out of sync, and then at the very end we perform // a "Repair" to recalculate all the container values. That way we never popcount() // an entire bitmap container more than once per bulk union operation. - target.Containers.Repair() + target.Containers.RepairBitmaps() } // Difference returns the difference of b and other. From 56b3d7d5db91d8a233823776ce503cd8db9a027a Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 16:31:23 -0500 Subject: [PATCH 034/125] fix comment --- roaring/roaring.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index c242bf94f..abd81c59f 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -413,9 +413,9 @@ func (b *Bitmap) UnionInPlace(others ...*Bitmap) { b.unionIntoTarget(b, others...) } -// unionIntoTarget stores the union of b and other into target. b and other will -// be left unchanged, but target will be modified in place. Used to share -// the union logic between the copy-on-write and in-place functions. +// unionIntoTarget stores the union of b and others into target. b and others will +// be left unchanged (unless one of them is also target), but target will be modified +// in place. // // This function performs an n-way union of n bitmaps. It performs this in an // optimized manner looping through all the bitmaps and performing unions one From f97ff4b5a0fe2a0a5d011b6f38cf8afd6dd34012 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 16:34:17 -0500 Subject: [PATCH 035/125] special case individual union --- roaring/roaring.go | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index abd81c59f..80d548456 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -399,9 +399,14 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { return output } -// Union returns the bitwise union of b and other. +// Union returns the bitwise union of b and other as a new bitmap. func (b *Bitmap) Union(others ...*Bitmap) *Bitmap { output := NewBitmap() + if len(others) == 1 { + b.unionIntoTargetSingle(output, others[0]) + return output + } + output.UnionInPlace(b) output.UnionInPlace(others...) return output @@ -413,6 +418,30 @@ func (b *Bitmap) UnionInPlace(others ...*Bitmap) { b.unionIntoTarget(b, others...) } +func (b *Bitmap) unionIntoTargetSingle(target *Bitmap, other *Bitmap) { + iiter, _ := b.Containers.Iterator(0) + jiter, _ := other.Containers.Iterator(0) + i, j := iiter.Next(), jiter.Next() + ki, ci := iiter.Value() + kj, cj := jiter.Value() + for i || j { + if i && (!j || ki < kj) { + target.Containers.Put(ki, ci.Clone()) + i = iiter.Next() + ki, ci = iiter.Value() + } else if j && (!i || ki > kj) { + target.Containers.Put(kj, cj.Clone()) + j = jiter.Next() + kj, cj = jiter.Value() + } else { // ki == kj + target.Containers.Put(ki, union(ci, cj)) + i, j = iiter.Next(), jiter.Next() + ki, ci = iiter.Value() + kj, cj = jiter.Value() + } + } +} + // unionIntoTarget stores the union of b and others into target. b and others will // be left unchanged (unless one of them is also target), but target will be modified // in place. From 038e3d4304f9cc21273c35138a594c735ce7fa21 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 16:34:43 -0500 Subject: [PATCH 036/125] fix diagram --- roaring/roaring.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 80d548456..180768ea8 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -517,7 +517,7 @@ func (b *Bitmap) unionIntoTargetSingle(target *Bitmap, other *Bitmap) { // interpreted from left -> right, top -> bottom. The ^ symbol represents the bitmaps current container iteration position, // and the - symbol represents a container that is at the current iteration position, but has been marked as "handled". // -// ---------------------------- | ---------------------------- | ---------------------------- +// ---------------------------- | ---------------------------- | ---------------------------- // Bitmap 1 |___X____________X__________| | |___X____________X__________| | |___X____________X__________| // ^ | _ | // ---------------------------- | ---------------------------- | ---------------------------- From efd6116d3f279626953ec675f0ba21a2941fc92e Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 16:35:35 -0500 Subject: [PATCH 037/125] replace word in comment for clarity --- roaring/roaring.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 180768ea8..584a614df 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -454,7 +454,7 @@ func (b *Bitmap) unionIntoTargetSingle(target *Bitmap, other *Bitmap) { // participate in the union. This significantly reduces allocations. In addition, // because we perform the unions one container at a time accross all the bitmaps, we // can calculate summary statistics that allow us to make more efficient decisions -// up front. For example, imagine trying to combine a union accross the following three +// up front. For example, imagine trying to perform a union accross the following three // bitsets: // // 1. Bitmap A: Single array container at key 0 with 400 values in it. From 3f3fec38249d95953f80be289f7d5f5e7900a5a2 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 16:36:38 -0500 Subject: [PATCH 038/125] refactor comment for clarity --- roaring/roaring.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 584a614df..b295ebec0 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -485,7 +485,7 @@ func (b *Bitmap) unionIntoTargetSingle(target *Bitmap, other *Bitmap) { // In the naive approach, we had to allocate two containers, whereas in the optimized approach // we only had to allocate one container, and we also had to perform less union operations. This // example is simplistic, but the impact in terms of CPU cycles and memory allocations achieved -// by using the optimized alogorithm when working with a large number of large bitmaps is huge. +// by using the optimized alogorithm when unioning many large bitmaps can be huge. // // An additional optimization that this function makes is that it recognizes that even when // CPU support is present, performing the popcount() operation isn't free. Imagine a scenario From 3193fc98ab94db68877718eb8486828bcf5c721d Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 16:38:13 -0500 Subject: [PATCH 039/125] refactor comment for clarity --- roaring/roaring.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index b295ebec0..54af56b2c 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -502,7 +502,7 @@ func (b *Bitmap) unionIntoTargetSingle(target *Bitmap, other *Bitmap) { // At every "tick" of the outermost loop, we increment our pointer into the bitmaps list of containers // by 1 (if we haven't reached the end of the containers for that bitmap.) // -// We then loop through all of the "current" values of the current container for all of the bitmaps +// We then loop through all of the "current" values(containers) for all of the bitmaps // and for each container with a specific key that we encounter, we scan forward to see if any of the // other bitmaps have a container for the same key. If so, we calculate some summary statistics and // then use that information to make a decision about how to union all of the containers with the same From d6e2d0768783930a140fee56d4113c611a80f495 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 16:39:07 -0500 Subject: [PATCH 040/125] refactor comment for clarity --- roaring/roaring.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 54af56b2c..dc80d0dcb 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -506,8 +506,8 @@ func (b *Bitmap) unionIntoTargetSingle(target *Bitmap, other *Bitmap) { // and for each container with a specific key that we encounter, we scan forward to see if any of the // other bitmaps have a container for the same key. If so, we calculate some summary statistics and // then use that information to make a decision about how to union all of the containers with the same -// key together, perform the union, and then move on to the next batch of containers that share the same -// key. +// key together, perform the union, mark the unioned containers as "handled" and then move on to the next +// batch of containers that share the same key. // // We repeat this process until every single bitmaps current container has been "handled". Then we start the // outer loop over again and the process repeats until we've iterated through every container in every bitmap From d35aabfa86701ea6696aaeea63cc68090dab8976 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 16:39:27 -0500 Subject: [PATCH 041/125] remove double space --- roaring/roaring.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index dc80d0dcb..2bbe19feb 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -509,7 +509,7 @@ func (b *Bitmap) unionIntoTargetSingle(target *Bitmap, other *Bitmap) { // key together, perform the union, mark the unioned containers as "handled" and then move on to the next // batch of containers that share the same key. // -// We repeat this process until every single bitmaps current container has been "handled". Then we start the +// We repeat this process until every single bitmaps current container has been "handled". Then we start the // outer loop over again and the process repeats until we've iterated through every container in every bitmap // and unioned everything into a single target bitmap. // From 5ddeb0f6a07a92f009c8ef1d8bef1fd177d90744 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 16:40:21 -0500 Subject: [PATCH 042/125] refactor comment for clarity --- roaring/roaring.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 2bbe19feb..04f6bed69 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -513,8 +513,9 @@ func (b *Bitmap) unionIntoTargetSingle(target *Bitmap, other *Bitmap) { // outer loop over again and the process repeats until we've iterated through every container in every bitmap // and unioned everything into a single target bitmap. // -// The diagram below shows the iteration state of four different maps as the algorithm progresses. The diagrams should be -// interpreted from left -> right, top -> bottom. The ^ symbol represents the bitmaps current container iteration position, +// The diagram below shows the iteration state of four different bitmaps as the algorithm progresses them. +// The diagrams should BE interpreted from left -> right, top -> bottom. The X's represent a container in +// the bitmap at a specific key, ^ symbol represents the bitmaps current container iteration position, // and the - symbol represents a container that is at the current iteration position, but has been marked as "handled". // // ---------------------------- | ---------------------------- | ---------------------------- From c1c1121e51179217bee47e34a361e88ec10812e0 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 16:41:30 -0500 Subject: [PATCH 043/125] more comment refactoring --- roaring/roaring.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 04f6bed69..d790bb24f 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -699,7 +699,7 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { // Performing the popcount() operation with every union is wasteful because // its likely the value will be invalidated by the next union operation. As - // a result, when we're performing all our in-place unions we allow the value of + // a result, when we're performing all of our in-place unions we allow the value of // n (container cardinality) to fall out of sync, and then at the very end we perform // a "Repair" to recalculate all the container values. That way we never popcount() // an entire bitmap container more than once per bulk union operation. From 76aea6d9bc361f27d0c80192c106fe91bd521e3d Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 16:53:37 -0500 Subject: [PATCH 044/125] rename structs --- roaring/roaring.go | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index d790bb24f..51e26f037 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -544,11 +544,19 @@ func (b *Bitmap) unionIntoTargetSingle(target *Bitmap, other *Bitmap) { // Bitmap 4 |___X_______________________| | |___X_______________________| | |___X_______________________| // _ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { - otherIters := make(wrappedIters, 0, len(others)+1) + var ( + wrappedArray = [20]handledIter{} + otherIters handledIters + ) + if len(others)+1 < 20 { + otherIters = wrappedArray[:0] + } else { + otherIters = make(handledIters, 0, len(others)+1) + } bIter, _ := b.Containers.Iterator(0) next := bIter.Next() if next { - otherIters = append(otherIters, wrapperIter{ + otherIters = append(otherIters, handledIter{ iter: bIter, hasNext: true, handled: false, @@ -559,7 +567,7 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { otherIter, _ := other.Containers.Iterator(0) next := otherIter.Next() if next { - otherIters = append(otherIters, wrapperIter{ + otherIters = append(otherIters, handledIter{ iter: otherIter, hasNext: true, handled: false, @@ -3981,15 +3989,15 @@ func readWithRuns(b *Bitmap, data []byte, pos int, keyN uint32) { } } -type wrapperIter struct { +type handledIter struct { iter ContainerIterator hasNext bool handled bool } -type wrappedIters []wrapperIter +type handledIters []handledIter -func (w wrappedIters) next() bool { +func (w handledIters) next() bool { hasNext := false for i, wrapped := range w { @@ -4004,7 +4012,7 @@ func (w wrappedIters) next() bool { return hasNext } -func (w wrappedIters) markItersWithCurrentKeyAsHandled(key uint64) { +func (w handledIters) markItersWithCurrentKeyAsHandled(key uint64) { for i, wrapped := range w { currKey, _ := wrapped.iter.Value() if currKey == key { From 3d0d0db2e94b197968bdd781d869ae276ec6e077 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 17:01:52 -0500 Subject: [PATCH 045/125] more micro-optimizations --- roaring/roaring.go | 42 ++++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 51e26f037..ba5998b72 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -545,22 +545,36 @@ func (b *Bitmap) unionIntoTargetSingle(target *Bitmap, other *Bitmap) { // _ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { var ( - wrappedArray = [20]handledIter{} - otherIters handledIters + requiredSliceSize = len(others) + // To avoid having to allocate a slice everytime, if the number of bitmaps + // being unioned is small enough we can just use this stack-allocated array. + staticHandledIters = [20]handledIter{} + otherIters handledIters ) - if len(others)+1 < 20 { - otherIters = wrappedArray[:0] - } else { - otherIters = make(handledIters, 0, len(others)+1) + if b != target { + // If b and target are not the same, we will need to union b into target which + // means we need room for one more iter. + requiredSliceSize++ } - bIter, _ := b.Containers.Iterator(0) - next := bIter.Next() - if next { - otherIters = append(otherIters, handledIter{ - iter: bIter, - hasNext: true, - handled: false, - }) + + if requiredSliceSize <= 20 { + otherIters = staticHandledIters[:0] + } else { + otherIters = make(handledIters, 0, requiredSliceSize) + } + + // Only include b in the list of iters if its not the same as target to avoid + // a wasteful self union. + if b != target { + bIter, _ := b.Containers.Iterator(0) + next := bIter.Next() + if next { + otherIters = append(otherIters, handledIter{ + iter: bIter, + hasNext: true, + handled: false, + }) + } } for _, other := range others { From d96bde179d37c9261f3a96f6e3cbb1dd293bde34 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 17:19:48 -0500 Subject: [PATCH 046/125] factor out summary stats calculation into helper --- roaring/roaring.go | 89 ++++++++++++++++++++++++++-------------------- 1 file changed, 50 insertions(+), 39 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index ba5998b72..6cf4313f6 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -606,45 +606,9 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { // Summary statistics about all the containers in the other bitmaps // that share the same key so we can make smarter union strategy // decisions later. - var ( - // Estimated cardinality of the union of all containers with the same - // key as iKey across all bitmaps. This calculation is very rough as - // we just sum the cardinality of the container across the different - // bitmaps which could result in very inflated values, but it allows - // us to avoid allocating expensive bitmaps when unioning many low - // density containers. - n = iContainer.n - // Whether iContainer is the only container across all the bitmaps - // with the key iKey. If true, we can skip all the unioning logic - // and just clone the container into target. - isOnlyContainerWithKey = true - // Whether any of the containers are storing every possible value that - // they can. If so, we can short-circuit all the unioning logic and use - // a RLE container with a single value in it. This is an optimization to - // avoid using an expensive bitmap container for bitmaps that have some - // extremely dense containers. - hasMaxRange = iContainer.n == maxContainerVal+1 - ) - for _, jIter := range otherIters[i:] { - if hasMaxRange { - // If we already know that we're going to use a max range RLE container, - // then there is no reason to continue calculating statistics. - continue - } + summaryStats := otherIters[i:].calculateSummaryStats(iKey) - // Calculate key-level statistics here - jKey, jContainer := jIter.iter.Value() - - if iKey == jKey { - isOnlyContainerWithKey = false - n += jContainer.n - if !hasMaxRange { - hasMaxRange = jContainer.n == maxContainerVal+1 - } - } - } - - if isOnlyContainerWithKey { + if summaryStats.isOnlyContainerWithKey { // TODO(rartoul): We can avoid these clones if we can determine // if the container is coming from an immutable bitmap and we // know that we can mark the target bitmap as immutable as well. @@ -655,7 +619,7 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { // There was more than one container across the bitmaps with key iKey // so we need to calculate a union. - if hasMaxRange { + if summaryStats.hasMaxRange { // One (or more) of the containers represented the maximum possible // range that a container can store, so instead of calculating a // union we can generate an RLE container that represents the entire @@ -4034,3 +3998,50 @@ func (w handledIters) markItersWithCurrentKeyAsHandled(key uint64) { } } } + +func (w handledIters) calculateSummaryStats(key uint64) containerUnionSummaryStats { + summary := containerUnionSummaryStats{} + + for _, iter := range w { + if summary.hasMaxRange { + // If we already know that we're going to use a max range RLE container, + // then there is no reason to continue calculating statistics. + continue + } + + // Calculate key-level statistics here + currKey, currContainer := iter.iter.Value() + + if key == currKey { + summary.isOnlyContainerWithKey = false + summary.n += currContainer.n + if !summary.hasMaxRange { + summary.hasMaxRange = (currContainer.n == maxContainerVal+1) + } + } + } + + return summary +} + +// Summary statistics about all the containers in the other bitmaps +// that share the same key so we can make smarter union strategy +// decisions. +type containerUnionSummaryStats struct { + // Estimated cardinality of the union of all containers with the same + // key across all bitmaps. This calculation is very rough as we just sum + // the cardinality of the container across the different bitmaps which could + // result in very inflated values, but it allows us to avoid allocating + // expensive bitmaps when unioning many low density containers. + n int32 + // Whether any other is the only container across all the bitmaps + // with the specified key. If true, we can skip all the unioning logic + // and just clone the container into target. + isOnlyContainerWithKey bool + // Whether any of the containers with the specified keys are storing every possible + // value that they can. If so, we can short-circuit all the unioning logic and use + // a RLE container with a single value in it. This is an optimization to + // avoid using an expensive bitmap container for bitmaps that have some + // extremely dense containers. + hasMaxRange bool +} From a2bb87771dc93f59b42ad172b33273eec83c2d7c Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 17:21:38 -0500 Subject: [PATCH 047/125] Refactor --- roaring/roaring.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 6cf4313f6..776e2d614 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -601,12 +601,14 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { continue } - iKey, iContainer := iIter.iter.Value() - - // Summary statistics about all the containers in the other bitmaps - // that share the same key so we can make smarter union strategy - // decisions later. - summaryStats := otherIters[i:].calculateSummaryStats(iKey) + var ( + iKey, iContainer = iIter.iter.Value() + // Summary statistics about all the containers in the other bitmaps + // that share the same key so we can make smarter union strategy + // decisions later. Note that we slice to [i:] not [i+1:] because we + // want to include the current containers information in the stats. + summaryStats = otherIters[i:].calculateSummaryStats(iKey) + ) if summaryStats.isOnlyContainerWithKey { // TODO(rartoul): We can avoid these clones if we can determine From 7a57b24b46a8dad349aa8b5fa4096df8bcb0f451 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 17:21:58 -0500 Subject: [PATCH 048/125] Fix comment --- roaring/roaring.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 776e2d614..1e684ba6f 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -656,7 +656,7 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { } } - // Once we've acquire a bitmap container (either by reusing the existing one + // Once we've acquired a bitmap container (either by reusing the existing one // or allocating a new one) then the last step is to iterate through all the // other containers to see which ones have the same key, and union all of them // into the target bitmap container. From ddfc95070b5f8f92dd374d80aa5475b561d1934c Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 17:23:18 -0500 Subject: [PATCH 049/125] rename var --- roaring/roaring.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 1e684ba6f..7b382ec6f 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -549,7 +549,7 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { // To avoid having to allocate a slice everytime, if the number of bitmaps // being unioned is small enough we can just use this stack-allocated array. staticHandledIters = [20]handledIter{} - otherIters handledIters + bitmapIters handledIters ) if b != target { // If b and target are not the same, we will need to union b into target which @@ -558,9 +558,9 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { } if requiredSliceSize <= 20 { - otherIters = staticHandledIters[:0] + bitmapIters = staticHandledIters[:0] } else { - otherIters = make(handledIters, 0, requiredSliceSize) + bitmapIters = make(handledIters, 0, requiredSliceSize) } // Only include b in the list of iters if its not the same as target to avoid @@ -569,7 +569,7 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { bIter, _ := b.Containers.Iterator(0) next := bIter.Next() if next { - otherIters = append(otherIters, handledIter{ + bitmapIters = append(bitmapIters, handledIter{ iter: bIter, hasNext: true, handled: false, @@ -581,7 +581,7 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { otherIter, _ := other.Containers.Iterator(0) next := otherIter.Next() if next { - otherIters = append(otherIters, handledIter{ + bitmapIters = append(bitmapIters, handledIter{ iter: otherIter, hasNext: true, handled: false, @@ -593,7 +593,7 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { hasNext := true for hasNext { // Loop until every iters current value has been handled. - for i, iIter := range otherIters { + for i, iIter := range bitmapIters { if !iIter.hasNext || iIter.handled { // Either we've exhausted this iter (it has no more containers), or // we've already handled the current container by unioning it with @@ -607,7 +607,7 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { // that share the same key so we can make smarter union strategy // decisions later. Note that we slice to [i:] not [i+1:] because we // want to include the current containers information in the stats. - summaryStats = otherIters[i:].calculateSummaryStats(iKey) + summaryStats = bitmapIters[i:].calculateSummaryStats(iKey) ) if summaryStats.isOnlyContainerWithKey { @@ -615,7 +615,7 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { // if the container is coming from an immutable bitmap and we // know that we can mark the target bitmap as immutable as well. target.Containers.Put(iKey, iContainer.Clone()) - otherIters[i].handled = true + bitmapIters[i].handled = true continue } @@ -632,7 +632,7 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { n: maxContainerVal + 1, } target.Containers.Put(iKey, container) - otherIters[i:].markItersWithCurrentKeyAsHandled(iKey) + bitmapIters[i:].markItersWithCurrentKeyAsHandled(iKey) } else { // Use a bitmap container for the target bitmap, and union everything // into it. @@ -660,7 +660,7 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { // or allocating a new one) then the last step is to iterate through all the // other containers to see which ones have the same key, and union all of them // into the target bitmap container. - for j, jIter := range otherIters { + for j, jIter := range bitmapIters { jKey, jContainer := jIter.iter.Value() if iKey == jKey { @@ -671,7 +671,7 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { } else { unionBitmapBitmapInPlace(container, jContainer) } - otherIters[j].handled = true + bitmapIters[j].handled = true } } @@ -682,7 +682,7 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { } } - hasNext = otherIters.next() + hasNext = bitmapIters.next() } // Performing the popcount() operation with every union is wasteful because From 12d45415bb6986c6fd3b689fb15d21d5e892238f Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 17:29:32 -0500 Subject: [PATCH 050/125] more refactoring and micro optimizations --- roaring/roaring.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 7b382ec6f..575515743 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -659,8 +659,9 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { // Once we've acquired a bitmap container (either by reusing the existing one // or allocating a new one) then the last step is to iterate through all the // other containers to see which ones have the same key, and union all of them - // into the target bitmap container. - for j, jIter := range bitmapIters { + // into the target bitmap container. Only need to loop starting from i because + // anything previous to that has already been handled. + for j, jIter := range bitmapIters[i:] { jKey, jContainer := jIter.iter.Value() if iKey == jKey { From 741f8e8b84fc6f8b02e9b62b1c5cb7dbd5c90263 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 17:35:04 -0500 Subject: [PATCH 051/125] remove repairBitmaps from public iface --- roaring/containers.go | 2 +- roaring/roaring.go | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/roaring/containers.go b/roaring/containers.go index 102ffd3bd..0e8fdf988 100644 --- a/roaring/containers.go +++ b/roaring/containers.go @@ -156,7 +156,7 @@ func (sc *sliceContainers) Iterator(key uint64) (citer ContainerIterator, found return &sliceIterator{e: sc, i: i}, found } -func (sc *sliceContainers) RepairBitmaps() { +func (sc *sliceContainers) repairBitmaps() { for _, c := range sc.containers { if c.isBitmap() { c.bitmapRepair() diff --git a/roaring/roaring.go b/roaring/roaring.go index 575515743..3cec6cf85 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -98,10 +98,6 @@ type Containers interface { Count() uint64 - // RepairBitmaps will repair cardinality(n) values on bitmap containers after - // in-place operations. - RepairBitmaps() - //Reset will clear the containers collection to allow for recycling during snapshot Reset() } @@ -692,7 +688,7 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { // n (container cardinality) to fall out of sync, and then at the very end we perform // a "Repair" to recalculate all the container values. That way we never popcount() // an entire bitmap container more than once per bulk union operation. - target.Containers.RepairBitmaps() + target.Containers.(*sliceContainers).repairBitmaps() } // Difference returns the difference of b and other. From 549595cd2e8b2887cca998051124e73a3792ce4b Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 17:45:10 -0500 Subject: [PATCH 052/125] fix lint issues --- roaring/roaring.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 3cec6cf85..31dd18ddf 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -448,9 +448,9 @@ func (b *Bitmap) unionIntoTargetSingle(target *Bitmap, other *Bitmap) { // containers for each union operation for a given container key, only one // new container needs to be allocated (or re-used) regardless of how many bitmaps // participate in the union. This significantly reduces allocations. In addition, -// because we perform the unions one container at a time accross all the bitmaps, we +// because we perform the unions one container at a time across all the bitmaps, we // can calculate summary statistics that allow us to make more efficient decisions -// up front. For example, imagine trying to perform a union accross the following three +// up front. For example, imagine trying to perform a union across the following three // bitsets: // // 1. Bitmap A: Single array container at key 0 with 400 values in it. From 6d021fe870d815349009c82153a535518de0ff59 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 17:45:49 -0500 Subject: [PATCH 053/125] add comment --- roaring/roaring.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roaring/roaring.go b/roaring/roaring.go index 31dd18ddf..13f0efede 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -3966,6 +3966,9 @@ func readWithRuns(b *Bitmap, data []byte, pos int, keyN uint32) { } } +// handledIter and handledIters are wrappers around Bitmap Container iterators +// and assist with the unionIntoTarget algorithm by abstracting away some tedious +// operations. type handledIter struct { iter ContainerIterator hasNext bool From 42e756b316818b4987080cffbfb3d8c05f7d2d1e Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 20:45:26 -0500 Subject: [PATCH 054/125] Add benchmark --- roaring/roaring_internal_test.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 78b686371..223953658 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -3267,6 +3267,27 @@ func TestUnmarshalOfficialRoaring(t *testing.T) { } +func BenchmarkUnionBitmapBitmapInPlace(b *testing.B) { + b1 := newTestBitmapContainer() + b2 := newTestBitmapContainer() + for n := 0; n < b.N; n++ { + unionBitmapBitmapInPlace(b1, b2) + } +} + +func newTestBitmapContainer() *Container { + var ( + buf = make([]uint64, bitmapN) + ob = buf[:bitmapN] + container = &Container{ + bitmap: ob, + n: 0, + containerType: containerBitmap, + } + ) + return container +} + /* // This function exercises an arcane edge case in dead code. // It doesn't need to be run right now. From c412bc595d62cf17396727c30a6b4dab6e2faff9 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 21:00:35 -0500 Subject: [PATCH 055/125] add benchmark --- roaring/roaring_internal_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 223953658..a5157a97d 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -3275,6 +3275,13 @@ func BenchmarkUnionBitmapBitmapInPlace(b *testing.B) { } } +func BenchmarkBitmapRepair(b *testing.B) { + b1 := newTestBitmapContainer() + for n := 0; n < b.N; n++ { + b1.bitmapRepair() + } +} + func newTestBitmapContainer() *Container { var ( buf = make([]uint64, bitmapN) From 069c2a281daa8cd24ea9c2e3d31b3df79c2809c7 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Thu, 29 Nov 2018 21:04:04 -0500 Subject: [PATCH 056/125] unroll to make a little faster --- roaring/roaring.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 13f0efede..e6ed7d4f4 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2094,8 +2094,14 @@ func (c *Container) check() error { func (c *Container) bitmapRepair() { n := int32(0) - for i := 0; i < bitmapN; i++ { + // Manually unroll loop to make it a little faster. + // TODO(rartoul): Can probably make this a few x faster using + // SIMD instructions. + for i := 0; i < bitmapN; i += 4 { n += int32(popcount(c.bitmap[i])) + n += int32(popcount(c.bitmap[i+1])) + n += int32(popcount(c.bitmap[i+2])) + n += int32(popcount(c.bitmap[i+3])) } c.n = n } @@ -2852,8 +2858,14 @@ func unionBitmapBitmapInPlace(a, b *Container) { bb = b.bitmap[:bitmapN] ) - for i := 0; i < bitmapN; i++ { + // Manually unroll loop to make it a little faster. + // TODO(rartoul): Can probably make this a few x faster using + // SIMD instructions. + for i := 0; i < bitmapN; i += 4 { ab[i] = ab[i] | bb[i] + ab[i+1] = ab[i+1] | bb[i+1] + ab[i+2] = ab[i+2] | bb[i+2] + ab[i+3] = ab[i+3] | bb[i+3] } } From 531b9d616dbfa3b8f21e7495f7cb7bd6f524c918 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Fri, 30 Nov 2018 14:40:06 -0500 Subject: [PATCH 057/125] Refactor code and comment for clarity --- roaring/roaring.go | 95 +++++++++++++++++++++++----------------------- 1 file changed, 48 insertions(+), 47 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index e6ed7d4f4..4aed09141 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -629,54 +629,55 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { } target.Containers.Put(iKey, container) bitmapIters[i:].markItersWithCurrentKeyAsHandled(iKey) - } else { - // Use a bitmap container for the target bitmap, and union everything - // into it. - // - // TODO(rartoul): Add another conditional case for n < ArrayMaxSize - // (or some fraction of that value) to avoid allocating expensive - // bitmaps when unioning many low-density array containers, but this - // will require writing a union in place algorithm for an array container - // that accepts multiple different containers to union into it for - // efficiency. - container := target.Containers.Get(iKey) - // If target already has a bitmap container for iKey then we can reuse that, - // otherwise we have to allocate a new one. - if container == nil || container.containerType != containerBitmap { - buf := make([]uint64, bitmapN) - ob := buf[:bitmapN] - container = &Container{ - bitmap: ob, - n: 0, - containerType: containerBitmap, - } - } - - // Once we've acquired a bitmap container (either by reusing the existing one - // or allocating a new one) then the last step is to iterate through all the - // other containers to see which ones have the same key, and union all of them - // into the target bitmap container. Only need to loop starting from i because - // anything previous to that has already been handled. - for j, jIter := range bitmapIters[i:] { - jKey, jContainer := jIter.iter.Value() - - if iKey == jKey { - if jContainer.isArray() { - unionBitmapArrayInPlace(container, jContainer) - } else if jContainer.isRun() { - unionBitmapRunInPlace(container, jContainer) - } else { - unionBitmapBitmapInPlace(container, jContainer) - } - bitmapIters[j].handled = true - } - } - - // Now that we've calculated a container is that a union of all the containers - // with the same key across all the bitmaps, we store it in the list of containers - // for the target. - target.Containers.Put(iKey, container) + continue } + + // Use a bitmap container for the target bitmap, and union everything + // into it. + // + // TODO(rartoul): Add another conditional case for n < ArrayMaxSize + // (or some fraction of that value) to avoid allocating expensive + // bitmaps when unioning many low-density array containers, but this + // will require writing a union in place algorithm for an array container + // that accepts multiple different containers to union into it for + // efficiency. + container := target.Containers.Get(iKey) + // If target already has a bitmap container for iKey then we can reuse that, + // otherwise we have to allocate a new one. + if container == nil || container.containerType != containerBitmap { + buf := make([]uint64, bitmapN) + ob := buf[:bitmapN] + container = &Container{ + bitmap: ob, + n: 0, + containerType: containerBitmap, + } + } + + // Once we've acquired a bitmap container (either by reusing the existing one + // or allocating a new one) then the last step is to iterate through all the + // other containers to see which ones have the same key, and union all of them + // into the target bitmap container. Only need to loop starting from i because + // anything previous to that has already been handled. + for j, jIter := range bitmapIters[i:] { + jKey, jContainer := jIter.iter.Value() + + if iKey == jKey { + if jContainer.isArray() { + unionBitmapArrayInPlace(container, jContainer) + } else if jContainer.isRun() { + unionBitmapRunInPlace(container, jContainer) + } else { + unionBitmapBitmapInPlace(container, jContainer) + } + bitmapIters[j].handled = true + } + } + + // Now that we've calculated a container that is a union of all the containers + // with the same key across all the bitmaps, we store it in the list of containers + // for the target. + target.Containers.Put(iKey, container) } hasNext = bitmapIters.next() From 3570ec7ab633e3ae6d29fd944b8d2d81959e533f Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Fri, 30 Nov 2018 14:41:30 -0500 Subject: [PATCH 058/125] collapse next calls into conditonals --- roaring/roaring.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 4aed09141..be6010b52 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -563,8 +563,7 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { // a wasteful self union. if b != target { bIter, _ := b.Containers.Iterator(0) - next := bIter.Next() - if next { + if bIter.Next() { bitmapIters = append(bitmapIters, handledIter{ iter: bIter, hasNext: true, @@ -575,8 +574,7 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { for _, other := range others { otherIter, _ := other.Containers.Iterator(0) - next := otherIter.Next() - if next { + if otherIter.Next() { bitmapIters = append(bitmapIters, handledIter{ iter: otherIter, hasNext: true, From 34b1f2199f16eadbef0e0cd2c1137d03b08c164b Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Fri, 30 Nov 2018 14:41:59 -0500 Subject: [PATCH 059/125] Fix comment --- roaring/roaring.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index be6010b52..2d88f2868 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -395,7 +395,7 @@ func (b *Bitmap) Intersect(other *Bitmap) *Bitmap { return output } -// Union returns the bitwise union of b and other as a new bitmap. +// Union returns the bitwise union of b and others as a new bitmap. func (b *Bitmap) Union(others ...*Bitmap) *Bitmap { output := NewBitmap() if len(others) == 1 { From 71621e60baab8ab5fe4033535d33df76c955b65e Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Fri, 30 Nov 2018 19:23:59 -0500 Subject: [PATCH 060/125] Refactor roaring repair operations --- roaring/containers.go | 6 ++---- roaring/roaring.go | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/roaring/containers.go b/roaring/containers.go index 0e8fdf988..b9928823a 100644 --- a/roaring/containers.go +++ b/roaring/containers.go @@ -156,11 +156,9 @@ func (sc *sliceContainers) Iterator(key uint64) (citer ContainerIterator, found return &sliceIterator{e: sc, i: i}, found } -func (sc *sliceContainers) repairBitmaps() { +func (sc *sliceContainers) Repair() { for _, c := range sc.containers { - if c.isBitmap() { - c.bitmapRepair() - } + c.Repair() } } diff --git a/roaring/roaring.go b/roaring/roaring.go index 2d88f2868..a1a535ed3 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -98,8 +98,12 @@ type Containers interface { Count() uint64 - //Reset will clear the containers collection to allow for recycling during snapshot + // Reset will clear the containers collection to allow for recycling during snapshot Reset() + + // Repair will repair the cardinality of any containers whose cardinality were corrupted + // due to optimized operations. + Repair() } type ContainerIterator interface { @@ -687,7 +691,7 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { // n (container cardinality) to fall out of sync, and then at the very end we perform // a "Repair" to recalculate all the container values. That way we never popcount() // an entire bitmap container more than once per bulk union operation. - target.Containers.(*sliceContainers).repairBitmaps() + target.Containers.Repair() } // Difference returns the difference of b and other. @@ -2091,6 +2095,14 @@ func (c *Container) check() error { return a } +// Repair repairs the cardinality of c if it has been corrupted by +// optimized operations. +func (c *Container) Repair() { + if c.isBitmap() { + c.bitmapRepair() + } +} + func (c *Container) bitmapRepair() { n := int32(0) // Manually unroll loop to make it a little faster. From bb20f058ba6c77e26029d963e974487273208529 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Fri, 30 Nov 2018 19:25:54 -0500 Subject: [PATCH 061/125] Add Repair operation to btree containers --- enterprise/b/containers_btree.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/enterprise/b/containers_btree.go b/enterprise/b/containers_btree.go index c2f95c0c7..fe357d933 100644 --- a/enterprise/b/containers_btree.go +++ b/enterprise/b/containers_btree.go @@ -177,6 +177,15 @@ func (btc *bTreeContainers) Iterator(key uint64) (citer roaring.ContainerIterato }, found } +func (btc *bTreeContainers) Repair() { + e, _ := btc.tree.Seek(0) + _, c, err := e.Next() + for err != io.EOF { + c.Repair() + _, c, err = e.Next() + } +} + type btcIterator struct { e *enumerator key uint64 From 5b72544d73ec316d7ef9cd7ca2be1a6d8907ff8b Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Mon, 3 Dec 2018 15:25:54 -0500 Subject: [PATCH 062/125] simplify helper with early return --- roaring/roaring.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index a1a535ed3..f78f98fba 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -4028,20 +4028,17 @@ func (w handledIters) calculateSummaryStats(key uint64) containerUnionSummarySta summary := containerUnionSummaryStats{} for _, iter := range w { - if summary.hasMaxRange { - // If we already know that we're going to use a max range RLE container, - // then there is no reason to continue calculating statistics. - continue - } - // Calculate key-level statistics here currKey, currContainer := iter.iter.Value() if key == currKey { summary.isOnlyContainerWithKey = false summary.n += currContainer.n - if !summary.hasMaxRange { - summary.hasMaxRange = (currContainer.n == maxContainerVal+1) + + if currContainer.n == maxContainerVal+1 { + summary.hasMaxRange = true + summary.n = maxContainerVal + 1 + return summary } } } From c495d08d1b9a42a46811c6dd907cac241e04a441 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Mon, 3 Dec 2018 15:30:54 -0500 Subject: [PATCH 063/125] simplify logic by removing concept or target --- roaring/roaring.go | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index f78f98fba..2c38cfea2 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -415,7 +415,7 @@ func (b *Bitmap) Union(others ...*Bitmap) *Bitmap { // UnionInPlace returns the bitwise union of b and others, modifying // b in place. func (b *Bitmap) UnionInPlace(others ...*Bitmap) { - b.unionIntoTarget(b, others...) + b.unionInPlace(others...) } func (b *Bitmap) unionIntoTargetSingle(target *Bitmap, other *Bitmap) { @@ -543,19 +543,15 @@ func (b *Bitmap) unionIntoTargetSingle(target *Bitmap, other *Bitmap) { // ---------------------------- | ---------------------------- | ---------------------------- // Bitmap 4 |___X_______________________| | |___X_______________________| | |___X_______________________| // _ -func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { +func (b *Bitmap) unionInPlace(others ...*Bitmap) { var ( requiredSliceSize = len(others) // To avoid having to allocate a slice everytime, if the number of bitmaps // being unioned is small enough we can just use this stack-allocated array. staticHandledIters = [20]handledIter{} bitmapIters handledIters + target = b ) - if b != target { - // If b and target are not the same, we will need to union b into target which - // means we need room for one more iter. - requiredSliceSize++ - } if requiredSliceSize <= 20 { bitmapIters = staticHandledIters[:0] @@ -563,19 +559,6 @@ func (b *Bitmap) unionIntoTarget(target *Bitmap, others ...*Bitmap) { bitmapIters = make(handledIters, 0, requiredSliceSize) } - // Only include b in the list of iters if its not the same as target to avoid - // a wasteful self union. - if b != target { - bIter, _ := b.Containers.Iterator(0) - if bIter.Next() { - bitmapIters = append(bitmapIters, handledIter{ - iter: bIter, - hasNext: true, - handled: false, - }) - } - } - for _, other := range others { otherIter, _ := other.Containers.Iterator(0) if otherIter.Next() { From 9da6d43b760293111dc95c8880af9e5a830f5489 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Mon, 3 Dec 2018 15:31:29 -0500 Subject: [PATCH 064/125] fix docstring nit --- roaring/roaring.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 2c38cfea2..e2fd3f893 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -98,7 +98,7 @@ type Containers interface { Count() uint64 - // Reset will clear the containers collection to allow for recycling during snapshot + // Reset clears the containers collection to allow for recycling during snapshot Reset() // Repair will repair the cardinality of any containers whose cardinality were corrupted From a11d04c061e258b045716ba793bc0afd41676fa8 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Fri, 7 Dec 2018 15:59:57 -0500 Subject: [PATCH 065/125] clarify comment --- roaring/roaring.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index e2fd3f893..93aa65add 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -478,7 +478,11 @@ func (b *Bitmap) unionIntoTargetSingle(target *Bitmap, other *Bitmap) { // 1. Detect that bitmaps A, B, and C all have containers for key 0. // 2. Estimate the resulting cardinality of the union of all their containers to be // 400 + 500 + 3500 > ArrayMaxSize and decide upfront to use a bitset for the target -// container. +// container. Note that this is just an approximation of the final cardinality and can +// be off by a wide margin if there is a lot of overlap between containers, but that is +// fine, we'll still get the same result at the end, we'll just be more biased towards +// using bitmap containers will still being able to use array containers when all the +// cardinalities are small. // 3. Union the containers from bitmaps A, B, and C into the new bitset container directly // using fast bitwise operations. // From dcfed6ebb63182c89fc72c3441306f2822e4868e Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Fri, 7 Dec 2018 16:00:49 -0500 Subject: [PATCH 066/125] fix grammar --- roaring/roaring.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 93aa65add..61827fc77 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -494,7 +494,7 @@ func (b *Bitmap) unionIntoTargetSingle(target *Bitmap, other *Bitmap) { // An additional optimization that this function makes is that it recognizes that even when // CPU support is present, performing the popcount() operation isn't free. Imagine a scenario // where 10 bitset containers are being unioned together one after the next. If every -// bitset<->bitset union operation needs to keep the containers cardinality up to date, then +// bitset<->bitset union operation needs to keep the containers' cardinality up to date, then // the algorithm will waste a lot of time performing intermediary popcount() operations that // will immediately be invalidated by the next union operation. As a result, we allow the cardinality // of containers to degrade when we perform the in-place union operations, and then when the algorithm From 082c8aba56108df22f4ade87e35b2000f4d17e92 Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Fri, 7 Dec 2018 16:13:42 -0500 Subject: [PATCH 067/125] switch to |= --- roaring/roaring.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 61827fc77..4df42afdc 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2860,10 +2860,10 @@ func unionBitmapBitmapInPlace(a, b *Container) { // TODO(rartoul): Can probably make this a few x faster using // SIMD instructions. for i := 0; i < bitmapN; i += 4 { - ab[i] = ab[i] | bb[i] - ab[i+1] = ab[i+1] | bb[i+1] - ab[i+2] = ab[i+2] | bb[i+2] - ab[i+3] = ab[i+3] | bb[i+3] + ab[i] |= bb[i] + ab[i+1] |= bb[i+1] + ab[i+2] |= bb[i+2] + ab[i+3] |= bb[i+3] } } From 9ac4d981fc837f31319ced8a90e4894435cc2aab Mon Sep 17 00:00:00 2001 From: Matthew Jaffee Date: Mon, 10 Dec 2018 17:24:23 -0600 Subject: [PATCH 068/125] wrap in backquotes so it gets displayed. previously was not rendering - I assume because it was being interpreted as an HTML tag. --- docs/query-language.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/query-language.md b/docs/query-language.md index 0f8feb795..babcc390c 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -762,7 +762,7 @@ Modifies the given query as follows: * `excludeRowAttrs`: Exclude row attributes from the result (Default: `false`). * `shards`: Run the query using only the data from the given shards. By default, the entire data set (i.e. data from all shards) is used. -**Result Type:** Same result type as . +**Result Type:** Same result type as ``. **Examples:** From c66daabc81b17ce8193d5f828590d9928ffad9d8 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 10 Dec 2018 20:37:26 -0600 Subject: [PATCH 069/125] convert the anti-entropy logic to use `ImportRoaring` instead of `QueryNode` --- api.go | 2 - fragment.go | 102 +++++++++++++++++++++++++++++++------------------ holder_test.go | 11 +++--- 3 files changed, 69 insertions(+), 46 deletions(-) diff --git a/api.go b/api.go index cf27522e6..ce4fbd0ca 100644 --- a/api.go +++ b/api.go @@ -333,8 +333,6 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, } return err }) - go func(node *Node) { - }(node) } else if !remote { // if remote == true we don't forward to other nodes // forward it on eg.Go(func() error { diff --git a/fragment.go b/fragment.go index 028d105b4..4f71809c9 100644 --- a/fragment.go +++ b/fragment.go @@ -28,6 +28,7 @@ import ( "math" "os" "sort" + "strings" "sync" "syscall" "time" @@ -2315,46 +2316,38 @@ func (s *fragmentSyncer) syncBlock(id int) error { // Write updates to remote blocks. for i := 0; i < len(uris); i++ { set, clear := sets[i], clears[i] - count := 0 - // Ignore if there are no differences. - if len(set.columnIDs) == 0 && len(clear.columnIDs) == 0 { - continue - } - - // Generate query with sets & clears, and group the requests to not exceed MaxWritesPerRequest. - total := len(set.columnIDs) + len(clear.columnIDs) - maxWrites := s.Cluster.maxWritesPerRequest - if maxWrites <= 0 { - maxWrites = 5000 - } - buffers := make([]bytes.Buffer, int(math.Ceil(float64(total)/float64(maxWrites)))) - - // Only sync the standard block. - for j := 0; j < len(set.columnIDs); j++ { - fmt.Fprintf(&(buffers[count/maxWrites]), "Set(%d, %s=%d)\n", (f.shard*ShardWidth)+set.columnIDs[j], f.field, set.rowIDs[j]) - count++ - } - for j := 0; j < len(clear.columnIDs); j++ { - fmt.Fprintf(&(buffers[count/maxWrites]), "Clear(%d, %s=%d)\n", (f.shard*ShardWidth)+clear.columnIDs[j], f.field, clear.rowIDs[j]) - count++ - } - - // Iterate over the buffers. - for k := 0; k < len(buffers); k++ { - // Verify sync is not prematurely closing. - if s.isClosing() { - return nil - } - - // Execute query. - queryRequest := &QueryRequest{ - Query: buffers[k].String(), - Remote: true, - } - _, err := s.Cluster.InternalClient.QueryNode(ctx, uris[i], f.index, queryRequest) + // Handle Sets. + if len(set.columnIDs) > 0 { + setData, err := bitsToRoaringData(set) if err != nil { - return errors.Wrap(err, "executing") + return errors.Wrap(err, "converting bits to roaring data (set)") + } + + setReq := &ImportRoaringRequest{ + Clear: false, + Views: map[string][]byte{cleanViewName(f.view): setData}, + } + + if err := s.Cluster.InternalClient.ImportRoaring(ctx, uris[i], f.index, f.field, f.shard, true, setReq); err != nil { + return errors.Wrap(err, "sending roaring data (set)") + } + } + + // Handle Clears. + if len(clear.columnIDs) > 0 { + clearData, err := bitsToRoaringData(clear) + if err != nil { + return errors.Wrap(err, "converting bits to roaring data (clear)") + } + + clearReq := &ImportRoaringRequest{ + Clear: true, + Views: map[string][]byte{"": clearData}, + } + + if err := s.Cluster.InternalClient.ImportRoaring(ctx, uris[i], f.index, f.field, f.shard, true, clearReq); err != nil { + return errors.Wrap(err, "sending roaring data (clear)") } } } @@ -2362,6 +2355,39 @@ func (s *fragmentSyncer) syncBlock(id int) error { return nil } +// cleanViewName converts a viewname into the equivalent +// string required by the external api. Because views are +// not exposed externally, the conversion looks like this: +// "standard" -> "" +// "standard_YYYYMMDD" -> "YYYYMMDD" +// "other" -> "other" (there is currently not a use for this) +func cleanViewName(v string) string { + viewPrefix := viewStandard + "_" + if strings.HasPrefix(v, viewPrefix) { + return v[len(viewPrefix):] + } else if v == viewStandard { + return "" + } + return v +} + +// bitsToRoaringData converts a pairSet into a roaring.Bitmap +// which represents the data within a single shard. +func bitsToRoaringData(ps pairSet) ([]byte, error) { + bmp := roaring.NewBitmap() + for j := 0; j < len(ps.columnIDs); j++ { + bmp.DirectAdd(ps.rowIDs[j]*ShardWidth + (ps.columnIDs[j] % ShardWidth)) + } + + var buf bytes.Buffer + _, err := bmp.WriteTo(&buf) + if err != nil { + return nil, errors.Wrap(err, "writing to buffer") + } + + return buf.Bytes(), nil +} + func madvise(b []byte, advice int) error { // nolint: unparam _, _, err := syscall.Syscall(syscall.SYS_MADVISE, uintptr(unsafe.Pointer(&b[0])), uintptr(len(b)), uintptr(advice)) if err != 0 { diff --git a/holder_test.go b/holder_test.go index 3920e4cc7..2835e5c4f 100644 --- a/holder_test.go +++ b/holder_test.go @@ -391,27 +391,26 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) { hldr0 := &test.Holder{Holder: c[0].Server.Holder()} hldr1 := &test.Holder{Holder: c[1].Server.Holder()} - // Set data on the local holder. + // Set data on the local holder for node0. t1 := time.Date(2018, 8, 1, 12, 30, 0, 0, time.UTC) t2 := time.Date(2018, 8, 2, 12, 30, 0, 0, time.UTC) hldr0.SetBitTime("i", "f", 0, 1, &t1) hldr0.SetBitTime("i", "f", 0, 2, &t2) + // Set data on node1 + hldr0.SetBitTime("i", "f", 0, 22, &t2) + err = c[0].Server.SyncData() if err != nil { t.Fatalf("syncing node 0: %v", err) } - err = c[1].Server.SyncData() - if err != nil { - t.Fatalf("syncing node 1: %v", err) - } // Verify data is the same on both nodes. for i, hldr := range []*test.Holder{hldr0, hldr1} { if a := hldr.RowTime("i", "f", 0, t1, quantum).Columns(); !reflect.DeepEqual(a, []uint64{1}) { t.Errorf("unexpected columns(%d/0): %+v", i, a) } - if a := hldr.RowTime("i", "f", 0, t2, quantum).Columns(); !reflect.DeepEqual(a, []uint64{2}) { + if a := hldr.RowTime("i", "f", 0, t2, quantum).Columns(); !reflect.DeepEqual(a, []uint64{2, 22}) { t.Errorf("unexpected columns(%d/0): %+v", i, a) } } From 7e6c406212521d0204e6b31755be65ff85ab3bdd Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 10 Dec 2018 14:53:06 -0600 Subject: [PATCH 070/125] fix bug where cluster goes into RESIZING instead of NORMAL running "make clustertests DOCKER_COMPOSE=internal/clustertests/docker-compose-replication2.yml" shows this issue (just remove the change in cluster.go). Also removed two unrelated lines of code that appear to be doing absolutely nothing. --- Makefile | 14 +++-- cluster.go | 2 +- .../docker-compose-replication2.yml | 58 +++++++++++++++++++ 3 files changed, 68 insertions(+), 6 deletions(-) create mode 100644 internal/clustertests/docker-compose-replication2.yml diff --git a/Makefile b/Makefile index af86b8241..8fa366137 100644 --- a/Makefile +++ b/Makefile @@ -68,20 +68,24 @@ release: check-clean $(MAKE) release-build GOOS=linux GOARCH=386 $(MAKE) release-build GOOS=linux GOARCH=386 ENTERPRISE=1 + +# try (e.g.) internal/clustertests/docker-compose-replication2.yml +DOCKER_COMPOSE=internal/clustertests/docker-compose.yml + # Run cluster integration tests using docker. Requires docker daemon to be # running. This will catch changes to internal/clustertests/*.go, but if you # make changes to Pilosa, you'll want to run clustertests-build to rebuild the # pilosa image. clustertests: - docker-compose -f internal/clustertests/docker-compose.yml down - docker-compose -f internal/clustertests/docker-compose.yml build client1 - docker-compose -f internal/clustertests/docker-compose.yml up --exit-code-from=client1 + docker-compose -f $(DOCKER_COMPOSE) down + docker-compose -f $(DOCKER_COMPOSE) build client1 + docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 # Like clustertests, but rebuilds all images. clustertests-build: - docker-compose -f internal/clustertests/docker-compose.yml down - docker-compose -f internal/clustertests/docker-compose.yml up --exit-code-from=client1 --build + docker-compose -f $(DOCKER_COMPOSE) down + docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build # Create prerelease builds prerelease: vendor diff --git a/cluster.go b/cluster.go index 155c4a4aa..af54132e8 100644 --- a/cluster.go +++ b/cluster.go @@ -983,7 +983,7 @@ func (c *cluster) markAsJoined() { // needTopologyAgreement is unprotected. func (c *cluster) needTopologyAgreement() bool { - return c.state == ClusterStateStarting && !stringSlicesAreEqual(c.Topology.nodeIDs, c.nodeIDs()) + return (c.state == ClusterStateStarting || c.state == ClusterStateDegraded) && !stringSlicesAreEqual(c.Topology.nodeIDs, c.nodeIDs()) } // haveTopologyAgreement is unprotected. diff --git a/internal/clustertests/docker-compose-replication2.yml b/internal/clustertests/docker-compose-replication2.yml new file mode 100644 index 000000000..291699aee --- /dev/null +++ b/internal/clustertests/docker-compose-replication2.yml @@ -0,0 +1,58 @@ +version: '2' +services: + pilosa1: + build: + context: ../.. + dockerfile: Dockerfile-clustertests + image: ptest + ports: + - "33455:10101" + environment: + - PILOSA_CLUSTER_COORDINATOR=true + - PILOSA_GOSSIP_SEEDS=pilosa1:14000 + - PILOSA_CLUSTER_REPLICAS=2 + networks: + - pilosanet + command: + - "/pilosa server --bind pilosa1:10101" + pilosa2: + build: + context: ../.. + dockerfile: Dockerfile-clustertests + image: ptest + ports: + - "33456:10101" + environment: + - PILOSA_GOSSIP_SEEDS=pilosa1:14000 + - PILOSA_CLUSTER_REPLICAS=2 + networks: + - pilosanet + command: + - "/pilosa server --bind pilosa2:10101" + pilosa3: + build: + context: ../.. + dockerfile: Dockerfile-clustertests + image: ptest + ports: + - "33457:10101" + environment: + - PILOSA_GOSSIP_SEEDS=pilosa1:14000,pilosa2:14000 + - PILOSA_CLUSTER_REPLICAS=2 + networks: + - pilosanet + command: + - "/pilosa server --bind pilosa3:10101" + client1: + build: + context: . + environment: + - ENABLE_PILOSA_CLUSTER_TESTS=1 + networks: + - pilosanet + volumes: + - /var/run/docker.sock:/var/run/docker.sock + command: + - "go test -v -count=1 github.com/pilosa/pilosa/internal/clustertests" +networks: + pilosanet: From 2c4401db8c72b3b6b2e88df7aada2bb1d5e1bc85 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 10 Dec 2018 15:48:25 -0600 Subject: [PATCH 071/125] hopefully fix data race --- gossip/gossip.go | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/gossip/gossip.go b/gossip/gossip.go index 8f19203f5..491e14cf2 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -333,15 +333,30 @@ func newEventReceiver(logger *log.Logger, papi *pilosa.API) *eventReceiver { } func (g *eventReceiver) NotifyJoin(n *memberlist.Node) { - g.ch <- memberlist.NodeEvent{Event: memberlist.NodeJoin, Node: n} + // copy node to avoid data race + n2 := *n + n2.Meta = make([]byte, len(n.Meta)) + copy(n2.Meta, n.Meta) + + g.ch <- memberlist.NodeEvent{Event: memberlist.NodeJoin, Node: &n2} } func (g *eventReceiver) NotifyLeave(n *memberlist.Node) { - g.ch <- memberlist.NodeEvent{Event: memberlist.NodeLeave, Node: n} + // copy node to avoid data race + n2 := *n + n2.Meta = make([]byte, len(n.Meta)) + copy(n2.Meta, n.Meta) + + g.ch <- memberlist.NodeEvent{Event: memberlist.NodeLeave, Node: &n2} } func (g *eventReceiver) NotifyUpdate(n *memberlist.Node) { - g.ch <- memberlist.NodeEvent{Event: memberlist.NodeUpdate, Node: n} + // copy node to avoid data race + n2 := *n + n2.Meta = make([]byte, len(n.Meta)) + copy(n2.Meta, n.Meta) + + g.ch <- memberlist.NodeEvent{Event: memberlist.NodeUpdate, Node: &n2} } func (g *eventReceiver) listen() { From 4b786e1057878fbaa40e3cd221be408693a51a2b Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 11 Dec 2018 15:00:46 -0600 Subject: [PATCH 072/125] attempt to fix deadlock by releasing view lock before broadcasting CreateShard --- http/handler.go | 10 +++++++++- view.go | 35 ++++++++++++++++++++++------------- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/http/handler.go b/http/handler.go index 68719fdfe..736a4b102 100644 --- a/http/handler.go +++ b/http/handler.go @@ -1504,7 +1504,15 @@ func GetHTTPClient(t *tls.Config) *http.Client { if t != nil { transport.TLSClientConfig = t } - return &http.Client{Transport: transport} + return &http.Client{ + Transport: transport, + // Internal queries will time out after 2h 7m by default. This is + // reduced from the old default of no timeout, so it was thought we + // should keep it fairly high, but it could probably be reduced in most + // cases. It is set to an odd number in the hopes that it will be + // recognizable in stats/traces/logs when this limit is being hit. + Timeout: 127 * time.Minute, + } } // handlPostRoaringImport diff --git a/view.go b/view.go index 128e3b828..52738fa42 100644 --- a/view.go +++ b/view.go @@ -206,37 +206,46 @@ func (v *view) recalculateCaches() { // CreateFragmentIfNotExists returns a fragment in the view by shard. func (v *view) CreateFragmentIfNotExists(shard uint64) (*fragment, error) { - v.mu.Lock() - defer v.mu.Unlock() - return v.createFragmentIfNotExists(shard) + frag, msg, err := v.createFragmentIfNotExists(shard) + + if err == nil && msg != nil { + // Broadcast a message that a new max shard was just created. + if err = v.broadcaster.SendSync(msg); err != nil { + v.mu.Lock() + delete(v.fragments, shard) + v.mu.Unlock() + frag.close() + return nil, errors.Wrap(err, "sending createshard message") + } + } + + return frag, err } -func (v *view) createFragmentIfNotExists(shard uint64) (*fragment, error) { +func (v *view) createFragmentIfNotExists(shard uint64) (*fragment, *CreateShardMessage, error) { + v.mu.Lock() + defer v.mu.Unlock() // Find fragment in cache first. if frag := v.fragments[shard]; frag != nil { - return frag, nil + return frag, nil, nil } // Initialize and open fragment. frag := v.newFragment(v.fragmentPath(shard), shard) if err := frag.Open(); err != nil { - return nil, errors.Wrap(err, "opening fragment") + return nil, nil, errors.Wrap(err, "opening fragment") } frag.RowAttrStore = v.rowAttrStore - // Broadcast a message that a new max shard was just created. - if err := v.broadcaster.SendSync(&CreateShardMessage{ + msg := &CreateShardMessage{ Index: v.index, Field: v.field, Shard: shard, - }); err != nil { - frag.close() - return nil, errors.Wrap(err, "sending createshard message") } + v.fragments[shard] = frag // Save to lookup. - v.fragments[shard] = frag - return frag, nil + return frag, msg, nil } func (v *view) newFragment(path string, shard uint64) *fragment { From 73042589670d35a8379a79f4457ad1f4208dd5a8 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 11 Dec 2018 15:09:55 -0600 Subject: [PATCH 073/125] improve comments --- view.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/view.go b/view.go index 52738fa42..50a2e02ff 100644 --- a/view.go +++ b/view.go @@ -208,6 +208,7 @@ func (v *view) recalculateCaches() { func (v *view) CreateFragmentIfNotExists(shard uint64) (*fragment, error) { frag, msg, err := v.createFragmentIfNotExists(shard) + // if msg is not nil, then a new shard was created if err == nil && msg != nil { // Broadcast a message that a new max shard was just created. if err = v.broadcaster.SendSync(msg); err != nil { @@ -242,9 +243,10 @@ func (v *view) createFragmentIfNotExists(shard uint64) (*fragment, *CreateShardM Field: v.field, Shard: shard, } - v.fragments[shard] = frag // Save to lookup. + v.fragments[shard] = frag + return frag, msg, nil } From 8b3e5b998aa87a77ac6571e8648f400a6610effb Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 11 Dec 2018 15:33:58 -0600 Subject: [PATCH 074/125] fix data race which appears to be unrelated to previous changes --- cluster.go | 7 +++++++ executor.go | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/cluster.go b/cluster.go index af54132e8..ad6cdbe6a 100644 --- a/cluster.go +++ b/cluster.go @@ -836,6 +836,13 @@ func (c *cluster) partition(index string, shard uint64) int { return int(h.Sum64() % uint64(c.partitionN)) } +// ShardNodes returns a list of nodes that own a fragment. Safe for concurrent use. +func (c *cluster) ShardNodes(index string, shard uint64) []*Node { + c.mu.RLock() + defer c.mu.RUnlock() + return c.shardNodes(index, shard) +} + // shardNodes returns a list of nodes that own a fragment. unprotected func (c *cluster) shardNodes(index string, shard uint64) []*Node { return c.partitionNodes(c.partition(index, shard)) diff --git a/executor.go b/executor.go index d8a24f8dd..dca0bd546 100644 --- a/executor.go +++ b/executor.go @@ -2138,7 +2138,7 @@ func (e *executor) shardsByNode(nodes []*Node, index string, shards []uint64) (m loop: for _, shard := range shards { - for _, node := range e.Cluster.shardNodes(index, shard) { + for _, node := range e.Cluster.ShardNodes(index, shard) { if Nodes(nodes).Contains(node) { m[node] = append(m[node], shard) continue loop From 9d4a6e2be7ee0de5818c8de1e2c2728740187491 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 11 Dec 2018 15:39:42 -0600 Subject: [PATCH 075/125] don't add the fragment and then remove it --- view.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/view.go b/view.go index 50a2e02ff..00dcf02c6 100644 --- a/view.go +++ b/view.go @@ -212,12 +212,12 @@ func (v *view) CreateFragmentIfNotExists(shard uint64) (*fragment, error) { if err == nil && msg != nil { // Broadcast a message that a new max shard was just created. if err = v.broadcaster.SendSync(msg); err != nil { - v.mu.Lock() - delete(v.fragments, shard) - v.mu.Unlock() frag.close() return nil, errors.Wrap(err, "sending createshard message") } + v.mu.Lock() + v.fragments[shard] = frag + v.mu.Unlock() } return frag, err @@ -244,9 +244,6 @@ func (v *view) createFragmentIfNotExists(shard uint64) (*fragment, *CreateShardM Shard: shard, } - // Save to lookup. - v.fragments[shard] = frag - return frag, msg, nil } From eadd77f901778f669995281c15ad2db24a96015f Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 10 Dec 2018 22:12:02 -0600 Subject: [PATCH 076/125] fixed a mistake in the test from PR 1780 --- holder_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/holder_test.go b/holder_test.go index 2835e5c4f..1bfe4615a 100644 --- a/holder_test.go +++ b/holder_test.go @@ -397,8 +397,8 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) { hldr0.SetBitTime("i", "f", 0, 1, &t1) hldr0.SetBitTime("i", "f", 0, 2, &t2) - // Set data on node1 - hldr0.SetBitTime("i", "f", 0, 22, &t2) + // Set data on node1. + hldr1.SetBitTime("i", "f", 0, 22, &t2) err = c[0].Server.SyncData() if err != nil { From dc01dff3d37c1fcd2df90b9cc3a3ca4190bf924d Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 6 Dec 2018 17:04:39 -0600 Subject: [PATCH 077/125] add import roaring and import w/update benchmarks --- fragment_internal_test.go | 185 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 185 insertions(+) diff --git a/fragment_internal_test.go b/fragment_internal_test.go index b36bf3380..c5546106c 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -20,6 +20,7 @@ import ( "fmt" "io/ioutil" "math" + "math/rand" "reflect" "sort" "testing" @@ -1746,6 +1747,190 @@ func BenchmarkFragment_Import(b *testing.B) { } } +func BenchmarkImportRoaring(b *testing.B) { + for _, cacheType := range []string{CacheTypeRanked} { // CacheTypeNone didn't seem to affect the results much + for _, numRows := range []uint64{10, 100, 1000, 10000, 100000} { + data := getZipfRowsSliceRoaring(numRows, 1) + name := fmt.Sprintf("Rows%dCache_%s", numRows, cacheType) + b.Logf("%s: %.2fMB\n", name, float64(len(data))/1024/1024) + b.Run(name, func(b *testing.B) { + b.StopTimer() + for i := 0; i < b.N; i++ { + f := mustOpenFragment("i", fmt.Sprintf("r%dc%s", numRows, cacheType), viewStandard, 0, cacheType) + b.StartTimer() + err := f.importRoaring(data, false) + if err != nil { + b.Fatalf("import error: %v", err) + } + b.StopTimer() + f.Close() + } + }) + } + } +} + +func BenchmarkImportStandard(b *testing.B) { + for _, cacheType := range []string{CacheTypeRanked} { + for _, numRows := range []uint64{10, 100, 1000, 10000, 100000} { + rowIDs, columnIDs := getZipfRowsSliceStandard(numRows, 1) + b.Run(fmt.Sprintf("Rows%dCache_%s", numRows, cacheType), func(b *testing.B) { + b.StopTimer() + for i := 0; i < b.N; i++ { + f := mustOpenFragment("i", fmt.Sprintf("r%dc%s", numRows, cacheType), viewStandard, 0, cacheType) + b.StartTimer() + err := f.bulkImport(rowIDs, columnIDs, &ImportOptions{}) + if err != nil { + b.Fatalf("import error: %v", err) + } + b.StopTimer() + f.Close() + } + }) + } + } +} + +func BenchmarkImportRoaringUpdate(b *testing.B) { + fileSize := make(map[string]int64) + names := []string{} + for _, cacheType := range []string{CacheTypeRanked} { + for _, numRows := range []uint64{10, 100, 1000, 10000, 100000} { + for _, numCols := range []uint64{20, 1000, 50000, 500000} { + data := getZipfRowsSliceRoaring(numRows, 1) + updata := getUpdataRoaring(numRows, numCols, 1) + name := fmt.Sprintf("%s%dRows%dCols", cacheType, numRows, numCols) + names = append(names, name) + b.Run(name, func(b *testing.B) { + b.StopTimer() + for i := 0; i < b.N; i++ { + f := mustOpenFragment("i", fmt.Sprintf("r%dc%s", numRows, cacheType), viewStandard, 0, cacheType) + err := f.importRoaring(data, false) + if err != nil { + b.Fatalf("import error: %v", err) + } + b.StartTimer() + err = f.importRoaring(updata, false) + if err != nil { + b.Fatalf("import error: %v", err) + } + b.StopTimer() + stat, _ := f.file.Stat() + fileSize[name] = stat.Size() + f.Close() + } + }) + + } + } + } + for _, name := range names { + b.Logf("%s: %.2fMB\n", name, float64(fileSize[name])/1024/1024) + } +} + +func TestGetZipfRowsSliceRoaring(t *testing.T) { + f := mustOpenFragment("i", "f", viewStandard, 0, DefaultCacheType) + data := getZipfRowsSliceRoaring(10, 1) + f.importRoaring(data, false) + if !reflect.DeepEqual(f.rows(0), []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) { + t.Fatalf("unexpected rows: %v", f.rows(0)) + } + for i := uint64(1); i < 10; i++ { + if f.row(i).Count() >= f.row(i-1).Count() { + t.Fatalf("suspect distribution from getZipfRowsSliceRoaring") + } + } +} + +// getZipfRowsSliceRoaring generates a random fragment with the given number of +// rows, and 1 bit set in each column. The row each bit is set in is chosen via +// the Zipf generator, and so will be skewed toward lower row numbers. If this +// is edited to change the data distribution, getZipfRowsSliceStandard should be +// edited as well. +func getZipfRowsSliceRoaring(numRows uint64, seed int64) []byte { + b := roaring.NewBitmap() + s := rand.NewSource(seed) + r := rand.New(s) + z := rand.NewZipf(r, 1.6, 50, numRows-1) + for i := uint64(0); i < ShardWidth; i++ { + row := z.Uint64() + b.DirectAdd(row*ShardWidth + i) + } + buf := bytes.NewBuffer(make([]byte, 0, 100000)) + _, err := b.WriteTo(buf) + if err != nil { + panic(err) + } + return buf.Bytes() +} + +// getUpdataRoaring gets a byte slice containing a roaring bitmap which +// represents numCols set bits distributed randomly throughout a shard's column +// space and zipfianly throughout numRows rows. +func getUpdataRoaring(numRows, numCols uint64, seed int64) []byte { + b := roaring.NewBitmap() + s := rand.NewSource(seed) + r := rand.New(s) + z := rand.NewZipf(r, 1.6, 50, numRows-1) + + for i := uint64(0); i < numCols; i++ { + col := uint64(r.Int63n(ShardWidth)) // assuming the number of repeats will be negligible + row := z.Uint64() + b.DirectAdd(row*ShardWidth + col) + } + buf := bytes.NewBuffer(make([]byte, 0, 100000)) + _, err := b.WriteTo(buf) + if err != nil { + panic(err) + } + return buf.Bytes() +} + +// getZipfRowsSliceStandard is the same as getZipfRowsSliceRoaring, but returns +// row and column ids instead of a byte slice containing roaring bitmap data. +func getZipfRowsSliceStandard(numRows uint64, seed int64) (rowIDs, columnIDs []uint64) { + s := rand.NewSource(seed) + r := rand.New(s) + z := rand.NewZipf(r, 1.6, 50, numRows-1) + rowIDs, columnIDs = make([]uint64, ShardWidth), make([]uint64, ShardWidth) + for i := uint64(0); i < ShardWidth; i++ { + rowIDs[i] = z.Uint64() + columnIDs[i] = i + } + return rowIDs, columnIDs +} + +func BenchmarkFileWrite(b *testing.B) { + for _, numRows := range []uint64{10, 100, 1000, 10000, 100000} { + data := getZipfRowsSliceRoaring(numRows, 1) + b.Run(fmt.Sprintf("Rows%d", numRows), func(b *testing.B) { + b.StopTimer() + for i := 0; i < b.N; i++ { + f, err := ioutil.TempFile("", "") + if err != nil { + b.Fatalf("getting temp file: %v", err) + } + b.StartTimer() + _, err = f.Write(data) + if err != nil { + b.Fatal(err) + } + err = f.Sync() + if err != nil { + b.Fatal(err) + } + err = f.Close() + if err != nil { + b.Fatal(err) + } + b.StopTimer() + } + }) + } + +} + ///////////////////////////////////////////////////////////////////// // mustOpenFragment returns a new instance of Fragment with a temporary path. From 054926fd6c5f223fda9d2fd60d123ebfad6f9449 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 7 Dec 2018 09:44:38 -0600 Subject: [PATCH 078/125] add concurrent import benchmark --- fragment_internal_test.go | 46 ++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/fragment_internal_test.go b/fragment_internal_test.go index c5546106c..017f10538 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -1748,12 +1748,11 @@ func BenchmarkFragment_Import(b *testing.B) { } func BenchmarkImportRoaring(b *testing.B) { - for _, cacheType := range []string{CacheTypeRanked} { // CacheTypeNone didn't seem to affect the results much - for _, numRows := range []uint64{10, 100, 1000, 10000, 100000} { - data := getZipfRowsSliceRoaring(numRows, 1) - name := fmt.Sprintf("Rows%dCache_%s", numRows, cacheType) - b.Logf("%s: %.2fMB\n", name, float64(len(data))/1024/1024) - b.Run(name, func(b *testing.B) { + for _, numRows := range []uint64{10, 100, 1000, 10000, 100000} { + data := getZipfRowsSliceRoaring(numRows, 1) + b.Logf("%dRows: %.2fMB\n", numRows, float64(len(data))/1024/1024) + for _, cacheType := range []string{CacheTypeRanked} { // CacheTypeNone didn't seem to affect the results much + b.Run(fmt.Sprintf("Rows%dCache_%s", numRows, cacheType), func(b *testing.B) { b.StopTimer() for i := 0; i < b.N; i++ { f := mustOpenFragment("i", fmt.Sprintf("r%dc%s", numRows, cacheType), viewStandard, 0, cacheType) @@ -1770,6 +1769,41 @@ func BenchmarkImportRoaring(b *testing.B) { } } +func BenchmarkImportRoaringConcurrent(b *testing.B) { + for _, numRows := range []uint64{10, 100, 1000, 10000, 100000} { + data := getZipfRowsSliceRoaring(numRows, 1) + b.Logf("%dRows: %.2fMB\n", numRows, float64(len(data))/1024/1024) + for _, concurrency := range []int{2, 4, 8} { + b.Run(fmt.Sprintf("%dRows%dConcurrency", numRows, concurrency), func(b *testing.B) { + b.StopTimer() + frags := make([]*fragment, concurrency) + for i := 0; i < b.N; i++ { + for j := 0; j < concurrency; j++ { + frags[j] = mustOpenFragment("i", "f", viewStandard, uint64(j), CacheTypeRanked) + } + eg := errgroup.Group{} + b.StartTimer() + for j := 0; j < concurrency; j++ { + j := j + eg.Go(func() error { + return frags[j].importRoaring(data, false) + }) + } + err := eg.Wait() + if err != nil { + b.Fatalf("importing fragment: %v", err) + } + b.StopTimer() + for j := 0; j < concurrency; j++ { + frags[j].Close() + } + } + }) + } + } + +} + func BenchmarkImportStandard(b *testing.B) { for _, cacheType := range []string{CacheTypeRanked} { for _, numRows := range []uint64{10, 100, 1000, 10000, 100000} { From 4f459028d9c3a2e83309595ec87fe223f6b88676 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 7 Dec 2018 13:07:37 -0600 Subject: [PATCH 079/125] add concurrent update benchmark and clean up temp frags --- fragment_internal_test.go | 194 ++++++++++++++++++++++++-------------- 1 file changed, 125 insertions(+), 69 deletions(-) diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 017f10538..1e30530cd 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -21,6 +21,7 @@ import ( "io/ioutil" "math" "math/rand" + "os" "reflect" "sort" "testing" @@ -43,7 +44,7 @@ var ( // Ensure a fragment can set a bit and retrieve it. func TestFragment_SetBit(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() // Set bits on the fragment. if _, err := f.setBit(120, 1); err != nil { @@ -74,7 +75,7 @@ func TestFragment_SetBit(t *testing.T) { // Ensure a fragment can clear a set bit. func TestFragment_ClearBit(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() // Set and then clear bits on the fragment. if _, err := f.setBit(1000, 1); err != nil { @@ -101,7 +102,7 @@ func TestFragment_ClearBit(t *testing.T) { // Ensure a fragment can clear a row. func TestFragment_ClearRow(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() // Set and then clear bits on the fragment. if _, err := f.setBit(1000, 1); err != nil { @@ -128,7 +129,7 @@ func TestFragment_ClearRow(t *testing.T) { // Ensure a fragment can set a row. func TestFragment_SetRow(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 7, "") - defer f.Close() + defer f.Clean() rowID := uint64(1000) @@ -177,7 +178,7 @@ func TestFragment_SetRow(t *testing.T) { func TestFragment_SetValue(t *testing.T) { t.Run("OK", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() // Set value. if changed, err := f.setValue(100, 16, 3829); err != nil { @@ -205,7 +206,7 @@ func TestFragment_SetValue(t *testing.T) { t.Run("Overwrite", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() // Set value. if changed, err := f.setValue(100, 16, 3829); err != nil { @@ -233,7 +234,7 @@ func TestFragment_SetValue(t *testing.T) { t.Run("Clear", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() // Set value. if changed, err := f.setValue(100, 16, 3829); err != nil { @@ -261,7 +262,7 @@ func TestFragment_SetValue(t *testing.T) { t.Run("NotExists", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() // Set value. if changed, err := f.setValue(100, 10, 20); err != nil { @@ -291,7 +292,7 @@ func TestFragment_SetValue(t *testing.T) { } f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() // Set values. m := make(map[uint64]int64) @@ -329,7 +330,7 @@ func TestFragment_Sum(t *testing.T) { const bitDepth = 16 f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() // Set values. if _, err := f.setValue(1000, bitDepth, 382); err != nil { @@ -368,7 +369,7 @@ func TestFragment_MinMax(t *testing.T) { const bitDepth = 16 f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() // Set values. if _, err := f.setValue(1000, bitDepth, 382); err != nil { @@ -442,7 +443,7 @@ func TestFragment_Range(t *testing.T) { t.Run("EQ", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() // Set values. if _, err := f.setValue(1000, bitDepth, 382); err != nil { @@ -465,7 +466,7 @@ func TestFragment_Range(t *testing.T) { t.Run("NEQ", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() // Set values. if _, err := f.setValue(1000, bitDepth, 382); err != nil { @@ -488,7 +489,7 @@ func TestFragment_Range(t *testing.T) { t.Run("LT", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() // Set values. if _, err := f.setValue(1000, bitDepth, 382); err != nil { @@ -536,7 +537,7 @@ func TestFragment_Range(t *testing.T) { t.Run("GT", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() // Set values. if _, err := f.setValue(1000, bitDepth, 382); err != nil { @@ -584,7 +585,7 @@ func TestFragment_Range(t *testing.T) { t.Run("BETWEEN", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() // Set values. if _, err := f.setValue(1000, bitDepth, 382); err != nil { @@ -634,7 +635,7 @@ func TestFragment_Range(t *testing.T) { // Ensure a fragment can snapshot correctly. func TestFragment_Snapshot(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() // Set and then clear bits on the fragment. if _, err := f.setBit(1000, 1); err != nil { @@ -663,7 +664,7 @@ func TestFragment_Snapshot(t *testing.T) { // Ensure a fragment can iterate over all bits in order. func TestFragment_ForEachBit(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() // Set bits on the fragment. if _, err := f.setBit(100, 20); err != nil { @@ -692,7 +693,7 @@ func TestFragment_ForEachBit(t *testing.T) { // Ensure a fragment can return the top n results. func TestFragment_Top(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Close() + defer f.Clean() // Set bits on the rows 100, 101, & 102. f.mustSetBits(100, 1, 3, 200) f.mustSetBits(101, 1) @@ -714,7 +715,7 @@ func TestFragment_Top(t *testing.T) { // Ensure a fragment can filter rows when retrieving the top n rows. func TestFragment_Top_Filter(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Close() + defer f.Clean() // Set bits on the rows 100, 101, & 102. f.mustSetBits(100, 1, 3, 200) @@ -744,7 +745,7 @@ func TestFragment_Top_Filter(t *testing.T) { // Ensure a fragment can return top rows that intersect with an input row. func TestFragment_TopN_Intersect(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Close() + defer f.Clean() // Create an intersecting input row. src := NewRow(1, 2, 3) @@ -775,7 +776,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { } f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Close() + defer f.Clean() // Create an intersecting input row. src := NewRow( @@ -813,7 +814,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { // Ensure a fragment can return top rows when specified by ID. func TestFragment_TopN_IDs(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Close() + defer f.Clean() // Set bits on various rows. f.mustSetBits(100, 1, 2, 3) @@ -834,7 +835,7 @@ func TestFragment_TopN_IDs(t *testing.T) { // Ensure a fragment return none if CacheTypeNone is set func TestFragment_TopN_NopCache(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeNone) - defer f.Close() + defer f.Clean() // Set bits on various rows. f.mustSetBits(100, 1, 2, 3) @@ -882,7 +883,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { if err := f.Open(); err != nil { panic(err) } - defer f.Close() + defer f.Clean() // Set bits on various rows. f.mustSetBits(100, 1, 2, 3) @@ -915,7 +916,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { // Ensure fragment can return a checksum for its blocks. func TestFragment_Checksum(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() // Retrieve checksum and set bits. orig := f.Checksum() @@ -934,7 +935,7 @@ func TestFragment_Checksum(t *testing.T) { // Ensure fragment can return a checksum for a given block. func TestFragment_Blocks(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() // Retrieve initial checksum. var prev []FragmentBlock @@ -972,7 +973,7 @@ func TestFragment_Blocks(t *testing.T) { // Ensure fragment returns an empty checksum if no data exists for a block. func TestFragment_Blocks_Empty(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() // Set bits on a different block. if _, err := f.setBit(100, 1); err != nil { @@ -990,7 +991,7 @@ func TestFragment_Blocks_Empty(t *testing.T) { // Ensure a fragment's cache can be persisted between restarts. func TestFragment_LRUCache_Persistence(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeLRU) - defer f.Close() + defer f.Clean() // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { @@ -1075,7 +1076,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { // Ensure a fragment can be copied to another fragment. func TestFragment_WriteTo_ReadFrom(t *testing.T) { f0 := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f0.Close() + defer f0.Clean() // Set and then clear bits on the fragment. if _, err := f0.setBit(1000, 1); err != nil { @@ -1136,7 +1137,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { if err := f.Open(); err != nil { b.Fatal(err) } - defer f.Close() + defer f.Clean() // Reset timer and execute benchmark. b.ResetTimer() @@ -1149,7 +1150,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { func BenchmarkFragment_IntersectionCount(b *testing.B) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() f.MaxOpN = math.MaxInt32 // Generate some intersecting data. @@ -1180,7 +1181,7 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) { func TestFragment_Tanimoto(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Close() + defer f.Clean() src := NewRow(1, 2, 3) @@ -1203,7 +1204,7 @@ func TestFragment_Tanimoto(t *testing.T) { func TestFragment_Zero_Tanimoto(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Close() + defer f.Clean() src := NewRow(1, 2, 3) @@ -1228,7 +1229,7 @@ func TestFragment_Zero_Tanimoto(t *testing.T) { func TestFragment_Snapshot_Run(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() // Set bits on the fragment. for i := uint64(1); i < 3; i++ { @@ -1255,7 +1256,7 @@ func TestFragment_Snapshot_Run(t *testing.T) { // Ensure a fragment can set mutually exclusive values. func TestFragment_SetMutex(t *testing.T) { f := mustOpenMutexFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() var cols []uint64 @@ -1369,7 +1370,7 @@ func TestFragment_ImportSet(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importset%d", i), func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() // Set import. err := f.bulkImport(test.setRowIDs, test.setColIDs, &ImportOptions{}) @@ -1405,7 +1406,7 @@ func TestFragment_ImportSet(t *testing.T) { func TestFragment_ConcurrentImport(t *testing.T) { t.Run("bulkImportStandard", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() eg := errgroup.Group{} eg.Go(func() error { return f.bulkImportStandard([]uint64{1, 2}, []uint64{1, 2}, &ImportOptions{}) }) @@ -1502,7 +1503,7 @@ func TestFragment_ImportMutex(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) { f := mustOpenMutexFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() // Set import. err := f.bulkImport(test.setRowIDs, test.setColIDs, &ImportOptions{}) @@ -1621,7 +1622,7 @@ func TestFragment_ImportBool(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) { f := mustOpenBoolFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() // Set import. err := f.bulkImport(test.setRowIDs, test.setColIDs, &ImportOptions{}) @@ -1665,7 +1666,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) { if err := f.Open(); err != nil { b.Fatal(err) } - defer f.Close() + defer f.Clean() b.ResetTimer() // Reset timer and execute benchmark. @@ -1681,7 +1682,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) { func BenchmarkFragment_FullSnapshot(b *testing.B) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() // Generate some intersecting data. maxX := 1048576 / 2 sz := maxX @@ -1719,7 +1720,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) { func BenchmarkFragment_Import(b *testing.B) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() maxX := 1048576 * 5 * 2 sz := maxX rows := make([]uint64, sz) @@ -1747,8 +1748,14 @@ func BenchmarkFragment_Import(b *testing.B) { } } +var ( + rowCases = []uint64{2, 50, 1000, 100000} + colCases = []uint64{20, 1000, 50000, 500000} + concurrencyCases = []int{2, 4, 8, 16} +) + func BenchmarkImportRoaring(b *testing.B) { - for _, numRows := range []uint64{10, 100, 1000, 10000, 100000} { + for _, numRows := range rowCases { data := getZipfRowsSliceRoaring(numRows, 1) b.Logf("%dRows: %.2fMB\n", numRows, float64(len(data))/1024/1024) for _, cacheType := range []string{CacheTypeRanked} { // CacheTypeNone didn't seem to affect the results much @@ -1759,10 +1766,11 @@ func BenchmarkImportRoaring(b *testing.B) { b.StartTimer() err := f.importRoaring(data, false) if err != nil { + f.Clean() b.Fatalf("import error: %v", err) } b.StopTimer() - f.Close() + f.Clean() } }) } @@ -1770,10 +1778,10 @@ func BenchmarkImportRoaring(b *testing.B) { } func BenchmarkImportRoaringConcurrent(b *testing.B) { - for _, numRows := range []uint64{10, 100, 1000, 10000, 100000} { + for _, numRows := range rowCases { data := getZipfRowsSliceRoaring(numRows, 1) b.Logf("%dRows: %.2fMB\n", numRows, float64(len(data))/1024/1024) - for _, concurrency := range []int{2, 4, 8} { + for _, concurrency := range concurrencyCases { b.Run(fmt.Sprintf("%dRows%dConcurrency", numRows, concurrency), func(b *testing.B) { b.StopTimer() frags := make([]*fragment, concurrency) @@ -1791,22 +1799,57 @@ func BenchmarkImportRoaringConcurrent(b *testing.B) { } err := eg.Wait() if err != nil { - b.Fatalf("importing fragment: %v", err) + b.Errorf("importing fragment: %v", err) } b.StopTimer() for j := 0; j < concurrency; j++ { - frags[j].Close() + frags[j].Clean() } } }) } } - +} +func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) { + for _, numRows := range rowCases { + for _, numCols := range colCases { + data := getZipfRowsSliceRoaring(numRows, 1) + updata := getUpdataRoaring(numRows, numCols, 1) + for _, concurrency := range concurrencyCases { + b.Run(fmt.Sprintf("%dRows%dCols%dConcurrency", numRows, numCols, concurrency), func(b *testing.B) { + b.StopTimer() + frags := make([]*fragment, concurrency) + for i := 0; i < b.N; i++ { + for j := 0; j < concurrency; j++ { + frags[j] = mustOpenFragment("i", "f", viewStandard, uint64(j), CacheTypeRanked) + frags[j].importRoaring(data, false) + } + eg := errgroup.Group{} + b.StartTimer() + for j := 0; j < concurrency; j++ { + j := j + eg.Go(func() error { + return frags[j].importRoaring(updata, false) + }) + } + err := eg.Wait() + if err != nil { + b.Errorf("importing fragment: %v", err) + } + b.StopTimer() + for j := 0; j < concurrency; j++ { + frags[j].Clean() + } + } + }) + } + } + } } func BenchmarkImportStandard(b *testing.B) { for _, cacheType := range []string{CacheTypeRanked} { - for _, numRows := range []uint64{10, 100, 1000, 10000, 100000} { + for _, numRows := range rowCases { rowIDs, columnIDs := getZipfRowsSliceStandard(numRows, 1) b.Run(fmt.Sprintf("Rows%dCache_%s", numRows, cacheType), func(b *testing.B) { b.StopTimer() @@ -1815,10 +1858,10 @@ func BenchmarkImportStandard(b *testing.B) { b.StartTimer() err := f.bulkImport(rowIDs, columnIDs, &ImportOptions{}) if err != nil { - b.Fatalf("import error: %v", err) + b.Errorf("import error: %v", err) } b.StopTimer() - f.Close() + f.Clean() } }) } @@ -1829,8 +1872,8 @@ func BenchmarkImportRoaringUpdate(b *testing.B) { fileSize := make(map[string]int64) names := []string{} for _, cacheType := range []string{CacheTypeRanked} { - for _, numRows := range []uint64{10, 100, 1000, 10000, 100000} { - for _, numCols := range []uint64{20, 1000, 50000, 500000} { + for _, numRows := range rowCases { + for _, numCols := range colCases { data := getZipfRowsSliceRoaring(numRows, 1) updata := getUpdataRoaring(numRows, numCols, 1) name := fmt.Sprintf("%s%dRows%dCols", cacheType, numRows, numCols) @@ -1841,17 +1884,18 @@ func BenchmarkImportRoaringUpdate(b *testing.B) { f := mustOpenFragment("i", fmt.Sprintf("r%dc%s", numRows, cacheType), viewStandard, 0, cacheType) err := f.importRoaring(data, false) if err != nil { - b.Fatalf("import error: %v", err) + b.Errorf("import error: %v", err) } b.StartTimer() err = f.importRoaring(updata, false) if err != nil { - b.Fatalf("import error: %v", err) + f.Clean() + b.Errorf("import error: %v", err) } b.StopTimer() stat, _ := f.file.Stat() fileSize[name] = stat.Size() - f.Close() + f.Clean() } }) @@ -1936,7 +1980,7 @@ func getZipfRowsSliceStandard(numRows uint64, seed int64) (rowIDs, columnIDs []u } func BenchmarkFileWrite(b *testing.B) { - for _, numRows := range []uint64{10, 100, 1000, 10000, 100000} { + for _, numRows := range rowCases { data := getZipfRowsSliceRoaring(numRows, 1) b.Run(fmt.Sprintf("Rows%d", numRows), func(b *testing.B) { b.StopTimer() @@ -1959,6 +2003,7 @@ func BenchmarkFileWrite(b *testing.B) { b.Fatal(err) } b.StopTimer() + os.Remove(f.Name()) } }) } @@ -1967,6 +2012,18 @@ func BenchmarkFileWrite(b *testing.B) { ///////////////////////////////////////////////////////////////////// +func (f *fragment) Clean() error { + errc := f.Close() + errf := os.Remove(f.path) + errp := os.Remove(f.cachePath()) + if errc != nil { + return errc + } else if errf != nil { + return errf + } + return errp +} + // mustOpenFragment returns a new instance of Fragment with a temporary path. func mustOpenFragment(index, field, view string, shard uint64, cacheType string) *fragment { file, err := ioutil.TempFile("", "pilosa-fragment-") @@ -2030,7 +2087,7 @@ func (f *fragment) mustSetBits(rowID uint64, columnIDs ...uint64) { func TestFragment_RowsIteration(t *testing.T) { t.Run("firstContainer", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() expectedAll := make([]uint64, 0) expectedOdd := make([]uint64, 0) @@ -2057,7 +2114,7 @@ func TestFragment_RowsIteration(t *testing.T) { t.Run("secondRow", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() expected := []uint64{1, 2} if _, err := f.setBit(1, 66000); err != nil { @@ -2081,7 +2138,7 @@ func TestFragment_RowsIteration(t *testing.T) { t.Run("combinations", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() expectedRows := make([]uint64, 0) for r := uint64(1); r < uint64(10000); r += 100 { @@ -2128,7 +2185,7 @@ func TestFragment_RoaringImport(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importroaring%d", i), func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Close() + defer f.Clean() for num, input := range test { buf := &bytes.Buffer{} bm := roaring.NewBitmap(input...) @@ -2171,7 +2228,7 @@ func TestFragment_RoaringImportTopN(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importroaring%d", i), func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Close() + defer f.Clean() options := &ImportOptions{} err := f.bulkImport(test.rowIDs, test.colIDs, options) @@ -2305,6 +2362,7 @@ func calcExpected(inputs ...[]uint64) [][]uint64 { func TestFragmentRowIterator(t *testing.T) { t.Run("basic", func(t *testing.T) { f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) + defer f.Clean() f.mustSetBits(0, 0) f.mustSetBits(1, 0) f.mustSetBits(2, 0) @@ -2333,11 +2391,11 @@ func TestFragmentRowIterator(t *testing.T) { if !wrapped { t.Fatalf("wrapped should be true after iterator is exhausted") } - f.Close() }) t.Run("skipped rows", func(t *testing.T) { f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) + defer f.Clean() f.mustSetBits(1, 0) f.mustSetBits(3, 0) f.mustSetBits(5, 0) @@ -2366,11 +2424,11 @@ func TestFragmentRowIterator(t *testing.T) { if !wrapped { t.Fatalf("wrapped should be true after iterator is exhausted") } - f.Close() }) t.Run("basic wrapped", func(t *testing.T) { f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) + defer f.Clean() f.mustSetBits(0, 0) f.mustSetBits(1, 0) f.mustSetBits(2, 0) @@ -2391,11 +2449,11 @@ func TestFragmentRowIterator(t *testing.T) { t.Fatalf("got wrong columns back on iteration %d - should just be 0 but %v", i, row.Columns()) } } - f.Close() }) t.Run("skipped rows wrapped", func(t *testing.T) { f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) + defer f.Clean() f.mustSetBits(1, 0) f.mustSetBits(3, 0) f.mustSetBits(5, 0) @@ -2416,7 +2474,5 @@ func TestFragmentRowIterator(t *testing.T) { t.Fatalf("got wrong columns back on iteration %d - should just be 0 but %v", i, row.Columns()) } } - f.Close() }) - } From 4f808c2028ce5ae965ecce9b8125e3e82a6215dd Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 7 Dec 2018 16:49:05 -0600 Subject: [PATCH 080/125] add flag or setting temp dir used for benchmarks --- fragment_internal_test.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 1e30530cd..fd6da6405 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -39,8 +39,13 @@ var ( // In order to generate the sample fragment file, // run an import and copy PILOSA_DATA_DIR/INDEX_NAME/FRAME_NAME/0 to testdata/sample_view FragmentPath = flag.String("fragment", "testdata/sample_view/0", "fragment path") + TempDir = "" ) +func init() { + flag.StringVar(&TempDir, "temp-dir", "", "Directory in which to place temporary data (e.g. for benchmarking). Useful if you are trying to benchmark different storage configurations.") +} + // Ensure a fragment can set a bit and retrieve it. func TestFragment_SetBit(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") @@ -1985,7 +1990,7 @@ func BenchmarkFileWrite(b *testing.B) { b.Run(fmt.Sprintf("Rows%d", numRows), func(b *testing.B) { b.StopTimer() for i := 0; i < b.N; i++ { - f, err := ioutil.TempFile("", "") + f, err := ioutil.TempFile(TempDir, "") if err != nil { b.Fatalf("getting temp file: %v", err) } @@ -2026,7 +2031,7 @@ func (f *fragment) Clean() error { // mustOpenFragment returns a new instance of Fragment with a temporary path. func mustOpenFragment(index, field, view string, shard uint64, cacheType string) *fragment { - file, err := ioutil.TempFile("", "pilosa-fragment-") + file, err := ioutil.TempFile(TempDir, "pilosa-fragment-") if err != nil { panic(err) } From f2578c401f9eeedabfabde43f90c0fd7e651db82 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 11 Dec 2018 15:24:49 -0600 Subject: [PATCH 081/125] clean up fragments more cleanly (in tests and benchmarks) --- fragment_internal_test.go | 131 +++++++++++++++++++------------------- 1 file changed, 66 insertions(+), 65 deletions(-) diff --git a/fragment_internal_test.go b/fragment_internal_test.go index fd6da6405..b0864ae4a 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -49,7 +49,7 @@ func init() { // Ensure a fragment can set a bit and retrieve it. func TestFragment_SetBit(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set bits on the fragment. if _, err := f.setBit(120, 1); err != nil { @@ -80,7 +80,7 @@ func TestFragment_SetBit(t *testing.T) { // Ensure a fragment can clear a set bit. func TestFragment_ClearBit(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set and then clear bits on the fragment. if _, err := f.setBit(1000, 1); err != nil { @@ -107,7 +107,7 @@ func TestFragment_ClearBit(t *testing.T) { // Ensure a fragment can clear a row. func TestFragment_ClearRow(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set and then clear bits on the fragment. if _, err := f.setBit(1000, 1); err != nil { @@ -134,7 +134,7 @@ func TestFragment_ClearRow(t *testing.T) { // Ensure a fragment can set a row. func TestFragment_SetRow(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 7, "") - defer f.Clean() + defer f.Clean(t) rowID := uint64(1000) @@ -183,7 +183,7 @@ func TestFragment_SetRow(t *testing.T) { func TestFragment_SetValue(t *testing.T) { t.Run("OK", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set value. if changed, err := f.setValue(100, 16, 3829); err != nil { @@ -211,7 +211,7 @@ func TestFragment_SetValue(t *testing.T) { t.Run("Overwrite", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set value. if changed, err := f.setValue(100, 16, 3829); err != nil { @@ -239,7 +239,7 @@ func TestFragment_SetValue(t *testing.T) { t.Run("Clear", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set value. if changed, err := f.setValue(100, 16, 3829); err != nil { @@ -267,7 +267,7 @@ func TestFragment_SetValue(t *testing.T) { t.Run("NotExists", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set value. if changed, err := f.setValue(100, 10, 20); err != nil { @@ -297,7 +297,7 @@ func TestFragment_SetValue(t *testing.T) { } f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set values. m := make(map[uint64]int64) @@ -335,7 +335,7 @@ func TestFragment_Sum(t *testing.T) { const bitDepth = 16 f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set values. if _, err := f.setValue(1000, bitDepth, 382); err != nil { @@ -374,7 +374,7 @@ func TestFragment_MinMax(t *testing.T) { const bitDepth = 16 f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set values. if _, err := f.setValue(1000, bitDepth, 382); err != nil { @@ -448,7 +448,7 @@ func TestFragment_Range(t *testing.T) { t.Run("EQ", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set values. if _, err := f.setValue(1000, bitDepth, 382); err != nil { @@ -471,7 +471,7 @@ func TestFragment_Range(t *testing.T) { t.Run("NEQ", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set values. if _, err := f.setValue(1000, bitDepth, 382); err != nil { @@ -494,7 +494,7 @@ func TestFragment_Range(t *testing.T) { t.Run("LT", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set values. if _, err := f.setValue(1000, bitDepth, 382); err != nil { @@ -542,7 +542,7 @@ func TestFragment_Range(t *testing.T) { t.Run("GT", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set values. if _, err := f.setValue(1000, bitDepth, 382); err != nil { @@ -590,7 +590,7 @@ func TestFragment_Range(t *testing.T) { t.Run("BETWEEN", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set values. if _, err := f.setValue(1000, bitDepth, 382); err != nil { @@ -640,7 +640,7 @@ func TestFragment_Range(t *testing.T) { // Ensure a fragment can snapshot correctly. func TestFragment_Snapshot(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set and then clear bits on the fragment. if _, err := f.setBit(1000, 1); err != nil { @@ -669,7 +669,7 @@ func TestFragment_Snapshot(t *testing.T) { // Ensure a fragment can iterate over all bits in order. func TestFragment_ForEachBit(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set bits on the fragment. if _, err := f.setBit(100, 20); err != nil { @@ -698,7 +698,7 @@ func TestFragment_ForEachBit(t *testing.T) { // Ensure a fragment can return the top n results. func TestFragment_Top(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Clean() + defer f.Clean(t) // Set bits on the rows 100, 101, & 102. f.mustSetBits(100, 1, 3, 200) f.mustSetBits(101, 1) @@ -720,7 +720,7 @@ func TestFragment_Top(t *testing.T) { // Ensure a fragment can filter rows when retrieving the top n rows. func TestFragment_Top_Filter(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Clean() + defer f.Clean(t) // Set bits on the rows 100, 101, & 102. f.mustSetBits(100, 1, 3, 200) @@ -750,7 +750,7 @@ func TestFragment_Top_Filter(t *testing.T) { // Ensure a fragment can return top rows that intersect with an input row. func TestFragment_TopN_Intersect(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Clean() + defer f.Clean(t) // Create an intersecting input row. src := NewRow(1, 2, 3) @@ -781,7 +781,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { } f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Clean() + defer f.Clean(t) // Create an intersecting input row. src := NewRow( @@ -819,7 +819,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { // Ensure a fragment can return top rows when specified by ID. func TestFragment_TopN_IDs(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Clean() + defer f.Clean(t) // Set bits on various rows. f.mustSetBits(100, 1, 2, 3) @@ -840,7 +840,7 @@ func TestFragment_TopN_IDs(t *testing.T) { // Ensure a fragment return none if CacheTypeNone is set func TestFragment_TopN_NopCache(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeNone) - defer f.Clean() + defer f.Clean(t) // Set bits on various rows. f.mustSetBits(100, 1, 2, 3) @@ -888,7 +888,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { if err := f.Open(); err != nil { panic(err) } - defer f.Clean() + defer f.Clean(t) // Set bits on various rows. f.mustSetBits(100, 1, 2, 3) @@ -921,7 +921,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { // Ensure fragment can return a checksum for its blocks. func TestFragment_Checksum(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Retrieve checksum and set bits. orig := f.Checksum() @@ -940,7 +940,7 @@ func TestFragment_Checksum(t *testing.T) { // Ensure fragment can return a checksum for a given block. func TestFragment_Blocks(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Retrieve initial checksum. var prev []FragmentBlock @@ -978,7 +978,7 @@ func TestFragment_Blocks(t *testing.T) { // Ensure fragment returns an empty checksum if no data exists for a block. func TestFragment_Blocks_Empty(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set bits on a different block. if _, err := f.setBit(100, 1); err != nil { @@ -996,7 +996,7 @@ func TestFragment_Blocks_Empty(t *testing.T) { // Ensure a fragment's cache can be persisted between restarts. func TestFragment_LRUCache_Persistence(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeLRU) - defer f.Clean() + defer f.Clean(t) // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { @@ -1081,7 +1081,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { // Ensure a fragment can be copied to another fragment. func TestFragment_WriteTo_ReadFrom(t *testing.T) { f0 := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f0.Clean() + defer f0.Clean(t) // Set and then clear bits on the fragment. if _, err := f0.setBit(1000, 1); err != nil { @@ -1142,7 +1142,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { if err := f.Open(); err != nil { b.Fatal(err) } - defer f.Clean() + defer f.Clean(b) // Reset timer and execute benchmark. b.ResetTimer() @@ -1155,7 +1155,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { func BenchmarkFragment_IntersectionCount(b *testing.B) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(b) f.MaxOpN = math.MaxInt32 // Generate some intersecting data. @@ -1186,7 +1186,7 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) { func TestFragment_Tanimoto(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Clean() + defer f.Clean(t) src := NewRow(1, 2, 3) @@ -1209,7 +1209,7 @@ func TestFragment_Tanimoto(t *testing.T) { func TestFragment_Zero_Tanimoto(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Clean() + defer f.Clean(t) src := NewRow(1, 2, 3) @@ -1234,7 +1234,7 @@ func TestFragment_Zero_Tanimoto(t *testing.T) { func TestFragment_Snapshot_Run(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set bits on the fragment. for i := uint64(1); i < 3; i++ { @@ -1261,7 +1261,7 @@ func TestFragment_Snapshot_Run(t *testing.T) { // Ensure a fragment can set mutually exclusive values. func TestFragment_SetMutex(t *testing.T) { f := mustOpenMutexFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) var cols []uint64 @@ -1375,7 +1375,7 @@ func TestFragment_ImportSet(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importset%d", i), func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set import. err := f.bulkImport(test.setRowIDs, test.setColIDs, &ImportOptions{}) @@ -1411,7 +1411,7 @@ func TestFragment_ImportSet(t *testing.T) { func TestFragment_ConcurrentImport(t *testing.T) { t.Run("bulkImportStandard", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) eg := errgroup.Group{} eg.Go(func() error { return f.bulkImportStandard([]uint64{1, 2}, []uint64{1, 2}, &ImportOptions{}) }) @@ -1508,7 +1508,7 @@ func TestFragment_ImportMutex(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) { f := mustOpenMutexFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set import. err := f.bulkImport(test.setRowIDs, test.setColIDs, &ImportOptions{}) @@ -1627,7 +1627,7 @@ func TestFragment_ImportBool(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) { f := mustOpenBoolFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) // Set import. err := f.bulkImport(test.setRowIDs, test.setColIDs, &ImportOptions{}) @@ -1671,7 +1671,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) { if err := f.Open(); err != nil { b.Fatal(err) } - defer f.Clean() + defer f.Clean(b) b.ResetTimer() // Reset timer and execute benchmark. @@ -1687,7 +1687,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) { func BenchmarkFragment_FullSnapshot(b *testing.B) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(b) // Generate some intersecting data. maxX := 1048576 / 2 sz := maxX @@ -1725,7 +1725,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) { func BenchmarkFragment_Import(b *testing.B) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(b) maxX := 1048576 * 5 * 2 sz := maxX rows := make([]uint64, sz) @@ -1771,11 +1771,11 @@ func BenchmarkImportRoaring(b *testing.B) { b.StartTimer() err := f.importRoaring(data, false) if err != nil { - f.Clean() + f.Clean(b) b.Fatalf("import error: %v", err) } b.StopTimer() - f.Clean() + f.Clean(b) } }) } @@ -1808,7 +1808,7 @@ func BenchmarkImportRoaringConcurrent(b *testing.B) { } b.StopTimer() for j := 0; j < concurrency; j++ { - frags[j].Clean() + frags[j].Clean(b) } } }) @@ -1843,7 +1843,7 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) { } b.StopTimer() for j := 0; j < concurrency; j++ { - frags[j].Clean() + frags[j].Clean(b) } } }) @@ -1866,7 +1866,7 @@ func BenchmarkImportStandard(b *testing.B) { b.Errorf("import error: %v", err) } b.StopTimer() - f.Clean() + f.Clean(b) } }) } @@ -1894,13 +1894,13 @@ func BenchmarkImportRoaringUpdate(b *testing.B) { b.StartTimer() err = f.importRoaring(updata, false) if err != nil { - f.Clean() + f.Clean(b) b.Errorf("import error: %v", err) } b.StopTimer() stat, _ := f.file.Stat() fileSize[name] = stat.Size() - f.Clean() + f.Clean(b) } }) @@ -2017,16 +2017,17 @@ func BenchmarkFileWrite(b *testing.B) { ///////////////////////////////////////////////////////////////////// -func (f *fragment) Clean() error { +func (f *fragment) Clean(t testing.TB) { errc := f.Close() errf := os.Remove(f.path) errp := os.Remove(f.cachePath()) - if errc != nil { - return errc - } else if errf != nil { - return errf + if errc != nil || errf != nil { + t.Fatal("cleaning up fragment: ", errc, errf, errp) + } + // not all fragments have cache files + if errp != nil && !os.IsNotExist(errp) { + t.Fatalf("cleaning up fragment cache: %v", errp) } - return errp } // mustOpenFragment returns a new instance of Fragment with a temporary path. @@ -2092,7 +2093,7 @@ func (f *fragment) mustSetBits(rowID uint64, columnIDs ...uint64) { func TestFragment_RowsIteration(t *testing.T) { t.Run("firstContainer", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) expectedAll := make([]uint64, 0) expectedOdd := make([]uint64, 0) @@ -2119,7 +2120,7 @@ func TestFragment_RowsIteration(t *testing.T) { t.Run("secondRow", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) expected := []uint64{1, 2} if _, err := f.setBit(1, 66000); err != nil { @@ -2143,7 +2144,7 @@ func TestFragment_RowsIteration(t *testing.T) { t.Run("combinations", func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) expectedRows := make([]uint64, 0) for r := uint64(1); r < uint64(10000); r += 100 { @@ -2190,7 +2191,7 @@ func TestFragment_RoaringImport(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importroaring%d", i), func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") - defer f.Clean() + defer f.Clean(t) for num, input := range test { buf := &bytes.Buffer{} bm := roaring.NewBitmap(input...) @@ -2233,7 +2234,7 @@ func TestFragment_RoaringImportTopN(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprintf("importroaring%d", i), func(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - defer f.Clean() + defer f.Clean(t) options := &ImportOptions{} err := f.bulkImport(test.rowIDs, test.colIDs, options) @@ -2367,7 +2368,7 @@ func calcExpected(inputs ...[]uint64) [][]uint64 { func TestFragmentRowIterator(t *testing.T) { t.Run("basic", func(t *testing.T) { f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) - defer f.Clean() + defer f.Clean(t) f.mustSetBits(0, 0) f.mustSetBits(1, 0) f.mustSetBits(2, 0) @@ -2400,7 +2401,7 @@ func TestFragmentRowIterator(t *testing.T) { t.Run("skipped rows", func(t *testing.T) { f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) - defer f.Clean() + defer f.Clean(t) f.mustSetBits(1, 0) f.mustSetBits(3, 0) f.mustSetBits(5, 0) @@ -2433,7 +2434,7 @@ func TestFragmentRowIterator(t *testing.T) { t.Run("basic wrapped", func(t *testing.T) { f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) - defer f.Clean() + defer f.Clean(t) f.mustSetBits(0, 0) f.mustSetBits(1, 0) f.mustSetBits(2, 0) @@ -2458,7 +2459,7 @@ func TestFragmentRowIterator(t *testing.T) { t.Run("skipped rows wrapped", func(t *testing.T) { f := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) - defer f.Clean() + defer f.Clean(t) f.mustSetBits(1, 0) f.mustSetBits(3, 0) f.mustSetBits(5, 0) From 7e90917c3c2d12344aa63934d30e23af2bf58140 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 11 Dec 2018 15:43:28 -0600 Subject: [PATCH 082/125] suppress linter for init call --- fragment_internal_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fragment_internal_test.go b/fragment_internal_test.go index b0864ae4a..cca88dda1 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -42,7 +42,7 @@ var ( TempDir = "" ) -func init() { +func init() { // nolint: gochecknoinits flag.StringVar(&TempDir, "temp-dir", "", "Directory in which to place temporary data (e.g. for benchmarking). Useful if you are trying to benchmark different storage configurations.") } From e7ca4562ea902f7c426b1ad168fa749ff9b2595c Mon Sep 17 00:00:00 2001 From: Richard Artoul Date: Wed, 12 Dec 2018 14:26:46 -0800 Subject: [PATCH 083/125] Fix bug in unionInPlaceImplementation --- roaring/roaring.go | 19 ++++++--- roaring/roaring_test.go | 89 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 6 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 4df42afdc..a2ddecb62 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -617,7 +617,7 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) { n: maxContainerVal + 1, } target.Containers.Put(iKey, container) - bitmapIters[i:].markItersWithCurrentKeyAsHandled(iKey) + bitmapIters.markItersWithCurrentKeyAsHandled(i, iKey) continue } @@ -632,8 +632,9 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) { // efficiency. container := target.Containers.Get(iKey) // If target already has a bitmap container for iKey then we can reuse that, - // otherwise we have to allocate a new one. - if container == nil || container.containerType != containerBitmap { + // otherwise we have to allocate a new one or convert the existing container + // into a bitmap container. + if container == nil { buf := make([]uint64, bitmapN) ob := buf[:bitmapN] container = &Container{ @@ -641,6 +642,10 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) { n: 0, containerType: containerBitmap, } + } else if container.isArray() { + container.arrayToBitmap() + } else if container.isRun() { + container.runToBitmap() } // Once we've acquired a bitmap container (either by reusing the existing one @@ -648,7 +653,8 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) { // other containers to see which ones have the same key, and union all of them // into the target bitmap container. Only need to loop starting from i because // anything previous to that has already been handled. - for j, jIter := range bitmapIters[i:] { + for j := i; j < len(bitmapIters); j++ { + jIter := bitmapIters[j] jKey, jContainer := jIter.iter.Value() if iKey == jKey { @@ -4002,8 +4008,9 @@ func (w handledIters) next() bool { return hasNext } -func (w handledIters) markItersWithCurrentKeyAsHandled(key uint64) { - for i, wrapped := range w { +func (w handledIters) markItersWithCurrentKeyAsHandled(startIdx int, key uint64) { + for i := startIdx; i < len(w); i++ { + wrapped := w[i] currKey, _ := wrapped.iter.Value() if currKey == key { w[i].handled = true diff --git a/roaring/roaring_test.go b/roaring/roaring_test.go index 72153538f..4c3d15a4d 100644 --- a/roaring/roaring_test.go +++ b/roaring/roaring_test.go @@ -23,6 +23,7 @@ import ( "sort" "testing" "testing/quick" + "time" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/roaring" @@ -436,6 +437,94 @@ func TestBitmap_UnionInPlace1(t *testing.T) { } } +// TestBitmap_UnionInPlaceProp is a manual property test that randomly generates +// a number of different bitmaps with random vals and unions them together. It +// then compares the result against a reference implementation (golang map) to +// ensure that all the unions were handled correctly. +func TestBitmap_UnionInPlaceProp(t *testing.T) { + var ( + seed = time.Now().UnixNano() + source = rand.NewSource(seed) + rng = rand.New(source) + numTests = 100 + maxNumIntsPerBatch = 100 + maxNumBatches = 100 + maxRangePercent = 2 + // Need to limit the range of possible numbers that we generate + // otherwise two randomly generated numbers landing in the same + // container would be extremely unlikely, leaving container merging + // behavior untested. + maxUint64Val = 1000000 + ) + + for i := 0; i < numTests; i++ { + var ( + // We will use sets as the "reference" implementation. + sets = []map[uint64]struct{}{} + bitmaps = []*roaring.Bitmap{} + ) + + // Ensure there are at least two batches. + numBatches := rng.Intn(maxNumBatches) + 2 + for j := 0; j < numBatches; j++ { + // For each "batch" create the equivalent set and bitmap. + var ( + set = map[uint64]struct{}{} + bitmap = roaring.NewBitmap() + ) + + if rng.Intn(100) <= maxRangePercent { + // Generate max range RLE containers with a configurable + // probability to ensure that code-path is exercised. + start := rng.Intn((maxUint64Val)) + // Add a continuous sequence of numbers that is 2x as long as the maximum + // size of a container to ensure we generate a maxRange container. + for x := start; x < (start + 2*(0xffff+1)); x++ { + set[uint64(x)] = struct{}{} + bitmap.Add(uint64(x)) + } + } + + // Generate and add a bunch of random values. + numIntsPerBatch := rng.Intn(maxNumIntsPerBatch) + for x := 0; x < numIntsPerBatch; x++ { + num := uint64(rng.Intn(maxUint64Val)) + set[num] = struct{}{} + bitmap.Add(num) + } + + sets = append(sets, set) + bitmaps = append(bitmaps, bitmap) + } + + // "Union" all the sets into the first one. + set0 := sets[0] + for _, set := range sets[1:] { + for val := range set { + set0[val] = struct{}{} + } + } + + // Union all the bitmaps into the first one. + bitmap0 := bitmaps[0] + bitmap0.UnionInPlace(bitmaps[1:]...) + + // Ensure the unioned set and bitmap have the same cardinality. + if len(set0) != int(bitmap0.Count()) { + t.Fatalf("cardinality of set is: %d, but bitmap is: %d, failed with seed: %d", + len(set0), bitmap0.Count(), seed) + } + + // Ensure the unioned set and bitmap have the exact same values. + for val := range set0 { + if !bitmap0.Contains(val) { + t.Fatalf("set contained %d, but bitmap did not, failed with seed: %d", + val, seed) + } + } + } +} + func TestBitmap_Intersection_Empty(t *testing.T) { bm0 := roaring.NewFileBitmap(0, 2683177) bm1 := roaring.NewFileBitmap() From 88c18ca1fe3c99acc5fd5e70211ccdb415a393ad Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Sat, 8 Dec 2018 16:14:02 -0700 Subject: [PATCH 084/125] Cancel queries on Context.Done() This commit periodicially checks if the context has been cancelled or if a deadline has been reached. If so, it returns a query-related error message depending on the cause. --- executor.go | 48 ++++++++++++++++++++++++++++++++++++++++++++++-- pilosa.go | 2 ++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/executor.go b/executor.go index dca0bd546..bbb8c2297 100644 --- a/executor.go +++ b/executor.go @@ -87,6 +87,11 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar resp := QueryResponse{} + // Check for query cancellation. + if err := validateQueryContext(ctx); err != nil { + return resp, err + } + // Verify that an index is set. if index == "" { return resp, ErrIndexRequired @@ -112,12 +117,16 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar if !opt.Remote { if err := e.translateCalls(ctx, index, idx, q.Calls); err != nil { return resp, err + } else if err := validateQueryContext(ctx); err != nil { + return resp, err } } results, err := e.execute(ctx, index, q, shards, opt) if err != nil { return resp, err + } else if err := validateQueryContext(ctx); err != nil { + return resp, err } resp.Results = results @@ -159,6 +168,8 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar if !opt.Remote { if err := e.translateResults(ctx, index, idx, q.Calls, results); err != nil { return resp, err + } else if err := validateQueryContext(ctx); err != nil { + return resp, err } } @@ -217,6 +228,10 @@ func (e *executor) execute(ctx context.Context, index string, q *pql.Query, shar // Execute each call serially. results := make([]interface{}, 0, len(q.Calls)) for _, call := range q.Calls { + if err := validateQueryContext(ctx); err != nil { + return nil, err + } + v, err := e.executeCall(ctx, index, call, shards, opt) if err != nil { return nil, err @@ -231,7 +246,9 @@ func (e *executor) executeCall(ctx context.Context, index string, c *pql.Call, s span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeCall") defer span.Finish() - if err := e.validateCallArgs(c); err != nil { + if err := validateQueryContext(ctx); err != nil { + return nil, err + } else if err := e.validateCallArgs(c); err != nil { return nil, errors.Wrap(err, "validating args") } indexTag := fmt.Sprintf("index:%s", index) @@ -521,6 +538,10 @@ func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.C // executeBitmapCallShard executes a bitmap call for a single shard. func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + if err := validateQueryContext(ctx); err != nil { + return nil, err + } + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapCallShard") defer span.Finish() @@ -1977,7 +1998,13 @@ func (e *executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal // Collect attributes by field/id. m := make(map[string]map[uint64]map[string]interface{}) - for _, c := range calls { + for i, c := range calls { + if i%10 == 0 { + if err := validateQueryContext(ctx); err != nil { + return nil, err + } + } + field, ok := c.Args["_field"].(string) if !ok { return nil, errors.New("SetRowAttrs() field required") @@ -2560,6 +2587,23 @@ func (e *executor) translateResult(index string, idx *Index, call *pql.Call, res return result, nil } +// validateQueryContext returns a query-appropriate error if the context is done. +func validateQueryContext(ctx context.Context) error { + select { + case <-ctx.Done(): + switch err := ctx.Err(); err { + case context.Canceled: + return ErrQueryCancelled + case context.DeadlineExceeded: + return ErrQueryTimeout + default: + return err + } + default: + return nil + } +} + // errShardUnavailable is a marker error if no nodes are available. var errShardUnavailable = errors.New("shard unavailable") diff --git a/pilosa.go b/pilosa.go index d2fdf8144..56e8e0f9e 100644 --- a/pilosa.go +++ b/pilosa.go @@ -54,6 +54,8 @@ var ( // ErrFragmentNotFound is returned when a fragment does not exist. ErrFragmentNotFound = errors.New("fragment not found") ErrQueryRequired = errors.New("query required") + ErrQueryCancelled = errors.New("query cancelled") + ErrQueryTimeout = errors.New("query timeout") ErrTooManyWrites = errors.New("too many write commands") ErrClusterDoesNotOwnShard = errors.New("cluster does not own shard") From 2047ebe377de9cd902b5964d53b7fcbd31905a98 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 14 Dec 2018 11:31:23 -0600 Subject: [PATCH 085/125] revert client Timeout addition suspect that this is somehow causing "cannot assign requested address" bugs for some users. removing since it wasn't a necessary part of the deadlock fix, but just seemed like a prudent thing to have. --- http/handler.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/http/handler.go b/http/handler.go index 736a4b102..68719fdfe 100644 --- a/http/handler.go +++ b/http/handler.go @@ -1504,15 +1504,7 @@ func GetHTTPClient(t *tls.Config) *http.Client { if t != nil { transport.TLSClientConfig = t } - return &http.Client{ - Transport: transport, - // Internal queries will time out after 2h 7m by default. This is - // reduced from the old default of no timeout, so it was thought we - // should keep it fairly high, but it could probably be reduced in most - // cases. It is set to an odd number in the hopes that it will be - // recognizable in stats/traces/logs when this limit is being hit. - Timeout: 127 * time.Minute, - } + return &http.Client{Transport: transport} } // handlPostRoaringImport From 45cd48c1b7a6c7a5bbfe65c9c4cf3554fbf78734 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 17 Dec 2018 15:44:50 -0600 Subject: [PATCH 086/125] change exists field to _exists --- executor_test.go | 12 ++++++------ holder.go | 2 +- http/client_test.go | 4 ++-- pilosa.go | 5 ----- pilosa_internal_test.go | 5 ++--- 5 files changed, 11 insertions(+), 17 deletions(-) diff --git a/executor_test.go b/executor_test.go index 3c54fbefb..141533d7f 100644 --- a/executor_test.go +++ b/executor_test.go @@ -2163,10 +2163,10 @@ func TestExecutor_Execute_Existence(t *testing.T) { t.Fatalf("unexpected columns: %+v", bits) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(exists=0)`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Row(f=10))`}); err != nil { t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1, ShardWidth + 2}) { - t.Fatalf("unexpected existence columns: %+v", bits) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{ShardWidth + 2}) { + t.Fatalf("unexpected columns after Not: %+v", bits) } // Reopen cluster to ensure existence field is reloaded. @@ -2174,10 +2174,10 @@ func TestExecutor_Execute_Existence(t *testing.T) { t.Fatal(err) } - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(exists=0)`}); err != nil { + if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Row(f=10))`}); err != nil { t.Fatal(err) - } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, ShardWidth + 1, ShardWidth + 2}) { - t.Fatalf("unexpected existence columns after reopen: %+v", bits) + } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{ShardWidth + 2}) { + t.Fatalf("unexpected columns after reopen: %+v", bits) } }) } diff --git a/holder.go b/holder.go index 9e58e8057..39fef3595 100644 --- a/holder.go +++ b/holder.go @@ -43,7 +43,7 @@ const ( fileLimit = 262144 // (512^2) // existenceFieldName is the name of the internal field used to store existence values. - existenceFieldName = "exists" + existenceFieldName = "_exists" ) // Holder represents a container for indexes. diff --git a/http/client_test.go b/http/client_test.go index fee3754df..79aa2a2ea 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -900,7 +900,7 @@ func TestClient_ImportExistence(t *testing.T) { } // Verify existence. - if a := hldr.ReadRow(idxName, "exists", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5, 6}) { + if a := hldr.ReadRow(idxName, "_exists", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5, 6}) { t.Fatalf("unexpected existence columns: %+v", a) } }) @@ -935,7 +935,7 @@ func TestClient_ImportExistence(t *testing.T) { } // Verify existence. - if a := hldr.ReadRow(idxName, "exists", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3}) { + if a := hldr.ReadRow(idxName, "_exists", 0).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3}) { t.Fatalf("unexpected existence columns: %+v", a) } }) diff --git a/pilosa.go b/pilosa.go index 56e8e0f9e..fe61d8969 100644 --- a/pilosa.go +++ b/pilosa.go @@ -49,8 +49,6 @@ var ( ErrName = errors.New("invalid index or field name, must match [a-z0-9_-]") ErrLabel = errors.New("invalid row or column label, must match [A-Za-z0-9_-]") - ErrReservedName = errors.New("reserved index or field name") - // ErrFragmentNotFound is returned when a fragment does not exist. ErrFragmentNotFound = errors.New("fragment not found") ErrQueryRequired = errors.New("query required") @@ -131,9 +129,6 @@ const TimeFormat = "2006-01-02T15:04" // validateName ensures that the name is a valid format. func validateName(name string) error { - if name == existenceFieldName { - return ErrReservedName - } if !nameRegexp.Match([]byte(name)) { return ErrName } diff --git a/pilosa_internal_test.go b/pilosa_internal_test.go index e1bea3199..40e33ac4b 100644 --- a/pilosa_internal_test.go +++ b/pilosa_internal_test.go @@ -20,7 +20,7 @@ import ( func TestValidateName(t *testing.T) { names := []string{ - "a", "ab", "ab1", "b-c", "d_e", + "a", "ab", "ab1", "b-c", "d_e", "exists", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", } for _, name := range names { @@ -33,8 +33,7 @@ func TestValidateName(t *testing.T) { func TestValidateNameInvalid(t *testing.T) { names := []string{ "", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "1", "_", "-", - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", - "exists", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", "_exists", } for _, name := range names { if validateName(name) == nil { From ef7f04c09d01d3bcfe298bbdaa5beb63d86cc688 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 17 Dec 2018 16:01:32 -0600 Subject: [PATCH 087/125] schema endpoint doesn't return internal fields --- api_test.go | 19 ++++++++++++++++++- holder.go | 3 +++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/api_test.go b/api_test.go index e569896f0..ecb4265ed 100644 --- a/api_test.go +++ b/api_test.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "reflect" + "strings" "testing" "github.com/pilosa/pilosa" @@ -48,7 +49,7 @@ func TestAPI_Import(t *testing.T) { index := "rick" field := "f" - _, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true}) + _, err := m0.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true, TrackExistence: true}) if err != nil { t.Fatalf("creating index: %v", err) } @@ -99,6 +100,22 @@ func TestAPI_Import(t *testing.T) { } else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) { t.Fatalf("unexpected column keys: %+v", keys) } + + }) + + // Relies on the previous test creating an index with TrackExistence and + // adding some data. + t.Run("SchemaHasNoExists", func(t *testing.T) { + schema := m1.API.Schema(context.Background()) + for _, f := range schema[0].Fields { + if f.Name == "_exists" { + t.Fatalf("found _exists field in schema") + } + if strings.HasPrefix(f.Name, "_") { + t.Fatalf("found internal field '%s' in schema output", f.Name) + } + } + }) t.Run("RowKeyColumnID", func(t *testing.T) { diff --git a/holder.go b/holder.go index 39fef3595..471d4bd3e 100644 --- a/holder.go +++ b/holder.go @@ -289,6 +289,9 @@ func (h *Holder) limitedSchema() []*IndexInfo { for _, index := range h.Indexes() { di := &IndexInfo{Name: index.Name(), Options: index.Options()} for _, field := range index.Fields() { + if strings.HasPrefix(field.name, "_") { + continue + } fi := &FieldInfo{Name: field.Name(), Options: field.Options()} di.Fields = append(di.Fields, fi) } From 84fddbc67f245d4dc95a3ad823f65c293b87684d Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 12 Dec 2018 12:03:57 -0600 Subject: [PATCH 088/125] Replace the /fragment/data endpoint to support cluster resizing --- api.go | 21 ++++++++++++++++++++- client.go | 4 ++-- cluster.go | 4 ++-- http/client.go | 11 +++-------- http/handler.go | 23 +++++++++++++++++++++++ 5 files changed, 50 insertions(+), 13 deletions(-) diff --git a/api.go b/api.go index ce4fbd0ca..c6a1f3725 100644 --- a/api.go +++ b/api.go @@ -559,6 +559,23 @@ func (api *API) FragmentBlocks(ctx context.Context, indexName, fieldName, viewNa return blocks, nil } +// FragmentData returns all data in the specified fragment. +func (api *API) FragmentData(ctx context.Context, indexName, fieldName, viewName string, shard uint64) (io.WriterTo, error) { + span, _ := tracing.StartSpanFromContext(ctx, "API.FragmentBlocks") + defer span.Finish() + + if err := api.validate(apiFragmentData); err != nil { + return nil, errors.Wrap(err, "validating api method") + } + + // Retrieve fragment from holder. + f := api.holder.fragment(indexName, fieldName, viewName, shard) + if f == nil { + return nil, ErrFragmentNotFound + } + return f, nil +} + // Hosts returns a list of the hosts in the cluster including their ID, // URL, and which is the coordinator. func (api *API) Hosts(ctx context.Context) []*Node { @@ -1203,6 +1220,7 @@ const ( apiExportCSV apiFragmentBlockData apiFragmentBlocks + apiFragmentData apiField apiFieldAttrDiff //apiHosts // not implemented @@ -1232,7 +1250,8 @@ var methodsCommon = map[apiMethod]struct{}{ } var methodsResizing = map[apiMethod]struct{}{ - apiResizeAbort: {}, + apiFragmentData: {}, + apiResizeAbort: {}, } var methodsNormal = map[apiMethod]struct{}{ diff --git a/client.go b/client.go index b16222b7a..3762a27b9 100644 --- a/client.go +++ b/client.go @@ -52,7 +52,7 @@ type InternalClient interface { ColumnAttrDiff(ctx context.Context, uri *URI, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) SendMessage(ctx context.Context, uri *URI, msg []byte) error - RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) + RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error) ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error } @@ -149,6 +149,6 @@ func (n nopInternalClient) RowAttrDiff(ctx context.Context, uri *URI, index, fie func (n nopInternalClient) SendMessage(ctx context.Context, uri *URI, msg []byte) error { return nil } -func (n nopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) { +func (n nopInternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri URI) (io.ReadCloser, error) { return nil, nil } diff --git a/cluster.go b/cluster.go index ad6cdbe6a..4c1c0853a 100644 --- a/cluster.go +++ b/cluster.go @@ -1309,7 +1309,7 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error { // Stream shard from remote node. c.logger.Printf("retrieve shard %d for index %s from host %s", src.Shard, src.Index, src.Node.URI) - rd, err := c.InternalClient.RetrieveShardFromURI(ctx, src.Index, src.Field, src.Shard, srcURI) + rd, err := c.InternalClient.RetrieveShardFromURI(ctx, src.Index, src.Field, src.View, src.Shard, srcURI) if err != nil { // For now it is an acceptable error if the fragment is not found // on the remote node. This occurs when a shard has been skipped and @@ -1318,7 +1318,7 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error { // TODO: figure out a way to distinguish from "fragment not found" errors // which are true errors and which simply mean the fragment doesn't have data. if err == ErrFragmentNotFound { - return nil + continue } return errors.Wrap(err, "retrieving shard") } else if rd == nil { diff --git a/http/client.go b/http/client.go index b18757a46..46800fd70 100644 --- a/http/client.go +++ b/http/client.go @@ -705,24 +705,19 @@ func (c *InternalClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, i return nil } -func (c *InternalClient) RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri pilosa.URI) (io.ReadCloser, error) { +func (c *InternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri pilosa.URI) (io.ReadCloser, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.RetrieveShardFromURI") defer span.Finish() node := &pilosa.Node{ URI: uri, } - return c.backupShardNode(ctx, index, field, shard, node) -} -func (c *InternalClient) backupShardNode(ctx context.Context, index, field string, shard uint64, node *pilosa.Node) (io.ReadCloser, error) { - span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.backupShardNode") - defer span.Finish() - - u := nodePathToURL(node, "/fragment/data") + u := nodePathToURL(node, "/internal/fragment/data") u.RawQuery = url.Values{ "index": {index}, "field": {field}, + "view": {view}, "shard": {strconv.FormatUint(shard, 10)}, }.Encode() diff --git a/http/handler.go b/http/handler.go index 68719fdfe..c370656fa 100644 --- a/http/handler.go +++ b/http/handler.go @@ -192,6 +192,7 @@ func (h *Handler) populateValidators() { h.validators["PostClusterMessage"] = queryValidationSpecRequired() h.validators["GetFragmentBlockData"] = queryValidationSpecRequired() h.validators["GetFragmentBlocks"] = queryValidationSpecRequired("index", "field", "view", "shard") + h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "field", "view", "shard") h.validators["GetFragmentNodes"] = queryValidationSpecRequired("shard", "index") h.validators["PostIndexAttrDiff"] = queryValidationSpecRequired() h.validators["PostFieldAttrDiff"] = queryValidationSpecRequired() @@ -262,6 +263,7 @@ func newRouter(handler *Handler) *mux.Router { router.HandleFunc("/internal/cluster/message", handler.handlePostClusterMessage).Methods("POST").Name("PostClusterMessage") router.HandleFunc("/internal/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET").Name("GetFragmentBlockData") router.HandleFunc("/internal/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks") + router.HandleFunc("/internal/fragment/data", handler.handleGetFragmentData).Methods("GET").Name("GetFragmentData") router.HandleFunc("/internal/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes") router.HandleFunc("/internal/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST").Name("PostIndexAttrDiff") router.HandleFunc("/internal/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST").Name("PostFieldAttrDiff") @@ -1214,6 +1216,27 @@ type getFragmentBlocksResponse struct { Blocks []pilosa.FragmentBlock `json:"blocks"` } +// handleGetFragmentData handles GET /internal/fragment/data requests. +func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request) { + // Read shard parameter. + q := r.URL.Query() + shard, err := strconv.ParseUint(q.Get("shard"), 10, 64) + if err != nil { + http.Error(w, "shard required", http.StatusBadRequest) + return + } + // Retrieve fragment data from holder. + f, err := h.api.FragmentData(r.Context(), q.Get("index"), q.Get("field"), q.Get("view"), shard) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + // Stream fragment to response body. + if _, err := f.WriteTo(w); err != nil { + h.logger.Printf("error streaming fragment data: %s", err) + } +} + // handleGetVersion handles /version requests. func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) { if !validHeaderAcceptJSON(r.Header) { From d28170ddc65f6d869b38963124a5b493da16b0e6 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 12 Dec 2018 15:12:49 -0600 Subject: [PATCH 089/125] Syncs AvailableShards when handling a ResizeInstruction. There was a situation where availableShards on a new node were not in sync with the cluster, so queries following a resize were incorrect. - Start a one-node cluster. - Write data to shards 0 and 1 - Start a second node. In the case where the hash algo was moving shard 0 to node1, then node1 only knew about shard 0, so queries to node1 would be incomplete. This PR modifies the ResizeInstruction message to replace `Schema` with `NodeStatus` (which contains both `Schema` and `AvailableShards`). So now when a resize instruction is received, the receiving node is able to sync its schema and availableShards. --- cluster.go | 48 +++++- encoding/proto/proto.go | 6 +- internal/private.pb.go | 313 ++++++++++++++++++++-------------------- internal/private.proto | 2 +- server/cluster_test.go | 44 ++++++ utils_internal_test.go | 19 ++- 6 files changed, 267 insertions(+), 165 deletions(-) diff --git a/cluster.go b/cluster.go index 4c1c0853a..c3a6b2f69 100644 --- a/cluster.go +++ b/cluster.go @@ -1218,7 +1218,7 @@ func (c *cluster) unprotectedGenerateResizeJobByAction(nodeAction nodeAction) (* Node: toCluster.unprotectedNodeByID(id), Coordinator: c.unprotectedCoordinatorNode(), Sources: sources, - Schema: &Schema{Indexes: c.holder.Schema()}, // Include the schema to ensure it's in sync on the receiving node. + NodeStatus: c.nodeStatus(), // Include the NodeStatus in order to ensure that schema and availableShards are in sync on the receiving node. ClusterStatus: c.unprotectedStatus(), } j.Instructions = append(j.Instructions, instr) @@ -1277,12 +1277,30 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error { span, ctx := tracing.StartSpanFromContext(context.Background(), "Cluster.followResizeInstruction") defer span.Finish() - // Sync the schema received in the resize instruction. + // Sync the NodeStatus received in the resize instruction. + // Sync schema. c.logger.Debugf("holder applySchema") - if err := c.holder.applySchema(instr.Schema); err != nil { + if err := c.holder.applySchema(instr.NodeStatus.Schema); err != nil { return errors.Wrap(err, "applying schema") } + // Sync available shards. + for _, is := range instr.NodeStatus.Indexes { + for _, fs := range is.Fields { + f := c.holder.Field(is.Name, fs.Name) + + // if we don't know about a field locally, log an error because + // fields should be created and synced prior to shard creation + if f == nil { + c.logger.Printf("local field not found: %s/%s", is.Name, fs.Name) + continue + } + if err := f.AddRemoteAvailableShards(fs.AvailableShards); err != nil { + return errors.Wrap(err, "adding remote available shards") + } + } + } + // Request each source file in ResizeSources. for _, src := range instr.Sources { c.logger.Printf("get shard %d for index %s from host %s", src.Shard, src.Index, src.Node.URI) @@ -1817,6 +1835,28 @@ func (c *cluster) nodeLeave(nodeID string) error { return nil } +func (c *cluster) nodeStatus() *NodeStatus { + ns := &NodeStatus{ + Node: c.Node, + Schema: &Schema{Indexes: c.holder.Schema()}, + } + for _, idx := range ns.Schema.Indexes { + is := &IndexStatus{Name: idx.Name} + for _, f := range idx.Fields { + availableShards := roaring.NewBitmap() + if field := c.holder.Field(idx.Name, f.Name); field != nil { + availableShards = field.AvailableShards() + } + is.Fields = append(is.Fields, &FieldStatus{ + Name: f.Name, + AvailableShards: availableShards, + }) + } + ns.Indexes = append(ns.Indexes, is) + } + return ns +} + func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error { c.mu.Lock() defer c.mu.Unlock() @@ -1918,7 +1958,7 @@ type ResizeInstruction struct { Node *Node Coordinator *Node Sources []*ResizeSource - Schema *Schema + NodeStatus *NodeStatus ClusterStatus *ClusterStatus } diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 25e95ebac..f796f2f3e 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -457,7 +457,7 @@ func encodeResizeInstruction(m *pilosa.ResizeInstruction) *internal.ResizeInstru Node: encodeNode(m.Node), Coordinator: encodeNode(m.Coordinator), Sources: encodeResizeSources(m.Sources), - Schema: encodeSchema(m.Schema), + NodeStatus: encodeNodeStatus(m.NodeStatus), ClusterStatus: encodeClusterStatus(m.ClusterStatus), } } @@ -737,8 +737,8 @@ func decodeResizeInstruction(ri *internal.ResizeInstruction, m *pilosa.ResizeIns decodeNode(ri.Coordinator, m.Coordinator) m.Sources = make([]*pilosa.ResizeSource, len(ri.Sources)) decodeResizeSources(ri.Sources, m.Sources) - m.Schema = &pilosa.Schema{} - decodeSchema(ri.Schema, m.Schema) + m.NodeStatus = &pilosa.NodeStatus{} + decodeNodeStatus(ri.NodeStatus, m.NodeStatus) m.ClusterStatus = &pilosa.ClusterStatus{} decodeClusterStatus(ri.ClusterStatus, m.ClusterStatus) } diff --git a/internal/private.pb.go b/internal/private.pb.go index 0c5862b13..c5a51741b 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -32,7 +32,7 @@ func (m *IndexMeta) Reset() { *m = IndexMeta{} } func (m *IndexMeta) String() string { return proto.CompactTextString(m) } func (*IndexMeta) ProtoMessage() {} func (*IndexMeta) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{0} + return fileDescriptor_private_8095a89af06a70de, []int{0} } func (m *IndexMeta) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -93,7 +93,7 @@ func (m *FieldOptions) Reset() { *m = FieldOptions{} } func (m *FieldOptions) String() string { return proto.CompactTextString(m) } func (*FieldOptions) ProtoMessage() {} func (*FieldOptions) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{1} + return fileDescriptor_private_8095a89af06a70de, []int{1} } func (m *FieldOptions) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -189,7 +189,7 @@ func (m *ImportResponse) Reset() { *m = ImportResponse{} } func (m *ImportResponse) String() string { return proto.CompactTextString(m) } func (*ImportResponse) ProtoMessage() {} func (*ImportResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{2} + return fileDescriptor_private_8095a89af06a70de, []int{2} } func (m *ImportResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -240,7 +240,7 @@ func (m *BlockDataRequest) Reset() { *m = BlockDataRequest{} } func (m *BlockDataRequest) String() string { return proto.CompactTextString(m) } func (*BlockDataRequest) ProtoMessage() {} func (*BlockDataRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{3} + return fileDescriptor_private_8095a89af06a70de, []int{3} } func (m *BlockDataRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -316,7 +316,7 @@ func (m *BlockDataResponse) Reset() { *m = BlockDataResponse{} } func (m *BlockDataResponse) String() string { return proto.CompactTextString(m) } func (*BlockDataResponse) ProtoMessage() {} func (*BlockDataResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{4} + return fileDescriptor_private_8095a89af06a70de, []int{4} } func (m *BlockDataResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -370,7 +370,7 @@ func (m *Cache) Reset() { *m = Cache{} } func (m *Cache) String() string { return proto.CompactTextString(m) } func (*Cache) ProtoMessage() {} func (*Cache) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{5} + return fileDescriptor_private_8095a89af06a70de, []int{5} } func (m *Cache) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -417,7 +417,7 @@ func (m *MaxShards) Reset() { *m = MaxShards{} } func (m *MaxShards) String() string { return proto.CompactTextString(m) } func (*MaxShards) ProtoMessage() {} func (*MaxShards) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{6} + return fileDescriptor_private_8095a89af06a70de, []int{6} } func (m *MaxShards) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -466,7 +466,7 @@ func (m *CreateShardMessage) Reset() { *m = CreateShardMessage{} } func (m *CreateShardMessage) String() string { return proto.CompactTextString(m) } func (*CreateShardMessage) ProtoMessage() {} func (*CreateShardMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{7} + return fileDescriptor_private_8095a89af06a70de, []int{7} } func (m *CreateShardMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -527,7 +527,7 @@ func (m *DeleteIndexMessage) Reset() { *m = DeleteIndexMessage{} } func (m *DeleteIndexMessage) String() string { return proto.CompactTextString(m) } func (*DeleteIndexMessage) ProtoMessage() {} func (*DeleteIndexMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{8} + return fileDescriptor_private_8095a89af06a70de, []int{8} } func (m *DeleteIndexMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -575,7 +575,7 @@ func (m *CreateIndexMessage) Reset() { *m = CreateIndexMessage{} } func (m *CreateIndexMessage) String() string { return proto.CompactTextString(m) } func (*CreateIndexMessage) ProtoMessage() {} func (*CreateIndexMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{9} + return fileDescriptor_private_8095a89af06a70de, []int{9} } func (m *CreateIndexMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -631,7 +631,7 @@ func (m *CreateFieldMessage) Reset() { *m = CreateFieldMessage{} } func (m *CreateFieldMessage) String() string { return proto.CompactTextString(m) } func (*CreateFieldMessage) ProtoMessage() {} func (*CreateFieldMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{10} + return fileDescriptor_private_8095a89af06a70de, []int{10} } func (m *CreateFieldMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -693,7 +693,7 @@ func (m *DeleteFieldMessage) Reset() { *m = DeleteFieldMessage{} } func (m *DeleteFieldMessage) String() string { return proto.CompactTextString(m) } func (*DeleteFieldMessage) ProtoMessage() {} func (*DeleteFieldMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{11} + return fileDescriptor_private_8095a89af06a70de, []int{11} } func (m *DeleteFieldMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -749,7 +749,7 @@ func (m *DeleteAvailableShardMessage) Reset() { *m = DeleteAvailableShar func (m *DeleteAvailableShardMessage) String() string { return proto.CompactTextString(m) } func (*DeleteAvailableShardMessage) ProtoMessage() {} func (*DeleteAvailableShardMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{12} + return fileDescriptor_private_8095a89af06a70de, []int{12} } func (m *DeleteAvailableShardMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -812,7 +812,7 @@ func (m *Field) Reset() { *m = Field{} } func (m *Field) String() string { return proto.CompactTextString(m) } func (*Field) ProtoMessage() {} func (*Field) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{13} + return fileDescriptor_private_8095a89af06a70de, []int{13} } func (m *Field) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -873,7 +873,7 @@ func (m *Schema) Reset() { *m = Schema{} } func (m *Schema) String() string { return proto.CompactTextString(m) } func (*Schema) ProtoMessage() {} func (*Schema) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{14} + return fileDescriptor_private_8095a89af06a70de, []int{14} } func (m *Schema) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -921,7 +921,7 @@ func (m *Index) Reset() { *m = Index{} } func (m *Index) String() string { return proto.CompactTextString(m) } func (*Index) ProtoMessage() {} func (*Index) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{15} + return fileDescriptor_private_8095a89af06a70de, []int{15} } func (m *Index) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -977,7 +977,7 @@ func (m *URI) Reset() { *m = URI{} } func (m *URI) String() string { return proto.CompactTextString(m) } func (*URI) ProtoMessage() {} func (*URI) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{16} + return fileDescriptor_private_8095a89af06a70de, []int{16} } func (m *URI) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1041,7 +1041,7 @@ func (m *Node) Reset() { *m = Node{} } func (m *Node) String() string { return proto.CompactTextString(m) } func (*Node) ProtoMessage() {} func (*Node) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{17} + return fileDescriptor_private_8095a89af06a70de, []int{17} } func (m *Node) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1110,7 +1110,7 @@ func (m *NodeStateMessage) Reset() { *m = NodeStateMessage{} } func (m *NodeStateMessage) String() string { return proto.CompactTextString(m) } func (*NodeStateMessage) ProtoMessage() {} func (*NodeStateMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{18} + return fileDescriptor_private_8095a89af06a70de, []int{18} } func (m *NodeStateMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1165,7 +1165,7 @@ func (m *NodeEventMessage) Reset() { *m = NodeEventMessage{} } func (m *NodeEventMessage) String() string { return proto.CompactTextString(m) } func (*NodeEventMessage) ProtoMessage() {} func (*NodeEventMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{19} + return fileDescriptor_private_8095a89af06a70de, []int{19} } func (m *NodeEventMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1221,7 +1221,7 @@ func (m *NodeStatus) Reset() { *m = NodeStatus{} } func (m *NodeStatus) String() string { return proto.CompactTextString(m) } func (*NodeStatus) ProtoMessage() {} func (*NodeStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{20} + return fileDescriptor_private_8095a89af06a70de, []int{20} } func (m *NodeStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1283,7 +1283,7 @@ func (m *IndexStatus) Reset() { *m = IndexStatus{} } func (m *IndexStatus) String() string { return proto.CompactTextString(m) } func (*IndexStatus) ProtoMessage() {} func (*IndexStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{21} + return fileDescriptor_private_8095a89af06a70de, []int{21} } func (m *IndexStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1338,7 +1338,7 @@ func (m *FieldStatus) Reset() { *m = FieldStatus{} } func (m *FieldStatus) String() string { return proto.CompactTextString(m) } func (*FieldStatus) ProtoMessage() {} func (*FieldStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{22} + return fileDescriptor_private_8095a89af06a70de, []int{22} } func (m *FieldStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1394,7 +1394,7 @@ func (m *ClusterStatus) Reset() { *m = ClusterStatus{} } func (m *ClusterStatus) String() string { return proto.CompactTextString(m) } func (*ClusterStatus) ProtoMessage() {} func (*ClusterStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{23} + return fileDescriptor_private_8095a89af06a70de, []int{23} } func (m *ClusterStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1458,7 +1458,7 @@ func (m *BSIGroup) Reset() { *m = BSIGroup{} } func (m *BSIGroup) String() string { return proto.CompactTextString(m) } func (*BSIGroup) ProtoMessage() {} func (*BSIGroup) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{24} + return fileDescriptor_private_8095a89af06a70de, []int{24} } func (m *BSIGroup) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1528,7 +1528,7 @@ func (m *CreateViewMessage) Reset() { *m = CreateViewMessage{} } func (m *CreateViewMessage) String() string { return proto.CompactTextString(m) } func (*CreateViewMessage) ProtoMessage() {} func (*CreateViewMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{25} + return fileDescriptor_private_8095a89af06a70de, []int{25} } func (m *CreateViewMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1591,7 +1591,7 @@ func (m *DeleteViewMessage) Reset() { *m = DeleteViewMessage{} } func (m *DeleteViewMessage) String() string { return proto.CompactTextString(m) } func (*DeleteViewMessage) ProtoMessage() {} func (*DeleteViewMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{26} + return fileDescriptor_private_8095a89af06a70de, []int{26} } func (m *DeleteViewMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1646,7 +1646,7 @@ type ResizeInstruction struct { Node *Node `protobuf:"bytes,2,opt,name=Node" json:"Node,omitempty"` Coordinator *Node `protobuf:"bytes,3,opt,name=Coordinator" json:"Coordinator,omitempty"` Sources []*ResizeSource `protobuf:"bytes,4,rep,name=Sources" json:"Sources,omitempty"` - Schema *Schema `protobuf:"bytes,5,opt,name=Schema" json:"Schema,omitempty"` + NodeStatus *NodeStatus `protobuf:"bytes,7,opt,name=NodeStatus" json:"NodeStatus,omitempty"` ClusterStatus *ClusterStatus `protobuf:"bytes,6,opt,name=ClusterStatus" json:"ClusterStatus,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -1657,7 +1657,7 @@ func (m *ResizeInstruction) Reset() { *m = ResizeInstruction{} } func (m *ResizeInstruction) String() string { return proto.CompactTextString(m) } func (*ResizeInstruction) ProtoMessage() {} func (*ResizeInstruction) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{27} + return fileDescriptor_private_8095a89af06a70de, []int{27} } func (m *ResizeInstruction) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1714,9 +1714,9 @@ func (m *ResizeInstruction) GetSources() []*ResizeSource { return nil } -func (m *ResizeInstruction) GetSchema() *Schema { +func (m *ResizeInstruction) GetNodeStatus() *NodeStatus { if m != nil { - return m.Schema + return m.NodeStatus } return nil } @@ -1743,7 +1743,7 @@ func (m *ResizeSource) Reset() { *m = ResizeSource{} } func (m *ResizeSource) String() string { return proto.CompactTextString(m) } func (*ResizeSource) ProtoMessage() {} func (*ResizeSource) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{28} + return fileDescriptor_private_8095a89af06a70de, []int{28} } func (m *ResizeSource) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1820,7 +1820,7 @@ func (m *ResizeInstructionComplete) Reset() { *m = ResizeInstructionComp func (m *ResizeInstructionComplete) String() string { return proto.CompactTextString(m) } func (*ResizeInstructionComplete) ProtoMessage() {} func (*ResizeInstructionComplete) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{29} + return fileDescriptor_private_8095a89af06a70de, []int{29} } func (m *ResizeInstructionComplete) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1881,7 +1881,7 @@ func (m *SetCoordinatorMessage) Reset() { *m = SetCoordinatorMessage{} } func (m *SetCoordinatorMessage) String() string { return proto.CompactTextString(m) } func (*SetCoordinatorMessage) ProtoMessage() {} func (*SetCoordinatorMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{30} + return fileDescriptor_private_8095a89af06a70de, []int{30} } func (m *SetCoordinatorMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1928,7 +1928,7 @@ func (m *UpdateCoordinatorMessage) Reset() { *m = UpdateCoordinatorMessa func (m *UpdateCoordinatorMessage) String() string { return proto.CompactTextString(m) } func (*UpdateCoordinatorMessage) ProtoMessage() {} func (*UpdateCoordinatorMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{31} + return fileDescriptor_private_8095a89af06a70de, []int{31} } func (m *UpdateCoordinatorMessage) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1976,7 +1976,7 @@ func (m *Topology) Reset() { *m = Topology{} } func (m *Topology) String() string { return proto.CompactTextString(m) } func (*Topology) ProtoMessage() {} func (*Topology) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{32} + return fileDescriptor_private_8095a89af06a70de, []int{32} } func (m *Topology) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2029,7 +2029,7 @@ func (m *RecalculateCaches) Reset() { *m = RecalculateCaches{} } func (m *RecalculateCaches) String() string { return proto.CompactTextString(m) } func (*RecalculateCaches) ProtoMessage() {} func (*RecalculateCaches) Descriptor() ([]byte, []int) { - return fileDescriptor_private_08d4c0c27f7a355f, []int{33} + return fileDescriptor_private_8095a89af06a70de, []int{33} } func (m *RecalculateCaches) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3270,21 +3270,21 @@ func (m *ResizeInstruction) MarshalTo(dAtA []byte) (int, error) { i += n } } - if m.Schema != nil { - dAtA[i] = 0x2a + if m.ClusterStatus != nil { + dAtA[i] = 0x32 i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.Schema.Size())) - n18, err := m.Schema.MarshalTo(dAtA[i:]) + i = encodeVarintPrivate(dAtA, i, uint64(m.ClusterStatus.Size())) + n18, err := m.ClusterStatus.MarshalTo(dAtA[i:]) if err != nil { return 0, err } i += n18 } - if m.ClusterStatus != nil { - dAtA[i] = 0x32 + if m.NodeStatus != nil { + dAtA[i] = 0x3a i++ - i = encodeVarintPrivate(dAtA, i, uint64(m.ClusterStatus.Size())) - n19, err := m.ClusterStatus.MarshalTo(dAtA[i:]) + i = encodeVarintPrivate(dAtA, i, uint64(m.NodeStatus.Size())) + n19, err := m.NodeStatus.MarshalTo(dAtA[i:]) if err != nil { return 0, err } @@ -4169,14 +4169,14 @@ func (m *ResizeInstruction) Size() (n int) { n += 1 + l + sovPrivate(uint64(l)) } } - if m.Schema != nil { - l = m.Schema.Size() - n += 1 + l + sovPrivate(uint64(l)) - } if m.ClusterStatus != nil { l = m.ClusterStatus.Size() n += 1 + l + sovPrivate(uint64(l)) } + if m.NodeStatus != nil { + l = m.NodeStatus.Size() + n += 1 + l + sovPrivate(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -8034,39 +8034,6 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex - case 5: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Schema", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Schema == nil { - m.Schema = &Schema{} - } - if err := m.Schema.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex case 6: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field ClusterStatus", wireType) @@ -8100,6 +8067,39 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field NodeStatus", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.NodeStatus == nil { + m.NodeStatus = &NodeStatus{} + } + if err := m.NodeStatus.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -8877,79 +8877,80 @@ var ( ErrIntOverflowPrivate = fmt.Errorf("proto: integer overflow") ) -func init() { proto.RegisterFile("private.proto", fileDescriptor_private_08d4c0c27f7a355f) } +func init() { proto.RegisterFile("private.proto", fileDescriptor_private_8095a89af06a70de) } -var fileDescriptor_private_08d4c0c27f7a355f = []byte{ - // 1131 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdd, 0x6e, 0x1b, 0xc5, - 0x17, 0xff, 0xef, 0x87, 0x1d, 0xfb, 0xb8, 0x4e, 0x93, 0xed, 0xbf, 0x61, 0x0b, 0x28, 0x84, 0x51, - 0x45, 0x43, 0x25, 0x42, 0xd5, 0xde, 0xf0, 0x55, 0xa9, 0x24, 0x0e, 0x65, 0x29, 0x09, 0x65, 0x9c, - 0xe4, 0x8e, 0x8b, 0x89, 0x3d, 0x6a, 0x56, 0x59, 0xef, 0x98, 0xdd, 0xd9, 0x24, 0xee, 0x05, 0xb7, - 0x20, 0xf1, 0x02, 0x88, 0x27, 0xe2, 0x92, 0x47, 0xa8, 0xc2, 0x8b, 0xa0, 0x39, 0x33, 0xb3, 0xbb, - 0x76, 0x1c, 0x12, 0x05, 0xee, 0xe6, 0xfc, 0xce, 0x99, 0xf3, 0x7d, 0xce, 0xec, 0x42, 0x77, 0x9c, - 0xc5, 0x27, 0x4c, 0xf2, 0x8d, 0x71, 0x26, 0xa4, 0x08, 0x5a, 0x71, 0x2a, 0x79, 0x96, 0xb2, 0x84, - 0x3c, 0x87, 0x76, 0x94, 0x0e, 0xf9, 0xd9, 0x0e, 0x97, 0x2c, 0x08, 0xc0, 0x7f, 0xc1, 0x27, 0x79, - 0xe8, 0xad, 0x39, 0xeb, 0x2d, 0x8a, 0xe7, 0xe0, 0x03, 0x58, 0xdc, 0xcb, 0xd8, 0xe0, 0x78, 0xfb, - 0x2c, 0xce, 0x25, 0x4f, 0x07, 0x3c, 0xf4, 0x91, 0x3b, 0x83, 0x92, 0x37, 0x0e, 0xdc, 0xfa, 0x2a, - 0xe6, 0xc9, 0xf0, 0xbb, 0xb1, 0x8c, 0x45, 0x9a, 0x07, 0xef, 0x42, 0x7b, 0x8b, 0x0d, 0x8e, 0xf8, - 0xde, 0x64, 0xcc, 0x51, 0x63, 0x9b, 0x56, 0x40, 0xc9, 0xed, 0xc7, 0xaf, 0xb5, 0xc6, 0x2e, 0xad, - 0x80, 0x60, 0x0d, 0x3a, 0x7b, 0xf1, 0x88, 0x7f, 0x5f, 0xb0, 0x54, 0x16, 0xa3, 0xb0, 0x81, 0xb7, - 0xeb, 0x90, 0x72, 0x15, 0x15, 0xb7, 0x90, 0x85, 0xe7, 0x60, 0x09, 0xbc, 0x9d, 0x38, 0x0d, 0xdb, - 0x6b, 0xce, 0xba, 0x47, 0xd5, 0x11, 0x11, 0x76, 0x16, 0x82, 0x41, 0xd8, 0x59, 0x19, 0x62, 0x67, - 0x3a, 0xc4, 0x5d, 0xd1, 0x97, 0x2c, 0x1d, 0xb2, 0x6c, 0x78, 0x10, 0xf3, 0xd3, 0xf0, 0x96, 0x0e, - 0x71, 0x1a, 0x25, 0x04, 0x16, 0xa3, 0xd1, 0x58, 0x64, 0x92, 0xf2, 0x7c, 0x2c, 0xd2, 0x1c, 0x2d, - 0x6e, 0x67, 0x59, 0xe8, 0xa0, 0x13, 0xea, 0x48, 0x7e, 0x82, 0xa5, 0xcd, 0x44, 0x0c, 0x8e, 0x7b, - 0x4c, 0x32, 0xca, 0x7f, 0x2c, 0x78, 0x2e, 0x83, 0xff, 0x43, 0x03, 0x73, 0x6c, 0xe4, 0x34, 0xa1, - 0x50, 0xcc, 0x57, 0xe8, 0x6a, 0x14, 0x09, 0x85, 0xe2, 0x7d, 0xcc, 0x98, 0x4f, 0x35, 0xa1, 0xd0, - 0xfe, 0x11, 0xcb, 0x86, 0x98, 0x29, 0x9f, 0x6a, 0x42, 0xc5, 0x82, 0xde, 0xea, 0xf4, 0xe0, 0x99, - 0x44, 0xb0, 0x5c, 0xb3, 0x6f, 0xdc, 0x5c, 0x81, 0x26, 0x15, 0xa7, 0x51, 0x2f, 0x0f, 0x9d, 0x35, - 0x6f, 0xdd, 0xa7, 0x86, 0xc2, 0x22, 0x88, 0xa4, 0x18, 0xa5, 0x8a, 0xe5, 0x22, 0xab, 0x02, 0xc8, - 0x3d, 0x68, 0x60, 0x45, 0x54, 0x94, 0xd5, 0x5d, 0x75, 0x24, 0x3f, 0x3b, 0xd0, 0xde, 0x61, 0x67, - 0xe8, 0x46, 0x1e, 0x3c, 0x85, 0x96, 0xcd, 0x13, 0x0a, 0x75, 0x1e, 0xbf, 0xbf, 0x61, 0x1b, 0x6c, - 0xa3, 0x14, 0xdb, 0xb0, 0x32, 0xdb, 0xa9, 0xcc, 0x26, 0xb4, 0xbc, 0xf2, 0xf6, 0xe7, 0xd0, 0x9d, - 0x62, 0x29, 0x7b, 0xc7, 0x7c, 0x62, 0xb3, 0x7a, 0xcc, 0x27, 0x2a, 0xfe, 0x13, 0x96, 0x14, 0x1c, - 0x73, 0xe5, 0x53, 0x4d, 0x7c, 0xe6, 0x7e, 0xe2, 0x90, 0x03, 0x08, 0xb6, 0x32, 0xce, 0x24, 0x47, - 0x23, 0x3b, 0x3c, 0xcf, 0xd9, 0x2b, 0x7e, 0x79, 0xc6, 0x75, 0x16, 0xdd, 0x7a, 0x16, 0xcb, 0x3a, - 0x78, 0xb5, 0x3a, 0x90, 0x87, 0x10, 0xf4, 0x78, 0xc2, 0x25, 0x37, 0xd3, 0xf1, 0x0f, 0x7a, 0x49, - 0xdf, 0xfa, 0x70, 0xb5, 0x6c, 0xf0, 0x00, 0x7c, 0x35, 0x6a, 0xe8, 0x42, 0xe7, 0xf1, 0x9d, 0x2a, - 0x4f, 0xe5, 0x14, 0x52, 0x14, 0x20, 0x89, 0x55, 0x8a, 0xfe, 0x5c, 0x19, 0xd8, 0x9c, 0x56, 0x7a, - 0x68, 0x4c, 0x79, 0x68, 0x6a, 0xa5, 0x32, 0x55, 0x1f, 0x53, 0x63, 0xed, 0x99, 0x0d, 0xf7, 0xa6, - 0xd6, 0xc8, 0x00, 0xde, 0xd1, 0x1a, 0xbe, 0x3c, 0x61, 0x71, 0xc2, 0x0e, 0x93, 0x6b, 0x56, 0x64, - 0x8e, 0xe3, 0x21, 0x2c, 0xe0, 0xdd, 0xa8, 0x67, 0xa6, 0xc0, 0x92, 0xe4, 0x07, 0x23, 0xaf, 0x5a, - 0x7f, 0x97, 0x8d, 0xb8, 0xd1, 0x86, 0xe7, 0x32, 0x5e, 0xf7, 0xea, 0x78, 0x95, 0x61, 0x35, 0x2e, - 0x6a, 0xd5, 0x79, 0xca, 0x30, 0x12, 0xe4, 0x09, 0x34, 0xfb, 0x83, 0x23, 0x3e, 0x62, 0xc1, 0x87, - 0xb0, 0x80, 0x1e, 0xf2, 0xdc, 0x74, 0xf4, 0xed, 0x99, 0x4a, 0x51, 0xcb, 0x27, 0x3d, 0x13, 0xd9, - 0x5c, 0x9f, 0x1e, 0x40, 0x13, 0xad, 0xe7, 0xa1, 0x3f, 0xab, 0x06, 0x71, 0x6a, 0xd8, 0x64, 0x1b, - 0xbc, 0x7d, 0x1a, 0xa9, 0x49, 0x45, 0x0f, 0xac, 0x16, 0x43, 0x29, 0xdd, 0x5f, 0x8b, 0x5c, 0x9a, - 0x3c, 0xe1, 0x59, 0x61, 0x2f, 0x45, 0x26, 0x31, 0x47, 0x5d, 0x8a, 0x67, 0x92, 0x83, 0xbf, 0x2b, - 0x86, 0x3c, 0x58, 0x04, 0x37, 0xea, 0x19, 0x1d, 0x6e, 0xd4, 0x0b, 0xde, 0x43, 0xf5, 0x26, 0x35, - 0xdd, 0xca, 0x89, 0x7d, 0x1a, 0x51, 0x34, 0x7c, 0x1f, 0xba, 0x51, 0xbe, 0x25, 0x44, 0x36, 0x8c, - 0x53, 0x26, 0x45, 0x66, 0xde, 0x80, 0x69, 0x10, 0x27, 0x48, 0x32, 0xa9, 0x37, 0x76, 0x9b, 0x6a, - 0x82, 0x3c, 0x83, 0x25, 0x65, 0x14, 0x09, 0x5b, 0xef, 0x15, 0x68, 0x2a, 0xac, 0x74, 0xc2, 0x50, - 0x95, 0x06, 0xb7, 0xae, 0xe1, 0x5b, 0xad, 0x61, 0xfb, 0x84, 0xa7, 0xb2, 0xd6, 0x31, 0x48, 0xa3, - 0x82, 0x2e, 0xd5, 0x44, 0x40, 0x74, 0x80, 0x26, 0x92, 0xc5, 0x2a, 0x12, 0x85, 0x52, 0xe4, 0x91, - 0x5f, 0x1d, 0x00, 0xeb, 0x50, 0x91, 0x97, 0x57, 0x9c, 0xcb, 0xaf, 0x04, 0xeb, 0xb6, 0xf2, 0x66, - 0x5a, 0x96, 0x2a, 0x29, 0x8d, 0x53, 0xdb, 0x19, 0x1f, 0x57, 0x9d, 0xa1, 0x4b, 0x7a, 0x77, 0xa6, - 0x33, 0xb4, 0xd5, 0xaa, 0x3f, 0x5e, 0x42, 0xa7, 0x86, 0xcf, 0xed, 0x92, 0x8f, 0xca, 0x2e, 0x71, - 0x67, 0x55, 0x22, 0x6e, 0x54, 0xda, 0x5e, 0x79, 0x01, 0x9d, 0x1a, 0x3c, 0x57, 0xe3, 0x3a, 0xdc, - 0x9e, 0x9e, 0x43, 0xbb, 0xdf, 0x67, 0x61, 0x12, 0x43, 0x77, 0x2b, 0x29, 0x72, 0xc9, 0x33, 0xa3, - 0x4e, 0x3d, 0x0a, 0x1a, 0x28, 0x8b, 0x57, 0x01, 0xf3, 0xeb, 0x17, 0xdc, 0x87, 0x86, 0x4a, 0xa3, - 0x1e, 0xa7, 0x8b, 0x39, 0xd6, 0x4c, 0x72, 0x00, 0xad, 0xcd, 0x7e, 0xf4, 0x3c, 0x13, 0xc5, 0x78, - 0xae, 0xd3, 0xf6, 0x4d, 0x77, 0x2f, 0xbe, 0xe9, 0xde, 0x85, 0x37, 0xdd, 0x2f, 0xdf, 0x74, 0xd2, - 0x87, 0x65, 0xbd, 0x2a, 0xd5, 0x14, 0xdf, 0x64, 0xe1, 0xd8, 0x87, 0xd4, 0xab, 0x3d, 0xa4, 0x7d, - 0x58, 0xd6, 0xfb, 0xec, 0xbf, 0x54, 0xfa, 0xbb, 0x0b, 0xcb, 0x94, 0xe7, 0xf1, 0x6b, 0x1e, 0xa5, - 0xb9, 0xcc, 0x8a, 0x81, 0xda, 0x49, 0xea, 0xfe, 0x37, 0xe2, 0xd0, 0x64, 0xdb, 0xa3, 0x9a, 0xb8, - 0x4e, 0xa7, 0x07, 0x8f, 0xa0, 0x33, 0x3b, 0xb3, 0x17, 0x45, 0xeb, 0x22, 0xc1, 0x23, 0x58, 0xe8, - 0x8b, 0x22, 0x1b, 0x94, 0xed, 0x5b, 0xdb, 0x93, 0xda, 0x33, 0xcd, 0xa6, 0x56, 0xac, 0x36, 0x1a, - 0x8d, 0x2b, 0x46, 0xe3, 0xe9, 0x4c, 0x2b, 0x85, 0x4d, 0xbc, 0xf0, 0x56, 0x75, 0x61, 0x8a, 0x4d, - 0xa7, 0xa5, 0xc9, 0x2f, 0x0e, 0xdc, 0xaa, 0xbb, 0x70, 0xad, 0xc1, 0x2d, 0x2b, 0xe2, 0xce, 0xad, - 0x88, 0x37, 0xaf, 0x22, 0x7e, 0x55, 0x91, 0xea, 0x9b, 0xa0, 0x51, 0xfb, 0x26, 0x20, 0xc7, 0x70, - 0xef, 0x42, 0x99, 0xb6, 0xc4, 0x68, 0xac, 0xfa, 0xe1, 0x5f, 0x94, 0x4b, 0xad, 0xb4, 0x2c, 0x33, - 0x85, 0x6a, 0x53, 0x4d, 0x90, 0x4f, 0xe1, 0x6e, 0x9f, 0xcb, 0x5a, 0x91, 0x6c, 0xb7, 0xad, 0x81, - 0xb7, 0xcb, 0x4f, 0x2f, 0x09, 0x5f, 0xb1, 0xc8, 0x17, 0x10, 0xee, 0x8f, 0x87, 0x4c, 0xf2, 0x1b, - 0xdd, 0xde, 0x84, 0xd6, 0x9e, 0x18, 0x8b, 0x44, 0xbc, 0x9a, 0x5c, 0x31, 0xf5, 0x21, 0x2c, 0xe8, - 0xfd, 0xad, 0xd7, 0x48, 0x9b, 0x5a, 0x92, 0xdc, 0x51, 0x0d, 0x3d, 0x60, 0xc9, 0xa0, 0x48, 0x94, - 0x1b, 0xea, 0x7b, 0x31, 0xdf, 0x5c, 0xfa, 0xe3, 0x7c, 0xd5, 0xf9, 0xf3, 0x7c, 0xd5, 0x79, 0x73, - 0xbe, 0xea, 0xfc, 0xf6, 0xd7, 0xea, 0xff, 0x0e, 0x9b, 0xf8, 0xdf, 0xf1, 0xe4, 0xef, 0x00, 0x00, - 0x00, 0xff, 0xff, 0x61, 0x80, 0xe4, 0xef, 0x88, 0x0c, 0x00, 0x00, +var fileDescriptor_private_8095a89af06a70de = []byte{ + // 1139 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xdb, 0x6e, 0x1b, 0xc5, + 0x1b, 0xff, 0xef, 0x21, 0x8e, 0xfd, 0x39, 0x4e, 0x93, 0x6d, 0x9b, 0xff, 0x16, 0x50, 0x08, 0xa3, + 0x8a, 0x86, 0x4a, 0x84, 0xaa, 0xe5, 0x82, 0x53, 0xa5, 0x92, 0x38, 0x94, 0xa5, 0x24, 0x94, 0x71, + 0x92, 0x3b, 0x2e, 0x26, 0xf6, 0xa8, 0x59, 0x65, 0xbd, 0x63, 0x76, 0x67, 0x93, 0xb8, 0x17, 0xdc, + 0x82, 0xc4, 0x0b, 0xf0, 0x04, 0x3c, 0x0b, 0x97, 0x3c, 0x42, 0x15, 0x5e, 0x04, 0xcd, 0x37, 0x33, + 0xbb, 0x6b, 0xc7, 0x21, 0x51, 0xe0, 0x6e, 0xbe, 0xdf, 0x77, 0x3e, 0xae, 0x0d, 0x9d, 0x51, 0x16, + 0x9f, 0x30, 0xc9, 0x37, 0x46, 0x99, 0x90, 0x22, 0x68, 0xc6, 0xa9, 0xe4, 0x59, 0xca, 0x12, 0xf2, + 0x1c, 0x5a, 0x51, 0x3a, 0xe0, 0x67, 0x3b, 0x5c, 0xb2, 0x20, 0x00, 0xff, 0x05, 0x1f, 0xe7, 0xa1, + 0xb7, 0xe6, 0xac, 0x37, 0x29, 0xbe, 0x83, 0xf7, 0x61, 0x71, 0x2f, 0x63, 0xfd, 0xe3, 0xed, 0xb3, + 0x38, 0x97, 0x3c, 0xed, 0xf3, 0xd0, 0x47, 0xee, 0x14, 0x4a, 0xde, 0x38, 0xb0, 0xf0, 0x55, 0xcc, + 0x93, 0xc1, 0x77, 0x23, 0x19, 0x8b, 0x34, 0x0f, 0xde, 0x81, 0xd6, 0x16, 0xeb, 0x1f, 0xf1, 0xbd, + 0xf1, 0x88, 0xa3, 0xc5, 0x16, 0xad, 0x80, 0x92, 0xdb, 0x8b, 0x5f, 0x6b, 0x8b, 0x1d, 0x5a, 0x01, + 0xc1, 0x1a, 0xb4, 0xf7, 0xe2, 0x21, 0xff, 0xbe, 0x60, 0xa9, 0x2c, 0x86, 0xe1, 0x1c, 0x6a, 0xd7, + 0x21, 0x15, 0x2a, 0x1a, 0x6e, 0x22, 0x0b, 0xdf, 0xc1, 0x12, 0x78, 0x3b, 0x71, 0x1a, 0xb6, 0xd6, + 0x9c, 0x75, 0x8f, 0xaa, 0x27, 0x22, 0xec, 0x2c, 0x04, 0x83, 0xb0, 0xb3, 0x32, 0xc5, 0xf6, 0x64, + 0x8a, 0xbb, 0xa2, 0x27, 0x59, 0x3a, 0x60, 0xd9, 0xe0, 0x20, 0xe6, 0xa7, 0xe1, 0x82, 0x4e, 0x71, + 0x12, 0x25, 0x04, 0x16, 0xa3, 0xe1, 0x48, 0x64, 0x92, 0xf2, 0x7c, 0x24, 0xd2, 0x1c, 0x3d, 0x6e, + 0x67, 0x59, 0xe8, 0x60, 0x10, 0xea, 0x49, 0x7e, 0x82, 0xa5, 0xcd, 0x44, 0xf4, 0x8f, 0xbb, 0x4c, + 0x32, 0xca, 0x7f, 0x2c, 0x78, 0x2e, 0x83, 0x3b, 0x30, 0x87, 0x35, 0x36, 0x72, 0x9a, 0x50, 0x28, + 0xd6, 0x2b, 0x74, 0x35, 0x8a, 0x84, 0x42, 0x51, 0x1f, 0x2b, 0xe6, 0x53, 0x4d, 0x28, 0xb4, 0x77, + 0xc4, 0xb2, 0x01, 0x56, 0xca, 0xa7, 0x9a, 0x50, 0xb9, 0x60, 0xb4, 0xba, 0x3c, 0xf8, 0x26, 0x11, + 0x2c, 0xd7, 0xfc, 0x9b, 0x30, 0x57, 0xa0, 0x41, 0xc5, 0x69, 0xd4, 0xcd, 0x43, 0x67, 0xcd, 0x5b, + 0xf7, 0xa9, 0xa1, 0xb0, 0x09, 0x22, 0x29, 0x86, 0xa9, 0x62, 0xb9, 0xc8, 0xaa, 0x00, 0x72, 0x0f, + 0xe6, 0xb0, 0x23, 0x2a, 0xcb, 0x4a, 0x57, 0x3d, 0xc9, 0xcf, 0x0e, 0xb4, 0x76, 0xd8, 0x19, 0x86, + 0x91, 0x07, 0x4f, 0xa1, 0x69, 0xeb, 0x84, 0x42, 0xed, 0xc7, 0xef, 0x6d, 0xd8, 0x01, 0xdb, 0x28, + 0xc5, 0x36, 0xac, 0xcc, 0x76, 0x2a, 0xb3, 0x31, 0x2d, 0x55, 0xde, 0xfa, 0x1c, 0x3a, 0x13, 0x2c, + 0xe5, 0xef, 0x98, 0x8f, 0x6d, 0x55, 0x8f, 0xf9, 0x58, 0xe5, 0x7f, 0xc2, 0x92, 0x82, 0x63, 0xad, + 0x7c, 0xaa, 0x89, 0xcf, 0xdc, 0x4f, 0x1c, 0x72, 0x00, 0xc1, 0x56, 0xc6, 0x99, 0xe4, 0xe8, 0x64, + 0x87, 0xe7, 0x39, 0x7b, 0xc5, 0x2f, 0xaf, 0xb8, 0xae, 0xa2, 0x5b, 0xaf, 0x62, 0xd9, 0x07, 0xaf, + 0xd6, 0x07, 0xf2, 0x10, 0x82, 0x2e, 0x4f, 0xb8, 0xe4, 0x66, 0x3b, 0xfe, 0xc1, 0x2e, 0xe9, 0xd9, + 0x18, 0xae, 0x96, 0x0d, 0x1e, 0x80, 0xaf, 0x56, 0x0d, 0x43, 0x68, 0x3f, 0xbe, 0x5d, 0xd5, 0xa9, + 0xdc, 0x42, 0x8a, 0x02, 0x24, 0xb1, 0x46, 0x31, 0x9e, 0x2b, 0x13, 0x9b, 0x31, 0x4a, 0x0f, 0x8d, + 0x2b, 0x0f, 0x5d, 0xad, 0x54, 0xae, 0xea, 0x6b, 0x6a, 0xbc, 0x3d, 0xb3, 0xe9, 0xde, 0xd4, 0x1b, + 0xe9, 0xc3, 0xdb, 0xda, 0xc2, 0x97, 0x27, 0x2c, 0x4e, 0xd8, 0x61, 0x72, 0xcd, 0x8e, 0xcc, 0x08, + 0x3c, 0x84, 0x79, 0xd4, 0x8d, 0xba, 0x66, 0x0b, 0x2c, 0x49, 0x7e, 0x30, 0xf2, 0x6a, 0xf4, 0x77, + 0xd9, 0x90, 0x1b, 0x6b, 0xf8, 0x2e, 0xf3, 0x75, 0xaf, 0xce, 0x57, 0x39, 0x56, 0xeb, 0xa2, 0x4e, + 0x9d, 0xa7, 0x1c, 0x23, 0x41, 0x9e, 0x40, 0xa3, 0xd7, 0x3f, 0xe2, 0x43, 0x16, 0x7c, 0x00, 0xf3, + 0x18, 0x21, 0xcf, 0xcd, 0x44, 0xdf, 0x9a, 0xea, 0x14, 0xb5, 0x7c, 0xd2, 0x35, 0x99, 0xcd, 0x8c, + 0xe9, 0x01, 0x34, 0xd0, 0x7b, 0x1e, 0xfa, 0xd3, 0x66, 0x10, 0xa7, 0x86, 0x4d, 0xb6, 0xc1, 0xdb, + 0xa7, 0x91, 0xda, 0x54, 0x8c, 0xc0, 0x5a, 0x31, 0x94, 0xb2, 0xfd, 0xb5, 0xc8, 0xa5, 0xa9, 0x13, + 0xbe, 0x15, 0xf6, 0x52, 0x64, 0x12, 0x6b, 0xd4, 0xa1, 0xf8, 0x26, 0x39, 0xf8, 0xbb, 0x62, 0xc0, + 0x83, 0x45, 0x70, 0xa3, 0xae, 0xb1, 0xe1, 0x46, 0xdd, 0xe0, 0x5d, 0x34, 0x6f, 0x4a, 0xd3, 0xa9, + 0x82, 0xd8, 0xa7, 0x11, 0x45, 0xc7, 0xf7, 0xa1, 0x13, 0xe5, 0x5b, 0x42, 0x64, 0x83, 0x38, 0x65, + 0x52, 0x64, 0xe6, 0x1b, 0x30, 0x09, 0xe2, 0x06, 0x49, 0x26, 0xf5, 0xc5, 0x6e, 0x51, 0x4d, 0x90, + 0x67, 0xb0, 0xa4, 0x9c, 0x22, 0x61, 0xfb, 0xbd, 0x02, 0x0d, 0x85, 0x95, 0x41, 0x18, 0xaa, 0xb2, + 0xe0, 0xd6, 0x2d, 0x7c, 0xab, 0x2d, 0x6c, 0x9f, 0xf0, 0x54, 0xd6, 0x26, 0x06, 0x69, 0x34, 0xd0, + 0xa1, 0x9a, 0x08, 0x88, 0x4e, 0xd0, 0x64, 0xb2, 0x58, 0x65, 0xa2, 0x50, 0x8a, 0x3c, 0xf2, 0xab, + 0x03, 0x60, 0x03, 0x2a, 0xf2, 0x52, 0xc5, 0xb9, 0x5c, 0x25, 0x58, 0xb7, 0x9d, 0x37, 0xdb, 0xb2, + 0x54, 0x49, 0x69, 0x9c, 0xda, 0xc9, 0xf8, 0xa8, 0x9a, 0x0c, 0xdd, 0xd2, 0xbb, 0x53, 0x93, 0xa1, + 0xbd, 0x56, 0xf3, 0xf1, 0x12, 0xda, 0x35, 0x7c, 0xe6, 0x94, 0x7c, 0x58, 0x4e, 0x89, 0x3b, 0x6d, + 0x12, 0x71, 0x63, 0xd2, 0xce, 0xca, 0x0b, 0x68, 0xd7, 0xe0, 0x99, 0x16, 0xd7, 0xe1, 0xd6, 0xe4, + 0x1e, 0xda, 0xfb, 0x3e, 0x0d, 0x93, 0x18, 0x3a, 0x5b, 0x49, 0x91, 0x4b, 0x9e, 0x19, 0x73, 0xea, + 0xa3, 0xa0, 0x81, 0xb2, 0x79, 0x15, 0x30, 0xbb, 0x7f, 0xc1, 0x7d, 0x98, 0x53, 0x65, 0xd4, 0xeb, + 0x74, 0xb1, 0xc6, 0x9a, 0x49, 0x0e, 0xa0, 0xb9, 0xd9, 0x8b, 0x9e, 0x67, 0xa2, 0x18, 0xcd, 0x0c, + 0xda, 0x7e, 0xd3, 0xdd, 0x8b, 0xdf, 0x74, 0xef, 0xc2, 0x37, 0xdd, 0x2f, 0xbf, 0xe9, 0xa4, 0x07, + 0xcb, 0xfa, 0x54, 0xaa, 0x2d, 0xbe, 0xc9, 0xc1, 0xb1, 0x1f, 0x52, 0xaf, 0xf6, 0x21, 0xed, 0xc1, + 0xb2, 0xbe, 0x67, 0xff, 0xa5, 0xd1, 0xdf, 0x5d, 0x58, 0xa6, 0x3c, 0x8f, 0x5f, 0xf3, 0x28, 0xcd, + 0x65, 0x56, 0xf4, 0xd5, 0x4d, 0x52, 0xfa, 0xdf, 0x88, 0x43, 0x53, 0x6d, 0x8f, 0x6a, 0xe2, 0x3a, + 0x93, 0x1e, 0x3c, 0x82, 0xf6, 0xf4, 0xce, 0x5e, 0x14, 0xad, 0x8b, 0x04, 0x8f, 0x60, 0xbe, 0x27, + 0x8a, 0xac, 0x5f, 0x8e, 0x6f, 0xed, 0x4e, 0xea, 0xc8, 0x34, 0x9b, 0x5a, 0xb1, 0xe0, 0xe9, 0xd4, + 0x80, 0x84, 0x0d, 0xf4, 0xf2, 0xff, 0x4a, 0x6f, 0x82, 0x4d, 0xa7, 0xc6, 0xe9, 0xe3, 0xfa, 0x2e, + 0x86, 0xf3, 0xa8, 0x7b, 0x67, 0x32, 0x42, 0xa3, 0x58, 0x93, 0x23, 0xbf, 0x38, 0xb0, 0x50, 0x0f, + 0xe7, 0x5a, 0x4b, 0x5c, 0x76, 0xc7, 0x9d, 0xd9, 0x1d, 0x6f, 0x56, 0x77, 0xfc, 0xaa, 0x3b, 0xd5, + 0xef, 0x83, 0xb9, 0xda, 0xef, 0x03, 0x72, 0x0c, 0xf7, 0x2e, 0xb4, 0x6c, 0x4b, 0x0c, 0x47, 0x6a, + 0x36, 0xfe, 0x45, 0xeb, 0xd4, 0x79, 0xcb, 0x32, 0xd3, 0xb4, 0x16, 0xd5, 0x04, 0xf9, 0x14, 0xee, + 0xf6, 0xb8, 0xac, 0x35, 0xcc, 0x4e, 0xde, 0x1a, 0x78, 0xbb, 0xfc, 0xf4, 0x92, 0xf4, 0x15, 0x8b, + 0x7c, 0x01, 0xe1, 0xfe, 0x68, 0xc0, 0x24, 0xbf, 0x91, 0xf6, 0x26, 0x34, 0xf7, 0xc4, 0x48, 0x24, + 0xe2, 0xd5, 0xf8, 0x8a, 0x0b, 0x10, 0xc2, 0xbc, 0xbe, 0xe5, 0xfa, 0xa4, 0xb4, 0xa8, 0x25, 0xc9, + 0x6d, 0x35, 0xdc, 0x7d, 0x96, 0xf4, 0x8b, 0x44, 0x85, 0xa1, 0x7e, 0x3b, 0xe6, 0x9b, 0x4b, 0x7f, + 0x9c, 0xaf, 0x3a, 0x7f, 0x9e, 0xaf, 0x3a, 0x6f, 0xce, 0x57, 0x9d, 0xdf, 0xfe, 0x5a, 0xfd, 0xdf, + 0x61, 0x03, 0xff, 0x83, 0x3c, 0xf9, 0x3b, 0x00, 0x00, 0xff, 0xff, 0x1b, 0xd8, 0x6d, 0x1f, 0x94, + 0x0c, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 1cd88c82c..971bb5f69 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -159,7 +159,7 @@ message ResizeInstruction { Node Node = 2; Node Coordinator = 3; repeated ResizeSource Sources = 4; - Schema Schema = 5; + NodeStatus NodeStatus = 7; ClusterStatus ClusterStatus = 6; } diff --git a/server/cluster_test.go b/server/cluster_test.go index 9227a2b53..90467f4b2 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -206,6 +206,16 @@ func TestClusterResize_AddNode(t *testing.T) { t.Fatal(err) } + // exp is the expected result for the Row queries that follow. + exp := `{"results":[{"attrs":{},"columns":[1,1300000]}]}` + "\n" + + // Verify the data exists on the single node. + if res, err := m0.Query("i", "", `Row(f=1)`); err != nil { + t.Fatal(err) + } else if res != exp { + t.Fatalf("unexpected result: %s", res) + } + // Configure node1 m1 := test.NewCommandNode(false) m1.Config.Gossip.Port = "0" @@ -221,6 +231,18 @@ func TestClusterResize_AddNode(t *testing.T) { } else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) { t.Fatalf("unexpected node1 cluster state: %s", m1.API.State()) } + + // Verify the data exists on both nodes. + if res, err := m0.Query("i", "", `Row(f=1)`); err != nil { + t.Fatal(err) + } else if res != exp { + t.Fatalf("unexpected result: %s", res) + } + if res, err := m1.Query("i", "", `Row(f=1)`); err != nil { + t.Fatal(err) + } else if res != exp { + t.Fatalf("unexpected result: %s", res) + } }) t.Run("SkippedShard", func(t *testing.T) { // Configure node0 @@ -247,6 +269,16 @@ func TestClusterResize_AddNode(t *testing.T) { t.Fatal(err) } + // exp is the expected result for the Row queries that follow. + exp := `{"results":[{"attrs":{},"columns":[1,2400000]}]}` + "\n" + + // Verify the data exists on the single node. + if res, err := m0.Query("i", "", `Row(f=1)`); err != nil { + t.Fatal(err) + } else if res != exp { + t.Fatalf("unexpected result: %s", res) + } + // Configure node1 m1 := test.NewCommandNode(false) m1.Config.Gossip.Port = "0" @@ -262,6 +294,18 @@ func TestClusterResize_AddNode(t *testing.T) { } else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) { t.Fatalf("unexpected node1 cluster state: %s", m1.API.State()) } + + // Verify the data exists on both nodes. + if res, err := m0.Query("i", "", `Row(f=1)`); err != nil { + t.Fatal(err) + } else if res != exp { + t.Fatalf("unexpected result: %s", res) + } + if res, err := m1.Query("i", "", `Row(f=1)`); err != nil { + t.Fatal(err) + } else if res != exp { + t.Fatalf("unexpected result: %s", res) + } }) } diff --git a/utils_internal_test.go b/utils_internal_test.go index ff149e8cf..21d8d0780 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -24,6 +24,7 @@ import ( "time" "github.com/gogo/protobuf/proto" + "github.com/pkg/errors" ) // NewTestCluster returns a cluster with n nodes and uses a mod-based hasher. @@ -371,10 +372,26 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error destCluster := t.clusterByID(instrNode.ID) // Sync the schema received in the resize instruction. - if err := destCluster.holder.applySchema(instr.Schema); err != nil { + if err := destCluster.holder.applySchema(instr.NodeStatus.Schema); err != nil { return err } + // Sync available shards. + for _, is := range instr.NodeStatus.Indexes { + for _, fs := range is.Fields { + f := destCluster.holder.Field(is.Name, fs.Name) + + // if we don't know about a field locally, log an error because + // fields should be created and synced prior to shard creation + if f == nil { + continue + } + if err := f.AddRemoteAvailableShards(fs.AvailableShards); err != nil { + return errors.Wrap(err, "adding remote available shards") + } + } + } + for _, src := range instr.Sources { srcCluster := t.clusterByID(src.Node.ID) From ca2241731d20eceebb0b7176c1571f8a18a1e37a Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 14 Dec 2018 17:29:48 -0600 Subject: [PATCH 090/125] fix tracing message. prevent reallocation of availableShards --- api.go | 2 +- cluster.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/api.go b/api.go index c6a1f3725..d3fb03615 100644 --- a/api.go +++ b/api.go @@ -561,7 +561,7 @@ func (api *API) FragmentBlocks(ctx context.Context, indexName, fieldName, viewNa // FragmentData returns all data in the specified fragment. func (api *API) FragmentData(ctx context.Context, indexName, fieldName, viewName string, shard uint64) (io.WriterTo, error) { - span, _ := tracing.StartSpanFromContext(ctx, "API.FragmentBlocks") + span, _ := tracing.StartSpanFromContext(ctx, "API.FragmentData") defer span.Finish() if err := api.validate(apiFragmentData); err != nil { diff --git a/cluster.go b/cluster.go index c3a6b2f69..e966c57a1 100644 --- a/cluster.go +++ b/cluster.go @@ -1840,12 +1840,14 @@ func (c *cluster) nodeStatus() *NodeStatus { Node: c.Node, Schema: &Schema{Indexes: c.holder.Schema()}, } + var availableShards *roaring.Bitmap for _, idx := range ns.Schema.Indexes { is := &IndexStatus{Name: idx.Name} for _, f := range idx.Fields { - availableShards := roaring.NewBitmap() if field := c.holder.Field(idx.Name, f.Name); field != nil { availableShards = field.AvailableShards() + } else { + availableShards = roaring.NewBitmap() } is.Fields = append(is.Fields, &FieldStatus{ Name: f.Name, From 827e5ea5fcf07385447c2306411dc067668f35e0 Mon Sep 17 00:00:00 2001 From: WaaX Date: Tue, 18 Dec 2018 09:34:52 -0500 Subject: [PATCH 091/125] Removed the mention of measurements Change `measurements` field to `patients index` Fix import command by removing `-f measurements` --- docs/tutorials.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/tutorials.md b/docs/tutorials.md index 47344db4f..4cb8d069c 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -455,7 +455,7 @@ curl localhost:10101/index/patients \ {"success":true} ``` -In addition to storing rows of bits, a field can also store integer values. The next steps creates three fields (`age`, `weight`, `tcells`) in the `measurements` field. +In addition to storing rows of bits, a field can also store integer values. The next steps creates three fields (`age`, `weight`, `tcells`) in the `patients` index. ``` request curl localhost:10101/index/patients/field/age \ -X POST \ @@ -529,7 +529,7 @@ Assuming we have a file called `ages.csv` that is structured like this: ``` where the first column of the CSV represents the patient `ID` and the second column represents the patient's `age`, then we can import the data into our `age` field by running this command: ``` -pilosa import -i patients -f measurements --field age ages.csv +pilosa import -i patients --field age ages.csv ``` Now that we have some data in our index, let's run a few queries to demonstrate how to use that data. From 95f05ca4d00431e981d9d6bfb52335021b5948a7 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 14 Dec 2018 16:47:28 -0600 Subject: [PATCH 092/125] WIP: allow translate log entry buffer to grow In the case where a translate log entry contained many key/id pairs, it was possible for the read buffer (which was allocated at 65536 bytes) to fail to handle it. This happened when the serialized LogEntry was larger than 65536 bytes. This PR adds logic which returns a custom error called ErrTranslateReadTargetUndersized notifying the reader to reallocate a larger read buffer and try the read again. TODO: - [ ] Add a max buffer size check to prevent this from doubling the buffer size with no limit. - [ ] Add tests. --- http/handler.go | 11 ++++++++++- translate.go | 18 ++++++++++++------ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/http/handler.go b/http/handler.go index c370656fa..f8188e8fd 100644 --- a/http/handler.go +++ b/http/handler.go @@ -1449,16 +1449,25 @@ func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request) } // Copy from reader to client until store or client disconnect. - buf := make([]byte, translateStoreBufferSize) + useBufferSize := translateStoreBufferSize + buf := make([]byte, useBufferSize) for { // Read from store. n, err := rdr.Read(buf) if err == io.EOF { return + } else if err == pilosa.ErrTranslateReadTargetUndersized { + // Increase the buffer size and try to read again. + useBufferSize *= 2 + buf = make([]byte, useBufferSize) + continue } else if err != nil { h.logger.Printf("http: translate store read error: %s", err) return } else if n == 0 { + // Reset the default buffer size. + useBufferSize = translateStoreBufferSize + buf = make([]byte, useBufferSize) continue } diff --git a/translate.go b/translate.go index 2c3125ad9..b86041e3d 100644 --- a/translate.go +++ b/translate.go @@ -29,10 +29,11 @@ const ( ) var ( - ErrTranslateStoreClosed = errors.New("pilosa: translate store closed") - ErrTranslateStoreReaderClosed = errors.New("pilosa: translate store reader closed") - ErrReplicationNotSupported = errors.New("pilosa: replication not supported") - ErrTranslateStoreReadOnly = errors.New("pilosa: translate store could not find or create key, translate store read only") + ErrTranslateStoreClosed = errors.New("pilosa: translate store closed") + ErrTranslateStoreReaderClosed = errors.New("pilosa: translate store reader closed") + ErrReplicationNotSupported = errors.New("pilosa: replication not supported") + ErrTranslateStoreReadOnly = errors.New("pilosa: translate store could not find or create key, translate store read only") + ErrTranslateReadTargetUndersized = errors.New("pilosa: translate read target is undersized") ) // TranslateStore is the storage for translation string-to-uint64 values. @@ -1089,8 +1090,13 @@ func (r *translateFileReader) read(p []byte) (n int, err error) { return 0, nil } - // Shorten buffer to maximum read size. - if max := sz - r.offset; int64(len(p)) > max { + if max := sz - r.offset; max > int64(len(p)) { + // If p is not large enough to hold a single entry, + // return an error so the client can increase the + // size of p and try again. + return 0, ErrTranslateReadTargetUndersized + } else if int64(len(p)) > max { + // Shorten buffer to maximum read size. p = p[:max] } From ab34df351d7b13f30bd75983ce492fc217080eb3 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 18 Dec 2018 12:27:32 -0600 Subject: [PATCH 093/125] add Gopkg.lock as a dependency for vendor target --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8fa366137..e7af8e119 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ clean: rm -rf vendor build # Set up vendor directory using `dep` -vendor: Gopkg.toml +vendor: Gopkg.toml Gopkg.lock $(MAKE) require-dep dep ensure -vendor-only touch vendor From 358c32a165ab5ed90cfb1ca070da34682b07773b Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 18 Dec 2018 12:48:20 -0600 Subject: [PATCH 094/125] add test for translate store buffer growth logic. add max limit to buffer size. --- ctl/import_test.go | 65 +++++++++++++++++++++++++++++++++++ http/handler.go | 11 +++++- http/handler_internal_test.go | 3 -- test/pilosa.go | 2 +- 4 files changed, 76 insertions(+), 5 deletions(-) diff --git a/ctl/import_test.go b/ctl/import_test.go index ad4fd6cf0..e701af1b2 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -18,9 +18,11 @@ import ( "bufio" "bytes" "context" + "fmt" "io" "io/ioutil" "net/http" + "reflect" "strings" "testing" @@ -184,6 +186,69 @@ func TestImportCommand_RunKeys(t *testing.T) { } } +// Ensure that import with keys runs with key replication. +func TestImportCommand_KeyReplication(t *testing.T) { + buf := bytes.Buffer{} + stdin, stdout, stderr := GetIO(buf) + cm := NewImportCommand(stdin, stdout, stderr) + file, err := ioutil.TempFile("", "import-key.csv") + + // create a large import file in order to test the + // translateStoreBufferSize growth logic. + keyBytes := []byte{} + for row := 0; row < 100; row++ { + for col := 0; col < 100; col++ { + x := fmt.Sprintf("foo%d,bar%d\n", row, col) + keyBytes = append(keyBytes, x...) + } + } + x := "fooEND,barEND" + keyBytes = append(keyBytes, x...) + + file.Write(keyBytes) + ctx := context.Background() + if err != nil { + t.Fatal(err) + } + + c := test.MustRunCluster(t, 2) + cmd0 := c[0] + cmd1 := c[1] + + host0 := cmd0.API.Node().URI.HostPort() + host1 := cmd1.API.Node().URI.HostPort() + + cm.Host = host0 + + http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader(`{"options":{"keys": true}}`))) + http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"keys": true}}`))) + + cm.Index = "i" + cm.Field = "f" + cm.Paths = []string{file.Name()} + err = cm.Run(ctx) + if err != nil { + t.Fatalf("Import Run with key replication doesn't work: %s", err) + } + + // Verify that the data is available on both nodes. + for _, host := range []string{host0, host1} { + qry := "Count(Row(f=foo0))" + resp, err := http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+host+"/index/i/query", strings.NewReader(qry))) + if err != nil { + t.Fatalf("Querying data for validation: %s", err) + } + + // Read body and unmarshal response. + exp := `{"results":[100]}` + "\n" + if body, err := ioutil.ReadAll(resp.Body); err != nil { + t.Fatalf("reading: %s", err) + } else if !reflect.DeepEqual(body, []byte(exp)) { + t.Fatalf("expected: %s, but got: %s", exp, body) + } + } +} + // Ensure that integer import with keys runs. func TestImportCommand_RunValueKeys(t *testing.T) { buf := bytes.Buffer{} diff --git a/http/handler.go b/http/handler.go index f8188e8fd..b85be6b0e 100644 --- a/http/handler.go +++ b/http/handler.go @@ -1426,7 +1426,11 @@ func (h *Handler) handlePostClusterMessage(w http.ResponseWriter, r *http.Reques type defaultClusterMessageResponse struct{} // translateStoreBufferSize is the buffer size used for streaming data. -const translateStoreBufferSize = 65536 +const translateStoreBufferSize = 1 << 16 // 64k + +// translateStoreBufferSizeMax is the maximum size that the buffer is allowed +// to grow before raising an error. +const translateStoreBufferSizeMax = 1 << 22 // 4Mb func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() @@ -1459,6 +1463,11 @@ func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request) } else if err == pilosa.ErrTranslateReadTargetUndersized { // Increase the buffer size and try to read again. useBufferSize *= 2 + // Prevent the buffer from growing without bound. + if useBufferSize > translateStoreBufferSizeMax { + h.logger.Printf("http: translate store buffer exceeded max size: %s", err) + return + } buf = make([]byte, useBufferSize) continue } else if err != nil { diff --git a/http/handler_internal_test.go b/http/handler_internal_test.go index 7ca9c0401..ecbf8ff87 100644 --- a/http/handler_internal_test.go +++ b/http/handler_internal_test.go @@ -57,7 +57,6 @@ func TestPostIndexRequestUnmarshalJSON(t *testing.T) { t.Errorf("expected: %v, but got: %v for JSON: %s", test.expected, *actual, test.json) } } - } } @@ -93,7 +92,6 @@ func TestPostFieldRequestUnmarshalJSON(t *testing.T) { t.Errorf("test %d: expected: %v, but got: %v", i, test.expected, *actual) } } - } } @@ -178,6 +176,5 @@ func TestFieldOptionValidation(t *testing.T) { t.Errorf("test %d: expected: %v, but got: %v", i, test.expected, *actual) } } - } } diff --git a/test/pilosa.go b/test/pilosa.go index 3bc43d593..e3858f5e0 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -69,7 +69,7 @@ func newCommand(opts ...server.CommandOption) *Command { m.Config.DataDir = path m.Config.Bind = "http://localhost:0" m.Config.Cluster.Disabled = true - m.Config.Translation.MapSize = 100000 + m.Config.Translation.MapSize = 140000 if testing.Verbose() { m.Command.Stdout = os.Stdout From ff800131cb3b2cc8b8643c6722fac471323465a0 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 19 Dec 2018 11:57:47 -0600 Subject: [PATCH 095/125] ensure internal client closes all response bodies to avoid leaking connections/goroutines --- http/client.go | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/http/client.go b/http/client.go index 46800fd70..78382b81b 100644 --- a/http/client.go +++ b/http/client.go @@ -164,7 +164,7 @@ func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilo } return err } - return nil + return errors.Wrap(resp.Body.Close(), "closing response body") } // FragmentNodes returns a list of nodes that own a shard. @@ -705,6 +705,9 @@ func (c *InternalClient) exportNodeCSV(ctx context.Context, node *pilosa.Node, i return nil } +// RetrieveShardFromURI returns a ReadCloser which contains the data of the +// specified shard from the specified node. Caller *must* close the returned +// ReadCloser or risk leaking goroutines/tcp connections. func (c *InternalClient) RetrieveShardFromURI(ctx context.Context, index, field, view string, shard uint64, uri pilosa.URI) (io.ReadCloser, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.RetrieveShardFromURI") defer span.Finish() @@ -800,7 +803,7 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel return err } - return nil + return errors.Wrap(resp.Body.Close(), "closing response body") } // FragmentBlocks returns a list of block checksums for a fragment on a host. @@ -994,15 +997,24 @@ func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, msg [ req.Header.Set("Accept", "application/json") // Execute request. - _, err = c.executeRequest(req.WithContext(ctx)) - return err + resp, err := c.executeRequest(req.WithContext(ctx)) + if err != nil { + return errors.Wrap(err, "executing request") + } + return errors.Wrap(resp.Body.Close(), "closing response body") } -// executeRequest executes the given request and checks the Response +// executeRequest executes the given request and checks the Response. For +// responses with non-2XX status, the body is read and closed, and an error is +// returned. If the error is nil, the caller must ensure that the response body +// is closed. func (c *InternalClient) executeRequest(req *http.Request) (*http.Response, error) { tracing.GlobalTracer.InjectHTTPHeaders(req) resp, err := c.httpClient.Do(req) if err != nil { + if resp != nil { + resp.Body.Close() + } return nil, errors.Wrap(err, "executing request") } if resp.StatusCode < 200 || resp.StatusCode >= 300 { From 31ab3d37b8e17ae6418493b3d9811c03c3992d15 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 19 Dec 2018 17:21:57 -0600 Subject: [PATCH 096/125] add stress tests these produce an issue on master, but it is fixed on this branch --- pilosa.go | 4 +- server/server_test.go | 105 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 1 deletion(-) diff --git a/pilosa.go b/pilosa.go index fe61d8969..550696a31 100644 --- a/pilosa.go +++ b/pilosa.go @@ -56,7 +56,9 @@ var ( ErrQueryTimeout = errors.New("query timeout") ErrTooManyWrites = errors.New("too many write commands") - ErrClusterDoesNotOwnShard = errors.New("cluster does not own shard") + // TODO(2.0) poorly named - used when a *node* doesn't own a shard. Probably + // we won't need this error at all by 2.0 though. + ErrClusterDoesNotOwnShard = errors.New("node does not own shard") ErrNodeIDNotExists = errors.New("node with provided ID does not exist") ErrNodeNotCoordinator = errors.New("node is not the coordinator") diff --git a/server/server_test.go b/server/server_test.go index 627e83862..044ae4720 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -15,8 +15,10 @@ package server_test import ( + "bytes" "context" "encoding/json" + "flag" "fmt" "io/ioutil" "math/rand" @@ -28,13 +30,22 @@ import ( "testing/quick" "time" + "golang.org/x/sync/errgroup" + "github.com/pelletier/go-toml" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/http" + "github.com/pilosa/pilosa/roaring" "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" ) +var runStress bool + +func init() { // nolint: gochecknoinits + flag.BoolVar(&runStress, "stress", false, "Enable stress tests (time consuming)") +} + // Ensure program can process queries and maintain consistency. func TestMain_Set_Quick(t *testing.T) { if testing.Short() { @@ -754,3 +765,97 @@ func TestClusterQueriesAfterRestart(t *testing.T) { } // TODO: confirm that things keep working if a node is hard-closed (no nodeLeave event) and immediately restarted with a different address. + +func TestClusterExhaustingConnections(t *testing.T) { + if !runStress { + t.Skip("stress") + } + cluster := test.MustRunCluster(t, 5) + defer cluster.Close() + cmd1 := cluster[1] + + for _, com := range cluster { + nodes := com.API.Hosts(context.Background()) + for _, n := range nodes { + if n.State != "READY" { + t.Fatalf("unexpected node state after upping cluster: %v", nodes) + } + } + } + + cmd1.MustCreateIndex(t, "testidx", pilosa.IndexOptions{}) + cmd1.MustCreateField(t, "testidx", "testfield", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 10)) + + eg := errgroup.Group{} + for i := 0; i < 20; i++ { + i := i + eg.Go(func() error { + for j := i; j < 10000; j += 20 { + _, err := cluster[i%5].API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "testidx", + Query: fmt.Sprintf("Set(%d, testfield=0)", j*pilosa.ShardWidth), + }) + if err != nil { + return err + } + } + return nil + }) + } + err := eg.Wait() + if err != nil { + t.Fatalf("setting lots of shards: %v", err) + } +} + +func TestClusterExhaustingConnectionsImport(t *testing.T) { + if !runStress { + t.Skip("stress") + } + cluster := test.MustRunCluster(t, 5) + defer cluster.Close() + cmd1 := cluster[1] + + for _, com := range cluster { + nodes := com.API.Hosts(context.Background()) + for _, n := range nodes { + if n.State != "READY" { + t.Fatalf("unexpected node state after upping cluster: %v", nodes) + } + } + } + + cmd1.MustCreateIndex(t, "testidx", pilosa.IndexOptions{}) + cmd1.MustCreateField(t, "testidx", "testfield", pilosa.OptFieldTypeSet(pilosa.CacheTypeRanked, 10)) + + bm := roaring.NewBitmap() + bm.DirectAdd(0) + buf := &bytes.Buffer{} + bm.WriteTo(buf) + data := buf.Bytes() + + eg := errgroup.Group{} + for i := uint64(0); i < 20; i++ { + i := i + eg.Go(func() error { + for j := i; j < 10000; j += 20 { + if (j-i)%1000 == 0 { + fmt.Printf("%d is %.2f%% done.\n", i, float64(j-i)*100/100000) + } + err := cluster[i%5].API.ImportRoaring(context.Background(), "testidx", "testfield", j, false, &pilosa.ImportRoaringRequest{ + Views: map[string][]byte{ + "": data, + }, + }) + if err != nil { + return err + } + } + return nil + }) + } + err := eg.Wait() + if err != nil { + t.Fatalf("setting lots of shards: %v", err) + } +} From 78ba259d26b1ba6bf6d5994eed24c93310dce40d Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 18 Dec 2018 17:00:50 -0600 Subject: [PATCH 097/125] Release v1.2.0 --- CHANGELOG.md | 87 ++++++++++++++++++++++++++++++++++++++++++++ Dockerfile | 2 +- docs/installation.md | 22 +++++------ 3 files changed, 99 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca283b2c7..8c369ccb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,92 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [v1.2.0] - 2018-12-18 + +This version contains 154 contributions from 11 contributors. There are 113 files changed; 18,968 insertions; and 4,325 deletions. + +### Added + +- Cancel queries on Context.Done() ([#1773](https://github.com/pilosa/pilosa/pull/1773)) +- Union In Place ([#1766](https://github.com/pilosa/pilosa/pull/1766)), ([#1774](https://github.com/pilosa/pilosa/pull/1774)) +- Import benchmarking ([#1771](https://github.com/pilosa/pilosa/pull/1771)) +- Add GroupBy() Filter ([#1753](https://github.com/pilosa/pilosa/pull/1753)) +- Added /internal/translate/keys endpoint ([#1751](https://github.com/pilosa/pilosa/pull/1751)) +- CircleCI: Add race detector to parallel build. Default to Go 1.11. ([#1756](https://github.com/pilosa/pilosa/pull/1756)) +- Add distributed tracing. ([#1684](https://github.com/pilosa/pilosa/pull/1684)) +- Adds NoStandardView field option. Fixes #1710 ([#1733](https://github.com/pilosa/pilosa/pull/1733)) +- Import roaring endpoint accepts a list of views ([#1738](https://github.com/pilosa/pilosa/pull/1738)) +- Add some stat tracking to roaring/ implementation. ([#1743](https://github.com/pilosa/pilosa/pull/1743)) +- Cluster tests ([#1717](https://github.com/pilosa/pilosa/pull/1717)) +- Allow backslash, carriage return in PQL strings ([#1713](https://github.com/pilosa/pilosa/pull/1713)) +- Add base system, curl and jq for debug and checks. ([#1707](https://github.com/pilosa/pilosa/pull/1707)) +- Add "Rows" and "GroupBy" functionality ([#1647](https://github.com/pilosa/pilosa/pull/1647)) +- Add `clear` functional option for imports ([#1699](https://github.com/pilosa/pilosa/pull/1699)) +- Maintain Available Shards ([#1600](https://github.com/pilosa/pilosa/pull/1600)), ([#1695](https://github.com/pilosa/pilosa/pull/1695)), ([#1624](https://github.com/pilosa/pilosa/pull/1624)), ([#1663](https://github.com/pilosa/pilosa/pull/1663)) +- Adds missing rowID/Key columnID/Key tests ([#1683](https://github.com/pilosa/pilosa/pull/1683)) +- Store() function ([#1666](https://github.com/pilosa/pilosa/pull/1666)) +- Adds diagnostics CPUArch field ([#1671](https://github.com/pilosa/pilosa/pull/1671)) +- Add CircleCI step to generate Docker image and push to Docker hub ([#1673](https://github.com/pilosa/pilosa/pull/1673)) +- Implement ClearRow() query ([#1645](https://github.com/pilosa/pilosa/pull/1645)) +- Add support for Bool fields ([#1658](https://github.com/pilosa/pilosa/pull/1658)) +- Make translate map size configurable ([#1653](https://github.com/pilosa/pilosa/pull/1653)) +- Adds DirectAdd function to roaring.Bitmap ([#1646](https://github.com/pilosa/pilosa/pull/1646)) +- Roaring import ([#1622](https://github.com/pilosa/pilosa/pull/1622)) +- Add Not() query ([#1635](https://github.com/pilosa/pilosa/pull/1635)) +- Implements Options call and excludeRowAttrs, excludeColumns, columnAttrs and shards args ([#1631](https://github.com/pilosa/pilosa/pull/1631)) +- Added field options to pilosa import ([#1625](https://github.com/pilosa/pilosa/pull/1625)) +- Existence Tracking ([#1788](https://github.com/pilosa/pilosa/pull/1788)), ([#1672](https://github.com/pilosa/pilosa/pull/1672)), ([#1628](https://github.com/pilosa/pilosa/pull/1628)) + +### Changed + +- Convert the anti-entropy logic to use `ImportRoaring` instead of `QueryNode` ([#1780](https://github.com/pilosa/pilosa/pull/1780)) +- Simplify "require-*" logic in Makefile ([#1755](https://github.com/pilosa/pilosa/pull/1755)) +- Logging cleanup ([#1748](https://github.com/pilosa/pilosa/pull/1748)) +- Improve benchmarking and performance. ([#1741](https://github.com/pilosa/pilosa/pull/1741)) +- Remove TravisCI, add CircleCI shield ([#1740](https://github.com/pilosa/pilosa/pull/1740)) +- Upgrade Peg dependency and regenerate grammar ([#1725](https://github.com/pilosa/pilosa/pull/1725)) +- Upgrade to protoc 3.6.1. (also updated protoc-gen-gofast). ([#1724](https://github.com/pilosa/pilosa/pull/1724)) +- Move column attrs logic to executor ([#1677](https://github.com/pilosa/pilosa/pull/1677)) +- Shrank n(container bit count cache) to int32 ([#1664](https://github.com/pilosa/pilosa/pull/1664)) +- Removing bounds check ([#1619](https://github.com/pilosa/pilosa/pull/1619)) + + +### Fixed +- Allow translate log entry buffer to grow ([#1787](https://github.com/pilosa/pilosa/pull/1787)) +- Add Gopkg.lock as a dependency for vendor target ([#1790](https://github.com/pilosa/pilosa/pull/1790)) +- Cluster resize fix ([#1785](https://github.com/pilosa/pilosa/pull/1785)) +- Attempt to fix deadlock by releasing view lock before broadcasting Cr… ([#1782](https://github.com/pilosa/pilosa/pull/1782)) +- Fix bug where cluster goes into RESIZING instead of NORMAL ([#1777](https://github.com/pilosa/pilosa/pull/1777)) +- Wrap `` in backquotes so it gets displayed. ([#1779](https://github.com/pilosa/pilosa/pull/1779)) +- Propogate updates to node details (not just additions and deletions) ([#1769](https://github.com/pilosa/pilosa/pull/1769)) +- Fix arm64 support ([#1764](https://github.com/pilosa/pilosa/pull/1764)) +- More races ([#1750](https://github.com/pilosa/pilosa/pull/1750)) +- Import cmd field type flag ([#1732](https://github.com/pilosa/pilosa/pull/1732)) +- Increase the translate file size for tests/benchmarks ([#1744](https://github.com/pilosa/pilosa/pull/1744)) +- Prevent panic in Bitmap.UnmarshalBinary when there is no data ([#1742](https://github.com/pilosa/pilosa/pull/1742)) +- Removed unused rule from peg grammar ([#1737](https://github.com/pilosa/pilosa/pull/1737)) +- Improve Internal Client errors. Fixes #1697 ([#1729](https://github.com/pilosa/pilosa/pull/1729)) +- Forward imports to non-coordinator shards ([#1719](https://github.com/pilosa/pilosa/pull/1719)) +- Fix double escapes in PQL grammar ([#1727](https://github.com/pilosa/pilosa/pull/1727)) +- Ensure btree comparison doesn't fail for smallish N ([#1712](https://github.com/pilosa/pilosa/pull/1712)) +- Drop now-superfluous methodNotAllowedHandler ([#1711](https://github.com/pilosa/pilosa/pull/1711)) +- Use pilosa.Logger everywhere ([#1674](https://github.com/pilosa/pilosa/pull/1674)) +- Ensure view closes fragment on broadcast error ([#1675](https://github.com/pilosa/pilosa/pull/1675)) +- Prevent closing os.Stderr (used in verbose test logging) ([#1696](https://github.com/pilosa/pilosa/pull/1696)) +- Allow holder to close/open/close without panic on closing closed channel ([#1686](https://github.com/pilosa/pilosa/pull/1686)) +- Fix bug with Range() queries with field keys ([#1679](https://github.com/pilosa/pilosa/pull/1679)) +- Synced query validation for handlers ([#1676](https://github.com/pilosa/pilosa/pull/1676)) +- Wrap translation store errors, decrease test map size to prevent failure on 32-bit ([#1665](https://github.com/pilosa/pilosa/pull/1665)) +- Fixes pass-by-value issue in proto decode ([#1662](https://github.com/pilosa/pilosa/pull/1662)) +- Do not run prerelease in CI if this is a pull request ([#1655](https://github.com/pilosa/pilosa/pull/1655)) +- Ensure mutex imports unset previous columns ([#1656](https://github.com/pilosa/pilosa/pull/1656)) +- Treat import timestamps as UTC ([#1651](https://github.com/pilosa/pilosa/pull/1651)) +- Remove unused log buffers from test cluster, fixes race ([#1612](https://github.com/pilosa/pilosa/pull/1612)) +- Adds --field-keys and --index-keys options to pilosa import ([#1621](https://github.com/pilosa/pilosa/pull/1621)) +- Use passed stdin, stdout and stderr in the cmd package. Fixes #1538 ([#1620](https://github.com/pilosa/pilosa/pull/1620)) +- Updated Go client sample to match latest master ([#1614](https://github.com/pilosa/pilosa/pull/1614)) + + ## [v1.1.0] - 2018-08-21 This version contains 32 contributions from 5 contributors. There are 89 files changed; 2,752 insertions; and 1,013 deletions. @@ -28,6 +114,7 @@ This version contains 32 contributions from 5 contributors. There are 89 files c - Add view parameter to sync logic for syncing time fields ([#1602](https://github.com/pilosa/pilosa/pull/1602)) - Fix translator in cluster environment ([#1552](https://github.com/pilosa/pilosa/pull/1552)) - Use string prefix instead of equality so json error message will pass on all Go versions ([#1558](https://github.com/pilosa/pilosa/pull/1558)) +- Fix fragment checksums race condition ([#1749](https://github.com/pilosa/pilosa/pull/1749)) ## [v1.0.2] - 2018-08-01 diff --git a/Dockerfile b/Dockerfile index e4f057564..02454a9c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.10.3 as builder +FROM golang:1.11.4 as builder COPY . /go/src/github.com/pilosa/pilosa/ diff --git a/docs/installation.md b/docs/installation.md index b2e94d1a0..9bd6f9bef 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -42,7 +42,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) backing up, and more. Complete documentation is available at https://www.pilosa.com/docs/. - Version: v1.1.0 + Version: v1.2.0 Build Time: 2018-05-14T22:14:01+0000 Usage: @@ -71,19 +71,19 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) 1. Download the latest release: ``` - curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.1.0/pilosa-v1.1.0-darwin-amd64.tar.gz + curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.2.0/pilosa-v1.2.0-darwin-amd64.tar.gz ``` Other releases can be downloaded from our Releases page on Github. 2. Extract the binary: ``` - tar xfz pilosa-v1.1.0-darwin-amd64.tar.gz + tar xfz pilosa-v1.2.0-darwin-amd64.tar.gz ``` 3. Move the binary into your PATH so you can run `pilosa` from any shell: ``` - cp -i pilosa-v1.1.0-darwin-amd64/pilosa /usr/local/bin + cp -i pilosa-v1.2.0-darwin-amd64/pilosa /usr/local/bin ``` 4. Make sure Pilosa is installed successfully: @@ -100,7 +100,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) backing up, and more. Complete documentation is available at https://www.pilosa.com/docs/. - Version: v1.1.0 + Version: v1.2.0 Build Time: 2018-05-14T22:14:01+0000 Usage: @@ -163,7 +163,7 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) backing up, and more. Complete documentation is available at https://www.pilosa.com/docs/. - Version: v1.1.0 + Version: v1.2.0 Build Time: 2018-05-14T22:14:01+0000 Usage: @@ -222,19 +222,19 @@ There are three ways to install Pilosa on Linux: download the binary (recommende 1. To install the latest version of Pilosa, download the latest release: ``` - curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.1.0/pilosa-v1.1.0-linux-amd64.tar.gz + curl -L -O https://github.com/pilosa/pilosa/releases/download/v1.2.0/pilosa-v1.2.0-linux-amd64.tar.gz ``` Note: This assumes you are using an `amd64` compatible architecture. Other releases can be downloaded from our Releases page on Github. 2. Extract the binary: ``` - tar xfz pilosa-v1.1.0-linux-amd64.tar.gz + tar xfz pilosa-v1.2.0-linux-amd64.tar.gz ``` 3. Move the binary into your PATH so you can run `pilosa` from any shell: ``` - cp -i pilosa-v1.1.0-linux-amd64/pilosa /usr/local/bin + cp -i pilosa-v1.2.0-linux-amd64/pilosa /usr/local/bin ``` 4. Make sure Pilosa is installed successfully: @@ -251,7 +251,7 @@ There are three ways to install Pilosa on Linux: download the binary (recommende backing up, and more. Complete documentation is available at https://www.pilosa.com/docs/. - Version: v1.1.0 + Version: v1.2.0 Build Time: 2018-05-14T22:14:01+0000 Usage: @@ -314,7 +314,7 @@ There are three ways to install Pilosa on Linux: download the binary (recommende backing up, and more. Complete documentation is available at https://www.pilosa.com/docs/. - Version: v1.1.0 + Version: v1.2.0 Build Time: 2018-05-14T22:14:01+0000 Usage: From fc09b180c3a2580bc5338141ad9addf3f0579172 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 19 Dec 2018 12:45:15 -0600 Subject: [PATCH 098/125] fix a bunch of changelog stuff post-review --- CHANGELOG.md | 77 +++++++++++++++++++++++++++------------------------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c369ccb1..53fd811aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,71 +5,74 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -## [v1.2.0] - 2018-12-18 +## [1.2.0] - 2018-12-19 This version contains 154 contributions from 11 contributors. There are 113 files changed; 18,968 insertions; and 4,325 deletions. ### Added - Cancel queries on Context.Done() ([#1773](https://github.com/pilosa/pilosa/pull/1773)) -- Union In Place ([#1766](https://github.com/pilosa/pilosa/pull/1766)), ([#1774](https://github.com/pilosa/pilosa/pull/1774)) +- Union In Place ([#1766](https://github.com/pilosa/pilosa/pull/1766), [#1774](https://github.com/pilosa/pilosa/pull/1774)) - Import benchmarking ([#1771](https://github.com/pilosa/pilosa/pull/1771)) - Add GroupBy() Filter ([#1753](https://github.com/pilosa/pilosa/pull/1753)) -- Added /internal/translate/keys endpoint ([#1751](https://github.com/pilosa/pilosa/pull/1751)) -- CircleCI: Add race detector to parallel build. Default to Go 1.11. ([#1756](https://github.com/pilosa/pilosa/pull/1756)) +- Add /internal/translate/keys endpoint ([#1751](https://github.com/pilosa/pilosa/pull/1751)) +- CircleCI: Add race detector to parallel build, default to Go 1.11. ([#1756](https://github.com/pilosa/pilosa/pull/1756)) - Add distributed tracing. ([#1684](https://github.com/pilosa/pilosa/pull/1684)) -- Adds NoStandardView field option. Fixes #1710 ([#1733](https://github.com/pilosa/pilosa/pull/1733)) +- Add NoStandardView field option ([#1733](https://github.com/pilosa/pilosa/pull/1733)) - Import roaring endpoint accepts a list of views ([#1738](https://github.com/pilosa/pilosa/pull/1738)) -- Add some stat tracking to roaring/ implementation. ([#1743](https://github.com/pilosa/pilosa/pull/1743)) +- Add some stat tracking to roaring implementation ([#1743](https://github.com/pilosa/pilosa/pull/1743)) - Cluster tests ([#1717](https://github.com/pilosa/pilosa/pull/1717)) - Allow backslash, carriage return in PQL strings ([#1713](https://github.com/pilosa/pilosa/pull/1713)) -- Add base system, curl and jq for debug and checks. ([#1707](https://github.com/pilosa/pilosa/pull/1707)) +- Add base system, curl and jq for debug and checks ([#1707](https://github.com/pilosa/pilosa/pull/1707)) - Add "Rows" and "GroupBy" functionality ([#1647](https://github.com/pilosa/pilosa/pull/1647)) - Add `clear` functional option for imports ([#1699](https://github.com/pilosa/pilosa/pull/1699)) -- Maintain Available Shards ([#1600](https://github.com/pilosa/pilosa/pull/1600)), ([#1695](https://github.com/pilosa/pilosa/pull/1695)), ([#1624](https://github.com/pilosa/pilosa/pull/1624)), ([#1663](https://github.com/pilosa/pilosa/pull/1663)) -- Adds missing rowID/Key columnID/Key tests ([#1683](https://github.com/pilosa/pilosa/pull/1683)) -- Store() function ([#1666](https://github.com/pilosa/pilosa/pull/1666)) -- Adds diagnostics CPUArch field ([#1671](https://github.com/pilosa/pilosa/pull/1671)) +- Implement tracking of available shards to help support sparse datasets ([#1600](https://github.com/pilosa/pilosa/pull/1600), [#1695](https://github.com/pilosa/pilosa/pull/1695), [#1624](https://github.com/pilosa/pilosa/pull/1624), [#1663](https://github.com/pilosa/pilosa/pull/1663)) +- Add missing rowID/Key columnID/Key tests ([#1683](https://github.com/pilosa/pilosa/pull/1683)) +- Add Store() operation to PQL ([#1666](https://github.com/pilosa/pilosa/pull/1666)) +- Add diagnostics CPUArch field ([#1671](https://github.com/pilosa/pilosa/pull/1671)) - Add CircleCI step to generate Docker image and push to Docker hub ([#1673](https://github.com/pilosa/pilosa/pull/1673)) - Implement ClearRow() query ([#1645](https://github.com/pilosa/pilosa/pull/1645)) - Add support for Bool fields ([#1658](https://github.com/pilosa/pilosa/pull/1658)) - Make translate map size configurable ([#1653](https://github.com/pilosa/pilosa/pull/1653)) -- Adds DirectAdd function to roaring.Bitmap ([#1646](https://github.com/pilosa/pilosa/pull/1646)) +- Add DirectAdd function to roaring.Bitmap ([#1646](https://github.com/pilosa/pilosa/pull/1646)) - Roaring import ([#1622](https://github.com/pilosa/pilosa/pull/1622)) - Add Not() query ([#1635](https://github.com/pilosa/pilosa/pull/1635)) -- Implements Options call and excludeRowAttrs, excludeColumns, columnAttrs and shards args ([#1631](https://github.com/pilosa/pilosa/pull/1631)) -- Added field options to pilosa import ([#1625](https://github.com/pilosa/pilosa/pull/1625)) -- Existence Tracking ([#1788](https://github.com/pilosa/pilosa/pull/1788)), ([#1672](https://github.com/pilosa/pilosa/pull/1672)), ([#1628](https://github.com/pilosa/pilosa/pull/1628)) +- Implement Options call and excludeRowAttrs, excludeColumns, columnAttrs and shards args ([#1631](https://github.com/pilosa/pilosa/pull/1631)) +- Add field options to pilosa import ([#1625](https://github.com/pilosa/pilosa/pull/1625)) +- Implement column existence tracking ([#1788](https://github.com/pilosa/pilosa/pull/1788), [#1672](https://github.com/pilosa/pilosa/pull/1672), [#1628](https://github.com/pilosa/pilosa/pull/1628)) ### Changed - Convert the anti-entropy logic to use `ImportRoaring` instead of `QueryNode` ([#1780](https://github.com/pilosa/pilosa/pull/1780)) - Simplify "require-*" logic in Makefile ([#1755](https://github.com/pilosa/pilosa/pull/1755)) -- Logging cleanup ([#1748](https://github.com/pilosa/pilosa/pull/1748)) -- Improve benchmarking and performance. ([#1741](https://github.com/pilosa/pilosa/pull/1741)) +- Cleanup logging ([#1748](https://github.com/pilosa/pilosa/pull/1748)) - Remove TravisCI, add CircleCI shield ([#1740](https://github.com/pilosa/pilosa/pull/1740)) - Upgrade Peg dependency and regenerate grammar ([#1725](https://github.com/pilosa/pilosa/pull/1725)) -- Upgrade to protoc 3.6.1. (also updated protoc-gen-gofast). ([#1724](https://github.com/pilosa/pilosa/pull/1724)) +- Upgrade to protoc 3.6.1 (also updated protoc-gen-gofast) ([#1724](https://github.com/pilosa/pilosa/pull/1724)) - Move column attrs logic to executor ([#1677](https://github.com/pilosa/pilosa/pull/1677)) - Shrank n(container bit count cache) to int32 ([#1664](https://github.com/pilosa/pilosa/pull/1664)) -- Removing bounds check ([#1619](https://github.com/pilosa/pilosa/pull/1619)) +### Performance + +- Remove bounds check ([#1619](https://github.com/pilosa/pilosa/pull/1619)) +- Improve benchmarking and performance ([#1741](https://github.com/pilosa/pilosa/pull/1741)) ### Fixed + - Allow translate log entry buffer to grow ([#1787](https://github.com/pilosa/pilosa/pull/1787)) - Add Gopkg.lock as a dependency for vendor target ([#1790](https://github.com/pilosa/pilosa/pull/1790)) - Cluster resize fix ([#1785](https://github.com/pilosa/pilosa/pull/1785)) -- Attempt to fix deadlock by releasing view lock before broadcasting Cr… ([#1782](https://github.com/pilosa/pilosa/pull/1782)) +- Attempt to fix deadlock by releasing view lock before broadcasting ([#1782](https://github.com/pilosa/pilosa/pull/1782)) - Fix bug where cluster goes into RESIZING instead of NORMAL ([#1777](https://github.com/pilosa/pilosa/pull/1777)) -- Wrap `` in backquotes so it gets displayed. ([#1779](https://github.com/pilosa/pilosa/pull/1779)) - Propogate updates to node details (not just additions and deletions) ([#1769](https://github.com/pilosa/pilosa/pull/1769)) - Fix arm64 support ([#1764](https://github.com/pilosa/pilosa/pull/1764)) -- More races ([#1750](https://github.com/pilosa/pilosa/pull/1750)) +- Fix data races ([#1750](https://github.com/pilosa/pilosa/pull/1750)) +- Fix fragment checksums race condition ([#1749](https://github.com/pilosa/pilosa/pull/1749)) - Import cmd field type flag ([#1732](https://github.com/pilosa/pilosa/pull/1732)) - Increase the translate file size for tests/benchmarks ([#1744](https://github.com/pilosa/pilosa/pull/1744)) - Prevent panic in Bitmap.UnmarshalBinary when there is no data ([#1742](https://github.com/pilosa/pilosa/pull/1742)) -- Removed unused rule from peg grammar ([#1737](https://github.com/pilosa/pilosa/pull/1737)) -- Improve Internal Client errors. Fixes #1697 ([#1729](https://github.com/pilosa/pilosa/pull/1729)) +- Remove unused rule from peg grammar ([#1737](https://github.com/pilosa/pilosa/pull/1737)) +- Improve Internal Client errors ([#1729](https://github.com/pilosa/pilosa/pull/1729)) - Forward imports to non-coordinator shards ([#1719](https://github.com/pilosa/pilosa/pull/1719)) - Fix double escapes in PQL grammar ([#1727](https://github.com/pilosa/pilosa/pull/1727)) - Ensure btree comparison doesn't fail for smallish N ([#1712](https://github.com/pilosa/pilosa/pull/1712)) @@ -79,19 +82,19 @@ This version contains 154 contributions from 11 contributors. There are 113 file - Prevent closing os.Stderr (used in verbose test logging) ([#1696](https://github.com/pilosa/pilosa/pull/1696)) - Allow holder to close/open/close without panic on closing closed channel ([#1686](https://github.com/pilosa/pilosa/pull/1686)) - Fix bug with Range() queries with field keys ([#1679](https://github.com/pilosa/pilosa/pull/1679)) -- Synced query validation for handlers ([#1676](https://github.com/pilosa/pilosa/pull/1676)) +- Sync query validation for handlers ([#1676](https://github.com/pilosa/pilosa/pull/1676)) - Wrap translation store errors, decrease test map size to prevent failure on 32-bit ([#1665](https://github.com/pilosa/pilosa/pull/1665)) -- Fixes pass-by-value issue in proto decode ([#1662](https://github.com/pilosa/pilosa/pull/1662)) +- Fix pass-by-value issue in proto decode ([#1662](https://github.com/pilosa/pilosa/pull/1662)) - Do not run prerelease in CI if this is a pull request ([#1655](https://github.com/pilosa/pilosa/pull/1655)) - Ensure mutex imports unset previous columns ([#1656](https://github.com/pilosa/pilosa/pull/1656)) - Treat import timestamps as UTC ([#1651](https://github.com/pilosa/pilosa/pull/1651)) - Remove unused log buffers from test cluster, fixes race ([#1612](https://github.com/pilosa/pilosa/pull/1612)) -- Adds --field-keys and --index-keys options to pilosa import ([#1621](https://github.com/pilosa/pilosa/pull/1621)) -- Use passed stdin, stdout and stderr in the cmd package. Fixes #1538 ([#1620](https://github.com/pilosa/pilosa/pull/1620)) -- Updated Go client sample to match latest master ([#1614](https://github.com/pilosa/pilosa/pull/1614)) +- Add --field-keys and --index-keys options to pilosa import ([#1621](https://github.com/pilosa/pilosa/pull/1621)) +- Use passed stdin, stdout, and stderr in the cmd package ([#1620](https://github.com/pilosa/pilosa/pull/1620)) +- Update Go client sample to match latest master ([#1614](https://github.com/pilosa/pilosa/pull/1614)) -## [v1.1.0] - 2018-08-21 +## [1.1.0] - 2018-08-21 This version contains 32 contributions from 5 contributors. There are 89 files changed; 2,752 insertions; and 1,013 deletions. @@ -114,9 +117,8 @@ This version contains 32 contributions from 5 contributors. There are 89 files c - Add view parameter to sync logic for syncing time fields ([#1602](https://github.com/pilosa/pilosa/pull/1602)) - Fix translator in cluster environment ([#1552](https://github.com/pilosa/pilosa/pull/1552)) - Use string prefix instead of equality so json error message will pass on all Go versions ([#1558](https://github.com/pilosa/pilosa/pull/1558)) -- Fix fragment checksums race condition ([#1749](https://github.com/pilosa/pilosa/pull/1749)) -## [v1.0.2] - 2018-08-01 +## [1.0.2] - 2018-08-01 This version contains 11 contributions from 3 contributors. There are 30 files changed; 1,569 insertions; and 1,215 deletions. @@ -131,7 +133,7 @@ This version contains 11 contributions from 3 contributors. There are 30 files c - Re-export erroneously unexported func Row.Intersect ([#1502](https://github.com/pilosa/pilosa/pull/1502)) - Update parser to handle row keys on SetRowAttrs() ([#1555](https://github.com/pilosa/pilosa/pull/1555)) -## [v1.0.1] - 2018-07-11 +## [1.0.1] - 2018-07-11 This version contains 12 contributions from 4 contributors. There are 11 files changed; 133 insertions; and 39 deletions. @@ -143,7 +145,7 @@ This version contains 12 contributions from 4 contributors. There are 11 files c - Add gossip Closer ([#1483](https://github.com/pilosa/pilosa/pull/1483)) - Update docs references to WebUI naming (console) and installation ([#1493](https://github.com/pilosa/pilosa/pull/1493)) -## [v1.0.0] - 2018-07-09 +## [1.0.0] - 2018-07-09 This version contains 218 contributions from 7 contributors. There are 184 files changed; 21,769 insertions; and 20,275 deletions. @@ -191,7 +193,7 @@ This version contains 218 contributions from 7 contributors. There are 184 files - Allow dashes in frame names ([#1415](https://github.com/pilosa/pilosa/pull/1415)) - Fix generate-config command, use single toml lib ([#1350](https://github.com/pilosa/pilosa/pull/1350)) -## [v0.10.0] - 2018-05-15 +## [0.10.0] - 2018-05-15 This version contains 93 contributions from 8 contributors. There are 93 files changed; 4,495 insertions; and 5,392 deletions. @@ -222,7 +224,7 @@ This version contains 93 contributions from 8 contributors. There are 93 files c - Avoid creating a slice of nil timestamps on Import() ([#1234](https://github.com/pilosa/pilosa/pull/1234)) - Fixup internal client ([#1253](https://github.com/pilosa/pilosa/pull/1253)) -## [v0.9.0] - 2018-05-04 +## [0.9.0] - 2018-05-04 This version contains 188 contributions from 12 contributors. There are 141 files changed; 17,832 insertions; and 7,503 deletions. @@ -576,7 +578,7 @@ This version contains 53 contributions from 13 contributors (including 4 volunte - Rewrite intersectCountArrayBitmap for perf test ([#577](https://github.com/pilosa/pilosa/pull/577)) - Check for duplicate attributes under read lock on insert ([#562](https://github.com/pilosa/pilosa/pull/562)) -[Unreleased]: https://github.com/pilosa/pilosa/compare/v0.5...HEAD +[Unreleased]: https://github.com/pilosa/pilosa/compare/v1.2...HEAD [0.4.0]: https://github.com/pilosa/pilosa/compare/v0.3...v0.4 [0.5.0]: https://github.com/pilosa/pilosa/compare/v0.4...v0.5 [0.6.0]: https://github.com/pilosa/pilosa/compare/v0.5...v0.6 @@ -586,3 +588,4 @@ This version contains 53 contributions from 13 contributors (including 4 volunte [0.10.0]: https://github.com/pilosa/pilosa/compare/v0.9...v0.10 [1.0.0]: https://github.com/pilosa/pilosa/compare/v0.10...v1.0 [1.1.0]: https://github.com/pilosa/pilosa/compare/v1.0...v1.1 +[1.2.0]: https://github.com/pilosa/pilosa/compare/v1.1...v1.2 From 100cabf8f20fc5a41497f6f268e650c1d01cd77a Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 20 Dec 2018 10:54:37 -0600 Subject: [PATCH 099/125] add final change to the changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53fd811aa..ee9476c48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## [1.2.0] - 2018-12-19 -This version contains 154 contributions from 11 contributors. There are 113 files changed; 18,968 insertions; and 4,325 deletions. +This version contains 155 contributions from 11 contributors. There are 113 files changed; 19,085 insertions; and 4,323 deletions. ### Added @@ -59,6 +59,7 @@ This version contains 154 contributions from 11 contributors. There are 113 file ### Fixed +- Ensure internal client closes all response bodies ([#1795](https://github.com/pilosa/pilosa/pull/1795)) - Allow translate log entry buffer to grow ([#1787](https://github.com/pilosa/pilosa/pull/1787)) - Add Gopkg.lock as a dependency for vendor target ([#1790](https://github.com/pilosa/pilosa/pull/1790)) - Cluster resize fix ([#1785](https://github.com/pilosa/pilosa/pull/1785)) From 1ebe68fd182051f3a3c392325c96ad4cf6dc62ad Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 20 Dec 2018 11:13:00 -0600 Subject: [PATCH 100/125] code review fixup for changelog --- CHANGELOG.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee9476c48..b7b558ac7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -## [1.2.0] - 2018-12-19 +## [1.2.0] - 2018-12-20 This version contains 155 contributions from 11 contributors. There are 113 files changed; 19,085 insertions; and 4,323 deletions. @@ -19,12 +19,11 @@ This version contains 155 contributions from 11 contributors. There are 113 file - CircleCI: Add race detector to parallel build, default to Go 1.11. ([#1756](https://github.com/pilosa/pilosa/pull/1756)) - Add distributed tracing. ([#1684](https://github.com/pilosa/pilosa/pull/1684)) - Add NoStandardView field option ([#1733](https://github.com/pilosa/pilosa/pull/1733)) -- Import roaring endpoint accepts a list of views ([#1738](https://github.com/pilosa/pilosa/pull/1738)) - Add some stat tracking to roaring implementation ([#1743](https://github.com/pilosa/pilosa/pull/1743)) -- Cluster tests ([#1717](https://github.com/pilosa/pilosa/pull/1717)) +- Add cluster fault testing using docker-compose and pumba ([#1717](https://github.com/pilosa/pilosa/pull/1717)) - Allow backslash, carriage return in PQL strings ([#1713](https://github.com/pilosa/pilosa/pull/1713)) - Add base system, curl and jq for debug and checks ([#1707](https://github.com/pilosa/pilosa/pull/1707)) -- Add "Rows" and "GroupBy" functionality ([#1647](https://github.com/pilosa/pilosa/pull/1647)) +- Add `Rows` and `GroupBy` functionality ([#1647](https://github.com/pilosa/pilosa/pull/1647)) - Add `clear` functional option for imports ([#1699](https://github.com/pilosa/pilosa/pull/1699)) - Implement tracking of available shards to help support sparse datasets ([#1600](https://github.com/pilosa/pilosa/pull/1600), [#1695](https://github.com/pilosa/pilosa/pull/1695), [#1624](https://github.com/pilosa/pilosa/pull/1624), [#1663](https://github.com/pilosa/pilosa/pull/1663)) - Add missing rowID/Key columnID/Key tests ([#1683](https://github.com/pilosa/pilosa/pull/1683)) @@ -35,7 +34,7 @@ This version contains 155 contributions from 11 contributors. There are 113 file - Add support for Bool fields ([#1658](https://github.com/pilosa/pilosa/pull/1658)) - Make translate map size configurable ([#1653](https://github.com/pilosa/pilosa/pull/1653)) - Add DirectAdd function to roaring.Bitmap ([#1646](https://github.com/pilosa/pilosa/pull/1646)) -- Roaring import ([#1622](https://github.com/pilosa/pilosa/pull/1622)) +- Implement Roaring import ([#1622](https://github.com/pilosa/pilosa/pull/1622), [#1738](https://github.com/pilosa/pilosa/pull/1738)) - Add Not() query ([#1635](https://github.com/pilosa/pilosa/pull/1635)) - Implement Options call and excludeRowAttrs, excludeColumns, columnAttrs and shards args ([#1631](https://github.com/pilosa/pilosa/pull/1631)) - Add field options to pilosa import ([#1625](https://github.com/pilosa/pilosa/pull/1625)) @@ -44,13 +43,13 @@ This version contains 155 contributions from 11 contributors. There are 113 file ### Changed - Convert the anti-entropy logic to use `ImportRoaring` instead of `QueryNode` ([#1780](https://github.com/pilosa/pilosa/pull/1780)) -- Simplify "require-*" logic in Makefile ([#1755](https://github.com/pilosa/pilosa/pull/1755)) +- Simplify `require-*` logic in Makefile ([#1755](https://github.com/pilosa/pilosa/pull/1755)) - Cleanup logging ([#1748](https://github.com/pilosa/pilosa/pull/1748)) - Remove TravisCI, add CircleCI shield ([#1740](https://github.com/pilosa/pilosa/pull/1740)) - Upgrade Peg dependency and regenerate grammar ([#1725](https://github.com/pilosa/pilosa/pull/1725)) - Upgrade to protoc 3.6.1 (also updated protoc-gen-gofast) ([#1724](https://github.com/pilosa/pilosa/pull/1724)) - Move column attrs logic to executor ([#1677](https://github.com/pilosa/pilosa/pull/1677)) -- Shrank n(container bit count cache) to int32 ([#1664](https://github.com/pilosa/pilosa/pull/1664)) +- Shrink container bit count to int32 ([#1664](https://github.com/pilosa/pilosa/pull/1664)) ### Performance From 97eb94397f09064093cd458cec0cadbcbffaa3fe Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 21 Dec 2018 11:55:25 -0600 Subject: [PATCH 101/125] fix Rows bug where Pilosa would crash without 'field' argument. --- executor.go | 5 ++++- executor_test.go | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/executor.go b/executor.go index bbb8c2297..57aadf3a5 100644 --- a/executor.go +++ b/executor.go @@ -2761,7 +2761,10 @@ func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, filter *Row, inde ignorePrev := false for i, call := range children { - fieldName := call.Args["field"].(string) // this has already been validated by this point + fieldName, ok := call.Args["field"].(string) + if !ok { + return nil, errors.Errorf("%s call must have 'field' argument", call.Name) + } gbi.fields[i].Field = fieldName // Fetch fragment. frag := holder.fragment(index, fieldName, viewStandard, shard) diff --git a/executor_test.go b/executor_test.go index 141533d7f..3924657da 100644 --- a/executor_test.go +++ b/executor_test.go @@ -2672,6 +2672,39 @@ func TestExecutor_Execute_Rows(t *testing.T) { if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) { t.Fatalf("unexpected rows: %+v", rows) } + +} + +func TestExecutor_Execute_Rows_Error(t *testing.T) { + c := test.MustRunCluster(t, 3) + defer c.Close() + c.CreateField(t, "i", pilosa.IndexOptions{}, "general") + + tests := []struct { + query string + error string + }{ + { + query: "GroupBy(Rows())", + error: "Rows call must have 'field' argument", + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + r, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "i", + Query: test.query, + }) + if err == nil { + t.Fatalf("should have gotten an error on invalid rows query, but got %#v", r) + } + if !strings.Contains(err.Error(), test.error) { + t.Fatalf("unexpected error message: %s", err.Error()) + } + }) + } + } func TestExecutor_Execute_Rows_Keys(t *testing.T) { From 19696e3085ccf6311b7538c20d6d69a5914d842f Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 21 Dec 2018 11:56:05 -0600 Subject: [PATCH 102/125] add GroupBy and Rows docs --- docs/query-language.md | 108 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/docs/query-language.md b/docs/query-language.md index babcc390c..4b93b6311 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -781,3 +781,111 @@ Options(Row(f1=10), shards=[0, 2]) ```response {"attrs":{},"columns":[100, 2097152]} ``` + +**Spec:** + +``` +Rows(field=, previous=, limit=, column=) +``` + +**Description:** + +Rows returns a list of row IDs in the given field which have at least one bit +set. The field argument is mandatory, the others are optional. + +If `previous` is given, rows prior to and including the specified row ID or +key will not be returned. If `column` is given, only rows which have a set bit +in the given column will be returned. `previous` or `column` must be strings if +and only if the field or index respectively is using key translation. If `limit` +is given, the number of rowIDs returned will be less than or equal to +`limit`. The combination of `limit` and `previous` allows for paging over large +result sets. Results are always ordered, so setting `previous` as the last +result of the previous request will start from the next available row. + + +**Result Type:** Object with `"rows" or "keys" and an array of integers or strings respectively.` + +**Examples:** + +Without keys: +```request +Rows(field=blah) +``` +```response +{"rows":[1,9,39]} +``` + +With keys: +```request +Rows(field=blahk) +``` +```response +{"rows":null,"keys":["haha","zaaa","traa"]} +``` + + +**Spec:** + +``` +GroupBy(, [RowsCall...], limit=, filter=) +``` + +**Description:** + +GroupBy returns the count of the intersection of every combination of rows +taking one row each from the specified `Rows` calls. It returns only those +combinations for which the count is greater than 0. + +The optional `filter` argument takes any type of `Row` query (e.g. Row, Union, + Intersect, etc.) which will be intersected with each result prior to returning + the count. This is analagous to a WHERE clause applied to a relational GROUP BY + query. + +The optional `limit` argument limits the number of results returned. The results +are ordered, so as long as the data isn't changing, the same query will return +the same result set. + +Paging through results is supported by passing the `previous` argument to each +of the `Rows` calls in the GroupBy. Take the last result from your previous +`GroupBy` query, and pass each row ID in that result as the `previous` argument +to each of the respective `Rows` queries in your next `GroupBy` query. + +**Result Type:** Array of "groups". Each group is an object with a group key and +a count key. The count is an integer, and the group is an array of objects which +specify the field and row for each row that was intersected to get that result. + +**Examples:** + +A single `Rows` query. +```request +GroupBy(Rows(field=blah)) +``` +```response +[{"group":[{"field":"blah","rowID":1}],"count":1}, +{"group":[{"field":"blah","rowID":9}],"count":1}, +{"group":[{"field":"blah","rowID":39}],"count":1}] +``` + +With two `Rows` queries - one with IDs and one with keys. +```request +GroupBy(Rows(field=blah), Rows(field=blahk), limit=7) +``` +```response +[{"group":[{"field":"blah","rowID":1},{"field":"blahk","rowKey":"haha"}],"count":1}, + {"group":[{"field":"blah","rowID":1},{"field":"blahk","rowKey":"zaaa"}],"count":1}, + {"group":[{"field":"blah","rowID":1},{"field":"blahk","rowKey":"traa"}],"count":1}, + {"group":[{"field":"blah","rowID":9},{"field":"blahk","rowKey":"haha"}],"count":1}, + {"group":[{"field":"blah","rowID":9},{"field":"blahk","rowKey":"zaaa"}],"count":1}, + {"group":[{"field":"blah","rowID":9},{"field":"blahk","rowKey":"traa"}],"count":1}, + {"group":[{"field":"blah","rowID":39},{"field":"blahk","rowKey":"haha"}],"count":1}] +``` + +Getting the rest of the results from the previous example (paging). +```request +GroupBy(Rows(field=blah, previous=39), Rows(field=blahk, previous="haha"), limit=7) +``` + +```response +[{"group":[{"field":"blah","rowID":39},{"field":"blahk","rowKey":"zaaa"}],"count":1}, + {"group":[{"field":"blah","rowID":39},{"field":"blahk","rowKey":"traa"}],"count":1}] +``` From 81d08be0440b24fbfd30e794615a70d1b33c1d62 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 21 Dec 2018 14:16:55 -0600 Subject: [PATCH 103/125] fix PQL, Rows and Group By problems make sure that args which are Uints are positive and return an error if not. improve group by error messages if field for Rows query is invalid --- executor.go | 5 ++++- executor_test.go | 27 +++++++++++++++++++++++++-- pql/ast.go | 3 +++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/executor.go b/executor.go index 57aadf3a5..08eae1d49 100644 --- a/executor.go +++ b/executor.go @@ -2763,7 +2763,10 @@ func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, filter *Row, inde for i, call := range children { fieldName, ok := call.Args["field"].(string) if !ok { - return nil, errors.Errorf("%s call must have 'field' argument", call.Name) + return nil, errors.Errorf("%s call must have 'field' argument with valid (string) field name. Got %v of type %[2]T", call.Name, call.Args["field"]) + } + if holder.Field(index, fieldName) == nil { + return nil, ErrFieldNotFound } gbi.fields[i].Field = fieldName // Fetch fragment. diff --git a/executor_test.go b/executor_test.go index 3924657da..73872abff 100644 --- a/executor_test.go +++ b/executor_test.go @@ -2672,10 +2672,9 @@ func TestExecutor_Execute_Rows(t *testing.T) { if !reflect.DeepEqual(rows, pilosa.RowIdentifiers{Rows: []uint64{11, 12}}) { t.Fatalf("unexpected rows: %+v", rows) } - } -func TestExecutor_Execute_Rows_Error(t *testing.T) { +func TestExecutor_Execute_Query_Error(t *testing.T) { c := test.MustRunCluster(t, 3) defer c.Close() c.CreateField(t, "i", pilosa.IndexOptions{}, "general") @@ -2688,6 +2687,30 @@ func TestExecutor_Execute_Rows_Error(t *testing.T) { query: "GroupBy(Rows())", error: "Rows call must have 'field' argument", }, + { + query: "GroupBy(Rows(field=true))", + error: "Rows call must have 'field' argument", + }, + { + query: "GroupBy(Rows(field=\"true\"))", + error: "field not found", + }, + { + query: "GroupBy(Rows(field=1))", + error: "Rows call must have 'field' argument", + }, + { + query: "GroupBy(Rows(field))", + error: "parse error", + }, + { + query: "GroupBy(Rows(field=general, limit=-1))", + error: "must be positive, but got", + }, + { + query: "GroupBy(Rows(field=general), limit=-1)", + error: "must be positive, but got", + }, } for i, test := range tests { diff --git a/pql/ast.go b/pql/ast.go index 32255b6a8..03b360099 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -291,6 +291,9 @@ func (c *Call) UintArg(key string) (uint64, bool, error) { } switch tval := val.(type) { case int64: + if tval < 0 { + return 0, true, fmt.Errorf("value for '%s' must be positive, but got %v", key, tval) + } return uint64(tval), true, nil case uint64: return tval, true, nil From ed637e69219342937617ba31d10c9e5b2cb3d3c4 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 21 Dec 2018 17:24:44 -0600 Subject: [PATCH 104/125] add test for group by with invalid filter --- executor_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/executor_test.go b/executor_test.go index 73872abff..8ac7008ab 100644 --- a/executor_test.go +++ b/executor_test.go @@ -2675,7 +2675,7 @@ func TestExecutor_Execute_Rows(t *testing.T) { } func TestExecutor_Execute_Query_Error(t *testing.T) { - c := test.MustRunCluster(t, 3) + c := test.MustRunCluster(t, 1) defer c.Close() c.CreateField(t, "i", pilosa.IndexOptions{}, "general") @@ -2711,6 +2711,10 @@ func TestExecutor_Execute_Query_Error(t *testing.T) { query: "GroupBy(Rows(field=general), limit=-1)", error: "must be positive, but got", }, + { + query: "GroupBy(Rows(field=general), filter=Rows(field=general))", + error: "unknown call: Rows", + }, } for i, test := range tests { From aa0d64047d06da328d8ed931a14b638625bcf33d Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 21 Dec 2018 17:40:50 -0600 Subject: [PATCH 105/125] add horrifying code to skip rows with count 0 in Group By --- executor.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/executor.go b/executor.go index 08eae1d49..a6a5fcf6b 100644 --- a/executor.go +++ b/executor.go @@ -2836,6 +2836,7 @@ func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, filter *Row, inde // nextAtIdx is a recursive helper method for getting the next row for the field // at index i, and then updating the rows in the "higher" fields if it wraps. func (gbi *groupByIterator) nextAtIdx(i int) { +TOP: nr, rowID, wrapped := gbi.rowIters[i].Next() if nr == nil { gbi.done = true @@ -2852,11 +2853,18 @@ func (gbi *groupByIterator) nextAtIdx(i int) { gbi.rows[i].row = nr.Intersect(gbi.rows[i-1].row) } gbi.rows[i].id = rowID + + if gbi.rows[i].row.Count() == 0 { + goto TOP // I wanted to just call nextAtIdx again, but if a bunch of + // rows in a row were 0, I was worried we'd get into a stack + // overflow situation + } } // Next returns a GroupCount representing the next group by record. When there // are no more records it will return an empty GroupCount and done==true. func (gbi *groupByIterator) Next() (ret GroupCount, done bool) { +TOPNEXT: if gbi.done { return ret, true } @@ -2865,6 +2873,10 @@ func (gbi *groupByIterator) Next() (ret GroupCount, done bool) { } else { ret.Count = gbi.rows[len(gbi.rows)-1].row.intersectionCount(gbi.rows[len(gbi.rows)-2].row) } + if ret.Count == 0 { + gbi.nextAtIdx(len(gbi.rows) - 1) + goto TOPNEXT + } ret.Group = make([]FieldRow, len(gbi.rows)) copy(ret.Group, gbi.fields) @@ -2873,6 +2885,7 @@ func (gbi *groupByIterator) Next() (ret GroupCount, done bool) { } // set up for next call + gbi.nextAtIdx(len(gbi.rows) - 1) return ret, false From 216fd0964a42cdf1f007f2afedbca719411329e7 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Sat, 22 Dec 2018 06:12:06 -0600 Subject: [PATCH 106/125] a quicker empty check for group by --- executor.go | 2 +- row.go | 15 +++++++++++++++ row_test.go | 14 ++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/executor.go b/executor.go index a6a5fcf6b..0c61e0714 100644 --- a/executor.go +++ b/executor.go @@ -2854,7 +2854,7 @@ TOP: } gbi.rows[i].id = rowID - if gbi.rows[i].row.Count() == 0 { + if gbi.rows[i].row.IsEmpty() { goto TOP // I wanted to just call nextAtIdx again, but if a bunch of // rows in a row were 0, I was worried we'd get into a stack // overflow situation diff --git a/row.go b/row.go index 9f8f9a403..4a19dc051 100644 --- a/row.go +++ b/row.go @@ -16,6 +16,7 @@ package pilosa import ( "encoding/json" + "fmt" "sort" "github.com/pilosa/pilosa/roaring" @@ -42,6 +43,20 @@ func NewRow(columns ...uint64) *Row { return r } +func (r *Row) IsEmpty() bool { + fmt.Println("what", len(r.segments)) + if len(r.segments) == 0 { + return true + } + for i := range r.segments { + if r.segments[i].n > 0 { + return false + } + + } + return true +} + // Merge merges data from other into r. func (r *Row) Merge(other *Row) { var segments []rowSegment diff --git a/row_test.go b/row_test.go index 7f1279ceb..49559f131 100644 --- a/row_test.go +++ b/row_test.go @@ -111,3 +111,17 @@ func TestRow_Difference_Segment(t *testing.T) { t.Fatalf("Test 2 Difference Results %v != expected %v\n", res.Columns(), exp) } } + +func TestRow_IsEmpty(t *testing.T) { + r1 := pilosa.NewRow(1, ShardWidth) + r2 := pilosa.NewRow(0, 2*ShardWidth) + res := r2.Intersect(r1) + + if r1.IsEmpty() { + t.Fatal("r1 Should Not Be Empty\n") + } + if !res.IsEmpty() { + t.Fatal("Result Should Be Empty\n") + } + +} From 8b06ed95942b29b13a102568f7d811cda7b0f84f Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Sat, 22 Dec 2018 17:47:41 -0600 Subject: [PATCH 107/125] removed some debug --- row.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/row.go b/row.go index 4a19dc051..255c3e847 100644 --- a/row.go +++ b/row.go @@ -16,7 +16,6 @@ package pilosa import ( "encoding/json" - "fmt" "sort" "github.com/pilosa/pilosa/roaring" @@ -44,7 +43,6 @@ func NewRow(columns ...uint64) *Row { } func (r *Row) IsEmpty() bool { - fmt.Println("what", len(r.segments)) if len(r.segments) == 0 { return true } From 5b14227e08d033bd750784519077657cc559b9d5 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Wed, 2 Jan 2019 14:25:25 -0600 Subject: [PATCH 108/125] convert gotos to for loops --- executor.go | 69 ++++++++++++++++++++++++++++------------------------- 1 file changed, 36 insertions(+), 33 deletions(-) diff --git a/executor.go b/executor.go index 0c61e0714..eea9a76a2 100644 --- a/executor.go +++ b/executor.go @@ -2836,46 +2836,49 @@ func newGroupByIterator(rowIDs []RowIDs, children []*pql.Call, filter *Row, inde // nextAtIdx is a recursive helper method for getting the next row for the field // at index i, and then updating the rows in the "higher" fields if it wraps. func (gbi *groupByIterator) nextAtIdx(i int) { -TOP: - nr, rowID, wrapped := gbi.rowIters[i].Next() - if nr == nil { - gbi.done = true - return - } - if wrapped && i != 0 { - gbi.nextAtIdx(i - 1) - } - if i == 0 && gbi.filter != nil { - gbi.rows[i].row = nr.Intersect(gbi.filter) - } else if i == 0 || i == len(gbi.rows)-1 { - gbi.rows[i].row = nr - } else { - gbi.rows[i].row = nr.Intersect(gbi.rows[i-1].row) - } - gbi.rows[i].id = rowID + // loop until we find a non-empty row. This is an optimization - the loop and if/break can be removed. + for { + nr, rowID, wrapped := gbi.rowIters[i].Next() + if nr == nil { + gbi.done = true + return + } + if wrapped && i != 0 { + gbi.nextAtIdx(i - 1) + } + if i == 0 && gbi.filter != nil { + gbi.rows[i].row = nr.Intersect(gbi.filter) + } else if i == 0 || i == len(gbi.rows)-1 { + gbi.rows[i].row = nr + } else { + gbi.rows[i].row = nr.Intersect(gbi.rows[i-1].row) + } + gbi.rows[i].id = rowID - if gbi.rows[i].row.IsEmpty() { - goto TOP // I wanted to just call nextAtIdx again, but if a bunch of - // rows in a row were 0, I was worried we'd get into a stack - // overflow situation + if !gbi.rows[i].row.IsEmpty() { + break + } } } // Next returns a GroupCount representing the next group by record. When there // are no more records it will return an empty GroupCount and done==true. func (gbi *groupByIterator) Next() (ret GroupCount, done bool) { -TOPNEXT: - if gbi.done { - return ret, true - } - if len(gbi.rows) == 1 { - ret.Count = gbi.rows[len(gbi.rows)-1].row.Count() - } else { - ret.Count = gbi.rows[len(gbi.rows)-1].row.intersectionCount(gbi.rows[len(gbi.rows)-2].row) - } - if ret.Count == 0 { - gbi.nextAtIdx(len(gbi.rows) - 1) - goto TOPNEXT + // loop until we find a result with count > 0 + for { + if gbi.done { + return ret, true + } + if len(gbi.rows) == 1 { + ret.Count = gbi.rows[len(gbi.rows)-1].row.Count() + } else { + ret.Count = gbi.rows[len(gbi.rows)-1].row.intersectionCount(gbi.rows[len(gbi.rows)-2].row) + } + if ret.Count == 0 { + gbi.nextAtIdx(len(gbi.rows) - 1) + continue + } + break } ret.Group = make([]FieldRow, len(gbi.rows)) From 65c30283d73e131321c949e0e3c8cba3b0b4fcc4 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Thu, 3 Jan 2019 15:20:02 +0300 Subject: [PATCH 109/125] adds tests for GroupBy with keys; removes unused Bit message from proto --- encoding/proto/proto.go | 24 ++- executor.go | 2 +- executor_test.go | 72 ++++--- internal/public.pb.go | 430 ++++++++++++---------------------------- internal/public.proto | 7 +- server/server_test.go | 56 +++++- test/pilosa.go | 56 ++++++ 7 files changed, 302 insertions(+), 345 deletions(-) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index f796f2f3e..3b62cbc15 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -1147,8 +1147,14 @@ func decodeGroupCounts(a []*internal.GroupCount) []pilosa.GroupCount { func decodeFieldRows(a []*internal.FieldRow) []pilosa.FieldRow { other := make([]pilosa.FieldRow, len(a)) for i := range a { - other[i].Field = a[i].Field - other[i].RowID = a[i].RowID + fr := a[i] + other[i].Field = fr.Field + if fr.RowKey == "" { + other[i].RowID = fr.RowID + } else { + other[i].RowKey = fr.RowKey + } + fmt.Println("OTHER", other) } return other } @@ -1226,9 +1232,17 @@ func encodeGroupCounts(counts []pilosa.GroupCount) []*internal.GroupCount { func encodeFieldRows(a []pilosa.FieldRow) []*internal.FieldRow { other := make([]*internal.FieldRow, len(a)) for i := range a { - other[i] = &internal.FieldRow{ - Field: a[i].Field, - RowID: a[i].RowID, + fr := a[i] + if fr.RowKey == "" { + other[i] = &internal.FieldRow{ + Field: fr.Field, + RowID: fr.RowID, + } + } else { + other[i] = &internal.FieldRow{ + Field: fr.Field, + RowKey: fr.RowKey, + } } } return other diff --git a/executor.go b/executor.go index 08eae1d49..f2cb39cc0 100644 --- a/executor.go +++ b/executor.go @@ -1000,7 +1000,7 @@ func (fr FieldRow) MarshalJSON() ([]byte, error) { } func (fr FieldRow) String() string { - return fmt.Sprintf("%s.%d", fr.Field, fr.RowID) + return fmt.Sprintf("%s.%d.%s", fr.Field, fr.RowID, fr.RowKey) } type GroupCount struct { diff --git a/executor_test.go b/executor_test.go index 8ac7008ab..c37a0de88 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1907,7 +1907,7 @@ Set(4500001, fn=4) {Group: []pilosa.FieldRow{{Field: "f", RowID: 10}}, Count: 4}, } results := res.Results[0].([]pilosa.GroupCount) - checkGroupBy(t, expected, results) + test.CheckGroupBy(t, expected, results) } }) } @@ -2902,7 +2902,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { } results := c.Query(t, "i", `GroupBy(Rows(field=general), Rows(field=sub))`).Results[0].([]pilosa.GroupCount) - checkGroupBy(t, expected, results) + test.CheckGroupBy(t, expected, results) }) t.Run("Filter", func(t *testing.T) { @@ -2912,7 +2912,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { } results := c.Query(t, "i", `GroupBy(Rows(field=general), Rows(field=sub), filter=Row(general=10))`).Results[0].([]pilosa.GroupCount) - checkGroupBy(t, expected, results) + test.CheckGroupBy(t, expected, results) }) t.Run("check field offset no limit", func(t *testing.T) { @@ -2922,7 +2922,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { } results := c.Query(t, "i", `GroupBy(Rows(field=general, previous=10))`).Results[0].([]pilosa.GroupCount) - checkGroupBy(t, expected, results) + test.CheckGroupBy(t, expected, results) }) t.Run("check field offset limit", func(t *testing.T) { @@ -2931,7 +2931,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { } results := c.Query(t, "i", `GroupBy(Rows(field=general, previous=10), limit=1)`).Results[0].([]pilosa.GroupCount) - checkGroupBy(t, expected, results) + test.CheckGroupBy(t, expected, results) }) @@ -2952,7 +2952,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { } results := c.Query(t, "i", `GroupBy(Rows(field=a), Rows(field=b), limit=1)`).Results[0].([]pilosa.GroupCount) - checkGroupBy(t, expected, results) + test.CheckGroupBy(t, expected, results) }) // set the same bits in a single shard in three fields @@ -2985,7 +2985,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "wa", RowID: 0}, {Field: "wb", RowID: 1}, {Field: "wc", RowID: 0}}, Count: 1}, {Group: []pilosa.FieldRow{{Field: "wa", RowID: 0}, {Field: "wb", RowID: 1}, {Field: "wc", RowID: 1}}, Count: 1}, } - checkGroupBy(t, expected, results) + test.CheckGroupBy(t, expected, results) }) t.Run("test previous is last result", func(t *testing.T) { @@ -3000,7 +3000,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { expected := []pilosa.GroupCount{ {Group: []pilosa.FieldRow{{Field: "wa", RowID: 1}, {Field: "wb", RowID: 0}, {Field: "wc", RowID: 0}}, Count: 1}, } - checkGroupBy(t, expected, results) + test.CheckGroupBy(t, expected, results) }) // test multiple shards with distinct results (different rows) and same @@ -3028,7 +3028,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "ma", RowID: 1}, {Field: "mb", RowID: 3}}, Count: 1}, {Group: []pilosa.FieldRow{{Field: "ma", RowID: 2}, {Field: "mb", RowID: 0}}, Count: 1}, } - checkGroupBy(t, expected, results) + test.CheckGroupBy(t, expected, results) }) t.Run("distinct rows in different shards with row limit", func(t *testing.T) { @@ -3039,7 +3039,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "ma", RowID: 2}, {Field: "mb", RowID: 0}}, Count: 1}, {Group: []pilosa.FieldRow{{Field: "ma", RowID: 3}, {Field: "mb", RowID: 1}}, Count: 1}, } - checkGroupBy(t, expected, results) + test.CheckGroupBy(t, expected, results) }) t.Run("distinct rows in different shards with column arg", func(t *testing.T) { @@ -3050,7 +3050,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "ma", RowID: 3}, {Field: "mb", RowID: 1}}, Count: 1}, {Group: []pilosa.FieldRow{{Field: "ma", RowID: 3}, {Field: "mb", RowID: 3}}, Count: 1}, } - checkGroupBy(t, expected, results) + test.CheckGroupBy(t, expected, results) }) c.CreateField(t, "i", pilosa.IndexOptions{}, "na") @@ -3075,7 +3075,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "na", RowID: 1}, {Field: "nb", RowID: 0}}, Count: 2}, {Group: []pilosa.FieldRow{{Field: "na", RowID: 1}, {Field: "nb", RowID: 1}}, Count: 2}, } - checkGroupBy(t, expected, results) + test.CheckGroupBy(t, expected, results) }) @@ -3120,8 +3120,43 @@ func TestExecutor_Execute_GroupBy(t *testing.T) { } expected[63].Count = 5 - checkGroupBy(t, expected, totalResults) + test.CheckGroupBy(t, expected, totalResults) }) + + // test row keys + c.CreateField(t, "i", pilosa.IndexOptions{}, "generalk", pilosa.OptFieldKeys()) + c.CreateField(t, "i", pilosa.IndexOptions{}, "subk", pilosa.OptFieldKeys()) + c.Query(t, "i", ` + Set(0, generalk="ten") + Set(1, generalk="ten") + Set(1001, generalk="ten") + Set(2, generalk="eleven") + Set(1002, generalk="eleven") + Set(2, generalk="twelve") + Set(1002, generalk="twelve") + + Set(0, subk="one-hundred") + Set(1, subk="one-hundred") + Set(3, subk="one-hundred") + Set(1001, subk="one-hundred") + Set(2, subk="one-hundred-ten") + Set(0, subk="one-hundred-ten") + `) + + t.Run("test row keys", func(t *testing.T) { + // the execututor returns row IDs when the field has keys, so they should be included in the target. + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "generalk", RowID: 1, RowKey: "ten"}, {Field: "subk", RowID: 1, RowKey: "one-hundred"}}, Count: 3}, + {Group: []pilosa.FieldRow{{Field: "generalk", RowID: 1, RowKey: "ten"}, {Field: "subk", RowID: 2, RowKey: "one-hundred-ten"}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "generalk", RowID: 2, RowKey: "eleven"}, {Field: "subk", RowID: 2, RowKey: "one-hundred-ten"}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "generalk", RowID: 3, RowKey: "twelve"}, {Field: "subk", RowID: 2, RowKey: "one-hundred-ten"}}, Count: 1}, + } + + results := c.Query(t, "i", `GroupBy(Rows(field="generalk"), Rows(field="subk"))`).Results[0].([]pilosa.GroupCount) + test.CheckGroupBy(t, expected, results) + + }) + } for size := range []int{1, 3} { t.Run(fmt.Sprintf("%d_nodes", size), func(t *testing.T) { @@ -3184,17 +3219,6 @@ func BenchmarkGroupBy(b *testing.B) { } -func checkGroupBy(t *testing.T, expected, results []pilosa.GroupCount) { - if len(results) != len(expected) { - t.Fatalf("number of groupings mismatch:\n got:%+v\nwant:%+v\n", results, expected) - } - for i, result := range results { - if !reflect.DeepEqual(expected[i], result) { - t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) - } - } -} - func runCallTest(t *testing.T, writeQuery string, readQueries []string, indexOptions *pilosa.IndexOptions, fieldOption ...pilosa.FieldOption) []pilosa.QueryResponse { if indexOptions == nil { indexOptions = &pilosa.IndexOptions{} diff --git a/internal/public.pb.go b/internal/public.pb.go index c7378399f..5cd86a833 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -35,7 +35,7 @@ func (m *Row) Reset() { *m = Row{} } func (m *Row) String() string { return proto.CompactTextString(m) } func (*Row) ProtoMessage() {} func (*Row) Descriptor() ([]byte, []int) { - return fileDescriptor_public_7d901ba8e84abe50, []int{0} + return fileDescriptor_public_f65cfea24ac19f54, []int{0} } func (m *Row) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -97,7 +97,7 @@ func (m *RowIdentifiers) Reset() { *m = RowIdentifiers{} } func (m *RowIdentifiers) String() string { return proto.CompactTextString(m) } func (*RowIdentifiers) ProtoMessage() {} func (*RowIdentifiers) Descriptor() ([]byte, []int) { - return fileDescriptor_public_7d901ba8e84abe50, []int{1} + return fileDescriptor_public_f65cfea24ac19f54, []int{1} } func (m *RowIdentifiers) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -153,7 +153,7 @@ func (m *Pair) Reset() { *m = Pair{} } func (m *Pair) String() string { return proto.CompactTextString(m) } func (*Pair) ProtoMessage() {} func (*Pair) Descriptor() ([]byte, []int) { - return fileDescriptor_public_7d901ba8e84abe50, []int{2} + return fileDescriptor_public_f65cfea24ac19f54, []int{2} } func (m *Pair) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -206,6 +206,7 @@ func (m *Pair) GetCount() uint64 { type FieldRow struct { Field string `protobuf:"bytes,1,opt,name=Field,proto3" json:"Field,omitempty"` RowID uint64 `protobuf:"varint,2,opt,name=RowID,proto3" json:"RowID,omitempty"` + RowKey string `protobuf:"bytes,3,opt,name=RowKey,proto3" json:"RowKey,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -215,7 +216,7 @@ func (m *FieldRow) Reset() { *m = FieldRow{} } func (m *FieldRow) String() string { return proto.CompactTextString(m) } func (*FieldRow) ProtoMessage() {} func (*FieldRow) Descriptor() ([]byte, []int) { - return fileDescriptor_public_7d901ba8e84abe50, []int{3} + return fileDescriptor_public_f65cfea24ac19f54, []int{3} } func (m *FieldRow) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -258,6 +259,13 @@ func (m *FieldRow) GetRowID() uint64 { return 0 } +func (m *FieldRow) GetRowKey() string { + if m != nil { + return m.RowKey + } + return "" +} + type GroupCount struct { Group []*FieldRow `protobuf:"bytes,1,rep,name=Group" json:"Group,omitempty"` Count uint64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` @@ -270,7 +278,7 @@ func (m *GroupCount) Reset() { *m = GroupCount{} } func (m *GroupCount) String() string { return proto.CompactTextString(m) } func (*GroupCount) ProtoMessage() {} func (*GroupCount) Descriptor() ([]byte, []int) { - return fileDescriptor_public_7d901ba8e84abe50, []int{4} + return fileDescriptor_public_f65cfea24ac19f54, []int{4} } func (m *GroupCount) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -325,7 +333,7 @@ func (m *ValCount) Reset() { *m = ValCount{} } func (m *ValCount) String() string { return proto.CompactTextString(m) } func (*ValCount) ProtoMessage() {} func (*ValCount) Descriptor() ([]byte, []int) { - return fileDescriptor_public_7d901ba8e84abe50, []int{5} + return fileDescriptor_public_f65cfea24ac19f54, []int{5} } func (m *ValCount) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -368,69 +376,6 @@ func (m *ValCount) GetCount() int64 { return 0 } -type Bit struct { - RowID uint64 `protobuf:"varint,1,opt,name=RowID,proto3" json:"RowID,omitempty"` - ColumnID uint64 `protobuf:"varint,2,opt,name=ColumnID,proto3" json:"ColumnID,omitempty"` - Timestamp int64 `protobuf:"varint,3,opt,name=Timestamp,proto3" json:"Timestamp,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Bit) Reset() { *m = Bit{} } -func (m *Bit) String() string { return proto.CompactTextString(m) } -func (*Bit) ProtoMessage() {} -func (*Bit) Descriptor() ([]byte, []int) { - return fileDescriptor_public_7d901ba8e84abe50, []int{6} -} -func (m *Bit) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Bit) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Bit.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (dst *Bit) XXX_Merge(src proto.Message) { - xxx_messageInfo_Bit.Merge(dst, src) -} -func (m *Bit) XXX_Size() int { - return m.Size() -} -func (m *Bit) XXX_DiscardUnknown() { - xxx_messageInfo_Bit.DiscardUnknown(m) -} - -var xxx_messageInfo_Bit proto.InternalMessageInfo - -func (m *Bit) GetRowID() uint64 { - if m != nil { - return m.RowID - } - return 0 -} - -func (m *Bit) GetColumnID() uint64 { - if m != nil { - return m.ColumnID - } - return 0 -} - -func (m *Bit) GetTimestamp() int64 { - if m != nil { - return m.Timestamp - } - return 0 -} - type ColumnAttrSet struct { ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` Key string `protobuf:"bytes,3,opt,name=Key,proto3" json:"Key,omitempty"` @@ -444,7 +389,7 @@ func (m *ColumnAttrSet) Reset() { *m = ColumnAttrSet{} } func (m *ColumnAttrSet) String() string { return proto.CompactTextString(m) } func (*ColumnAttrSet) ProtoMessage() {} func (*ColumnAttrSet) Descriptor() ([]byte, []int) { - return fileDescriptor_public_7d901ba8e84abe50, []int{7} + return fileDescriptor_public_f65cfea24ac19f54, []int{6} } func (m *ColumnAttrSet) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -510,7 +455,7 @@ func (m *Attr) Reset() { *m = Attr{} } func (m *Attr) String() string { return proto.CompactTextString(m) } func (*Attr) ProtoMessage() {} func (*Attr) Descriptor() ([]byte, []int) { - return fileDescriptor_public_7d901ba8e84abe50, []int{8} + return fileDescriptor_public_f65cfea24ac19f54, []int{7} } func (m *Attr) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -592,7 +537,7 @@ func (m *AttrMap) Reset() { *m = AttrMap{} } func (m *AttrMap) String() string { return proto.CompactTextString(m) } func (*AttrMap) ProtoMessage() {} func (*AttrMap) Descriptor() ([]byte, []int) { - return fileDescriptor_public_7d901ba8e84abe50, []int{9} + return fileDescriptor_public_f65cfea24ac19f54, []int{8} } func (m *AttrMap) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -644,7 +589,7 @@ func (m *QueryRequest) Reset() { *m = QueryRequest{} } func (m *QueryRequest) String() string { return proto.CompactTextString(m) } func (*QueryRequest) ProtoMessage() {} func (*QueryRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_7d901ba8e84abe50, []int{10} + return fileDescriptor_public_f65cfea24ac19f54, []int{9} } func (m *QueryRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -728,7 +673,7 @@ func (m *QueryResponse) Reset() { *m = QueryResponse{} } func (m *QueryResponse) String() string { return proto.CompactTextString(m) } func (*QueryResponse) ProtoMessage() {} func (*QueryResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_public_7d901ba8e84abe50, []int{11} + return fileDescriptor_public_f65cfea24ac19f54, []int{10} } func (m *QueryResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -797,7 +742,7 @@ func (m *QueryResult) Reset() { *m = QueryResult{} } func (m *QueryResult) String() string { return proto.CompactTextString(m) } func (*QueryResult) ProtoMessage() {} func (*QueryResult) Descriptor() ([]byte, []int) { - return fileDescriptor_public_7d901ba8e84abe50, []int{12} + return fileDescriptor_public_f65cfea24ac19f54, []int{11} } func (m *QueryResult) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -907,7 +852,7 @@ func (m *ImportRequest) Reset() { *m = ImportRequest{} } func (m *ImportRequest) String() string { return proto.CompactTextString(m) } func (*ImportRequest) ProtoMessage() {} func (*ImportRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_7d901ba8e84abe50, []int{13} + return fileDescriptor_public_f65cfea24ac19f54, []int{12} } func (m *ImportRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1008,7 +953,7 @@ func (m *ImportValueRequest) Reset() { *m = ImportValueRequest{} } func (m *ImportValueRequest) String() string { return proto.CompactTextString(m) } func (*ImportValueRequest) ProtoMessage() {} func (*ImportValueRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_7d901ba8e84abe50, []int{14} + return fileDescriptor_public_f65cfea24ac19f54, []int{13} } func (m *ImportValueRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1092,7 +1037,7 @@ func (m *TranslateKeysRequest) Reset() { *m = TranslateKeysRequest{} } func (m *TranslateKeysRequest) String() string { return proto.CompactTextString(m) } func (*TranslateKeysRequest) ProtoMessage() {} func (*TranslateKeysRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_7d901ba8e84abe50, []int{15} + return fileDescriptor_public_f65cfea24ac19f54, []int{14} } func (m *TranslateKeysRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1153,7 +1098,7 @@ func (m *TranslateKeysResponse) Reset() { *m = TranslateKeysResponse{} } func (m *TranslateKeysResponse) String() string { return proto.CompactTextString(m) } func (*TranslateKeysResponse) ProtoMessage() {} func (*TranslateKeysResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_public_7d901ba8e84abe50, []int{16} + return fileDescriptor_public_f65cfea24ac19f54, []int{15} } func (m *TranslateKeysResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1201,7 +1146,7 @@ func (m *ImportRoaringRequestView) Reset() { *m = ImportRoaringRequestVi func (m *ImportRoaringRequestView) String() string { return proto.CompactTextString(m) } func (*ImportRoaringRequestView) ProtoMessage() {} func (*ImportRoaringRequestView) Descriptor() ([]byte, []int) { - return fileDescriptor_public_7d901ba8e84abe50, []int{17} + return fileDescriptor_public_f65cfea24ac19f54, []int{16} } func (m *ImportRoaringRequestView) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1256,7 +1201,7 @@ func (m *ImportRoaringRequest) Reset() { *m = ImportRoaringRequest{} } func (m *ImportRoaringRequest) String() string { return proto.CompactTextString(m) } func (*ImportRoaringRequest) ProtoMessage() {} func (*ImportRoaringRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_7d901ba8e84abe50, []int{18} + return fileDescriptor_public_f65cfea24ac19f54, []int{17} } func (m *ImportRoaringRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1306,7 +1251,6 @@ func init() { proto.RegisterType((*FieldRow)(nil), "internal.FieldRow") proto.RegisterType((*GroupCount)(nil), "internal.GroupCount") proto.RegisterType((*ValCount)(nil), "internal.ValCount") - proto.RegisterType((*Bit)(nil), "internal.Bit") proto.RegisterType((*ColumnAttrSet)(nil), "internal.ColumnAttrSet") proto.RegisterType((*Attr)(nil), "internal.Attr") proto.RegisterType((*AttrMap)(nil), "internal.AttrMap") @@ -1501,6 +1445,12 @@ func (m *FieldRow) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPublic(dAtA, i, uint64(m.RowID)) } + if len(m.RowKey) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.RowKey))) + i += copy(dAtA[i:], m.RowKey) + } if m.XXX_unrecognized != nil { i += copy(dAtA[i:], m.XXX_unrecognized) } @@ -1576,42 +1526,6 @@ func (m *ValCount) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *Bit) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *Bit) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if m.RowID != 0 { - dAtA[i] = 0x8 - i++ - i = encodeVarintPublic(dAtA, i, uint64(m.RowID)) - } - if m.ColumnID != 0 { - dAtA[i] = 0x10 - i++ - i = encodeVarintPublic(dAtA, i, uint64(m.ColumnID)) - } - if m.Timestamp != 0 { - dAtA[i] = 0x18 - i++ - i = encodeVarintPublic(dAtA, i, uint64(m.Timestamp)) - } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } - return i, nil -} - func (m *ColumnAttrSet) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -2465,6 +2379,10 @@ func (m *FieldRow) Size() (n int) { if m.RowID != 0 { n += 1 + sovPublic(uint64(m.RowID)) } + l = len(m.RowKey) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -2510,27 +2428,6 @@ func (m *ValCount) Size() (n int) { return n } -func (m *Bit) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.RowID != 0 { - n += 1 + sovPublic(uint64(m.RowID)) - } - if m.ColumnID != 0 { - n += 1 + sovPublic(uint64(m.ColumnID)) - } - if m.Timestamp != 0 { - n += 1 + sovPublic(uint64(m.Timestamp)) - } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } - return n -} - func (m *ColumnAttrSet) Size() (n int) { if m == nil { return 0 @@ -3451,6 +3348,35 @@ func (m *FieldRow) Unmarshal(dAtA []byte) error { break } } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RowKey", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RowKey = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -3663,114 +3589,6 @@ func (m *ValCount) Unmarshal(dAtA []byte) error { } return nil } -func (m *Bit) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPublic - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: Bit: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: Bit: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field RowID", wireType) - } - m.RowID = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPublic - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.RowID |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ColumnID", wireType) - } - m.ColumnID = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPublic - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.ColumnID |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Timestamp", wireType) - } - m.Timestamp = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPublic - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Timestamp |= (int64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - default: - iNdEx = preIndex - skippy, err := skipPublic(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} func (m *ColumnAttrSet) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -6167,65 +5985,63 @@ var ( ErrIntOverflowPublic = fmt.Errorf("proto: integer overflow") ) -func init() { proto.RegisterFile("public.proto", fileDescriptor_public_7d901ba8e84abe50) } +func init() { proto.RegisterFile("public.proto", fileDescriptor_public_f65cfea24ac19f54) } -var fileDescriptor_public_7d901ba8e84abe50 = []byte{ - // 902 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdf, 0x8e, 0xdb, 0xc4, - 0x17, 0xfe, 0x4d, 0xec, 0x24, 0xce, 0xc9, 0x26, 0xbf, 0x6a, 0x94, 0x16, 0x0b, 0x55, 0x21, 0xb2, - 0x10, 0x32, 0x37, 0x5b, 0x29, 0x48, 0x55, 0xaf, 0xf8, 0xb3, 0xdd, 0x2d, 0x8a, 0x0a, 0x2b, 0x98, - 0x5d, 0x82, 0xb8, 0x9c, 0x36, 0xd3, 0xd6, 0x92, 0xe3, 0x09, 0xf6, 0x98, 0x74, 0x9f, 0x83, 0x1b, - 0x1e, 0x81, 0x0b, 0x1e, 0xa4, 0x97, 0x88, 0x27, 0x80, 0xe5, 0x45, 0xd0, 0x39, 0xe3, 0xc9, 0x38, - 0xd9, 0xa5, 0x42, 0x88, 0xbb, 0xf9, 0xce, 0x99, 0x73, 0xfc, 0x7d, 0x73, 0xfe, 0x24, 0x70, 0xb4, - 0xa9, 0x9f, 0xe5, 0xd9, 0xf3, 0xe3, 0x4d, 0xa9, 0x8d, 0xe6, 0x51, 0x56, 0x18, 0x55, 0x16, 0x32, - 0x4f, 0xbe, 0x83, 0x40, 0xe8, 0x2d, 0x8f, 0xa1, 0xff, 0x58, 0xe7, 0xf5, 0xba, 0xa8, 0x62, 0x36, - 0x0b, 0xd2, 0x50, 0x38, 0xc8, 0xdf, 0x87, 0xee, 0x67, 0xc6, 0x94, 0x55, 0xdc, 0x99, 0x05, 0xe9, - 0x70, 0x3e, 0x3e, 0x76, 0xa1, 0xc7, 0x68, 0x16, 0xd6, 0xc9, 0x39, 0x84, 0x4f, 0xd5, 0x55, 0x15, - 0x07, 0xb3, 0x20, 0x1d, 0x08, 0x3a, 0x27, 0x8f, 0x60, 0x2c, 0xf4, 0x76, 0xb1, 0x52, 0x85, 0xc9, - 0x5e, 0x64, 0xca, 0xde, 0x12, 0x7a, 0xeb, 0x3e, 0x41, 0xe7, 0x5d, 0x64, 0xa7, 0x15, 0xf9, 0x31, - 0x84, 0x5f, 0xc9, 0xac, 0xe4, 0x63, 0xe8, 0x2c, 0x4e, 0x63, 0x36, 0x63, 0x69, 0x28, 0x3a, 0x8b, - 0x53, 0x3e, 0x81, 0xee, 0x63, 0x5d, 0x17, 0x26, 0xee, 0x90, 0xc9, 0x02, 0x7e, 0x07, 0x82, 0xa7, - 0xea, 0x2a, 0x0e, 0x66, 0x2c, 0x1d, 0x08, 0x3c, 0x26, 0x0f, 0x21, 0x7a, 0x92, 0xa9, 0x7c, 0x85, - 0xca, 0x26, 0xd0, 0xa5, 0x33, 0xa5, 0x19, 0x08, 0x0b, 0xd0, 0x8a, 0xdc, 0x4e, 0x5d, 0x26, 0x02, - 0xc9, 0x17, 0x00, 0x9f, 0x97, 0xba, 0xde, 0xd8, 0xbc, 0x29, 0x74, 0x09, 0x11, 0xdd, 0xe1, 0x9c, - 0x7b, 0xe5, 0x2e, 0xb9, 0xb0, 0x17, 0x6e, 0xe7, 0x95, 0xcc, 0x21, 0x5a, 0xca, 0x7c, 0xc7, 0x71, - 0x29, 0x73, 0xe2, 0x10, 0x08, 0x3c, 0xee, 0xc7, 0x04, 0x2e, 0xe6, 0x1b, 0x08, 0x4e, 0x32, 0xe3, - 0xe9, 0xb1, 0x16, 0x3d, 0xfe, 0x2e, 0x44, 0xb6, 0x2a, 0x3b, 0xde, 0x3b, 0xcc, 0xef, 0xc3, 0xe0, - 0x32, 0x5b, 0xab, 0xca, 0xc8, 0xf5, 0x86, 0x9e, 0x22, 0x10, 0xde, 0x90, 0x7c, 0x0b, 0x23, 0x7b, - 0x13, 0xab, 0x75, 0xa1, 0xcc, 0x8d, 0x97, 0xfd, 0x67, 0x55, 0xbe, 0xf9, 0xd2, 0x3f, 0x33, 0x08, - 0xd1, 0xe7, 0x5c, 0x6c, 0xe7, 0xc2, 0xc2, 0x5e, 0x5e, 0x6d, 0x54, 0xc3, 0x94, 0xce, 0x7c, 0x06, - 0xc3, 0x0b, 0x53, 0x66, 0xc5, 0xcb, 0xa5, 0xcc, 0x6b, 0xd5, 0x24, 0x6a, 0x9b, 0x50, 0xe3, 0xa2, - 0x30, 0xd6, 0x1d, 0x92, 0x8c, 0x1d, 0x46, 0x8d, 0x27, 0x5a, 0xe7, 0xd6, 0xd9, 0x9d, 0xb1, 0x34, - 0x12, 0xde, 0xc0, 0xa7, 0x00, 0x4f, 0x72, 0x2d, 0x9b, 0xd8, 0xde, 0x8c, 0xa5, 0x4c, 0xb4, 0x2c, - 0xc9, 0x03, 0xe8, 0x23, 0xd3, 0x2f, 0xe5, 0xc6, 0xab, 0x65, 0x6f, 0x51, 0x9b, 0xbc, 0x61, 0x70, - 0xf4, 0x75, 0xad, 0xca, 0x2b, 0xa1, 0xbe, 0xaf, 0x55, 0x45, 0x55, 0x21, 0xec, 0x5a, 0x89, 0x00, - 0xbf, 0x07, 0xbd, 0x8b, 0x57, 0xb2, 0x5c, 0xd9, 0xb7, 0x0b, 0x45, 0x83, 0x50, 0xab, 0x7f, 0xf3, - 0x8a, 0xb4, 0x46, 0xa2, 0x6d, 0xc2, 0x48, 0xa1, 0xd6, 0xda, 0x38, 0x31, 0x0d, 0xe2, 0x29, 0xfc, - 0xff, 0xec, 0xf5, 0xf3, 0xbc, 0x5e, 0x29, 0xa1, 0xb7, 0x36, 0xba, 0x47, 0x17, 0x0e, 0xcd, 0xfc, - 0x03, 0x18, 0x37, 0x26, 0x37, 0xbd, 0x7d, 0xba, 0x78, 0x60, 0x4d, 0x7e, 0x64, 0x30, 0x6a, 0xa4, - 0x54, 0x1b, 0x5d, 0x54, 0x0a, 0xeb, 0x75, 0x56, 0x96, 0xae, 0x5e, 0x67, 0x65, 0xc9, 0x1f, 0x40, - 0x5f, 0xa8, 0xaa, 0xce, 0x8d, 0x6b, 0x82, 0xbb, 0xfe, 0x59, 0x5c, 0x6c, 0x9d, 0x1b, 0xe1, 0x6e, - 0xf1, 0x4f, 0x60, 0xbc, 0xd7, 0x54, 0x76, 0xfa, 0x87, 0xf3, 0x77, 0x7c, 0xdc, 0x9e, 0x5f, 0x1c, - 0x5c, 0x4f, 0x7e, 0xeb, 0xc0, 0xb0, 0x95, 0x99, 0xbf, 0x47, 0xbb, 0x88, 0x38, 0x0d, 0xe7, 0x23, - 0x9f, 0x05, 0x27, 0x8d, 0xb6, 0xd4, 0x11, 0xb0, 0xf3, 0xa6, 0x9f, 0xd8, 0x39, 0x56, 0x11, 0xb7, - 0x84, 0xfb, 0x6c, 0xab, 0x8a, 0x68, 0x16, 0xd6, 0x49, 0x9b, 0xed, 0x95, 0x2c, 0x5e, 0xaa, 0x15, - 0xf5, 0x53, 0x24, 0x1c, 0xe4, 0xc7, 0x7e, 0x3e, 0xa9, 0x00, 0x7b, 0x23, 0xee, 0x3c, 0xc2, 0xcf, - 0xb0, 0x6b, 0x68, 0xac, 0xc5, 0xa8, 0x69, 0x68, 0x2c, 0x21, 0xce, 0x26, 0x3e, 0x3c, 0x15, 0xdf, - 0x22, 0xfe, 0x10, 0x86, 0x7e, 0x93, 0x54, 0x71, 0x44, 0x0c, 0x27, 0x3e, 0xbd, 0x77, 0x8a, 0xf6, - 0x45, 0xfe, 0xe9, 0xe1, 0xce, 0x8c, 0x07, 0xc4, 0x2c, 0xde, 0x7b, 0x8d, 0x96, 0x5f, 0x1c, 0xdc, - 0x4f, 0xfe, 0x60, 0x30, 0x5a, 0xac, 0x37, 0xba, 0x34, 0xad, 0xb6, 0x5d, 0x14, 0x2b, 0xf5, 0xda, - 0xb5, 0x2d, 0x01, 0xbf, 0x17, 0x3b, 0x07, 0x7b, 0x91, 0xda, 0x97, 0xda, 0x35, 0x14, 0x16, 0xb4, - 0x54, 0x86, 0x7b, 0x2a, 0xef, 0xc3, 0xc0, 0x2d, 0xa0, 0x2a, 0xee, 0x92, 0xcb, 0x1b, 0x70, 0x20, - 0x77, 0x1b, 0x08, 0x3b, 0x38, 0x48, 0x03, 0xd1, 0xb2, 0x60, 0x65, 0x84, 0xde, 0xd2, 0xf2, 0xef, - 0xd3, 0xf2, 0x77, 0x10, 0x23, 0x6d, 0x1a, 0x72, 0x46, 0xe4, 0x6c, 0x59, 0x92, 0x5f, 0x18, 0x70, - 0xab, 0x91, 0x46, 0xfb, 0xbf, 0x13, 0xfa, 0x76, 0x41, 0xf7, 0xa0, 0x47, 0xdf, 0x73, 0x62, 0x1a, - 0x74, 0x40, 0xb7, 0x7f, 0x83, 0xee, 0x12, 0x26, 0x97, 0xa5, 0x2c, 0xaa, 0x5c, 0x1a, 0x85, 0x86, - 0x7f, 0xc3, 0xf7, 0xb6, 0x1f, 0xd8, 0x0f, 0xe1, 0xee, 0x41, 0x5e, 0x3f, 0xdc, 0x28, 0x20, 0x20, - 0x01, 0x78, 0x4c, 0x4e, 0x20, 0x6e, 0x9a, 0x42, 0x4b, 0x5c, 0xb6, 0x0d, 0x85, 0x65, 0xa6, 0xb6, - 0x98, 0xfa, 0x5c, 0xae, 0x55, 0xc3, 0x82, 0xce, 0x68, 0x3b, 0x95, 0x46, 0x12, 0x87, 0x23, 0x41, - 0xe7, 0xe4, 0x05, 0x4c, 0x6e, 0xcb, 0x41, 0xbf, 0x64, 0xb9, 0x92, 0x76, 0x99, 0x44, 0xc2, 0x02, - 0xfe, 0x08, 0xba, 0x3f, 0x64, 0x6a, 0xeb, 0x96, 0x49, 0xe2, 0x1b, 0xf8, 0xef, 0x88, 0x08, 0x1b, - 0x70, 0x72, 0xe7, 0xcd, 0xf5, 0x94, 0xfd, 0x7a, 0x3d, 0x65, 0xbf, 0x5f, 0x4f, 0xd9, 0x4f, 0x7f, - 0x4e, 0xff, 0xf7, 0xac, 0x47, 0xff, 0x5a, 0x3e, 0xfa, 0x2b, 0x00, 0x00, 0xff, 0xff, 0xe8, 0x8e, - 0x85, 0x07, 0xc5, 0x08, 0x00, 0x00, +var fileDescriptor_public_f65cfea24ac19f54 = []byte{ + // 880 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcd, 0x8e, 0x1b, 0x45, + 0x10, 0xa6, 0x3d, 0x63, 0x7b, 0x5c, 0x5e, 0x9b, 0xa8, 0xe5, 0x84, 0x11, 0x8a, 0x8c, 0x35, 0x42, + 0x68, 0xb8, 0x6c, 0x24, 0x23, 0xa1, 0x9c, 0xf8, 0xd9, 0x78, 0x83, 0xac, 0xc0, 0x0a, 0x6a, 0x57, + 0x46, 0x1c, 0x3b, 0x71, 0x27, 0x19, 0x69, 0x3c, 0x6d, 0x66, 0x7a, 0x70, 0xf6, 0x39, 0xb8, 0xf0, + 0x08, 0x1c, 0x78, 0x90, 0x1c, 0x11, 0x4f, 0x00, 0xcb, 0x8b, 0xa0, 0xae, 0x9e, 0xde, 0x1e, 0x7b, + 0x97, 0x08, 0xa1, 0xdc, 0xea, 0xab, 0xea, 0xaa, 0xa9, 0xaf, 0xfe, 0x6c, 0x38, 0xda, 0xd6, 0x4f, + 0xf3, 0xec, 0xd9, 0xf1, 0xb6, 0x54, 0x5a, 0xf1, 0x28, 0x2b, 0xb4, 0x2c, 0x0b, 0x91, 0x27, 0x3f, + 0x40, 0x80, 0x6a, 0xc7, 0x63, 0xe8, 0x3f, 0x52, 0x79, 0xbd, 0x29, 0xaa, 0x98, 0xcd, 0x82, 0x34, + 0x44, 0x07, 0xf9, 0x87, 0xd0, 0xfd, 0x52, 0xeb, 0xb2, 0x8a, 0x3b, 0xb3, 0x20, 0x1d, 0xce, 0xc7, + 0xc7, 0xce, 0xf5, 0xd8, 0xa8, 0xd1, 0x1a, 0x39, 0x87, 0xf0, 0x89, 0xbc, 0xac, 0xe2, 0x60, 0x16, + 0xa4, 0x03, 0x24, 0x39, 0x79, 0x08, 0x63, 0x54, 0xbb, 0xe5, 0x5a, 0x16, 0x3a, 0x7b, 0x9e, 0x49, + 0xfb, 0x0a, 0xd5, 0xce, 0x7d, 0x82, 0xe4, 0x6b, 0xcf, 0x4e, 0xcb, 0xf3, 0x33, 0x08, 0xbf, 0x15, + 0x59, 0xc9, 0xc7, 0xd0, 0x59, 0x2e, 0x62, 0x36, 0x63, 0x69, 0x88, 0x9d, 0xe5, 0x82, 0x4f, 0xa0, + 0xfb, 0x48, 0xd5, 0x85, 0x8e, 0x3b, 0xa4, 0xb2, 0x80, 0xdf, 0x81, 0xe0, 0x89, 0xbc, 0x8c, 0x83, + 0x19, 0x4b, 0x07, 0x68, 0xc4, 0xe4, 0x0c, 0xa2, 0xc7, 0x99, 0xcc, 0xd7, 0x86, 0xd9, 0x04, 0xba, + 0x24, 0x53, 0x98, 0x01, 0x5a, 0x60, 0xb4, 0x26, 0xb7, 0x85, 0x8b, 0x44, 0x80, 0xdf, 0x83, 0x1e, + 0xaa, 0x9d, 0x0f, 0xd6, 0xa0, 0xe4, 0x6b, 0x80, 0xaf, 0x4a, 0x55, 0x6f, 0xed, 0xf7, 0x52, 0xe8, + 0x12, 0x22, 0x1a, 0xc3, 0x39, 0xf7, 0x15, 0x71, 0x1f, 0x45, 0xfb, 0xe0, 0xf6, 0x7c, 0x93, 0x39, + 0x44, 0x2b, 0x91, 0x5f, 0xe7, 0xbe, 0x12, 0x39, 0xe5, 0x16, 0xa0, 0x11, 0xf7, 0x7d, 0x02, 0xe7, + 0xf3, 0x3d, 0x8c, 0x6c, 0x43, 0x4c, 0xb9, 0xcf, 0xa5, 0xbe, 0x51, 0x9a, 0xff, 0xd6, 0xa6, 0x9b, + 0xa5, 0xfa, 0x95, 0x41, 0x68, 0x6c, 0xce, 0xc4, 0xae, 0x4d, 0xa6, 0x33, 0x17, 0x97, 0x5b, 0xd9, + 0x24, 0x4f, 0x32, 0x9f, 0xc1, 0xf0, 0x5c, 0x97, 0x59, 0xf1, 0x62, 0x25, 0xf2, 0x5a, 0x36, 0x81, + 0xda, 0x2a, 0xfe, 0x3e, 0x44, 0xcb, 0x42, 0x5b, 0x73, 0x48, 0x14, 0xae, 0x31, 0xbf, 0x0f, 0x83, + 0x13, 0xa5, 0x72, 0x6b, 0xec, 0xce, 0x58, 0x1a, 0xa1, 0x57, 0xf0, 0x29, 0xc0, 0xe3, 0x5c, 0x89, + 0xc6, 0xb7, 0x37, 0x63, 0x29, 0xc3, 0x96, 0x26, 0x79, 0x00, 0x7d, 0x93, 0xe9, 0x37, 0x62, 0xeb, + 0xd9, 0xb2, 0x37, 0xb0, 0x4d, 0x5e, 0x33, 0x38, 0xfa, 0xae, 0x96, 0xe5, 0x25, 0xca, 0x1f, 0x6b, + 0x59, 0x69, 0x53, 0x5b, 0xc2, 0x6e, 0x16, 0x08, 0x98, 0xae, 0x9f, 0xbf, 0x14, 0xe5, 0xda, 0xd6, + 0x2e, 0xc4, 0x06, 0x19, 0xae, 0xbe, 0xe6, 0x15, 0x71, 0x8d, 0xb0, 0xad, 0xa2, 0x79, 0x91, 0x1b, + 0xa5, 0x1d, 0x99, 0x06, 0xf1, 0x14, 0xde, 0x3d, 0x7d, 0xf5, 0x2c, 0xaf, 0xd7, 0x12, 0xd5, 0xce, + 0x7a, 0xf7, 0xe8, 0xc1, 0xa1, 0x9a, 0x7f, 0x04, 0xe3, 0x46, 0xe5, 0xd6, 0xaf, 0x4f, 0x0f, 0x0f, + 0xb4, 0xc9, 0xcf, 0x0c, 0x46, 0x0d, 0x95, 0x6a, 0xab, 0x8a, 0x4a, 0x9a, 0x7e, 0x9d, 0x96, 0xa5, + 0xeb, 0xd7, 0x69, 0x59, 0xf2, 0x07, 0xd0, 0x47, 0x59, 0xd5, 0xb9, 0x76, 0x43, 0x70, 0xd7, 0x97, + 0xc5, 0xf9, 0xd6, 0xb9, 0x46, 0xf7, 0x8a, 0x7f, 0x0e, 0xe3, 0xbd, 0xa1, 0xb2, 0xeb, 0x3b, 0x9c, + 0xbf, 0xe7, 0xfd, 0xf6, 0xec, 0x78, 0xf0, 0x3c, 0xf9, 0xa3, 0x03, 0xc3, 0x56, 0x64, 0xfe, 0x01, + 0x1d, 0x13, 0xca, 0x69, 0x38, 0x1f, 0xf9, 0x28, 0x66, 0x25, 0xe8, 0xcc, 0x1c, 0x01, 0x3b, 0x6b, + 0xe6, 0x89, 0x9d, 0x99, 0x2e, 0x9a, 0x35, 0x77, 0x9f, 0x6d, 0x75, 0xd1, 0xa8, 0xd1, 0x1a, 0xe9, + 0x34, 0xbd, 0x14, 0xc5, 0x0b, 0xb9, 0xa6, 0x79, 0x8a, 0xd0, 0x41, 0x7e, 0xec, 0x17, 0x89, 0x1a, + 0xb0, 0xb7, 0x8b, 0xce, 0x82, 0x7e, 0xd9, 0xdc, 0x40, 0x9b, 0x5e, 0x8c, 0x9a, 0x81, 0xb6, 0x2b, + 0xbf, 0x5c, 0x98, 0xc2, 0x53, 0xf3, 0x2d, 0xe2, 0x9f, 0xc2, 0xd0, 0xaf, 0x7c, 0x15, 0x47, 0x94, + 0xe1, 0xc4, 0x87, 0xf7, 0x46, 0x6c, 0x3f, 0xe4, 0x5f, 0x1c, 0x1e, 0xbd, 0x78, 0x40, 0x99, 0xc5, + 0x7b, 0xd5, 0x68, 0xd9, 0xf1, 0xe0, 0x7d, 0xf2, 0x17, 0x83, 0xd1, 0x72, 0xb3, 0x55, 0xa5, 0x6e, + 0x8d, 0xed, 0xb2, 0x58, 0xcb, 0x57, 0x6e, 0x6c, 0x09, 0xf8, 0xc3, 0xd6, 0x39, 0x38, 0x6c, 0x34, + 0xbe, 0x34, 0xae, 0x21, 0x5a, 0xd0, 0x62, 0x19, 0xee, 0xb1, 0xbc, 0x0f, 0x03, 0xdb, 0x52, 0x63, + 0xea, 0x92, 0xc9, 0x2b, 0xcc, 0x42, 0x5e, 0x64, 0x1b, 0x59, 0x69, 0xb1, 0xd9, 0x9a, 0x09, 0x0e, + 0xd2, 0x00, 0x5b, 0x1a, 0xd3, 0x19, 0x7b, 0x20, 0x6d, 0xf1, 0x06, 0xe8, 0xa0, 0xf1, 0xb4, 0x61, + 0xc8, 0x18, 0x91, 0xb1, 0xa5, 0x49, 0x7e, 0x63, 0xc0, 0x2d, 0x47, 0x5a, 0xed, 0xb7, 0x47, 0xf4, + 0xcd, 0x84, 0xee, 0x41, 0x8f, 0xbe, 0xe7, 0xc8, 0x34, 0xe8, 0x20, 0xdd, 0xfe, 0x8d, 0x74, 0x57, + 0x30, 0xb9, 0x28, 0x45, 0x51, 0xe5, 0x42, 0x4b, 0xa3, 0xf8, 0x3f, 0xf9, 0xde, 0xf6, 0x0b, 0xf9, + 0x31, 0xdc, 0x3d, 0x88, 0xeb, 0x97, 0xdb, 0x10, 0x08, 0x88, 0x80, 0x11, 0x93, 0x13, 0x88, 0x9b, + 0xa1, 0x50, 0xc2, 0x1c, 0xdb, 0x26, 0x85, 0x55, 0x26, 0x77, 0x26, 0xf4, 0x99, 0xd8, 0xc8, 0x26, + 0x0b, 0x92, 0x8d, 0x6e, 0x21, 0xb4, 0xa0, 0x1c, 0x8e, 0x90, 0xe4, 0xe4, 0x39, 0x4c, 0x6e, 0x8b, + 0x41, 0x3f, 0x39, 0xb9, 0x14, 0xf6, 0x98, 0x44, 0x68, 0x01, 0x7f, 0x08, 0xdd, 0x9f, 0x32, 0xb9, + 0x73, 0xc7, 0x24, 0xf1, 0x03, 0xfc, 0x6f, 0x89, 0xa0, 0x75, 0x38, 0xb9, 0xf3, 0xfa, 0x6a, 0xca, + 0x7e, 0xbf, 0x9a, 0xb2, 0x3f, 0xaf, 0xa6, 0xec, 0x97, 0xbf, 0xa7, 0xef, 0x3c, 0xed, 0xd1, 0xdf, + 0x8e, 0x4f, 0xfe, 0x09, 0x00, 0x00, 0xff, 0xff, 0x9f, 0x87, 0xba, 0x9b, 0x86, 0x08, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index d44dd2108..592874b95 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -22,6 +22,7 @@ message Pair { message FieldRow{ string Field = 1; uint64 RowID = 2; + string RowKey = 3; } message GroupCount{ @@ -34,12 +35,6 @@ message ValCount { int64 Count = 2; } -message Bit { - uint64 RowID = 1; - uint64 ColumnID = 2; - int64 Timestamp = 3; -} - message ColumnAttrSet { uint64 ID = 1; string Key = 3; diff --git a/server/server_test.go b/server/server_test.go index 044ae4720..a2fb5b5d7 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -30,14 +30,13 @@ import ( "testing/quick" "time" - "golang.org/x/sync/errgroup" - "github.com/pelletier/go-toml" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/roaring" "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" + "golang.org/x/sync/errgroup" ) var runStress bool @@ -248,6 +247,59 @@ func TestMain_SetColumnAttrs(t *testing.T) { } } +func TestMain_GroupBy(t *testing.T) { + m := test.MustRunCommand() + defer m.Close() + + // Create fields. + client := m.Client() + if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { + t.Fatal(err) + } + if err := client.CreateFieldWithOptions(context.Background(), "i", "generalk", pilosa.FieldOptions{Keys: true}); err != nil { + t.Fatal(err) + } + if err := client.CreateFieldWithOptions(context.Background(), "i", "subk", pilosa.FieldOptions{Keys: true}); err != nil { + t.Fatal(err) + } + + query := ` + Set(0, generalk="ten") + Set(1, generalk="ten") + Set(1001, generalk="ten") + Set(2, generalk="eleven") + Set(1002, generalk="eleven") + Set(2, generalk="twelve") + Set(1002, generalk="twelve") + + Set(0, subk="one-hundred") + Set(1, subk="one-hundred") + Set(3, subk="one-hundred") + Set(1001, subk="one-hundred") + Set(2, subk="one-hundred-ten") + Set(0, subk="one-hundred-ten") + ` + + // Set columns on row. + if _, err := m.Query("i", "", query); err != nil { + t.Fatal(err) + } + + expected := []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "generalk", RowKey: "ten"}, {Field: "subk", RowKey: "one-hundred"}}, Count: 3}, + {Group: []pilosa.FieldRow{{Field: "generalk", RowKey: "ten"}, {Field: "subk", RowKey: "one-hundred-ten"}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "generalk", RowKey: "eleven"}, {Field: "subk", RowKey: "one-hundred-ten"}}, Count: 1}, + {Group: []pilosa.FieldRow{{Field: "generalk", RowKey: "twelve"}, {Field: "subk", RowKey: "one-hundred-ten"}}, Count: 1}, + } + + // Query row. + if res, err := m.QueryProtobuf("i", `GroupBy(Rows(field="generalk"), Rows(field="subk"))`); err != nil { + t.Fatal(err) + } else { + test.CheckGroupBy(t, expected, res.Results[0].([]pilosa.GroupCount)) + } +} + // Ensure the host can be parsed. func TestConfig_Parse_Host(t *testing.T) { if c, err := ParseConfig(`bind = "local"`); err != nil { diff --git a/test/pilosa.go b/test/pilosa.go index e3858f5e0..dd3915406 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -22,12 +22,14 @@ import ( gohttp "net/http" "os" "path" + "reflect" "strconv" "strings" "testing" "time" "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/encoding/proto" "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/server" "github.com/pkg/errors" @@ -185,6 +187,49 @@ func (m *Command) Query(index, rawQuery, query string) (string, error) { return resp.Body, nil } +func (m *Command) QueryProtobuf(indexName string, query string) (*pilosa.QueryResponse, error) { + var ser proto.Serializer + queryReq := &pilosa.QueryRequest{ + Index: indexName, + Query: query, + } + body, err := ser.Marshal(queryReq) + if err != nil { + return nil, err + } + + req, err := gohttp.NewRequest( + "POST", + fmt.Sprintf("%s/index/%s/query", m.URL(), indexName), + bytes.NewReader(body), + ) + if err != nil { + return nil, err + } + + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("Accept", "application/x-protobuf") + + resp, err := gohttp.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + buf, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + response := &pilosa.QueryResponse{} + err = ser.Unmarshal(buf, response) + if err != nil { + return nil, err + } + + return response, nil +} + // RecalculateCaches is deprecated. Use MustRecalculateCaches. func (m *Command) RecalculateCaches() error { resp := MustDo("POST", fmt.Sprintf("%s/recalculate-caches", m.URL()), "") @@ -380,6 +425,17 @@ func MustDo(method, urlStr string, body string) *httpResponse { return &httpResponse{Response: resp, Body: string(buf)} } +func CheckGroupBy(t *testing.T, expected, results []pilosa.GroupCount) { + if len(results) != len(expected) { + t.Fatalf("number of groupings mismatch:\n got:%+v\nwant:%+v\n", results, expected) + } + for i, result := range results { + if !reflect.DeepEqual(expected[i], result) { + t.Fatalf("unexpected result at %d: \n got:%+v\nwant:%+v\n", i, result, expected[i]) + } + } +} + // httpResponse is a wrapper for http.Response that holds the Body as a string. type httpResponse struct { *gohttp.Response From 2babb3c51c037e3a7ba282d967c56c6f8639995f Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Fri, 4 Jan 2019 01:54:05 +0300 Subject: [PATCH 110/125] trivial --- encoding/proto/proto.go | 1 - 1 file changed, 1 deletion(-) diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 3b62cbc15..d40db1fb9 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -1154,7 +1154,6 @@ func decodeFieldRows(a []*internal.FieldRow) []pilosa.FieldRow { } else { other[i].RowKey = fr.RowKey } - fmt.Println("OTHER", other) } return other } From c30b03df14ed703fbaa506ea637577a99348491e Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Fri, 4 Jan 2019 01:56:10 +0300 Subject: [PATCH 111/125] trivial --- test/pilosa.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pilosa.go b/test/pilosa.go index dd3915406..31e93bb1d 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -427,7 +427,7 @@ func MustDo(method, urlStr string, body string) *httpResponse { func CheckGroupBy(t *testing.T, expected, results []pilosa.GroupCount) { if len(results) != len(expected) { - t.Fatalf("number of groupings mismatch:\n got:%+v\nwant:%+v\n", results, expected) + t.Fatalf("number of groupings mismatch:\n got:%+v\nwant:%+v\n", results, expected) } for i, result := range results { if !reflect.DeepEqual(expected[i], result) { From 7db655cb8c1863ed007f6f2e252c02a25b00e309 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 4 Jan 2019 08:23:19 -0600 Subject: [PATCH 112/125] fix incorrect error message --- field.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/field.go b/field.go index b71f9ca7e..669700574 100644 --- a/field.go +++ b/field.go @@ -1124,7 +1124,7 @@ func (f *Field) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time, opts frag, err := view.CreateFragmentIfNotExists(key.Shard) if err != nil { - return errors.Wrap(err, "creating view") + return errors.Wrap(err, "creating fragment") } if err := frag.bulkImport(data.RowIDs, data.ColumnIDs, options); err != nil { From af26473f77752d90fb21dd39e4f98faac2406dc7 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 7 Jan 2019 16:52:43 -0600 Subject: [PATCH 113/125] disable anti-entropy if not using replication (there will never be anything to sync) --- server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server.go b/server.go index 2e1e7091f..ca5bfcce4 100644 --- a/server.go +++ b/server.go @@ -428,7 +428,7 @@ func (s *Server) SyncData() error { } func (s *Server) monitorAntiEntropy() { - if s.antiEntropyInterval == 0 { + if s.antiEntropyInterval == 0 || s.cluster.ReplicaN <= 1 { return // anti entropy disabled } s.cluster.initializeAntiEntropy() From 9d1e5ca8ce6c1eb0d411b9f1e2848400ac6289fe Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Fri, 21 Dec 2018 16:44:01 -0700 Subject: [PATCH 114/125] Merge Range() into Row() call. This commit refactors the `Range()` call and merges its functionality into the `Row()` call. --- api_test.go | 2 +- docs/administration.md | 2 +- docs/data-model.md | 6 +- docs/glossary.md | 10 +- docs/query-language.md | 166 +-- docs/tutorials.md | 14 +- executor.go | 194 ++- executor_test.go | 371 +++++- http/client_test.go | 6 +- pql/ast.go | 14 +- pql/pql.peg | 8 +- pql/pql.peg.go | 2564 +++++++++++++++++++--------------------- pql/pqlpeg_test.go | 70 +- 13 files changed, 1841 insertions(+), 1586 deletions(-) diff --git a/api_test.go b/api_test.go index ecb4265ed..a33b064c1 100644 --- a/api_test.go +++ b/api_test.go @@ -223,7 +223,7 @@ func TestAPI_ImportValue(t *testing.T) { t.Fatal(err) } - pql := fmt.Sprintf("Range(%s>0)", field) + pql := fmt.Sprintf("Row(%s>0)", field) // Query node0. if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}); err != nil { diff --git a/docs/administration.md b/docs/administration.md index 3f07a5b68..4bfa5470a 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -323,7 +323,7 @@ We currently track the following events - **Xor:** Count of Xor queries. - **Not:** Count of Not queries. - **Count:** Count of Count queries. -- **Range:** Count of Range queries. +- **Range:** Count of ranged Row queries. - **Snapshot:** Event count when the snapshot process is triggered. - **BlockRepair:** Count of data blocks that were out of sync and repaired. - **GarbageCollection:** Event count when garbage collection occurs. diff --git a/docs/data-model.md b/docs/data-model.md index 9bb44bb99..1a0781ccc 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -64,7 +64,7 @@ Simple queries: Relational | Pilosa -----------------------------------------------|------------------------------------ `select ID from People where Name = 'Bob'` | `Row(Name="Bob")` - `select ID from People where Age > 30` | `Range(Age > 30)` + `select ID from People where Age > 30` | `Row(Age > 30)` `select ID from People where Member = true` | `Row(Member=0)` Note that `Row(Member=0)` selects all entities with a bit set in row 0 of the Member field. We could just as well use row 1 to store this, in which case we would use `Row(Member=1)`, which looks a bit more intuitive. In the relational model, joins are often necessary. Because Pilosa supports extremely high cardinality in both rows and columns, many types of joins are accomplished with basic Pilosa queries across multiple fields. For example, this SQL join: @@ -100,7 +100,7 @@ The LRU cache maintains the most recently accessed Rows. ### Time Quantum -Setting a time quantum on a field creates extra views which allow Range queries down to the time interval specified. For example, if the time quantum is set to `YMD`, Range queries down to the granularity of a day are supported. +Setting a time quantum on a field creates extra views which allow ranged Row queries down to the time interval specified. For example, if the time quantum is set to `YMD`, ranged Row queries down to the granularity of a day are supported. ### Attribute @@ -145,7 +145,7 @@ curl localhost:10101/index/repository/field/quantity \ ##### BSI Range-Encoding -Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded bit-sliced indexes of base-2, along with an additional row indicating "not null". This means that a 16-bit integer will require 17 rows: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null row. Pilosa can evaluate `Range`, `Min`, `Max`, and `Sum` queries on these BSI integers. The result of a `Sum` query includes a count, which can be used to compute an average with no other overhead. +Bit-Sliced Indexing (BSI) is the storage method Pilosa uses to represent multi-bit integers in a bitmap index. Integers are stored as n-bit, range-encoded bit-sliced indexes of base-2, along with an additional row indicating "not null". This means that a 16-bit integer will require 17 rows: one for each 0-bit of the 16 bit-slice components (the 1-bit does not need to be stored because with range-encoding the highest bit position is always 1) and one for the non-null row. Pilosa can evaluate `Row`, `Min`, `Max`, and `Sum` queries on these BSI integers. The result of a `Sum` query includes a count, which can be used to compute an average with no other overhead. Internally Pilosa stores each BSI `field` as a `view`. The rows of the `view` contain the base-2 representations of the integer values. Pilosa manages the base-2 offset and translation that efficiently packs the integer value within the minimum set of rows. diff --git a/docs/glossary.md b/docs/glossary.md index 86ae6bd70..9ff232454 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -46,16 +46,16 @@ nav = [] [Protobuf](https://developers.google.com/protocol-buffers/): Protocol Buffers is a binary serialization format which Pilosa uses for internal messages, and can be used by clients as an alternative to JSON. -[Range](../query-language/#range-queries):: A [PQL](#pql) query that returns bits based on comparison to timestamps, set according to the [time quantum](#time-quantum). - -[Range (BSI)](../query-language/#range-bsi):: A [PQL](#pql) query that returns bits based on comparison to integers stored in [BSI](#bsi) [fields](#field). - [Replica](../configuration/#cluster-replicas): A copy of a [fragment](#fragment) on a different [node](#node) than the original. The `cluster.replicas` configuration parameter determines how many replicas of a fragment exist in the cluster. This includes the original, so a value of 1 means no extra copies are made. [Roaring Bitmap](http://roaringbitmap.org): the compressed bitmap format which Pilosa uses to [implement bitmaps](../architecture/#roaring-bitmap-storage-format), for both storage and logical query operations. [Row](../data-model/#row): Rows are the fundamental vertical data axis within Pilosa. They are namespaced to each [field](#field) within an [index](#index). Represented as a [Bitmap](#bitmap). +[Row (Ranged)](../query-language/#range-queries):: A [PQL](#pql) query that returns bits based on comparison to timestamps, set according to the [time quantum](#time-quantum). + +[Row (BSI)](../query-language/#range-bsi):: A [PQL](#pql) query that returns bits based on comparison to integers stored in [BSI](#bsi) [fields](#field). + [Slice](../data-model/#shard): Prior to Pilosa 1.0, shards were known as slices. [Shard](../data-model/#shard): [Columns](#column) are [sharded](https://en.wikipedia.org/wiki/Shard_(database_architecture)) on a preset [width](#shardwidth). Shards are operated on in parallel and are evenly distributed across the cluster via a [consistent hash](#jump-consistent-hash). @@ -64,7 +64,7 @@ nav = [] [Sum](../query-language/#sum): A [PQL](#pql) query that returns the sum of integers stored in an [integer](#bsi) [field](#field). -[Time quantum](../data-model/#time-quantum): Defines the granularity to be used for [Range](#range) queries on time [fields](#field). +[Time quantum](../data-model/#time-quantum): Defines the granularity to be used for [ranged Row](#range) queries on time [fields](#field). [TOML](https://github.com/toml-lang/toml): the language used for Pilosa's [configuration file](../configuration/). diff --git a/docs/query-language.md b/docs/query-language.md index 4b93b6311..5eb4ef2d6 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -50,7 +50,7 @@ curl localhost:10101/index/repository/query \ * `ATTR_NAME` Must be a valid identifier `[A-Za-z][A-Za-z0-9._-]*` * `ATTR_VALUE` Can be a string, float, integer, or bool. * `CALL` Any query -* `ROW_CALL` Any query which returns a row, such as `Row`, `Union`, `Difference`, `Xor`, `Intersect`, `Range`, `Not` +* `ROW_CALL` Any query which returns a row, such as `Row`, `Union`, `Difference`, `Xor`, `Intersect`, `Not` * `[]ATTR_VALUE` Denotes an array of `ATTR_VALUE`s. (e.g. `["a", "b", "c"]`) ### Write Operations @@ -335,6 +335,89 @@ Row(stargazer=1) * attrs are the attributes for user 1 * columns are the repositories which user 1 has starred. + +#### Row (Range) + +**Spec:** + +``` +Row(=, , ) +``` + +**Description:** + +Similar to `Row`, but only returns bits which were set with timestamps +between the given `start` (first) and `end` (second) timestamps. + +**Result Type:** object with attrs and bits + + +**Examples:** + +Query all columns with a bit set in row 1 of a field (repositories that a user has starred), within a date range: +```request +Row(stargazer=1, 2010-01-01T00:00, 2017-03-02T03:00) +``` +```response +{{"attrs":{},"columns":[10]} +``` + +This example assumes timestamps have been set on some bits. + +* columns are repositories which were starred by user 1 in the time range 2010-01-01 to 2017-03-02. + + +#### Row (BSI) + +**Spec:** + +``` +Row([ ] ) +``` + +**Description:** + +The `Row` query is overloaded to work on `integer` values as well as `timestamp` values. +Returns bits that are true for the comparison operator. + +**Result Type:** object with attrs and columns + +**Examples:** + +In our source data, commitactivity was counted over the last year. +The following greater-than `Row` query returns all columns with a field value greater than 100 (repositories having more than 100 commits): + +```request +Row(commitactivity > 100) +``` +```response +{{"attrs":{},"columns":[10]} +``` + +* columns are repositories which had at least 100 commits in the last year. + +BSI range queries support the following operators: + + Operator | Name | Value +----------|-------------------------------|-------------------- + `>` | greater-than, GT | integer + `<` | less-than, LT | integer + `<=` | less-than-or-equal-to, LTE | integer + `>=` | greater-than-or-equal-to, GTE | integer + `==` | equal-to, EQ | integer + `!=` | not-equal-to, NEQ | integer or `null` + +`<`, and `<=` can be chained together to represent a bounded interval. For example: + +```request +Row(50 < commitactivity < 150) +``` +```response +{{"attrs":{},"columns":[10]} +``` + +As of Pilosa 1.0, the "between" syntax `Row(frame=stats, commitactivity >< [50, 150])` is no longer supported. + #### Union **Spec:** @@ -584,87 +667,6 @@ TopN(stargazer, n=2, attrName=active, attrValues=[true]) * Results are the top two users (rows) which have the "active" attribute set to "true", sorted by the number of bits set (repositories that they've starred). -#### Range Queries - -**Spec:** - -``` -Range(=, , ) -``` - -**Description:** - -Similar to `Row`, but only returns bits which were set with timestamps -between the given `start` (first) and `end` (second) timestamps. - -**Result Type:** object with attrs and bits - - -**Examples:** - -Query all columns with a bit set in row 1 of a field (repositories that a user has starred), within a date range: -```request -Range(stargazer=1, 2010-01-01T00:00, 2017-03-02T03:00) -``` -```response -{{"attrs":{},"columns":[10]} -``` - -This example assumes timestamps have been set on some bits. - -* columns are repositories which were starred by user 1 in the time range 2010-01-01 to 2017-03-02. - - -#### Range (BSI) - -**Spec:** - -``` -Range([ ] ) -``` - -**Description:** - -The `Range` query is overloaded to work on `integer` values as well as `timestamp` values. -Returns bits that are true for the comparison operator. - -**Result Type:** object with attrs and columns - -**Examples:** - -In our source data, commitactivity was counted over the last year. -The following greater-than `Range` query returns all columns with a field value greater than 100 (repositories having more than 100 commits): - -```request -Range(commitactivity > 100) -``` -```response -{{"attrs":{},"columns":[10]} -``` - -* columns are repositories which had at least 100 commits in the last year. - -BSI range queries support the following operators: - - Operator | Name | Value -----------|-------------------------------|-------------------- - `>` | greater-than, GT | integer - `<` | less-than, LT | integer - `<=` | less-than-or-equal-to, LTE | integer - `>=` | greater-than-or-equal-to, GTE | integer - `==` | equal-to, EQ | integer - `!=` | not-equal-to, NEQ | integer or `null` - -`<`, and `<=` can be chained together to represent a bounded interval. For example: - -```request -Range(50 < commitactivity < 150) -``` -```response -{{"attrs":{},"columns":[10]} -``` - -As of Pilosa 1.0, the "between" syntax `Range(frame=stats, commitactivity >< [50, 150])` is no longer supported. #### Min diff --git a/docs/tutorials.md b/docs/tutorials.md index 4cb8d069c..01e525362 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -444,7 +444,7 @@ Refer to the [Docker documentation](https://docs.docker.com) to see your options #### Introduction -Pilosa can store integer values associated to the columns in an index, and those values are used to support `Range`, `Min`, `Max`, and `Sum` queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients. +Pilosa can store integer values associated to the columns in an index, and those values are used to support `Row`, `Min`, `Max`, and `Sum` queries. In this tutorial we will show how to set up integer fields, populate those fields with data, and query the fields. The example index we're going to create will represent fictional patients at a medical facility and various bits of information about those patients. First, create an index called `patients`: ``` request @@ -534,17 +534,17 @@ pilosa import -i patients --field age ages.csv Now that we have some data in our index, let's run a few queries to demonstrate how to use that data. -In order to find all patients over the age of 40, then simply run a `Range` query against the `age` field. +In order to find all patients over the age of 40, then simply run a `Row` query against the `age` field. ``` request curl localhost:10101/index/patients/query \ -X POST \ - -d 'Range(age > 40)' + -d 'Row(age > 40)' ``` ``` response {"results":[{"attrs":{},"columns":[2,6,9]}]} ``` -You can find a list of supported range operators in the [Range Query](../query-language/#range-bsi) documentation. +You can find a list of supported range operators in the [Row (BSI) Query](../query-language/#range-bsi) documentation. To find the average age of all patients, run a `Sum` query: ``` request @@ -561,7 +561,7 @@ You can also provide a filter to the `Sum()` function to find the average age of ``` request curl localhost:10101/index/patients/query \ -X POST \ - -d 'Sum(Range(age > 40), field="age")' + -d 'Sum(Row(age > 40), field="age")' ``` ``` response {"results":[{"value":191,"count":3}]} @@ -583,7 +583,7 @@ You can also provide a filter to the `Min()` function to find the minimum age of ``` request curl localhost:10101/index/patients/query \ -X POST \ - -d 'Min(Range(age > 40), field="age")' + -d 'Min(Row(age > 40), field="age")' ``` ``` response {"results":[{"value":57,"count":1}]} @@ -604,7 +604,7 @@ You can also provide a filter to the `Max()` function to find the maximum age of ``` request curl localhost:10101/index/patients/query \ -X POST \ - -d 'Max(Range(age < 40), field="age")' + -d 'Max(Row(age < 40), field="age")' ``` ``` response {"results":[{"value":34,"count":1}]} diff --git a/executor.go b/executor.go index 8d29cd900..b9f7c31ca 100644 --- a/executor.go +++ b/executor.go @@ -18,6 +18,7 @@ import ( "context" "encoding/json" "fmt" + "log" "sort" "time" @@ -492,11 +493,11 @@ func (e *executor) executeBitmapCall(ctx context.Context, index string, c *pql.C return nil, errors.Wrap(err, "map reduce") } - // Attach attributes for Row() calls. + // Attach attributes for non-BSI Row() calls. // If the column label is used then return column attributes. // If the row label is used then return bitmap attributes. row, _ := other.(*Row) - if c.Name == "Row" { + if c.Name == "Row" && !c.HasConditionArg() { if opt.ExcludeRowAttrs { row.Attrs = map[string]interface{}{} } else { @@ -546,14 +547,12 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, index string, c * defer span.Finish() switch c.Name { - case "Row": - return e.executeBitmapShard(ctx, index, c, shard) + case "Row", "Range": + return e.executeRowShard(ctx, index, c, shard) case "Difference": return e.executeDifferenceShard(ctx, index, c, shard) case "Intersect": return e.executeIntersectShard(ctx, index, c, shard) - case "Range": - return e.executeRangeShard(ctx, index, c, shard) case "Union": return e.executeUnionShard(ctx, index, c, shard) case "Xor": @@ -1170,10 +1169,19 @@ func (e *executor) executeRowsShard(_ context.Context, index string, c *pql.Call return frag.rows(start, filters...), nil } -func (e *executor) executeBitmapShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { - span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapShard") +func (e *executor) executeRowShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeRowShard") defer span.Finish() + if c.Name == "Range" { + log.Print("DEPRECATED: Range() is deprecated, please use Row() instead.") + } + + // Handle bsiGroup ranges differently. + if c.HasConditionArg() { + return e.executeRowBSIGroupShard(ctx, index, c, shard) + } + // Fetch column label from index. idx := e.Holder.Index(index) if idx == nil { @@ -1197,93 +1205,43 @@ func (e *executor) executeBitmapShard(ctx context.Context, index string, c *pql. return nil, fmt.Errorf("Row() must specify %v", rowLabel) } - frag := e.Holder.fragment(index, fieldName, viewStandard, shard) - if frag == nil { - return NewRow(), nil - } - return frag.row(rowID), nil -} - -// executeIntersectShard executes a intersect() call for a local shard. -func (e *executor) executeIntersectShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { - span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeIntersectShard") - defer span.Finish() - - var other *Row - if len(c.Children) == 0 { - return nil, fmt.Errorf("empty Intersect query is currently not supported") - } - for i, input := range c.Children { - row, err := e.executeBitmapCallShard(ctx, index, input, shard) - if err != nil { - return nil, err - } - - if i == 0 { - other = row - } else { - other = other.Intersect(row) + // Parse "from" time, if set. + var fromTime time.Time + if _, ok := c.Args["from"]; ok { + switch v := c.Args["from"].(type) { + case string: + if fromTime, err = time.Parse(TimeFormat, v); err != nil { + return nil, errors.New("cannot parse Row() 'from' time") + } + case int64: + fromTime = time.Unix(v, 0).UTC() + default: + return nil, errors.New("Row() 'from' arg must be a timestamp") } } - other.invalidateCount() - return other, nil -} -// executeRangeShard executes a range() call for a local shard. -func (e *executor) executeRangeShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { - span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeRangeShard") - defer span.Finish() - - // Handle bsiGroup ranges differently. - if c.HasConditionArg() { - return e.executeBSIGroupRangeShard(ctx, index, c, shard) + // Parse "to" time, if set. + var toTime time.Time + if _, ok := c.Args["to"]; ok { + switch v := c.Args["to"].(type) { + case string: + if toTime, err = time.Parse(TimeFormat, v); err != nil { + return nil, errors.New("cannot parse Row() 'to' time") + } + case int64: + toTime = time.Unix(v, 0).UTC() + default: + return nil, errors.New("Row() 'to' arg must be a timestamp") + } } - // Parse field. - fieldName, err := c.FieldArg() - if err != nil { - return nil, errors.New("Range() argument required: field") - } - - // Retrieve column label. - idx := e.Holder.Index(index) - if idx == nil { - return nil, ErrIndexNotFound - } - - // Retrieve base field. - f := idx.Field(fieldName) - if f == nil { - return nil, ErrFieldNotFound - } - - // Read row & column id. - rowID, rowOK, err := c.UintArg(fieldName) - if err != nil { - return nil, fmt.Errorf("executeRangeShard - reading row: %v", err) - } - if !rowOK { - return nil, fmt.Errorf("Range() must specify %q", rowLabel) - } - - // Parse start time. - startTimeStr, ok := c.Args["_start"].(string) - if !ok { - return nil, errors.New("Range() start time required") - } - startTime, err := time.Parse(TimeFormat, startTimeStr) - if err != nil { - return nil, errors.New("cannot parse Range() start time") - } - - // Parse end time. - endTimeStr, ok := c.Args["_end"].(string) - if !ok { - return nil, errors.New("Range() end time required") - } - endTime, err := time.Parse(TimeFormat, endTimeStr) - if err != nil { - return nil, errors.New("cannot parse Range() end time") + // Simply return row if times are not set. + if c.Name == "Row" && fromTime.IsZero() && toTime.IsZero() { + frag := e.Holder.fragment(index, fieldName, viewStandard, shard) + if frag == nil { + return NewRow(), nil + } + return frag.row(rowID), nil } // If no quantum exists then return an empty bitmap. @@ -1292,9 +1250,17 @@ func (e *executor) executeRangeShard(ctx context.Context, index string, c *pql.C return &Row{}, nil } + // Set maximum "to" value if only "from" is set. We don't need to worry + // about setting the minimum "from" since it is the zero value if omitted. + if toTime.IsZero() { + // This is the maximum comparable time.Time value. + // https://stackoverflow.com/a/32620397 + toTime = time.Unix(1<<63-62135596801, 999999999) + } + // Union bitmaps across all time-based views. row := &Row{} - for _, view := range viewsByTimeRange(viewStandard, startTime, endTime, q) { + for _, view := range viewsByTimeRange(viewStandard, fromTime, toTime, q) { f := e.Holder.fragment(index, fieldName, view, shard) if f == nil { continue @@ -1303,18 +1269,19 @@ func (e *executor) executeRangeShard(ctx context.Context, index string, c *pql.C } f.Stats.Count("range", 1, 1.0) return row, nil + } -// executeBSIGroupRangeShard executes a range(bsiGroup) call for a local shard. -func (e *executor) executeBSIGroupRangeShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { - span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeBSIGroupRangeShard") +// executeRowBSIGroupShard executes a range(bsiGroup) call for a local shard. +func (e *executor) executeRowBSIGroupShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeRowBSIGroupShard") defer span.Finish() // Only one conditional should be present. if len(c.Args) == 0 { - return nil, errors.New("Range(): condition required") + return nil, errors.New("Row(): condition required") } else if len(c.Args) > 1 { - return nil, errors.New("Range(): too many arguments") + return nil, errors.New("Row(): too many arguments") } // Extract conditional. @@ -1323,7 +1290,7 @@ func (e *executor) executeBSIGroupRangeShard(ctx context.Context, index string, for k, v := range c.Args { vv, ok := v.(*pql.Condition) if !ok { - return nil, fmt.Errorf("Range(): %q: expected condition argument, got %v", k, v) + return nil, fmt.Errorf("Row(): %q: expected condition argument, got %v", k, v) } fieldName, cond = k, vv } @@ -1335,7 +1302,7 @@ func (e *executor) executeBSIGroupRangeShard(ctx context.Context, index string, // EQ null (not implemented: flip frag.NotNull with max ColumnID) // NEQ null frag.NotNull() - // BETWEEN a,b(in) BETWEEN/frag.RangeBetween() + // BETWEEN a,b(in) BETWEEN/frag.RowBetween() // BETWEEN a,b(out) BETWEEN/frag.NotNull() // EQ frag.RangeOp // NEQ frag.RangeOp @@ -1365,11 +1332,11 @@ func (e *executor) executeBSIGroupRangeShard(ctx context.Context, index string, // Only support two integers for the between operation. if len(predicates) != 2 { - return nil, errors.New("Range(): BETWEEN condition requires exactly two integer values") + return nil, errors.New("Row(): BETWEEN condition requires exactly two integer values") } // The reason we don't just call: - // return f.RangeBetween(fieldName, predicates[0], predicates[1]) + // return f.RowBetween(fieldName, predicates[0], predicates[1]) // here is because we need the call to be shard-specific. // Find bsiGroup. @@ -1402,7 +1369,7 @@ func (e *executor) executeBSIGroupRangeShard(ctx context.Context, index string, // Only support integers for now. value, ok := cond.Value.(int64) if !ok { - return nil, errors.New("Range(): conditions only support integer values") + return nil, errors.New("Row(): conditions only support integer values") } // Find bsiGroup. @@ -1438,6 +1405,31 @@ func (e *executor) executeBSIGroupRangeShard(ctx context.Context, index string, } } +// executeIntersectShard executes a intersect() call for a local shard. +func (e *executor) executeIntersectShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeIntersectShard") + defer span.Finish() + + var other *Row + if len(c.Children) == 0 { + return nil, fmt.Errorf("empty Intersect query is currently not supported") + } + for i, input := range c.Children { + row, err := e.executeBitmapCallShard(ctx, index, input, shard) + if err != nil { + return nil, err + } + + if i == 0 { + other = row + } else { + other = other.Intersect(row) + } + } + other.invalidateCount() + return other, nil +} + // executeUnionShard executes a union() call for a local shard. func (e *executor) executeUnionShard(ctx context.Context, index string, c *pql.Call, shard uint64) (*Row, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeUnionShard") diff --git a/executor_test.go b/executor_test.go index c37a0de88..11fcd32bd 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1478,7 +1478,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { } // Ensure a range query can be executed. -func TestExecutor_Execute_Range(t *testing.T) { +func TestExecutor_Execute_Row_Range(t *testing.T) { t.Run("RowIDColumnID", func(t *testing.T) { writeQuery := ` Set(2, f=1, 1999-12-31T00:00) @@ -1492,9 +1492,9 @@ func TestExecutor_Execute_Range(t *testing.T) { Set(2, f=1, 2002-02-01T00:00) Set(2, f=10, 2001-01-01T00:00)` readQueries := []string{ - `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`, + `Row(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`, `Clear( 2, f=1)`, - `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`, + `Row(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`, } responses := runCallTest(t, writeQuery, readQueries, nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) @@ -1525,9 +1525,9 @@ func TestExecutor_Execute_Range(t *testing.T) { Set("two", f=1, 2002-02-01T00:00) Set("two", f=10, 2001-01-01T00:00)` readQueries := []string{ - `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`, + `Row(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`, `Clear("two", f=1)`, - `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`, + `Row(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`, } responses := runCallTest(t, writeQuery, readQueries, &pilosa.IndexOptions{Keys: true}, @@ -1559,9 +1559,9 @@ func TestExecutor_Execute_Range(t *testing.T) { Set(2, f="foo", 2002-02-01T00:00) Set(2, f="bar", 2001-01-01T00:00)` readQueries := []string{ - `Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`, + `Row(f="foo", from=1999-12-31T00:00, to=2002-01-01T03:00)`, `Clear( 2, f="foo")`, - `Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`, + `Row(f="foo", from=1999-12-31T00:00, to=2002-01-01T03:00)`, } responses := runCallTest(t, writeQuery, readQueries, nil, @@ -1594,9 +1594,182 @@ func TestExecutor_Execute_Range(t *testing.T) { Set("two", f="foo", 2002-02-01T00:00) Set("two", f="bar", 2001-01-01T00:00)` readQueries := []string{ - `Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`, + `Row(f="foo", from=1999-12-31T00:00, to=2002-01-01T03:00)`, `Clear("two", f="foo")`, - `Range(f="foo", 1999-12-31T00:00, 2002-01-01T03:00)`, + `Row(f="foo", from=1999-12-31T00:00, to=2002-01-01T03:00)`, + } + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}, + pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")), + pilosa.OptFieldKeys()) + + t.Run("Standard", func(t *testing.T) { + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"two", "three", "four", "five", "six", "seven"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) + + t.Run("Clear", func(t *testing.T) { + if keys := responses[2].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"three", "four", "five", "six", "seven"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) + }) + + t.Run("UnixTimestamp", func(t *testing.T) { + writeQuery := ` + Set(2, f=1, 1999-12-31T00:00) + Set(3, f=1, 2000-01-01T00:00) + Set(4, f=1, 2000-01-02T00:00) + Set(5, f=1, 2000-02-01T00:00) + Set(6, f=1, 2001-01-01T00:00) + Set(7, f=1, 2002-01-01T02:00) + + Set(2, f=1, 1999-12-30T00:00) + Set(2, f=1, 2002-02-01T00:00) + Set(2, f=10, 2001-01-01T00:00)` + readQueries := []string{ + `Row(f=1, from=946598400, to=1009854000)`, + `Clear( 2, f=1)`, + `Row(f=1, from=946598400, to=1009854000)`, + } + responses := runCallTest(t, writeQuery, readQueries, + nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + + t.Run("Standard", func(t *testing.T) { + if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("Clear", func(t *testing.T) { + if columns := responses[2].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + }) +} + +// Ensure a range query can be executed. +func TestExecutor_Execute_Range_Deprecated(t *testing.T) { + t.Run("RowIDColumnID", func(t *testing.T) { + writeQuery := ` + Set(2, f=1, 1999-12-31T00:00) + Set(3, f=1, 2000-01-01T00:00) + Set(4, f=1, 2000-01-02T00:00) + Set(5, f=1, 2000-02-01T00:00) + Set(6, f=1, 2001-01-01T00:00) + Set(7, f=1, 2002-01-01T02:00) + + Set(2, f=1, 1999-12-30T00:00) + Set(2, f=1, 2002-02-01T00:00) + Set(2, f=10, 2001-01-01T00:00)` + readQueries := []string{ + `Range(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`, + `Clear( 2, f=1)`, + `Range(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`, + } + responses := runCallTest(t, writeQuery, readQueries, + nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + + t.Run("Standard", func(t *testing.T) { + if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("Clear", func(t *testing.T) { + if columns := responses[2].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + }) + + t.Run("RowIDColumnKey", func(t *testing.T) { + writeQuery := ` + Set("two", f=1, 1999-12-31T00:00) + Set("three", f=1, 2000-01-01T00:00) + Set("four", f=1, 2000-01-02T00:00) + Set("five", f=1, 2000-02-01T00:00) + Set("six", f=1, 2001-01-01T00:00) + Set("seven", f=1, 2002-01-01T02:00) + + Set("two", f=1, 1999-12-30T00:00) + Set("two", f=1, 2002-02-01T00:00) + Set("two", f=10, 2001-01-01T00:00)` + readQueries := []string{ + `Range(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`, + `Clear("two", f=1)`, + `Range(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`, + } + responses := runCallTest(t, writeQuery, readQueries, + &pilosa.IndexOptions{Keys: true}, + pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + + t.Run("Standard", func(t *testing.T) { + if keys := responses[0].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"two", "three", "four", "five", "six", "seven"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) + + t.Run("Clear", func(t *testing.T) { + if keys := responses[2].Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, []string{"three", "four", "five", "six", "seven"}) { + t.Fatalf("unexpected keys: %+v", keys) + } + }) + }) + + t.Run("RowKeyColumnID", func(t *testing.T) { + writeQuery := ` + Set(2, f="foo", 1999-12-31T00:00) + Set(3, f="foo", 2000-01-01T00:00) + Set(4, f="foo", 2000-01-02T00:00) + Set(5, f="foo", 2000-02-01T00:00) + Set(6, f="foo", 2001-01-01T00:00) + Set(7, f="foo", 2002-01-01T02:00) + + Set(2, f="foo", 1999-12-30T00:00) + Set(2, f="foo", 2002-02-01T00:00) + Set(2, f="bar", 2001-01-01T00:00)` + readQueries := []string{ + `Range(f="foo", from=1999-12-31T00:00, to=2002-01-01T03:00)`, + `Clear( 2, f="foo")`, + `Range(f="foo", from=1999-12-31T00:00, to=2002-01-01T03:00)`, + } + responses := runCallTest(t, writeQuery, readQueries, + nil, + pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH")), + pilosa.OptFieldKeys()) + + t.Run("Standard", func(t *testing.T) { + if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("Clear", func(t *testing.T) { + if columns := responses[2].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + }) + + t.Run("RowKeyColumnKey", func(t *testing.T) { + writeQuery := ` + Set("two", f="foo", 1999-12-31T00:00) + Set("three", f="foo", 2000-01-01T00:00) + Set("four", f="foo", 2000-01-02T00:00) + Set("five", f="foo", 2000-02-01T00:00) + Set("six", f="foo", 2001-01-01T00:00) + Set("seven", f="foo", 2002-01-01T02:00) + + Set("two", f="foo", 1999-12-30T00:00) + Set("two", f="foo", 2002-02-01T00:00) + Set("two", f="bar", 2001-01-01T00:00)` + readQueries := []string{ + `Range(f="foo", from=1999-12-31T00:00, to=2002-01-01T03:00)`, + `Clear("two", f="foo")`, + `Range(f="foo", from=1999-12-31T00:00, to=2002-01-01T03:00)`, } responses := runCallTest(t, writeQuery, readQueries, &pilosa.IndexOptions{Keys: true}, @@ -1617,8 +1790,174 @@ func TestExecutor_Execute_Range(t *testing.T) { }) } -// Ensure a Range(bsiGroup) query can be executed. -func TestExecutor_Execute_BSIGroupRange(t *testing.T) { +// Ensure a Row(bsiGroup) query can be executed. +func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + hldr := test.Holder{Holder: c[0].Server.Holder()} + + idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}) + if err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("foo", pilosa.OptFieldTypeInt(10, 100)); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("bar", pilosa.OptFieldTypeInt(0, 100000)); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("other", pilosa.OptFieldTypeInt(0, 1000)); err != nil { + t.Fatal(err) + } + + if _, err := idx.CreateField("edge", pilosa.OptFieldTypeInt(-100, 100)); err != nil { + t.Fatal(err) + } + + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: ` + Set(0, f=0) + Set(` + strconv.Itoa(ShardWidth+1) + `, f=0) + + Set(50, foo=20) + Set(50, bar=2000) + Set(` + strconv.Itoa(ShardWidth) + `, foo=30) + Set(` + strconv.Itoa(ShardWidth+2) + `, foo=10) + Set(` + strconv.Itoa((5*ShardWidth)+100) + `, foo=20) + Set(` + strconv.Itoa(ShardWidth+1) + `, foo=60) + Set(0, other=1000) + Set(0, edge=100) + Set(1, edge=-100) + `}); err != nil { + t.Fatal(err) + } + + t.Run("EQ", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo == 20)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{50, (5 * ShardWidth) + 100}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("NEQ", func(t *testing.T) { + // NEQ null + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(other != null)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + // NEQ + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo != 20)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1, ShardWidth + 2}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + // NEQ - + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(other != -20)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { + //t.Fatalf("unexpected result: %s", spew.Sdump(result)) + t.Fatalf("unexpected result: %v", result.Results[0].(*pilosa.Row).Columns()) + } + }) + + t.Run("LT", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo < 20)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{ShardWidth + 2}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("LTE", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo <= 20)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{50, ShardWidth + 2, (5 * ShardWidth) + 100}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("GT", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo > 20)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{ShardWidth, ShardWidth + 1}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("GTE", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo >= 20)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{50, ShardWidth, ShardWidth + 1, (5 * ShardWidth) + 100}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("BETWEEN", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(0 < other < 1000)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + // Ensure that the NotNull code path gets run. + t.Run("NotNull", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(-1 < other < 1000)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("BelowMin", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo == 0)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("AboveMax", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(foo == 200)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result)) + } + }) + + t.Run("LTAboveMax", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(edge < 200)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{0, 1}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result.Results[0].(*pilosa.Row).Columns())) + } + }) + + t.Run("GTBelowMin", func(t *testing.T) { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(edge > -200)`}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual([]uint64{0, 1}, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result.Results[0].(*pilosa.Row).Columns())) + } + }) + + t.Run("ErrFieldNotFound", func(t *testing.T) { + if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(bad_field >= 20)`}); errors.Cause(err) != pilosa.ErrFieldNotFound { + t.Fatal(err) + } + }) +} + +// Ensure a Range(bsiGroup) query can be executed. (Deprecated) +func TestExecutor_Execute_Range_BSIGroup_Deprecated(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() hldr := test.Holder{Holder: c[0].Server.Holder()} @@ -2010,7 +2349,7 @@ func TestExecutor_Time_Clear_Quantums(t *testing.T) { Set(2, f=10, 2001-01-01T00:00) ` clearColumn := `Clear( 2, f=1)` - rangeCheckQuery := `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)` + rangeCheckQuery := `Row(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)` for i, tt := range rangeTests { t.Run(fmt.Sprintf("#%d Quantum %s", i+1, tt.quantum), func(t *testing.T) { @@ -2346,10 +2685,10 @@ func TestExecutor_Execute_ClearRow(t *testing.T) { Set(2, f=1, 2002-02-01T00:00) Set(2, f=10, 2001-01-01T00:00)` readQueries := []string{ - `Range(f=1, 1999-12-31T00:00, 2003-01-01T03:00)`, + `Row(f=1, from=1999-12-31T00:00, to=2003-01-01T03:00)`, `ClearRow(f=1)`, - `Range(f=1, 1999-12-31T00:00, 2003-01-01T03:00)`, - `Range(f=10, 1999-12-31T00:00, 2003-01-01T03:00)`, + `Row(f=1, from=1999-12-31T00:00, to=2003-01-01T03:00)`, + `Row(f=10, from=1999-12-31T00:00, to=2003-01-01T03:00)`, } responses := runCallTest(t, writeQuery, readQueries, &pilosa.IndexOptions{TrackExistence: true}, @@ -3220,6 +3559,8 @@ func BenchmarkGroupBy(b *testing.B) { } func runCallTest(t *testing.T, writeQuery string, readQueries []string, indexOptions *pilosa.IndexOptions, fieldOption ...pilosa.FieldOption) []pilosa.QueryResponse { + t.Helper() + if indexOptions == nil { indexOptions = &pilosa.IndexOptions{} } diff --git a/http/client_test.go b/http/client_test.go index 79aa2a2ea..2b65f82d4 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -710,9 +710,9 @@ func TestClient_ImportKeys(t *testing.T) { t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=50, cnt=3", sum, cnt) } - // Verify Range. + // Verify range. queryRequest := &pilosa.QueryRequest{ - Query: fmt.Sprintf(`Range(%s>10)`, fldName), + Query: fmt.Sprintf(`Row(%s>10)`, fldName), Remote: false, } @@ -743,7 +743,7 @@ func TestClient_ImportKeys(t *testing.T) { // Verify Range. queryRequest = &pilosa.QueryRequest{ - Query: fmt.Sprintf(`Range(%s>10)`, fldName), + Query: fmt.Sprintf(`Row(%s>10)`, fldName), Remote: false, } diff --git a/pql/ast.go b/pql/ast.go index 03b360099..0985ab8a5 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -255,13 +255,25 @@ type Call struct { // Returns the field as a string if present, or an error if not. func (c *Call) FieldArg() (string, error) { for arg := range c.Args { - if !strings.HasPrefix(arg, "_") { + if !IsReservedArg(arg) { return arg, nil } } return "", fmt.Errorf("No field argument specified") } +func IsReservedArg(name string) bool { + if strings.HasPrefix(name, "_") { + return true + } + switch name { + case "from", "to": + return true + default: + return false + } +} + // BoolArg is for reading the value at key from call.Args as a bool. If the // key is not in Call.Args, the value of the returned bool will be false, and // the error will be nil. The value is assumed to be a bool. An error is diff --git a/pql/pql.peg b/pql/pql.peg index 13e900c33..66c1d3265 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -13,12 +13,12 @@ Call <- 'Set' {p.startCall("Set")} open col comma args (comma timestamp)? close / 'ClearRow' {p.startCall("ClearRow")} open arg close {p.endCall()} / 'Store' {p.startCall("Store")} open Call comma arg close {p.endCall()} / 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()} - / 'Range' {p.startCall("Range")} open (timerange / conditional / arg) close {p.endCall()} / < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close { p.endCall() } allargs <- Call (comma Call)* (comma args)? / args / sp args <- arg (comma args)? sp arg <- ( field sp '=' sp value / field sp COND sp value + / conditional ) COND <- ( '><' { p.addBTWN() } / '<=' { p.addLTE() } @@ -28,13 +28,12 @@ COND <- ( '><' { p.addBTWN() } / '<' { p.addLT() } / '>' { p.addGT() } ) + conditional <- {p.startConditional()} condint condLT condfield condLT condint {p.endConditional()} condint <- <'-'? [1-9] [0-9]* / '0'> sp {p.condAdd(buffer[begin:end])} condLT <- <('<=' / '<')> sp {p.condAdd(buffer[begin:end])} condfield <- sp {p.condAdd(buffer[begin:end])} -timerange <- field sp '=' sp value comma {p.addPosStr("_start", buffer[begin:end])} comma {p.addPosStr("_end", buffer[begin:end])} - value <- ( item / lbrack { p.startList() } list rbrack { p.endList() } ) @@ -42,6 +41,7 @@ list <- item (comma list)? item <- ( 'null' &(comma / sp close) { p.addVal(nil) } / 'true' &(comma / sp close) { p.addVal(true) } / 'false' &(comma / sp close) { p.addVal(false) } + / timestampfmt { p.addVal(buffer[begin:end]) } / < '-'? [0-9]+ ('.'[0-9]*)? > { p.addNumVal(buffer[begin:end]) } / < '-'? '.'[0-9]+ > { p.addNumVal(buffer[begin:end]) } / < IDENT > { p.startCall(buffer[begin:end]) } open allargs comma? close { p.addVal(p.endCall()) } @@ -77,5 +77,5 @@ IDENT <- [[A-Z]] ([[A-Z]] / [0-9])* timestampbasicfmt <- [0-9][0-9][0-9][0-9]'-'[01][0-9]'-'[0-3][0-9]'T'[0-9][0-9]':'[0-9][0-9] -timestampfmt <- '"' timestampbasicfmt '"' / '\'' timestampbasicfmt '\'' / timestampbasicfmt +timestampfmt <- '"' '"' / '\'' '\'' / timestamp <- {p.addPosStr("_timestamp", buffer[begin:end])} diff --git a/pql/pql.peg.go b/pql/pql.peg.go index edb6b6a70..8df2fcb72 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -26,7 +26,6 @@ const ( rulecondint rulecondLT rulecondfield - ruletimerange rulevalue rulelist ruleitem @@ -63,9 +62,9 @@ const ( ruleAction11 ruleAction12 ruleAction13 + rulePegText ruleAction14 ruleAction15 - rulePegText ruleAction16 ruleAction17 ruleAction18 @@ -100,9 +99,6 @@ const ( ruleAction47 ruleAction48 ruleAction49 - ruleAction50 - ruleAction51 - ruleAction52 ) var rul3s = [...]string{ @@ -117,7 +113,6 @@ var rul3s = [...]string{ "condint", "condLT", "condfield", - "timerange", "value", "list", "item", @@ -154,9 +149,9 @@ var rul3s = [...]string{ "Action11", "Action12", "Action13", + "PegText", "Action14", "Action15", - "PegText", "Action16", "Action17", "Action18", @@ -191,9 +186,6 @@ var rul3s = [...]string{ "Action47", "Action48", "Action49", - "Action50", - "Action51", - "Action52", } type token32 struct { @@ -310,7 +302,7 @@ type PQL struct { Buffer string buffer []rune - rules [88]func() bool + rules [84]func() bool parse func(rule ...int) error reset func() Pretty bool @@ -431,83 +423,77 @@ func (p *PQL) Execute() { case ruleAction13: p.endCall() case ruleAction14: - p.startCall("Range") + p.startCall(buffer[begin:end]) case ruleAction15: p.endCall() case ruleAction16: - p.startCall(buffer[begin:end]) - case ruleAction17: - p.endCall() - case ruleAction18: p.addBTWN() - case ruleAction19: + case ruleAction17: p.addLTE() - case ruleAction20: + case ruleAction18: p.addGTE() - case ruleAction21: + case ruleAction19: p.addEQ() - case ruleAction22: + case ruleAction20: p.addNEQ() - case ruleAction23: + case ruleAction21: p.addLT() - case ruleAction24: + case ruleAction22: p.addGT() - case ruleAction25: + case ruleAction23: p.startConditional() - case ruleAction26: + case ruleAction24: p.endConditional() + case ruleAction25: + p.condAdd(buffer[begin:end]) + case ruleAction26: + p.condAdd(buffer[begin:end]) case ruleAction27: p.condAdd(buffer[begin:end]) case ruleAction28: - p.condAdd(buffer[begin:end]) - case ruleAction29: - p.condAdd(buffer[begin:end]) - case ruleAction30: - p.addPosStr("_start", buffer[begin:end]) - case ruleAction31: - p.addPosStr("_end", buffer[begin:end]) - case ruleAction32: p.startList() - case ruleAction33: + case ruleAction29: p.endList() - case ruleAction34: + case ruleAction30: p.addVal(nil) - case ruleAction35: + case ruleAction31: p.addVal(true) - case ruleAction36: + case ruleAction32: p.addVal(false) - case ruleAction37: - p.addNumVal(buffer[begin:end]) - case ruleAction38: - p.addNumVal(buffer[begin:end]) - case ruleAction39: - p.startCall(buffer[begin:end]) - case ruleAction40: - p.addVal(p.endCall()) - case ruleAction41: + case ruleAction33: p.addVal(buffer[begin:end]) - case ruleAction42: + case ruleAction34: + p.addNumVal(buffer[begin:end]) + case ruleAction35: + p.addNumVal(buffer[begin:end]) + case ruleAction36: + p.startCall(buffer[begin:end]) + case ruleAction37: + p.addVal(p.endCall()) + case ruleAction38: + p.addVal(buffer[begin:end]) + case ruleAction39: s, _ := strconv.Unquote(buffer[begin:end]) p.addVal(s) - case ruleAction43: + case ruleAction40: p.addVal(buffer[begin:end]) - case ruleAction44: + case ruleAction41: p.addField(buffer[begin:end]) - case ruleAction45: + case ruleAction42: p.addPosStr("_field", buffer[begin:end]) - case ruleAction46: + case ruleAction43: p.addPosNum("_col", buffer[begin:end]) - case ruleAction47: + case ruleAction44: p.addPosStr("_col", buffer[begin:end]) - case ruleAction48: + case ruleAction45: p.addPosStr("_col", buffer[begin:end]) - case ruleAction49: + case ruleAction46: p.addPosNum("_row", buffer[begin:end]) - case ruleAction50: + case ruleAction47: p.addPosStr("_row", buffer[begin:end]) - case ruleAction51: + case ruleAction48: p.addPosStr("_row", buffer[begin:end]) - case ruleAction52: + case ruleAction49: p.addPosStr("_timestamp", buffer[begin:end]) } @@ -620,7 +606,7 @@ func (p *PQL) Init() { position, tokenIndex = position0, tokenIndex0 return false }, - /* 1 Call <- <(('S' 'e' 't' Action0 open col comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma row comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'u' 'm' 'n' 'A' 't' 't' 'r' 's' Action4 open col comma args close Action5) / ('C' 'l' 'e' 'a' 'r' Action6 open col comma args close Action7) / ('C' 'l' 'e' 'a' 'r' 'R' 'o' 'w' Action8 open arg close Action9) / ('S' 't' 'o' 'r' 'e' Action10 open Call comma arg close Action11) / ('T' 'o' 'p' 'N' Action12 open posfield (comma allargs)? close Action13) / ('R' 'a' 'n' 'g' 'e' Action14 open (timerange / conditional / arg) close Action15) / ( Action16 open allargs comma? close Action17))> */ + /* 1 Call <- <(('S' 'e' 't' Action0 open col comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma row comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'u' 'm' 'n' 'A' 't' 't' 'r' 's' Action4 open col comma args close Action5) / ('C' 'l' 'e' 'a' 'r' Action6 open col comma args close Action7) / ('C' 'l' 'e' 'a' 'r' 'R' 'o' 'w' Action8 open arg close Action9) / ('S' 't' 'o' 'r' 'e' Action10 open Call comma arg close Action11) / ('T' 'o' 'p' 'N' Action12 open posfield (comma allargs)? close Action13) / ( Action14 open allargs comma? close Action15))> */ func() bool { position5, tokenIndex5 := position, tokenIndex { @@ -669,7 +655,7 @@ func (p *PQL) Init() { add(rulePegText, position13) } { - add(ruleAction52, position) + add(ruleAction49, position) } add(ruletimestamp, position12) } @@ -755,7 +741,7 @@ func (p *PQL) Init() { add(rulePegText, position21) } { - add(ruleAction49, position) + add(ruleAction46, position) } goto l19 l20: @@ -776,7 +762,7 @@ func (p *PQL) Init() { } position++ { - add(ruleAction50, position) + add(ruleAction47, position) } goto l19 l23: @@ -797,7 +783,7 @@ func (p *PQL) Init() { } position++ { - add(ruleAction51, position) + add(ruleAction48, position) } } l19: @@ -1083,148 +1069,15 @@ func (p *PQL) Init() { goto l7 l41: position, tokenIndex = position7, tokenIndex7 - if buffer[position] != rune('R') { - goto l46 - } - position++ - if buffer[position] != rune('a') { - goto l46 - } - position++ - if buffer[position] != rune('n') { - goto l46 - } - position++ - if buffer[position] != rune('g') { - goto l46 - } - position++ - if buffer[position] != rune('e') { - goto l46 - } - position++ { - add(ruleAction14, position) - } - if !_rules[ruleopen]() { - goto l46 - } - { - position48, tokenIndex48 := position, tokenIndex - { - position50 := position - if !_rules[rulefield]() { - goto l49 - } - if !_rules[rulesp]() { - goto l49 - } - if buffer[position] != rune('=') { - goto l49 - } - position++ - if !_rules[rulesp]() { - goto l49 - } - if !_rules[rulevalue]() { - goto l49 - } - if !_rules[rulecomma]() { - goto l49 - } - { - position51 := position - if !_rules[ruletimestampfmt]() { - goto l49 - } - add(rulePegText, position51) - } - { - add(ruleAction30, position) - } - if !_rules[rulecomma]() { - goto l49 - } - { - position53 := position - if !_rules[ruletimestampfmt]() { - goto l49 - } - add(rulePegText, position53) - } - { - add(ruleAction31, position) - } - add(ruletimerange, position50) - } - goto l48 - l49: - position, tokenIndex = position48, tokenIndex48 - { - position56 := position - { - add(ruleAction25, position) - } - if !_rules[rulecondint]() { - goto l55 - } - if !_rules[rulecondLT]() { - goto l55 - } - { - position58 := position - { - position59 := position - if !_rules[rulefieldExpr]() { - goto l55 - } - add(rulePegText, position59) - } - if !_rules[rulesp]() { - goto l55 - } - { - add(ruleAction29, position) - } - add(rulecondfield, position58) - } - if !_rules[rulecondLT]() { - goto l55 - } - if !_rules[rulecondint]() { - goto l55 - } - { - add(ruleAction26, position) - } - add(ruleconditional, position56) - } - goto l48 - l55: - position, tokenIndex = position48, tokenIndex48 - if !_rules[rulearg]() { - goto l46 - } - } - l48: - if !_rules[ruleclose]() { - goto l46 - } - { - add(ruleAction15, position) - } - goto l7 - l46: - position, tokenIndex = position7, tokenIndex7 - { - position63 := position + position46 := position if !_rules[ruleIDENT]() { goto l5 } - add(rulePegText, position63) + add(rulePegText, position46) } { - add(ruleAction16, position) + add(ruleAction14, position) } if !_rules[ruleopen]() { goto l5 @@ -1233,20 +1086,20 @@ func (p *PQL) Init() { goto l5 } { - position65, tokenIndex65 := position, tokenIndex + position48, tokenIndex48 := position, tokenIndex if !_rules[rulecomma]() { - goto l65 + goto l48 } - goto l66 - l65: - position, tokenIndex = position65, tokenIndex65 + goto l49 + l48: + position, tokenIndex = position48, tokenIndex48 } - l66: + l49: if !_rules[ruleclose]() { goto l5 } { - add(ruleAction17, position) + add(ruleAction15, position) } } l7: @@ -1259,239 +1112,340 @@ func (p *PQL) Init() { }, /* 2 allargs <- <((Call (comma Call)* (comma args)?) / args / sp)> */ func() bool { - position68, tokenIndex68 := position, tokenIndex + position51, tokenIndex51 := position, tokenIndex { - position69 := position + position52 := position { - position70, tokenIndex70 := position, tokenIndex + position53, tokenIndex53 := position, tokenIndex if !_rules[ruleCall]() { - goto l71 + goto l54 } - l72: + l55: { - position73, tokenIndex73 := position, tokenIndex + position56, tokenIndex56 := position, tokenIndex if !_rules[rulecomma]() { - goto l73 + goto l56 } if !_rules[ruleCall]() { - goto l73 + goto l56 } - goto l72 - l73: - position, tokenIndex = position73, tokenIndex73 + goto l55 + l56: + position, tokenIndex = position56, tokenIndex56 } { - position74, tokenIndex74 := position, tokenIndex + position57, tokenIndex57 := position, tokenIndex if !_rules[rulecomma]() { - goto l74 + goto l57 } if !_rules[ruleargs]() { - goto l74 + goto l57 } - goto l75 - l74: - position, tokenIndex = position74, tokenIndex74 + goto l58 + l57: + position, tokenIndex = position57, tokenIndex57 } - l75: - goto l70 - l71: - position, tokenIndex = position70, tokenIndex70 + l58: + goto l53 + l54: + position, tokenIndex = position53, tokenIndex53 if !_rules[ruleargs]() { - goto l76 + goto l59 } - goto l70 - l76: - position, tokenIndex = position70, tokenIndex70 + goto l53 + l59: + position, tokenIndex = position53, tokenIndex53 if !_rules[rulesp]() { - goto l68 + goto l51 } } - l70: - add(ruleallargs, position69) + l53: + add(ruleallargs, position52) } return true - l68: - position, tokenIndex = position68, tokenIndex68 + l51: + position, tokenIndex = position51, tokenIndex51 return false }, /* 3 args <- <(arg (comma args)? sp)> */ func() bool { - position77, tokenIndex77 := position, tokenIndex + position60, tokenIndex60 := position, tokenIndex { - position78 := position + position61 := position if !_rules[rulearg]() { - goto l77 + goto l60 } { - position79, tokenIndex79 := position, tokenIndex + position62, tokenIndex62 := position, tokenIndex if !_rules[rulecomma]() { - goto l79 + goto l62 } if !_rules[ruleargs]() { - goto l79 + goto l62 } - goto l80 - l79: - position, tokenIndex = position79, tokenIndex79 + goto l63 + l62: + position, tokenIndex = position62, tokenIndex62 } - l80: + l63: if !_rules[rulesp]() { - goto l77 + goto l60 } - add(ruleargs, position78) + add(ruleargs, position61) } return true - l77: - position, tokenIndex = position77, tokenIndex77 + l60: + position, tokenIndex = position60, tokenIndex60 return false }, - /* 4 arg <- <((field sp '=' sp value) / (field sp COND sp value))> */ + /* 4 arg <- <((field sp '=' sp value) / (field sp COND sp value) / conditional)> */ func() bool { - position81, tokenIndex81 := position, tokenIndex + position64, tokenIndex64 := position, tokenIndex { - position82 := position + position65 := position { - position83, tokenIndex83 := position, tokenIndex + position66, tokenIndex66 := position, tokenIndex if !_rules[rulefield]() { - goto l84 + goto l67 } if !_rules[rulesp]() { - goto l84 + goto l67 } if buffer[position] != rune('=') { - goto l84 + goto l67 } position++ if !_rules[rulesp]() { - goto l84 + goto l67 } if !_rules[rulevalue]() { - goto l84 + goto l67 } - goto l83 - l84: - position, tokenIndex = position83, tokenIndex83 + goto l66 + l67: + position, tokenIndex = position66, tokenIndex66 if !_rules[rulefield]() { - goto l81 + goto l68 } if !_rules[rulesp]() { - goto l81 + goto l68 } { - position85 := position + position69 := position { - position86, tokenIndex86 := position, tokenIndex + position70, tokenIndex70 := position, tokenIndex if buffer[position] != rune('>') { - goto l87 + goto l71 } position++ if buffer[position] != rune('<') { - goto l87 + goto l71 + } + position++ + { + add(ruleAction16, position) + } + goto l70 + l71: + position, tokenIndex = position70, tokenIndex70 + if buffer[position] != rune('<') { + goto l73 + } + position++ + if buffer[position] != rune('=') { + goto l73 + } + position++ + { + add(ruleAction17, position) + } + goto l70 + l73: + position, tokenIndex = position70, tokenIndex70 + if buffer[position] != rune('>') { + goto l75 + } + position++ + if buffer[position] != rune('=') { + goto l75 } position++ { add(ruleAction18, position) } - goto l86 - l87: - position, tokenIndex = position86, tokenIndex86 - if buffer[position] != rune('<') { - goto l89 + goto l70 + l75: + position, tokenIndex = position70, tokenIndex70 + if buffer[position] != rune('=') { + goto l77 } position++ if buffer[position] != rune('=') { - goto l89 + goto l77 } position++ { add(ruleAction19, position) } - goto l86 - l89: - position, tokenIndex = position86, tokenIndex86 - if buffer[position] != rune('>') { - goto l91 + goto l70 + l77: + position, tokenIndex = position70, tokenIndex70 + if buffer[position] != rune('!') { + goto l79 } position++ if buffer[position] != rune('=') { - goto l91 + goto l79 } position++ { add(ruleAction20, position) } - goto l86 - l91: - position, tokenIndex = position86, tokenIndex86 - if buffer[position] != rune('=') { - goto l93 - } - position++ - if buffer[position] != rune('=') { - goto l93 + goto l70 + l79: + position, tokenIndex = position70, tokenIndex70 + if buffer[position] != rune('<') { + goto l81 } position++ { add(ruleAction21, position) } - goto l86 - l93: - position, tokenIndex = position86, tokenIndex86 - if buffer[position] != rune('!') { - goto l95 - } - position++ - if buffer[position] != rune('=') { - goto l95 + goto l70 + l81: + position, tokenIndex = position70, tokenIndex70 + if buffer[position] != rune('>') { + goto l68 } position++ { add(ruleAction22, position) } - goto l86 - l95: - position, tokenIndex = position86, tokenIndex86 - if buffer[position] != rune('<') { + } + l70: + add(ruleCOND, position69) + } + if !_rules[rulesp]() { + goto l68 + } + if !_rules[rulevalue]() { + goto l68 + } + goto l66 + l68: + position, tokenIndex = position66, tokenIndex66 + { + position84 := position + { + add(ruleAction23, position) + } + if !_rules[rulecondint]() { + goto l64 + } + if !_rules[rulecondLT]() { + goto l64 + } + { + position86 := position + { + position87 := position + if !_rules[rulefieldExpr]() { + goto l64 + } + add(rulePegText, position87) + } + if !_rules[rulesp]() { + goto l64 + } + { + add(ruleAction27, position) + } + add(rulecondfield, position86) + } + if !_rules[rulecondLT]() { + goto l64 + } + if !_rules[rulecondint]() { + goto l64 + } + { + add(ruleAction24, position) + } + add(ruleconditional, position84) + } + } + l66: + add(rulearg, position65) + } + return true + l64: + position, tokenIndex = position64, tokenIndex64 + return false + }, + /* 5 COND <- <(('>' '<' Action16) / ('<' '=' Action17) / ('>' '=' Action18) / ('=' '=' Action19) / ('!' '=' Action20) / ('<' Action21) / ('>' Action22))> */ + nil, + /* 6 conditional <- <(Action23 condint condLT condfield condLT condint Action24)> */ + nil, + /* 7 condint <- <(<(('-'? [1-9] [0-9]*) / '0')> sp Action25)> */ + func() bool { + position92, tokenIndex92 := position, tokenIndex + { + position93 := position + { + position94 := position + { + position95, tokenIndex95 := position, tokenIndex + { + position97, tokenIndex97 := position, tokenIndex + if buffer[position] != rune('-') { goto l97 } position++ - { - add(ruleAction23, position) - } - goto l86 + goto l98 l97: - position, tokenIndex = position86, tokenIndex86 - if buffer[position] != rune('>') { - goto l81 + position, tokenIndex = position97, tokenIndex97 + } + l98: + if c := buffer[position]; c < rune('1') || c > rune('9') { + goto l96 + } + position++ + l99: + { + position100, tokenIndex100 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l100 } position++ - { - add(ruleAction24, position) - } + goto l99 + l100: + position, tokenIndex = position100, tokenIndex100 } - l86: - add(ruleCOND, position85) - } - if !_rules[rulesp]() { - goto l81 - } - if !_rules[rulevalue]() { - goto l81 + goto l95 + l96: + position, tokenIndex = position95, tokenIndex95 + if buffer[position] != rune('0') { + goto l92 + } + position++ } + l95: + add(rulePegText, position94) } - l83: - add(rulearg, position82) + if !_rules[rulesp]() { + goto l92 + } + { + add(ruleAction25, position) + } + add(rulecondint, position93) } return true - l81: - position, tokenIndex = position81, tokenIndex81 + l92: + position, tokenIndex = position92, tokenIndex92 return false }, - /* 5 COND <- <(('>' '<' Action18) / ('<' '=' Action19) / ('>' '=' Action20) / ('=' '=' Action21) / ('!' '=' Action22) / ('<' Action23) / ('>' Action24))> */ - nil, - /* 6 conditional <- <(Action25 condint condLT condfield condLT condint Action26)> */ - nil, - /* 7 condint <- <(<(('-'? [1-9] [0-9]*) / '0')> sp Action27)> */ + /* 8 condLT <- <(<(('<' '=') / '<')> sp Action26)> */ func() bool { position102, tokenIndex102 := position, tokenIndex { @@ -1500,36 +1454,18 @@ func (p *PQL) Init() { position104 := position { position105, tokenIndex105 := position, tokenIndex - { - position107, tokenIndex107 := position, tokenIndex - if buffer[position] != rune('-') { - goto l107 - } - position++ - goto l108 - l107: - position, tokenIndex = position107, tokenIndex107 - } - l108: - if c := buffer[position]; c < rune('1') || c > rune('9') { + if buffer[position] != rune('<') { goto l106 } position++ - l109: - { - position110, tokenIndex110 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l110 - } - position++ - goto l109 - l110: - position, tokenIndex = position110, tokenIndex110 + if buffer[position] != rune('=') { + goto l106 } + position++ goto l105 l106: position, tokenIndex = position105, tokenIndex105 - if buffer[position] != rune('0') { + if buffer[position] != rune('<') { goto l102 } position++ @@ -1541,1462 +1477,1434 @@ func (p *PQL) Init() { goto l102 } { - add(ruleAction27, position) + add(ruleAction26, position) } - add(rulecondint, position103) + add(rulecondLT, position103) } return true l102: position, tokenIndex = position102, tokenIndex102 return false }, - /* 8 condLT <- <(<(('<' '=') / '<')> sp Action28)> */ - func() bool { - position112, tokenIndex112 := position, tokenIndex - { - position113 := position - { - position114 := position - { - position115, tokenIndex115 := position, tokenIndex - if buffer[position] != rune('<') { - goto l116 - } - position++ - if buffer[position] != rune('=') { - goto l116 - } - position++ - goto l115 - l116: - position, tokenIndex = position115, tokenIndex115 - if buffer[position] != rune('<') { - goto l112 - } - position++ - } - l115: - add(rulePegText, position114) - } - if !_rules[rulesp]() { - goto l112 - } - { - add(ruleAction28, position) - } - add(rulecondLT, position113) - } - return true - l112: - position, tokenIndex = position112, tokenIndex112 - return false - }, - /* 9 condfield <- <( sp Action29)> */ + /* 9 condfield <- <( sp Action27)> */ nil, - /* 10 timerange <- <(field sp '=' sp value comma Action30 comma Action31)> */ - nil, - /* 11 value <- <(item / (lbrack Action32 list rbrack Action33))> */ + /* 10 value <- <(item / (lbrack Action28 list rbrack Action29))> */ func() bool { - position120, tokenIndex120 := position, tokenIndex + position109, tokenIndex109 := position, tokenIndex { - position121 := position + position110 := position { - position122, tokenIndex122 := position, tokenIndex + position111, tokenIndex111 := position, tokenIndex if !_rules[ruleitem]() { - goto l123 + goto l112 } - goto l122 - l123: - position, tokenIndex = position122, tokenIndex122 + goto l111 + l112: + position, tokenIndex = position111, tokenIndex111 { - position124 := position + position113 := position if buffer[position] != rune('[') { - goto l120 + goto l109 } position++ if !_rules[rulesp]() { - goto l120 + goto l109 } - add(rulelbrack, position124) + add(rulelbrack, position113) + } + { + add(ruleAction28, position) + } + if !_rules[rulelist]() { + goto l109 + } + { + position115 := position + if !_rules[rulesp]() { + goto l109 + } + if buffer[position] != rune(']') { + goto l109 + } + position++ + if !_rules[rulesp]() { + goto l109 + } + add(rulerbrack, position115) + } + { + add(ruleAction29, position) + } + } + l111: + add(rulevalue, position110) + } + return true + l109: + position, tokenIndex = position109, tokenIndex109 + return false + }, + /* 11 list <- <(item (comma list)?)> */ + func() bool { + position117, tokenIndex117 := position, tokenIndex + { + position118 := position + if !_rules[ruleitem]() { + goto l117 + } + { + position119, tokenIndex119 := position, tokenIndex + if !_rules[rulecomma]() { + goto l119 + } + if !_rules[rulelist]() { + goto l119 + } + goto l120 + l119: + position, tokenIndex = position119, tokenIndex119 + } + l120: + add(rulelist, position118) + } + return true + l117: + position, tokenIndex = position117, tokenIndex117 + return false + }, + /* 12 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action30) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action31) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action32) / (timestampfmt Action33) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action34) / (<('-'? '.' [0-9]+)> Action35) / ( Action36 open allargs comma? close Action37) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action38) / (<('"' doublequotedstring '"')> Action39) / ('\'' '\'' Action40))> */ + func() bool { + position121, tokenIndex121 := position, tokenIndex + { + position122 := position + { + position123, tokenIndex123 := position, tokenIndex + if buffer[position] != rune('n') { + goto l124 + } + position++ + if buffer[position] != rune('u') { + goto l124 + } + position++ + if buffer[position] != rune('l') { + goto l124 + } + position++ + if buffer[position] != rune('l') { + goto l124 + } + position++ + { + position125, tokenIndex125 := position, tokenIndex + { + position126, tokenIndex126 := position, tokenIndex + if !_rules[rulecomma]() { + goto l127 + } + goto l126 + l127: + position, tokenIndex = position126, tokenIndex126 + if !_rules[rulesp]() { + goto l124 + } + if !_rules[ruleclose]() { + goto l124 + } + } + l126: + position, tokenIndex = position125, tokenIndex125 + } + { + add(ruleAction30, position) + } + goto l123 + l124: + position, tokenIndex = position123, tokenIndex123 + if buffer[position] != rune('t') { + goto l129 + } + position++ + if buffer[position] != rune('r') { + goto l129 + } + position++ + if buffer[position] != rune('u') { + goto l129 + } + position++ + if buffer[position] != rune('e') { + goto l129 + } + position++ + { + position130, tokenIndex130 := position, tokenIndex + { + position131, tokenIndex131 := position, tokenIndex + if !_rules[rulecomma]() { + goto l132 + } + goto l131 + l132: + position, tokenIndex = position131, tokenIndex131 + if !_rules[rulesp]() { + goto l129 + } + if !_rules[ruleclose]() { + goto l129 + } + } + l131: + position, tokenIndex = position130, tokenIndex130 + } + { + add(ruleAction31, position) + } + goto l123 + l129: + position, tokenIndex = position123, tokenIndex123 + if buffer[position] != rune('f') { + goto l134 + } + position++ + if buffer[position] != rune('a') { + goto l134 + } + position++ + if buffer[position] != rune('l') { + goto l134 + } + position++ + if buffer[position] != rune('s') { + goto l134 + } + position++ + if buffer[position] != rune('e') { + goto l134 + } + position++ + { + position135, tokenIndex135 := position, tokenIndex + { + position136, tokenIndex136 := position, tokenIndex + if !_rules[rulecomma]() { + goto l137 + } + goto l136 + l137: + position, tokenIndex = position136, tokenIndex136 + if !_rules[rulesp]() { + goto l134 + } + if !_rules[ruleclose]() { + goto l134 + } + } + l136: + position, tokenIndex = position135, tokenIndex135 } { add(ruleAction32, position) } - if !_rules[rulelist]() { - goto l120 - } - { - position126 := position - if !_rules[rulesp]() { - goto l120 - } - if buffer[position] != rune(']') { - goto l120 - } - position++ - if !_rules[rulesp]() { - goto l120 - } - add(rulerbrack, position126) + goto l123 + l134: + position, tokenIndex = position123, tokenIndex123 + if !_rules[ruletimestampfmt]() { + goto l139 } { add(ruleAction33, position) } - } - l122: - add(rulevalue, position121) - } - return true - l120: - position, tokenIndex = position120, tokenIndex120 - return false - }, - /* 12 list <- <(item (comma list)?)> */ - func() bool { - position128, tokenIndex128 := position, tokenIndex - { - position129 := position - if !_rules[ruleitem]() { - goto l128 - } - { - position130, tokenIndex130 := position, tokenIndex - if !_rules[rulecomma]() { - goto l130 - } - if !_rules[rulelist]() { - goto l130 - } - goto l131 - l130: - position, tokenIndex = position130, tokenIndex130 - } - l131: - add(rulelist, position129) - } - return true - l128: - position, tokenIndex = position128, tokenIndex128 - return false - }, - /* 13 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action34) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action35) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action36) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action37) / (<('-'? '.' [0-9]+)> Action38) / ( Action39 open allargs comma? close Action40) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action41) / (<('"' doublequotedstring '"')> Action42) / ('\'' '\'' Action43))> */ - func() bool { - position132, tokenIndex132 := position, tokenIndex - { - position133 := position - { - position134, tokenIndex134 := position, tokenIndex - if buffer[position] != rune('n') { - goto l135 - } - position++ - if buffer[position] != rune('u') { - goto l135 - } - position++ - if buffer[position] != rune('l') { - goto l135 - } - position++ - if buffer[position] != rune('l') { - goto l135 - } - position++ + goto l123 + l139: + position, tokenIndex = position123, tokenIndex123 { - position136, tokenIndex136 := position, tokenIndex + position142 := position { - position137, tokenIndex137 := position, tokenIndex - if !_rules[rulecomma]() { - goto l138 - } - goto l137 - l138: - position, tokenIndex = position137, tokenIndex137 - if !_rules[rulesp]() { - goto l135 - } - if !_rules[ruleclose]() { - goto l135 + position143, tokenIndex143 := position, tokenIndex + if buffer[position] != rune('-') { + goto l143 } + position++ + goto l144 + l143: + position, tokenIndex = position143, tokenIndex143 } - l137: - position, tokenIndex = position136, tokenIndex136 + l144: + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l141 + } + position++ + l145: + { + position146, tokenIndex146 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l146 + } + position++ + goto l145 + l146: + position, tokenIndex = position146, tokenIndex146 + } + { + position147, tokenIndex147 := position, tokenIndex + if buffer[position] != rune('.') { + goto l147 + } + position++ + l149: + { + position150, tokenIndex150 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l150 + } + position++ + goto l149 + l150: + position, tokenIndex = position150, tokenIndex150 + } + goto l148 + l147: + position, tokenIndex = position147, tokenIndex147 + } + l148: + add(rulePegText, position142) } { add(ruleAction34, position) } - goto l134 - l135: - position, tokenIndex = position134, tokenIndex134 - if buffer[position] != rune('t') { - goto l140 - } - position++ - if buffer[position] != rune('r') { - goto l140 - } - position++ - if buffer[position] != rune('u') { - goto l140 - } - position++ - if buffer[position] != rune('e') { - goto l140 - } - position++ + goto l123 + l141: + position, tokenIndex = position123, tokenIndex123 { - position141, tokenIndex141 := position, tokenIndex + position153 := position { - position142, tokenIndex142 := position, tokenIndex - if !_rules[rulecomma]() { - goto l143 - } - goto l142 - l143: - position, tokenIndex = position142, tokenIndex142 - if !_rules[rulesp]() { - goto l140 - } - if !_rules[ruleclose]() { - goto l140 + position154, tokenIndex154 := position, tokenIndex + if buffer[position] != rune('-') { + goto l154 } + position++ + goto l155 + l154: + position, tokenIndex = position154, tokenIndex154 } - l142: - position, tokenIndex = position141, tokenIndex141 + l155: + if buffer[position] != rune('.') { + goto l152 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l152 + } + position++ + l156: + { + position157, tokenIndex157 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l157 + } + position++ + goto l156 + l157: + position, tokenIndex = position157, tokenIndex157 + } + add(rulePegText, position153) } { add(ruleAction35, position) } - goto l134 - l140: - position, tokenIndex = position134, tokenIndex134 - if buffer[position] != rune('f') { - goto l145 - } - position++ - if buffer[position] != rune('a') { - goto l145 - } - position++ - if buffer[position] != rune('l') { - goto l145 - } - position++ - if buffer[position] != rune('s') { - goto l145 - } - position++ - if buffer[position] != rune('e') { - goto l145 - } - position++ + goto l123 + l152: + position, tokenIndex = position123, tokenIndex123 { - position146, tokenIndex146 := position, tokenIndex - { - position147, tokenIndex147 := position, tokenIndex - if !_rules[rulecomma]() { - goto l148 - } - goto l147 - l148: - position, tokenIndex = position147, tokenIndex147 - if !_rules[rulesp]() { - goto l145 - } - if !_rules[ruleclose]() { - goto l145 - } + position160 := position + if !_rules[ruleIDENT]() { + goto l159 } - l147: - position, tokenIndex = position146, tokenIndex146 + add(rulePegText, position160) } { add(ruleAction36, position) } - goto l134 - l145: - position, tokenIndex = position134, tokenIndex134 + if !_rules[ruleopen]() { + goto l159 + } + if !_rules[ruleallargs]() { + goto l159 + } { - position151 := position - { - position152, tokenIndex152 := position, tokenIndex - if buffer[position] != rune('-') { - goto l152 - } - position++ - goto l153 - l152: - position, tokenIndex = position152, tokenIndex152 + position162, tokenIndex162 := position, tokenIndex + if !_rules[rulecomma]() { + goto l162 } - l153: - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l150 - } - position++ - l154: - { - position155, tokenIndex155 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l155 - } - position++ - goto l154 - l155: - position, tokenIndex = position155, tokenIndex155 - } - { - position156, tokenIndex156 := position, tokenIndex - if buffer[position] != rune('.') { - goto l156 - } - position++ - l158: - { - position159, tokenIndex159 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l159 - } - position++ - goto l158 - l159: - position, tokenIndex = position159, tokenIndex159 - } - goto l157 - l156: - position, tokenIndex = position156, tokenIndex156 - } - l157: - add(rulePegText, position151) + goto l163 + l162: + position, tokenIndex = position162, tokenIndex162 + } + l163: + if !_rules[ruleclose]() { + goto l159 } { add(ruleAction37, position) } - goto l134 - l150: - position, tokenIndex = position134, tokenIndex134 + goto l123 + l159: + position, tokenIndex = position123, tokenIndex123 { - position162 := position + position166 := position { - position163, tokenIndex163 := position, tokenIndex - if buffer[position] != rune('-') { - goto l163 + position169, tokenIndex169 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l170 } position++ - goto l164 - l163: - position, tokenIndex = position163, tokenIndex163 - } - l164: - if buffer[position] != rune('.') { - goto l161 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l161 - } - position++ - l165: - { - position166, tokenIndex166 := position, tokenIndex + goto l169 + l170: + position, tokenIndex = position169, tokenIndex169 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l171 + } + position++ + goto l169 + l171: + position, tokenIndex = position169, tokenIndex169 if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l166 + goto l172 + } + position++ + goto l169 + l172: + position, tokenIndex = position169, tokenIndex169 + if buffer[position] != rune('-') { + goto l173 + } + position++ + goto l169 + l173: + position, tokenIndex = position169, tokenIndex169 + if buffer[position] != rune('_') { + goto l174 + } + position++ + goto l169 + l174: + position, tokenIndex = position169, tokenIndex169 + if buffer[position] != rune(':') { + goto l165 } position++ - goto l165 - l166: - position, tokenIndex = position166, tokenIndex166 } - add(rulePegText, position162) + l169: + l167: + { + position168, tokenIndex168 := position, tokenIndex + { + position175, tokenIndex175 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l176 + } + position++ + goto l175 + l176: + position, tokenIndex = position175, tokenIndex175 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l177 + } + position++ + goto l175 + l177: + position, tokenIndex = position175, tokenIndex175 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l178 + } + position++ + goto l175 + l178: + position, tokenIndex = position175, tokenIndex175 + if buffer[position] != rune('-') { + goto l179 + } + position++ + goto l175 + l179: + position, tokenIndex = position175, tokenIndex175 + if buffer[position] != rune('_') { + goto l180 + } + position++ + goto l175 + l180: + position, tokenIndex = position175, tokenIndex175 + if buffer[position] != rune(':') { + goto l168 + } + position++ + } + l175: + goto l167 + l168: + position, tokenIndex = position168, tokenIndex168 + } + add(rulePegText, position166) } { add(ruleAction38, position) } - goto l134 - l161: - position, tokenIndex = position134, tokenIndex134 + goto l123 + l165: + position, tokenIndex = position123, tokenIndex123 { - position169 := position - if !_rules[ruleIDENT]() { - goto l168 + position183 := position + if buffer[position] != rune('"') { + goto l182 } - add(rulePegText, position169) + position++ + if !_rules[ruledoublequotedstring]() { + goto l182 + } + if buffer[position] != rune('"') { + goto l182 + } + position++ + add(rulePegText, position183) } { add(ruleAction39, position) } - if !_rules[ruleopen]() { - goto l168 - } - if !_rules[ruleallargs]() { - goto l168 + goto l123 + l182: + position, tokenIndex = position123, tokenIndex123 + if buffer[position] != rune('\'') { + goto l121 } + position++ { - position171, tokenIndex171 := position, tokenIndex - if !_rules[rulecomma]() { - goto l171 + position185 := position + if !_rules[rulesinglequotedstring]() { + goto l121 } - goto l172 - l171: - position, tokenIndex = position171, tokenIndex171 + add(rulePegText, position185) } - l172: - if !_rules[ruleclose]() { - goto l168 + if buffer[position] != rune('\'') { + goto l121 } + position++ { add(ruleAction40, position) } - goto l134 - l168: - position, tokenIndex = position134, tokenIndex134 - { - position175 := position - { - position178, tokenIndex178 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l179 - } - position++ - goto l178 - l179: - position, tokenIndex = position178, tokenIndex178 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l180 - } - position++ - goto l178 - l180: - position, tokenIndex = position178, tokenIndex178 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l181 - } - position++ - goto l178 - l181: - position, tokenIndex = position178, tokenIndex178 - if buffer[position] != rune('-') { - goto l182 - } - position++ - goto l178 - l182: - position, tokenIndex = position178, tokenIndex178 - if buffer[position] != rune('_') { - goto l183 - } - position++ - goto l178 - l183: - position, tokenIndex = position178, tokenIndex178 - if buffer[position] != rune(':') { - goto l174 - } - position++ - } - l178: - l176: - { - position177, tokenIndex177 := position, tokenIndex - { - position184, tokenIndex184 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l185 - } - position++ - goto l184 - l185: - position, tokenIndex = position184, tokenIndex184 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l186 - } - position++ - goto l184 - l186: - position, tokenIndex = position184, tokenIndex184 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l187 - } - position++ - goto l184 - l187: - position, tokenIndex = position184, tokenIndex184 - if buffer[position] != rune('-') { - goto l188 - } - position++ - goto l184 - l188: - position, tokenIndex = position184, tokenIndex184 - if buffer[position] != rune('_') { - goto l189 - } - position++ - goto l184 - l189: - position, tokenIndex = position184, tokenIndex184 - if buffer[position] != rune(':') { - goto l177 - } - position++ - } - l184: - goto l176 - l177: - position, tokenIndex = position177, tokenIndex177 - } - add(rulePegText, position175) - } - { - add(ruleAction41, position) - } - goto l134 - l174: - position, tokenIndex = position134, tokenIndex134 - { - position192 := position - if buffer[position] != rune('"') { - goto l191 - } - position++ - if !_rules[ruledoublequotedstring]() { - goto l191 - } - if buffer[position] != rune('"') { - goto l191 - } - position++ - add(rulePegText, position192) - } - { - add(ruleAction42, position) - } - goto l134 - l191: - position, tokenIndex = position134, tokenIndex134 - if buffer[position] != rune('\'') { - goto l132 - } - position++ - { - position194 := position - if !_rules[rulesinglequotedstring]() { - goto l132 - } - add(rulePegText, position194) - } - if buffer[position] != rune('\'') { - goto l132 - } - position++ - { - add(ruleAction43, position) - } } - l134: - add(ruleitem, position133) + l123: + add(ruleitem, position122) } return true - l132: - position, tokenIndex = position132, tokenIndex132 + l121: + position, tokenIndex = position121, tokenIndex121 return false }, - /* 14 doublequotedstring <- <(('\\' '"') / ('\\' '\\') / (!'"' .))*> */ + /* 13 doublequotedstring <- <(('\\' '"') / ('\\' '\\') / (!'"' .))*> */ func() bool { { - position197 := position - l198: + position188 := position + l189: { - position199, tokenIndex199 := position, tokenIndex + position190, tokenIndex190 := position, tokenIndex { - position200, tokenIndex200 := position, tokenIndex + position191, tokenIndex191 := position, tokenIndex if buffer[position] != rune('\\') { - goto l201 + goto l192 } position++ if buffer[position] != rune('"') { - goto l201 + goto l192 } position++ - goto l200 - l201: - position, tokenIndex = position200, tokenIndex200 + goto l191 + l192: + position, tokenIndex = position191, tokenIndex191 if buffer[position] != rune('\\') { - goto l202 + goto l193 } position++ if buffer[position] != rune('\\') { - goto l202 + goto l193 } position++ - goto l200 - l202: - position, tokenIndex = position200, tokenIndex200 + goto l191 + l193: + position, tokenIndex = position191, tokenIndex191 { - position203, tokenIndex203 := position, tokenIndex + position194, tokenIndex194 := position, tokenIndex if buffer[position] != rune('"') { - goto l203 + goto l194 } position++ - goto l199 - l203: - position, tokenIndex = position203, tokenIndex203 + goto l190 + l194: + position, tokenIndex = position194, tokenIndex194 } if !matchDot() { - goto l199 + goto l190 } } - l200: - goto l198 - l199: - position, tokenIndex = position199, tokenIndex199 + l191: + goto l189 + l190: + position, tokenIndex = position190, tokenIndex190 } - add(ruledoublequotedstring, position197) + add(ruledoublequotedstring, position188) } return true }, - /* 15 singlequotedstring <- <(('\\' '\'') / ('\\' '\\') / (!'\'' .))*> */ + /* 14 singlequotedstring <- <(('\\' '\'') / ('\\' '\\') / (!'\'' .))*> */ func() bool { { - position205 := position - l206: + position196 := position + l197: { - position207, tokenIndex207 := position, tokenIndex + position198, tokenIndex198 := position, tokenIndex { - position208, tokenIndex208 := position, tokenIndex + position199, tokenIndex199 := position, tokenIndex if buffer[position] != rune('\\') { - goto l209 + goto l200 } position++ if buffer[position] != rune('\'') { - goto l209 + goto l200 } position++ - goto l208 - l209: - position, tokenIndex = position208, tokenIndex208 + goto l199 + l200: + position, tokenIndex = position199, tokenIndex199 if buffer[position] != rune('\\') { - goto l210 + goto l201 } position++ if buffer[position] != rune('\\') { - goto l210 + goto l201 } position++ - goto l208 - l210: - position, tokenIndex = position208, tokenIndex208 + goto l199 + l201: + position, tokenIndex = position199, tokenIndex199 { - position211, tokenIndex211 := position, tokenIndex + position202, tokenIndex202 := position, tokenIndex if buffer[position] != rune('\'') { - goto l211 + goto l202 } position++ - goto l207 - l211: - position, tokenIndex = position211, tokenIndex211 + goto l198 + l202: + position, tokenIndex = position202, tokenIndex202 } if !matchDot() { - goto l207 + goto l198 } } - l208: - goto l206 - l207: - position, tokenIndex = position207, tokenIndex207 + l199: + goto l197 + l198: + position, tokenIndex = position198, tokenIndex198 } - add(rulesinglequotedstring, position205) + add(rulesinglequotedstring, position196) } return true }, - /* 16 fieldExpr <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '_' / '-')*)> */ + /* 15 fieldExpr <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '_' / '-')*)> */ func() bool { - position212, tokenIndex212 := position, tokenIndex + position203, tokenIndex203 := position, tokenIndex { - position213 := position + position204 := position { - position214, tokenIndex214 := position, tokenIndex + position205, tokenIndex205 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l215 + goto l206 } position++ - goto l214 - l215: - position, tokenIndex = position214, tokenIndex214 + goto l205 + l206: + position, tokenIndex = position205, tokenIndex205 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l212 + goto l203 } position++ } - l214: - l216: + l205: + l207: { - position217, tokenIndex217 := position, tokenIndex + position208, tokenIndex208 := position, tokenIndex { - position218, tokenIndex218 := position, tokenIndex + position209, tokenIndex209 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l219 + goto l210 } position++ - goto l218 - l219: - position, tokenIndex = position218, tokenIndex218 + goto l209 + l210: + position, tokenIndex = position209, tokenIndex209 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l220 + goto l211 } position++ - goto l218 - l220: - position, tokenIndex = position218, tokenIndex218 + goto l209 + l211: + position, tokenIndex = position209, tokenIndex209 if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l221 + goto l212 } position++ - goto l218 - l221: - position, tokenIndex = position218, tokenIndex218 + goto l209 + l212: + position, tokenIndex = position209, tokenIndex209 if buffer[position] != rune('_') { - goto l222 + goto l213 } position++ - goto l218 - l222: - position, tokenIndex = position218, tokenIndex218 + goto l209 + l213: + position, tokenIndex = position209, tokenIndex209 if buffer[position] != rune('-') { - goto l217 + goto l208 } position++ } - l218: - goto l216 - l217: - position, tokenIndex = position217, tokenIndex217 + l209: + goto l207 + l208: + position, tokenIndex = position208, tokenIndex208 } - add(rulefieldExpr, position213) + add(rulefieldExpr, position204) } return true - l212: - position, tokenIndex = position212, tokenIndex212 + l203: + position, tokenIndex = position203, tokenIndex203 return false }, - /* 17 field <- <(<(fieldExpr / reserved)> Action44)> */ + /* 16 field <- <(<(fieldExpr / reserved)> Action41)> */ func() bool { - position223, tokenIndex223 := position, tokenIndex + position214, tokenIndex214 := position, tokenIndex { - position224 := position + position215 := position { - position225 := position + position216 := position { - position226, tokenIndex226 := position, tokenIndex + position217, tokenIndex217 := position, tokenIndex if !_rules[rulefieldExpr]() { - goto l227 + goto l218 } - goto l226 - l227: - position, tokenIndex = position226, tokenIndex226 + goto l217 + l218: + position, tokenIndex = position217, tokenIndex217 { - position228 := position + position219 := position { - position229, tokenIndex229 := position, tokenIndex + position220, tokenIndex220 := position, tokenIndex if buffer[position] != rune('_') { - goto l230 + goto l221 } position++ if buffer[position] != rune('r') { - goto l230 + goto l221 } position++ if buffer[position] != rune('o') { - goto l230 + goto l221 } position++ if buffer[position] != rune('w') { - goto l230 + goto l221 } position++ - goto l229 - l230: - position, tokenIndex = position229, tokenIndex229 + goto l220 + l221: + position, tokenIndex = position220, tokenIndex220 if buffer[position] != rune('_') { - goto l231 + goto l222 } position++ if buffer[position] != rune('c') { - goto l231 + goto l222 } position++ if buffer[position] != rune('o') { - goto l231 + goto l222 } position++ if buffer[position] != rune('l') { - goto l231 + goto l222 } position++ - goto l229 - l231: - position, tokenIndex = position229, tokenIndex229 + goto l220 + l222: + position, tokenIndex = position220, tokenIndex220 if buffer[position] != rune('_') { - goto l232 + goto l223 } position++ if buffer[position] != rune('s') { - goto l232 + goto l223 } position++ if buffer[position] != rune('t') { - goto l232 + goto l223 } position++ if buffer[position] != rune('a') { - goto l232 + goto l223 } position++ if buffer[position] != rune('r') { - goto l232 + goto l223 } position++ if buffer[position] != rune('t') { - goto l232 + goto l223 } position++ - goto l229 - l232: - position, tokenIndex = position229, tokenIndex229 + goto l220 + l223: + position, tokenIndex = position220, tokenIndex220 if buffer[position] != rune('_') { - goto l233 + goto l224 } position++ if buffer[position] != rune('e') { - goto l233 + goto l224 } position++ if buffer[position] != rune('n') { - goto l233 + goto l224 } position++ if buffer[position] != rune('d') { - goto l233 + goto l224 } position++ - goto l229 - l233: - position, tokenIndex = position229, tokenIndex229 + goto l220 + l224: + position, tokenIndex = position220, tokenIndex220 if buffer[position] != rune('_') { - goto l234 + goto l225 } position++ if buffer[position] != rune('t') { - goto l234 + goto l225 } position++ if buffer[position] != rune('i') { - goto l234 + goto l225 } position++ if buffer[position] != rune('m') { - goto l234 + goto l225 } position++ if buffer[position] != rune('e') { - goto l234 + goto l225 } position++ if buffer[position] != rune('s') { - goto l234 + goto l225 } position++ if buffer[position] != rune('t') { - goto l234 + goto l225 } position++ if buffer[position] != rune('a') { - goto l234 + goto l225 } position++ if buffer[position] != rune('m') { - goto l234 + goto l225 } position++ if buffer[position] != rune('p') { - goto l234 + goto l225 } position++ - goto l229 - l234: - position, tokenIndex = position229, tokenIndex229 + goto l220 + l225: + position, tokenIndex = position220, tokenIndex220 if buffer[position] != rune('_') { - goto l223 + goto l214 } position++ if buffer[position] != rune('f') { - goto l223 + goto l214 } position++ if buffer[position] != rune('i') { - goto l223 + goto l214 } position++ if buffer[position] != rune('e') { - goto l223 + goto l214 } position++ if buffer[position] != rune('l') { - goto l223 + goto l214 } position++ if buffer[position] != rune('d') { - goto l223 + goto l214 } position++ } - l229: - add(rulereserved, position228) + l220: + add(rulereserved, position219) } } - l226: - add(rulePegText, position225) + l217: + add(rulePegText, position216) } { - add(ruleAction44, position) + add(ruleAction41, position) } - add(rulefield, position224) + add(rulefield, position215) } return true - l223: - position, tokenIndex = position223, tokenIndex223 + l214: + position, tokenIndex = position214, tokenIndex214 return false }, - /* 18 reserved <- <(('_' 'r' 'o' 'w') / ('_' 'c' 'o' 'l') / ('_' 's' 't' 'a' 'r' 't') / ('_' 'e' 'n' 'd') / ('_' 't' 'i' 'm' 'e' 's' 't' 'a' 'm' 'p') / ('_' 'f' 'i' 'e' 'l' 'd'))> */ + /* 17 reserved <- <(('_' 'r' 'o' 'w') / ('_' 'c' 'o' 'l') / ('_' 's' 't' 'a' 'r' 't') / ('_' 'e' 'n' 'd') / ('_' 't' 'i' 'm' 'e' 's' 't' 'a' 'm' 'p') / ('_' 'f' 'i' 'e' 'l' 'd'))> */ nil, - /* 19 posfield <- <( Action45)> */ + /* 18 posfield <- <( Action42)> */ func() bool { - position237, tokenIndex237 := position, tokenIndex + position228, tokenIndex228 := position, tokenIndex { - position238 := position + position229 := position { - position239 := position + position230 := position if !_rules[rulefieldExpr]() { - goto l237 + goto l228 } - add(rulePegText, position239) + add(rulePegText, position230) } { - add(ruleAction45, position) + add(ruleAction42, position) } - add(ruleposfield, position238) + add(ruleposfield, position229) } return true - l237: - position, tokenIndex = position237, tokenIndex237 + l228: + position, tokenIndex = position228, tokenIndex228 return false }, - /* 20 uint <- <(([1-9] [0-9]*) / '0')> */ + /* 19 uint <- <(([1-9] [0-9]*) / '0')> */ func() bool { - position241, tokenIndex241 := position, tokenIndex + position232, tokenIndex232 := position, tokenIndex { - position242 := position + position233 := position { - position243, tokenIndex243 := position, tokenIndex + position234, tokenIndex234 := position, tokenIndex if c := buffer[position]; c < rune('1') || c > rune('9') { + goto l235 + } + position++ + l236: + { + position237, tokenIndex237 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l237 + } + position++ + goto l236 + l237: + position, tokenIndex = position237, tokenIndex237 + } + goto l234 + l235: + position, tokenIndex = position234, tokenIndex234 + if buffer[position] != rune('0') { + goto l232 + } + position++ + } + l234: + add(ruleuint, position233) + } + return true + l232: + position, tokenIndex = position232, tokenIndex232 + return false + }, + /* 20 col <- <(( Action43) / ('\'' '\'' Action44) / ('"' '"' Action45))> */ + func() bool { + position238, tokenIndex238 := position, tokenIndex + { + position239 := position + { + position240, tokenIndex240 := position, tokenIndex + { + position242 := position + if !_rules[ruleuint]() { + goto l241 + } + add(rulePegText, position242) + } + { + add(ruleAction43, position) + } + goto l240 + l241: + position, tokenIndex = position240, tokenIndex240 + if buffer[position] != rune('\'') { goto l244 } position++ - l245: { - position246, tokenIndex246 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l246 - } - position++ - goto l245 - l246: - position, tokenIndex = position246, tokenIndex246 - } - goto l243 - l244: - position, tokenIndex = position243, tokenIndex243 - if buffer[position] != rune('0') { - goto l241 - } - position++ - } - l243: - add(ruleuint, position242) - } - return true - l241: - position, tokenIndex = position241, tokenIndex241 - return false - }, - /* 21 col <- <(( Action46) / ('\'' '\'' Action47) / ('"' '"' Action48))> */ - func() bool { - position247, tokenIndex247 := position, tokenIndex - { - position248 := position - { - position249, tokenIndex249 := position, tokenIndex - { - position251 := position - if !_rules[ruleuint]() { - goto l250 - } - add(rulePegText, position251) - } - { - add(ruleAction46, position) - } - goto l249 - l250: - position, tokenIndex = position249, tokenIndex249 - if buffer[position] != rune('\'') { - goto l253 - } - position++ - { - position254 := position + position245 := position if !_rules[rulesinglequotedstring]() { - goto l253 + goto l244 } - add(rulePegText, position254) + add(rulePegText, position245) } if buffer[position] != rune('\'') { - goto l253 + goto l244 } position++ { - add(ruleAction47, position) + add(ruleAction44, position) } - goto l249 - l253: - position, tokenIndex = position249, tokenIndex249 + goto l240 + l244: + position, tokenIndex = position240, tokenIndex240 if buffer[position] != rune('"') { - goto l247 + goto l238 } position++ { - position256 := position + position247 := position if !_rules[ruledoublequotedstring]() { - goto l247 + goto l238 } - add(rulePegText, position256) + add(rulePegText, position247) } if buffer[position] != rune('"') { - goto l247 + goto l238 } position++ { - add(ruleAction48, position) + add(ruleAction45, position) } } - l249: - add(rulecol, position248) + l240: + add(rulecol, position239) } return true - l247: - position, tokenIndex = position247, tokenIndex247 + l238: + position, tokenIndex = position238, tokenIndex238 return false }, - /* 22 row <- <(( Action49) / ('\'' '\'' Action50) / ('"' '"' Action51))> */ + /* 21 row <- <(( Action46) / ('\'' '\'' Action47) / ('"' '"' Action48))> */ nil, - /* 23 open <- <('(' sp)> */ + /* 22 open <- <('(' sp)> */ func() bool { - position259, tokenIndex259 := position, tokenIndex + position250, tokenIndex250 := position, tokenIndex { - position260 := position + position251 := position if buffer[position] != rune('(') { - goto l259 + goto l250 } position++ if !_rules[rulesp]() { - goto l259 + goto l250 } - add(ruleopen, position260) + add(ruleopen, position251) } return true - l259: - position, tokenIndex = position259, tokenIndex259 + l250: + position, tokenIndex = position250, tokenIndex250 return false }, - /* 24 close <- <(')' sp)> */ + /* 23 close <- <(')' sp)> */ + func() bool { + position252, tokenIndex252 := position, tokenIndex + { + position253 := position + if buffer[position] != rune(')') { + goto l252 + } + position++ + if !_rules[rulesp]() { + goto l252 + } + add(ruleclose, position253) + } + return true + l252: + position, tokenIndex = position252, tokenIndex252 + return false + }, + /* 24 sp <- <(' ' / '\t' / '\n')*> */ + func() bool { + { + position255 := position + l256: + { + position257, tokenIndex257 := position, tokenIndex + { + position258, tokenIndex258 := position, tokenIndex + if buffer[position] != rune(' ') { + goto l259 + } + position++ + goto l258 + l259: + position, tokenIndex = position258, tokenIndex258 + if buffer[position] != rune('\t') { + goto l260 + } + position++ + goto l258 + l260: + position, tokenIndex = position258, tokenIndex258 + if buffer[position] != rune('\n') { + goto l257 + } + position++ + } + l258: + goto l256 + l257: + position, tokenIndex = position257, tokenIndex257 + } + add(rulesp, position255) + } + return true + }, + /* 25 comma <- <(sp ',' sp)> */ func() bool { position261, tokenIndex261 := position, tokenIndex { position262 := position - if buffer[position] != rune(')') { + if !_rules[rulesp]() { + goto l261 + } + if buffer[position] != rune(',') { goto l261 } position++ if !_rules[rulesp]() { goto l261 } - add(ruleclose, position262) + add(rulecomma, position262) } return true l261: position, tokenIndex = position261, tokenIndex261 return false }, - /* 25 sp <- <(' ' / '\t' / '\n')*> */ + /* 26 lbrack <- <('[' sp)> */ + nil, + /* 27 rbrack <- <(sp ']' sp)> */ + nil, + /* 28 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ func() bool { + position265, tokenIndex265 := position, tokenIndex { - position264 := position - l265: + position266 := position { - position266, tokenIndex266 := position, tokenIndex + position267, tokenIndex267 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l268 + } + position++ + goto l267 + l268: + position, tokenIndex = position267, tokenIndex267 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l265 + } + position++ + } + l267: + l269: + { + position270, tokenIndex270 := position, tokenIndex { - position267, tokenIndex267 := position, tokenIndex - if buffer[position] != rune(' ') { - goto l268 + position271, tokenIndex271 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l272 } position++ - goto l267 - l268: - position, tokenIndex = position267, tokenIndex267 - if buffer[position] != rune('\t') { - goto l269 + goto l271 + l272: + position, tokenIndex = position271, tokenIndex271 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l273 } position++ - goto l267 - l269: - position, tokenIndex = position267, tokenIndex267 - if buffer[position] != rune('\n') { - goto l266 + goto l271 + l273: + position, tokenIndex = position271, tokenIndex271 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l270 } position++ } - l267: - goto l265 - l266: - position, tokenIndex = position266, tokenIndex266 + l271: + goto l269 + l270: + position, tokenIndex = position270, tokenIndex270 } - add(rulesp, position264) + add(ruleIDENT, position266) } return true - }, - /* 26 comma <- <(sp ',' sp)> */ - func() bool { - position270, tokenIndex270 := position, tokenIndex - { - position271 := position - if !_rules[rulesp]() { - goto l270 - } - if buffer[position] != rune(',') { - goto l270 - } - position++ - if !_rules[rulesp]() { - goto l270 - } - add(rulecomma, position271) - } - return true - l270: - position, tokenIndex = position270, tokenIndex270 + l265: + position, tokenIndex = position265, tokenIndex265 return false }, - /* 27 lbrack <- <('[' sp)> */ - nil, - /* 28 rbrack <- <(sp ']' sp)> */ - nil, - /* 29 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ + /* 29 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ func() bool { position274, tokenIndex274 := position, tokenIndex { position275 := position + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l274 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l274 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l274 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l274 + } + position++ + if buffer[position] != rune('-') { + goto l274 + } + position++ { position276, tokenIndex276 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { + if buffer[position] != rune('0') { goto l277 } position++ goto l276 l277: position, tokenIndex = position276, tokenIndex276 - if c := buffer[position]; c < rune('A') || c > rune('Z') { + if buffer[position] != rune('1') { goto l274 } position++ } l276: - l278: - { - position279, tokenIndex279 := position, tokenIndex - { - position280, tokenIndex280 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l281 - } - position++ - goto l280 - l281: - position, tokenIndex = position280, tokenIndex280 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l282 - } - position++ - goto l280 - l282: - position, tokenIndex = position280, tokenIndex280 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l279 - } - position++ - } - l280: - goto l278 - l279: - position, tokenIndex = position279, tokenIndex279 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l274 } - add(ruleIDENT, position275) + position++ + if buffer[position] != rune('-') { + goto l274 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('3') { + goto l274 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l274 + } + position++ + if buffer[position] != rune('T') { + goto l274 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l274 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l274 + } + position++ + if buffer[position] != rune(':') { + goto l274 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l274 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l274 + } + position++ + add(ruletimestampbasicfmt, position275) } return true l274: position, tokenIndex = position274, tokenIndex274 return false }, - /* 30 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ + /* 30 timestampfmt <- <(('"' '"') / ('\'' '\'') / )> */ func() bool { - position283, tokenIndex283 := position, tokenIndex + position278, tokenIndex278 := position, tokenIndex { - position284 := position - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l283 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l283 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l283 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l283 - } - position++ - if buffer[position] != rune('-') { - goto l283 - } - position++ + position279 := position { - position285, tokenIndex285 := position, tokenIndex - if buffer[position] != rune('0') { - goto l286 + position280, tokenIndex280 := position, tokenIndex + if buffer[position] != rune('"') { + goto l281 } position++ - goto l285 - l286: - position, tokenIndex = position285, tokenIndex285 - if buffer[position] != rune('1') { + { + position282 := position + if !_rules[ruletimestampbasicfmt]() { + goto l281 + } + add(rulePegText, position282) + } + if buffer[position] != rune('"') { + goto l281 + } + position++ + goto l280 + l281: + position, tokenIndex = position280, tokenIndex280 + if buffer[position] != rune('\'') { goto l283 } position++ - } - l285: - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l283 - } - position++ - if buffer[position] != rune('-') { - goto l283 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('3') { - goto l283 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l283 - } - position++ - if buffer[position] != rune('T') { - goto l283 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l283 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l283 - } - position++ - if buffer[position] != rune(':') { - goto l283 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l283 - } - position++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l283 - } - position++ - add(ruletimestampbasicfmt, position284) - } - return true - l283: - position, tokenIndex = position283, tokenIndex283 - return false - }, - /* 31 timestampfmt <- <(('"' timestampbasicfmt '"') / ('\'' timestampbasicfmt '\'') / timestampbasicfmt)> */ - func() bool { - position287, tokenIndex287 := position, tokenIndex - { - position288 := position - { - position289, tokenIndex289 := position, tokenIndex - if buffer[position] != rune('"') { - goto l290 - } - position++ - if !_rules[ruletimestampbasicfmt]() { - goto l290 - } - if buffer[position] != rune('"') { - goto l290 - } - position++ - goto l289 - l290: - position, tokenIndex = position289, tokenIndex289 - if buffer[position] != rune('\'') { - goto l291 - } - position++ - if !_rules[ruletimestampbasicfmt]() { - goto l291 + { + position284 := position + if !_rules[ruletimestampbasicfmt]() { + goto l283 + } + add(rulePegText, position284) } if buffer[position] != rune('\'') { - goto l291 + goto l283 } position++ - goto l289 - l291: - position, tokenIndex = position289, tokenIndex289 - if !_rules[ruletimestampbasicfmt]() { - goto l287 + goto l280 + l283: + position, tokenIndex = position280, tokenIndex280 + { + position285 := position + if !_rules[ruletimestampbasicfmt]() { + goto l278 + } + add(rulePegText, position285) } } - l289: - add(ruletimestampfmt, position288) + l280: + add(ruletimestampfmt, position279) } return true - l287: - position, tokenIndex = position287, tokenIndex287 + l278: + position, tokenIndex = position278, tokenIndex278 return false }, - /* 32 timestamp <- <( Action52)> */ + /* 31 timestamp <- <( Action49)> */ nil, - /* 34 Action0 <- <{p.startCall("Set")}> */ + /* 33 Action0 <- <{p.startCall("Set")}> */ nil, - /* 35 Action1 <- <{p.endCall()}> */ + /* 34 Action1 <- <{p.endCall()}> */ nil, - /* 36 Action2 <- <{p.startCall("SetRowAttrs")}> */ + /* 35 Action2 <- <{p.startCall("SetRowAttrs")}> */ nil, - /* 37 Action3 <- <{p.endCall()}> */ + /* 36 Action3 <- <{p.endCall()}> */ nil, - /* 38 Action4 <- <{p.startCall("SetColumnAttrs")}> */ + /* 37 Action4 <- <{p.startCall("SetColumnAttrs")}> */ nil, - /* 39 Action5 <- <{p.endCall()}> */ + /* 38 Action5 <- <{p.endCall()}> */ nil, - /* 40 Action6 <- <{p.startCall("Clear")}> */ + /* 39 Action6 <- <{p.startCall("Clear")}> */ nil, - /* 41 Action7 <- <{p.endCall()}> */ + /* 40 Action7 <- <{p.endCall()}> */ nil, - /* 42 Action8 <- <{p.startCall("ClearRow")}> */ + /* 41 Action8 <- <{p.startCall("ClearRow")}> */ nil, - /* 43 Action9 <- <{p.endCall()}> */ + /* 42 Action9 <- <{p.endCall()}> */ nil, - /* 44 Action10 <- <{p.startCall("Store")}> */ + /* 43 Action10 <- <{p.startCall("Store")}> */ nil, - /* 45 Action11 <- <{p.endCall()}> */ + /* 44 Action11 <- <{p.endCall()}> */ nil, - /* 46 Action12 <- <{p.startCall("TopN")}> */ + /* 45 Action12 <- <{p.startCall("TopN")}> */ nil, - /* 47 Action13 <- <{p.endCall()}> */ - nil, - /* 48 Action14 <- <{p.startCall("Range")}> */ - nil, - /* 49 Action15 <- <{p.endCall()}> */ + /* 46 Action13 <- <{p.endCall()}> */ nil, nil, - /* 51 Action16 <- <{ p.startCall(buffer[begin:end] ) }> */ + /* 48 Action14 <- <{ p.startCall(buffer[begin:end] ) }> */ nil, - /* 52 Action17 <- <{ p.endCall() }> */ + /* 49 Action15 <- <{ p.endCall() }> */ nil, - /* 53 Action18 <- <{ p.addBTWN() }> */ + /* 50 Action16 <- <{ p.addBTWN() }> */ nil, - /* 54 Action19 <- <{ p.addLTE() }> */ + /* 51 Action17 <- <{ p.addLTE() }> */ nil, - /* 55 Action20 <- <{ p.addGTE() }> */ + /* 52 Action18 <- <{ p.addGTE() }> */ nil, - /* 56 Action21 <- <{ p.addEQ() }> */ + /* 53 Action19 <- <{ p.addEQ() }> */ nil, - /* 57 Action22 <- <{ p.addNEQ() }> */ + /* 54 Action20 <- <{ p.addNEQ() }> */ nil, - /* 58 Action23 <- <{ p.addLT() }> */ + /* 55 Action21 <- <{ p.addLT() }> */ nil, - /* 59 Action24 <- <{ p.addGT() }> */ + /* 56 Action22 <- <{ p.addGT() }> */ nil, - /* 60 Action25 <- <{p.startConditional()}> */ + /* 57 Action23 <- <{p.startConditional()}> */ nil, - /* 61 Action26 <- <{p.endConditional()}> */ + /* 58 Action24 <- <{p.endConditional()}> */ nil, - /* 62 Action27 <- <{p.condAdd(buffer[begin:end])}> */ + /* 59 Action25 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 63 Action28 <- <{p.condAdd(buffer[begin:end])}> */ + /* 60 Action26 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 64 Action29 <- <{p.condAdd(buffer[begin:end])}> */ + /* 61 Action27 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 65 Action30 <- <{p.addPosStr("_start", buffer[begin:end])}> */ + /* 62 Action28 <- <{ p.startList() }> */ nil, - /* 66 Action31 <- <{p.addPosStr("_end", buffer[begin:end])}> */ + /* 63 Action29 <- <{ p.endList() }> */ nil, - /* 67 Action32 <- <{ p.startList() }> */ + /* 64 Action30 <- <{ p.addVal(nil) }> */ nil, - /* 68 Action33 <- <{ p.endList() }> */ + /* 65 Action31 <- <{ p.addVal(true) }> */ nil, - /* 69 Action34 <- <{ p.addVal(nil) }> */ + /* 66 Action32 <- <{ p.addVal(false) }> */ nil, - /* 70 Action35 <- <{ p.addVal(true) }> */ + /* 67 Action33 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 71 Action36 <- <{ p.addVal(false) }> */ + /* 68 Action34 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 72 Action37 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 69 Action35 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 73 Action38 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 70 Action36 <- <{ p.startCall(buffer[begin:end]) }> */ nil, - /* 74 Action39 <- <{ p.startCall(buffer[begin:end]) }> */ + /* 71 Action37 <- <{ p.addVal(p.endCall()) }> */ nil, - /* 75 Action40 <- <{ p.addVal(p.endCall()) }> */ + /* 72 Action38 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 76 Action41 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 73 Action39 <- <{ s, _ := strconv.Unquote(buffer[begin:end]); p.addVal(s) }> */ nil, - /* 77 Action42 <- <{ s, _ := strconv.Unquote(buffer[begin:end]); p.addVal(s) }> */ + /* 74 Action40 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 78 Action43 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 75 Action41 <- <{ p.addField(buffer[begin:end]) }> */ nil, - /* 79 Action44 <- <{ p.addField(buffer[begin:end]) }> */ + /* 76 Action42 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ nil, - /* 80 Action45 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ + /* 77 Action43 <- <{p.addPosNum("_col", buffer[begin:end])}> */ nil, - /* 81 Action46 <- <{p.addPosNum("_col", buffer[begin:end])}> */ + /* 78 Action44 <- <{p.addPosStr("_col", buffer[begin:end])}> */ nil, - /* 82 Action47 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + /* 79 Action45 <- <{p.addPosStr("_col", buffer[begin:end])}> */ nil, - /* 83 Action48 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + /* 80 Action46 <- <{p.addPosNum("_row", buffer[begin:end])}> */ nil, - /* 84 Action49 <- <{p.addPosNum("_row", buffer[begin:end])}> */ + /* 81 Action47 <- <{p.addPosStr("_row", buffer[begin:end])}> */ nil, - /* 85 Action50 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + /* 82 Action48 <- <{p.addPosStr("_row", buffer[begin:end])}> */ nil, - /* 86 Action51 <- <{p.addPosStr("_row", buffer[begin:end])}> */ - nil, - /* 87 Action52 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ + /* 83 Action49 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ nil, } p.rules = _rules diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index f79780c8c..0c3a83bb2 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -8,7 +8,7 @@ import ( func TestPEG(t *testing.T) { p := PQL{Buffer: ` -SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9.com=\\'hello' and \"hello\"")), Hitmap(row=ag-bee)), a="4z", b=5) Count(Union(Witmap(row=5.73, frame=.10), Range(zztop><[2, 9]))) TopN(blah, fields=["hello", "goodbye", "zero"])`[1:]} +SetBit(Union(Zitmap(row==4), Intersect(Qitmap(blah>4), Ritmap(field="http://zoo9.com=\\'hello' and \"hello\"")), Hitmap(row=ag-bee)), a="4z", b=5) Count(Union(Witmap(row=5.73, frame=.10), Row(zztop><[2, 9]))) TopN(blah, fields=["hello", "goodbye", "zero"])`[1:]} p.Init() err := p.Parse() if err != nil { @@ -202,51 +202,51 @@ func TestPEGWorking(t *testing.T) { ncalls: 1}, { name: "RangeLT", - input: "Range(a < 4)", + input: "Row(a < 4)", ncalls: 1}, { name: "RangeGT", - input: "Range(a > 4)", + input: "Row(a > 4)", ncalls: 1}, { name: "RangeLTE", - input: "Range(a <= 4)", + input: "Row(a <= 4)", ncalls: 1}, { name: "RangeGTE", - input: "Range(a >= 4)", + input: "Row(a >= 4)", ncalls: 1}, { name: "RangeEQ", - input: "Range(a == 4)", + input: "Row(a == 4)", ncalls: 1}, { name: "RangeNEQ", - input: "Range(a != null)", + input: "Row(a != null)", ncalls: 1}, { name: "RangeLTLT", - input: "Range(4 < a < 9)", + input: "Row(4 < a < 9)", ncalls: 1}, { name: "RangeLTLTE", - input: "Range(4 < a <= 9)", + input: "Row(4 < a <= 9)", ncalls: 1}, { name: "RangeLTELT", - input: "Range(4 <= a < 9)", + input: "Row(4 <= a < 9)", ncalls: 1}, { name: "RangeLTELTE", - input: "Range(4 <= a <= 9)", + input: "Row(4 <= a <= 9)", ncalls: 1}, { name: "RangeTime", - input: "Range(a=4, 2010-07-04T00:00, 2010-08-04T00:00)", + input: "Row(a=4, from=2010-07-04T00:00, to=2010-08-04T00:00)", ncalls: 1}, { name: "RangeTimeQuotes", - input: `Range(a=4, '2010-07-04T00:00', "2010-08-04T00:00")`, + input: `Row(a=4, from='2010-07-04T00:00', to="2010-08-04T00:00")`, ncalls: 1}, { name: "Dashed Frame", @@ -302,10 +302,10 @@ func TestPEGErrors(t *testing.T) { input: "Clear(9)"}, { name: "RangeTimeGT", - input: "Range(a>4, 2010-07-04T00:00, 2010-08-04T00:00)"}, + input: "Row(a>4, 2010-07-04T00:00, 2010-08-04T00:00)"}, { name: "RangeTimeOneStamp", - input: "Range(a=4, 2010-07-04T00:00)"}, + input: "Row(a=4, 2010-07-04T00:00)"}, } for i, test := range tests { @@ -423,9 +423,9 @@ func TestPQLDeepEquality(t *testing.T) { }}, { name: "RangeEQ", - call: "Range(a==7)", + call: "Row(a==7)", exp: &Call{ - Name: "Range", + Name: "Row", Args: map[string]interface{}{ "a": &Condition{ Op: EQ, @@ -435,9 +435,9 @@ func TestPQLDeepEquality(t *testing.T) { }}, { name: "RangeLT", - call: "Range(a<7)", + call: "Row(a<7)", exp: &Call{ - Name: "Range", + Name: "Row", Args: map[string]interface{}{ "a": &Condition{ Op: LT, @@ -447,9 +447,9 @@ func TestPQLDeepEquality(t *testing.T) { }}, { name: "RangeLTE", - call: "Range(a<=7)", + call: "Row(a<=7)", exp: &Call{ - Name: "Range", + Name: "Row", Args: map[string]interface{}{ "a": &Condition{ Op: LTE, @@ -459,9 +459,9 @@ func TestPQLDeepEquality(t *testing.T) { }}, { name: "RangeGTE", - call: "Range(a>=7)", + call: "Row(a>=7)", exp: &Call{ - Name: "Range", + Name: "Row", Args: map[string]interface{}{ "a": &Condition{ Op: GTE, @@ -471,9 +471,9 @@ func TestPQLDeepEquality(t *testing.T) { }}, { name: "RangeGT", - call: "Range(a>7)", + call: "Row(a>7)", exp: &Call{ - Name: "Range", + Name: "Row", Args: map[string]interface{}{ "a": &Condition{ Op: GT, @@ -483,9 +483,9 @@ func TestPQLDeepEquality(t *testing.T) { }}, { name: "RangeNEQ", - call: "Range(a!=null)", + call: "Row(a!=null)", exp: &Call{ - Name: "Range", + Name: "Row", Args: map[string]interface{}{ "a": &Condition{ Op: NEQ, @@ -495,9 +495,9 @@ func TestPQLDeepEquality(t *testing.T) { }}, { name: "RangeLTELT", - call: "Range(4 <= a < 9)", + call: "Row(4 <= a < 9)", exp: &Call{ - Name: "Range", + Name: "Row", Args: map[string]interface{}{ "a": &Condition{ Op: BETWEEN, @@ -507,9 +507,9 @@ func TestPQLDeepEquality(t *testing.T) { }}, { name: "RangeLTLT", - call: "Range(4 < a < 9)", + call: "Row(4 < a < 9)", exp: &Call{ - Name: "Range", + Name: "Row", Args: map[string]interface{}{ "a": &Condition{ Op: BETWEEN, @@ -519,9 +519,9 @@ func TestPQLDeepEquality(t *testing.T) { }}, { name: "RangeLTELTE", - call: "Range(4 <= a <= 9)", + call: "Row(4 <= a <= 9)", exp: &Call{ - Name: "Range", + Name: "Row", Args: map[string]interface{}{ "a": &Condition{ Op: BETWEEN, @@ -531,9 +531,9 @@ func TestPQLDeepEquality(t *testing.T) { }}, { name: "RangeLTLTE", - call: "Range(4 < a <= 9)", + call: "Row(4 < a <= 9)", exp: &Call{ - Name: "Range", + Name: "Row", Args: map[string]interface{}{ "a": &Condition{ Op: BETWEEN, From e949a79d341f540755684f16a77a40bdaa3ac17f Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 9 Jan 2019 17:12:49 -0600 Subject: [PATCH 115/125] reverse the order of items in the github issue template --- .github/ISSUE_TEMPLATE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 1dce0ad40..74c6b9331 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,8 +1,8 @@ For bugs, please provide the following: -### Expected behavior +### Unxpected behavior -### Actual behavior +### Expected behavior ### Steps to reproduce the behavior From d1e938ef1ac8d19f4eaa7560800578dc8aff3e70 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 10 Jan 2019 08:07:48 -0600 Subject: [PATCH 116/125] provide friendlier prompts --- .github/ISSUE_TEMPLATE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 74c6b9331..05043a418 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,8 +1,8 @@ For bugs, please provide the following: -### Unxpected behavior +### What's going wrong? -### Expected behavior +### What was expected? ### Steps to reproduce the behavior From 07674d859cfbc9fa22c9122acfe1113bb4c6f7b5 Mon Sep 17 00:00:00 2001 From: Seebs Date: Thu, 3 Jan 2019 13:50:00 -0600 Subject: [PATCH 117/125] setValue test and benchmark updates This provides a simple benchmark that can be used for setValue, to give a way to compare results from adding BSI support to roaring. Use the BSIGroup prefix for the fragments, and specify a cache type of "none", to prevent the use of a LRU cache (which makes things more expensive). Add a parallel benchmark for ImportValue, so we can compare them. (Unsurprisingly, the bulk-import endpoint is quite a lot faster.) Also, add a test for clearing values to the TestFragment_Sum test; it turns out that this was never tested in this code, but the http client test would test it and verify it, it should probably also be tested here. --- fragment_internal_test.go | 79 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/fragment_internal_test.go b/fragment_internal_test.go index cca88dda1..c7a999848 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -367,6 +367,20 @@ func TestFragment_Sum(t *testing.T) { t.Fatalf("unexpected sum: %d", sum) } }) + + // verify that clearValue clears values + if _, err := f.clearValue(1000, bitDepth, 23); err != nil { + t.Fatal(err) + } + t.Run("ClearValue", func(t *testing.T) { + if sum, n, err := f.sum(nil, bitDepth); err != nil { + t.Fatal(err) + } else if n != 3 { + t.Fatalf("unexpected count: %d", n) + } else if sum != (3800 - 382) { + t.Fatalf("unexpected sum: got %d, expecting %d", sum, 3800-382) + } + }) } // Ensure a fragment can find the min and max of values. @@ -637,6 +651,71 @@ func TestFragment_Range(t *testing.T) { }) } +// benchmarkSetValues is a helper function to explore, very roughly, the cost +// of setting values. +func benchmarkSetValues(b *testing.B, bitDepth uint, f *fragment, cfunc func(uint64) uint64) { + column := uint64(0) + for i := 0; i < b.N; i++ { + f.setValue(column, bitDepth, uint64(i)) + column = cfunc(column) + } +} + +// Benchmark performance of setValue for BSI ranges. +func BenchmarkFragment_SetValue(b *testing.B) { + depths := []uint{4, 8, 16} + for _, bitDepth := range depths { + name := fmt.Sprintf("Depth%d", bitDepth) + f := mustOpenFragment("i", "f", viewBSIGroupPrefix+"foo", 0, "none") + b.Run(name+"_Sparse", func(b *testing.B) { + benchmarkSetValues(b, bitDepth, f, func(u uint64) uint64 { return (u + 70000) & (ShardWidth - 1) }) + }) + f.Clean(b) + f = mustOpenFragment("i", "f", viewBSIGroupPrefix+"foo", 0, "none") + b.Run(name+"_Dense", func(b *testing.B) { + benchmarkSetValues(b, bitDepth, f, func(u uint64) uint64 { return (u + 1) & (ShardWidth - 1) }) + }) + f.Clean(b) + } +} + +// benchmarkImportValues is a helper function to explore, very roughly, the cost +// of setting values using the special setter used for imports. +func benchmarkImportValues(b *testing.B, bitDepth uint, f *fragment, cfunc func(uint64) uint64) { + column := uint64(0) + b.StopTimer() + columns := make([]uint64, b.N) + values := make([]uint64, b.N) + for i := 0; i < b.N; i++ { + values[i] = uint64(i) + columns[i] = column + column = cfunc(column) + } + b.StartTimer() + err := f.importValue(columns, values, bitDepth, false) + if err != nil { + b.Fatalf("error importing values: %s", err) + } +} + +// Benchmark performance of setValue for BSI ranges. +func BenchmarkFragment_ImportValue(b *testing.B) { + depths := []uint{4, 8, 16} + for _, bitDepth := range depths { + name := fmt.Sprintf("Depth%d", bitDepth) + f := mustOpenFragment("i", "f", viewBSIGroupPrefix+"foo", 0, "none") + b.Run(name+"_Sparse", func(b *testing.B) { + benchmarkImportValues(b, bitDepth, f, func(u uint64) uint64 { return (u + 70000) & (ShardWidth - 1) }) + }) + f.Clean(b) + f = mustOpenFragment("i", "f", viewBSIGroupPrefix+"foo", 0, "none") + b.Run(name+"_Dense", func(b *testing.B) { + benchmarkImportValues(b, bitDepth, f, func(u uint64) uint64 { return (u + 1) & (ShardWidth - 1) }) + }) + f.Clean(b) + } +} + // Ensure a fragment can snapshot correctly. func TestFragment_Snapshot(t *testing.T) { f := mustOpenFragment("i", "f", viewStandard, 0, "") From 93b60482639ae0d874985bfe4a2e2e66068ffa51 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 10 Jan 2019 16:12:54 -0600 Subject: [PATCH 118/125] add verbose flag to circle ci race test to help debug timeout --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 08b615b3b..a27e34be1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -50,7 +50,7 @@ jobs: - *fast-checkout - run: sudo apt-get install lsof - run: - command: make test TESTFLAGS="-race -timeout=30m" + command: make test TESTFLAGS="-race -v -timeout=30m" no_output_timeout: 30m test-golang-1.11-386: <<: *base-test From 0b1fb73b145046aac324013f7ad17fb804df5ee5 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Thu, 10 Jan 2019 11:00:10 -0600 Subject: [PATCH 119/125] add a test for groupby filter with RangeLTLT --- pql/pqlpeg_test.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index 0c3a83bb2..c49c4647a 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -629,6 +629,26 @@ func TestPQLDeepEquality(t *testing.T) { {Name: "Rows"}, }, }}, + { + name: "GroupByFilterRangeLTLT", + call: "GroupBy(Rows(), filter=Row(4 < a < 9))", + exp: &Call{ + Name: "GroupBy", + Args: map[string]interface{}{ + "filter": &Call{ + Name: "Row", + Args: map[string]interface{}{ + "a": &Condition{ + Op: BETWEEN, + Value: []interface{}{int64(5), int64(9)}, + }, + }, + }, + }, + Children: []*Call{ + {Name: "Rows"}, + }, + }}, } for i, test := range tests { From f7e3296f620784e886a0f56de5d7815538605810 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 11 Jan 2019 09:20:09 -0600 Subject: [PATCH 120/125] fixes a bug on upper end of bsi range queries --- executor_test.go | 42 +++++++++++++++++++++++++++++++++++------- pql/ast.go | 4 ++-- pql/pqlpeg_test.go | 10 +++++----- 3 files changed, 42 insertions(+), 14 deletions(-) diff --git a/executor_test.go b/executor_test.go index 11fcd32bd..19f39dc7f 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1901,16 +1901,44 @@ func TestExecutor_Execute_Row_BSIGroup(t *testing.T) { }) t.Run("BETWEEN", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(0 < other < 1000)`}); err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { - t.Fatalf("unexpected result: %s", spew.Sdump(result)) + tests := []struct { + q string + exp bool + }{ + {q: `Row(0 < other < 1000)`, exp: false}, + {q: `Row(0 <= other < 1000)`, exp: false}, + {q: `Row(0 <= other <= 1000)`, exp: true}, + {q: `Row(0 < other <= 1000)`, exp: true}, + + {q: `Row(1000 < other < 1000)`, exp: false}, + {q: `Row(1000 <= other < 1000)`, exp: false}, + {q: `Row(1000 <= other <= 1000)`, exp: true}, + {q: `Row(1000 < other <= 1000)`, exp: false}, + + {q: `Row(1000 < other < 2000)`, exp: false}, + {q: `Row(1000 <= other < 2000)`, exp: true}, + {q: `Row(1000 <= other <= 2000)`, exp: true}, + {q: `Row(1000 < other <= 2000)`, exp: false}, } + for i, test := range tests { + t.Run(fmt.Sprintf("#%d_%s", i, test.q), func(t *testing.T) { + var expected = []uint64{} + if test.exp { + expected = []uint64{0} + } + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.q}); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(expected, result.Results[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result for query: %s", test.q) + } + }) + } + }) // Ensure that the NotNull code path gets run. t.Run("NotNull", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(-1 < other < 1000)`}); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Row(0 <= other <= 1000)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) @@ -2069,14 +2097,14 @@ func TestExecutor_Execute_Range_BSIGroup_Deprecated(t *testing.T) { t.Run("BETWEEN", func(t *testing.T) { if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(0 < other < 1000)`}); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { + } else if !reflect.DeepEqual([]uint64{}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) // Ensure that the NotNull code path gets run. t.Run("NotNull", func(t *testing.T) { - if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(-1 < other < 1000)`}); err != nil { + if result, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Range(0 <= other <= 1000)`}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual([]uint64{0}, result.Results[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) diff --git a/pql/ast.go b/pql/ast.go index 0985ab8a5..8b23c708b 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -91,8 +91,8 @@ func (q *Query) endConditional() { if q.conditional[1] == "<" { low++ } - if q.conditional[3] == "<=" { - high++ + if q.conditional[3] == "<" { + high-- } elem := q.lastCallStackElem() diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index c49c4647a..3785bb713 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -501,7 +501,7 @@ func TestPQLDeepEquality(t *testing.T) { Args: map[string]interface{}{ "a": &Condition{ Op: BETWEEN, - Value: []interface{}{int64(4), int64(9)}, + Value: []interface{}{int64(4), int64(8)}, }, }, }}, @@ -513,7 +513,7 @@ func TestPQLDeepEquality(t *testing.T) { Args: map[string]interface{}{ "a": &Condition{ Op: BETWEEN, - Value: []interface{}{int64(5), int64(9)}, + Value: []interface{}{int64(5), int64(8)}, }, }, }}, @@ -525,7 +525,7 @@ func TestPQLDeepEquality(t *testing.T) { Args: map[string]interface{}{ "a": &Condition{ Op: BETWEEN, - Value: []interface{}{int64(4), int64(10)}, + Value: []interface{}{int64(4), int64(9)}, }, }, }}, @@ -537,7 +537,7 @@ func TestPQLDeepEquality(t *testing.T) { Args: map[string]interface{}{ "a": &Condition{ Op: BETWEEN, - Value: []interface{}{int64(5), int64(10)}, + Value: []interface{}{int64(5), int64(9)}, }, }, }}, @@ -640,7 +640,7 @@ func TestPQLDeepEquality(t *testing.T) { Args: map[string]interface{}{ "a": &Condition{ Op: BETWEEN, - Value: []interface{}{int64(5), int64(9)}, + Value: []interface{}{int64(5), int64(8)}, }, }, }, From 9f6d489be85ca59fd39feab81dd16b6f0d0fa3f4 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Mon, 14 Jan 2019 14:39:08 +0300 Subject: [PATCH 121/125] fixes #1823. Updates tests and docs for row range --- docs/query-language.md | 9 ++++----- executor.go | 5 ++--- executor_test.go | 16 +++++++++++++++- pql/pqlpeg_test.go | 8 ++++++++ 4 files changed, 29 insertions(+), 9 deletions(-) diff --git a/docs/query-language.md b/docs/query-language.md index 5eb4ef2d6..73e1cab7d 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -341,13 +341,12 @@ Row(stargazer=1) **Spec:** ``` -Row(=, , ) +Row(=, from=, to=) ``` **Description:** -Similar to `Row`, but only returns bits which were set with timestamps -between the given `start` (first) and `end` (second) timestamps. +Similar to `Row`, but only returns bits which were set with timestamps between the given `from` (inclusive) and `to` (exclusive) timestamps. Both `from` and `to` parameters are optional. The default for `to` timestamp is current time + 1 day. If a later end timestamp is required, specify it explicitly. **Result Type:** object with attrs and bits @@ -356,7 +355,7 @@ between the given `start` (first) and `end` (second) timestamps. Query all columns with a bit set in row 1 of a field (repositories that a user has starred), within a date range: ```request -Row(stargazer=1, 2010-01-01T00:00, 2017-03-02T03:00) +Row(stargazer=1, from='2010-01-01T00:00', to='2017-03-02T03:00') ``` ```response {{"attrs":{},"columns":[10]} @@ -836,7 +835,7 @@ GroupBy(, [RowsCall...], limit=, filter=) GroupBy returns the count of the intersection of every combination of rows taking one row each from the specified `Rows` calls. It returns only those -combinations for which the count is greater than 0. +combinations for which the count is greater than 0. The optional `filter` argument takes any type of `Row` query (e.g. Row, Union, Intersect, etc.) which will be intersected with each result prior to returning diff --git a/executor.go b/executor.go index b9f7c31ca..f68fec726 100644 --- a/executor.go +++ b/executor.go @@ -1253,9 +1253,8 @@ func (e *executor) executeRowShard(ctx context.Context, index string, c *pql.Cal // Set maximum "to" value if only "from" is set. We don't need to worry // about setting the minimum "from" since it is the zero value if omitted. if toTime.IsZero() { - // This is the maximum comparable time.Time value. - // https://stackoverflow.com/a/32620397 - toTime = time.Unix(1<<63-62135596801, 999999999) + // Set the end timestamp to current time + 1 day, in order to account for timezone differences. + toTime = time.Now().AddDate(0, 0, 1) } // Union bitmaps across all time-based views. diff --git a/executor_test.go b/executor_test.go index 19f39dc7f..034208485 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1493,6 +1493,8 @@ func TestExecutor_Execute_Row_Range(t *testing.T) { Set(2, f=10, 2001-01-01T00:00)` readQueries := []string{ `Row(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`, + `Row(f=1, from=1999-12-31T00:00)`, + `Row(f=1, to=2002-01-01T02:00)`, `Clear( 2, f=1)`, `Row(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`, } @@ -1505,8 +1507,20 @@ func TestExecutor_Execute_Row_Range(t *testing.T) { } }) + t.Run("From", func(t *testing.T) { + if columns := responses[1].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + + t.Run("To", func(t *testing.T) { + if columns := responses[2].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) + t.Run("Clear", func(t *testing.T) { - if columns := responses[2].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) { + if columns := responses[4].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, 4, 5, 6, 7}) { t.Fatalf("unexpected columns: %+v", columns) } }) diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index 3785bb713..54f73b809 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -248,6 +248,14 @@ func TestPEGWorking(t *testing.T) { name: "RangeTimeQuotes", input: `Row(a=4, from='2010-07-04T00:00', to="2010-08-04T00:00")`, ncalls: 1}, + { + name: "RangeTimeFromQuotes", + input: `Row(a=4, from='2010-07-04T00:00')`, + ncalls: 1}, + { + name: "RangeTimeToQuotes", + input: `Row(a=4, to="2010-08-04T00:00")`, + ncalls: 1}, { name: "Dashed Frame", input: "Set(1, my-frame=9)", From 600b39e4e575e76bc18224f8ba4cac9480be3459 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Mon, 14 Jan 2019 23:40:31 +0300 Subject: [PATCH 122/125] updates row range test with a timestamp > the default end timestamp --- executor_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/executor_test.go b/executor_test.go index 034208485..e8472bbe4 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1480,17 +1480,21 @@ func TestExecutor_Execute_Sum(t *testing.T) { // Ensure a range query can be executed. func TestExecutor_Execute_Row_Range(t *testing.T) { t.Run("RowIDColumnID", func(t *testing.T) { - writeQuery := ` + // Create a timestamp just out of the current date + 1 day timestamp (default end timestamp). + nextDayExclusive := time.Now().AddDate(0, 0, 1).Add(1 * time.Second) + + writeQuery := fmt.Sprintf(` Set(2, f=1, 1999-12-31T00:00) Set(3, f=1, 2000-01-01T00:00) Set(4, f=1, 2000-01-02T00:00) Set(5, f=1, 2000-02-01T00:00) Set(6, f=1, 2001-01-01T00:00) Set(7, f=1, 2002-01-01T02:00) + Set(8, f=1, %s) Set(2, f=1, 1999-12-30T00:00) Set(2, f=1, 2002-02-01T00:00) - Set(2, f=10, 2001-01-01T00:00)` + Set(2, f=10, 2001-01-01T00:00)`, nextDayExclusive.Format("2006-01-02T15:04")) readQueries := []string{ `Row(f=1, from=1999-12-31T00:00, to=2002-01-01T03:00)`, `Row(f=1, from=1999-12-31T00:00)`, From d75e9eb772e9a39d3a22b00e3683a1a5972ef2a4 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Mon, 14 Jan 2019 23:45:58 +0300 Subject: [PATCH 123/125] updated row range test --- executor_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor_test.go b/executor_test.go index e8472bbe4..19a660b50 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1481,7 +1481,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { func TestExecutor_Execute_Row_Range(t *testing.T) { t.Run("RowIDColumnID", func(t *testing.T) { // Create a timestamp just out of the current date + 1 day timestamp (default end timestamp). - nextDayExclusive := time.Now().AddDate(0, 0, 1).Add(1 * time.Second) + nextDayExclusive := time.Now().AddDate(0, 0, 1).Add(1 * time.Hour) writeQuery := fmt.Sprintf(` Set(2, f=1, 1999-12-31T00:00) From 76de81dacffb36e113dfb66d99a7faa87b9c09b4 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Mon, 14 Jan 2019 23:46:34 +0300 Subject: [PATCH 124/125] updated row range test --- executor_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor_test.go b/executor_test.go index 19a660b50..0d42af8c5 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1481,7 +1481,7 @@ func TestExecutor_Execute_Sum(t *testing.T) { func TestExecutor_Execute_Row_Range(t *testing.T) { t.Run("RowIDColumnID", func(t *testing.T) { // Create a timestamp just out of the current date + 1 day timestamp (default end timestamp). - nextDayExclusive := time.Now().AddDate(0, 0, 1).Add(1 * time.Hour) + nextDayExclusive := time.Now().AddDate(0, 0, 2) writeQuery := fmt.Sprintf(` Set(2, f=1, 1999-12-31T00:00) From 0a8bd6548be945653880b942815ea9febcc3122b Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Mon, 14 Jan 2019 17:09:30 -0600 Subject: [PATCH 125/125] don't delete test fragment data (part of repo) --- fragment_internal_test.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/fragment_internal_test.go b/fragment_internal_test.go index cca88dda1..f35cec46d 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -1142,7 +1142,7 @@ func BenchmarkFragment_Blocks(b *testing.B) { if err := f.Open(); err != nil { b.Fatal(err) } - defer f.Clean(b) + defer f.CleanKeep(b) // Reset timer and execute benchmark. b.ResetTimer() @@ -1671,7 +1671,7 @@ func BenchmarkFragment_Snapshot(b *testing.B) { if err := f.Open(); err != nil { b.Fatal(err) } - defer f.Clean(b) + defer f.CleanKeep(b) b.ResetTimer() // Reset timer and execute benchmark. @@ -2030,6 +2030,20 @@ func (f *fragment) Clean(t testing.TB) { } } +// CleanKeep is just like Clean(), but it doesn't remove the +// fragment file (note that it DOES remove the cache file). +func (f *fragment) CleanKeep(t testing.TB) { + errc := f.Close() + errp := os.Remove(f.cachePath()) + if errc != nil { + t.Fatal("closing fragment: ", errc, errp) + } + // not all fragments have cache files + if errp != nil && !os.IsNotExist(errp) { + t.Fatalf("cleaning up fragment cache: %v", errp) + } +} + // mustOpenFragment returns a new instance of Fragment with a temporary path. func mustOpenFragment(index, field, view string, shard uint64, cacheType string) *fragment { file, err := ioutil.TempFile(TempDir, "pilosa-fragment-")