diff --git a/api.go b/api.go index a55e23f7b..13900f258 100644 --- a/api.go +++ b/api.go @@ -268,21 +268,11 @@ func setUpImportOptions(opts ...ImportOption) (*ImportOptions, error) { // (shard*ShardWidth)+(i%ShardWidth). That is to say that "data" represents all // of the rows in this shard of this field concatenated together in one long // bitmap. -func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, shard uint64, remote bool, data []byte, opts ...ImportOption) (err error) { - if len(data) == 0 { - return errors.New("no data to import") - } - +func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, shard uint64, remote bool, req *ImportRoaringRequest) (err error) { if err = api.validate(apiField); err != nil { return errors.Wrap(err, "validating api method") } - // Set up import options. - options, err := setUpImportOptions(opts...) - if err != nil { - return errors.Wrap(err, "setting up import options") - } - nodes := api.cluster.shardNodes(indexName, shard) var eg errgroup.Group @@ -291,26 +281,42 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, return newNotFoundError(ErrFieldNotFound) } - // only set fields are supported - if field.Type() != FieldTypeSet { - return NewBadRequestError(errors.New("roaring import is only supported for set fields")) + // only set and time fields are supported + if field.Type() != FieldTypeSet && field.Type() != FieldTypeTime { + return NewBadRequestError(errors.New("roaring import is only supported for set and time fields")) } for _, node := range nodes { node := node if node.ID == api.server.nodeID { - // must make a copy of data to operate on locally. field.importRoaring changes data - d2 := make([]byte, len(data)) - copy(d2, data) eg.Go(func() error { - return field.importRoaring(d2, shard, options.Clear) + var err error + for viewName, viewData := range req.Views { + if viewName == "" { + viewName = viewStandard + } else { + viewName = fmt.Sprintf("%s_%s", viewStandard, viewName) + } + if len(viewData) == 0 { + return fmt.Errorf("no data to import for view: %s", viewName) + } + // must make a copy of data to operate on locally. + // field.importRoaring changes data + data := make([]byte, len(viewData)) + copy(data, viewData) + err = field.importRoaring(data, shard, viewName, req.Clear) + if err != nil { + return err + } + } + return err }) go func(node *Node) { }(node) } else if !remote { // if remote == true we don't forward to other nodes // forward it on eg.Go(func() error { - return api.server.defaultClient.ImportRoaring(ctx, &node.URI, indexName, fieldName, shard, true, data, opts...) + return api.server.defaultClient.ImportRoaring(ctx, &node.URI, indexName, fieldName, shard, true, req) }) } } diff --git a/client.go b/client.go index f6c49d14b..b16222b7a 100644 --- a/client.go +++ b/client.go @@ -53,7 +53,7 @@ type InternalClient interface { RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) SendMessage(ctx context.Context, uri *URI, msg []byte) error RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) - ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, data []byte, opts ...ImportOption) error + ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error } //=============== @@ -109,7 +109,7 @@ func (n nopInternalClient) Import(ctx context.Context, index, field string, shar func (n nopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit, opts ...ImportOption) error { return nil } -func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, data []byte, opts ...ImportOption) error { +func (n nopInternalClient) ImportRoaring(ctx context.Context, uri *URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error { return nil } func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error { diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 9826c7ef9..3c7629a65 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -217,6 +217,14 @@ func (Serializer) Unmarshal(buf []byte, m pilosa.Message) error { } decodeImportValueRequest(msg, mt) return nil + case *pilosa.ImportRoaringRequest: + msg := &internal.ImportRoaringRequest{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshaling ImportRoaringRequest") + } + decodeImportRoaringRequest(msg, mt) + return nil case *pilosa.ImportResponse: msg := &internal.ImportResponse{} err := proto.Unmarshal(buf, msg) @@ -292,6 +300,8 @@ func encodeToProto(m pilosa.Message) proto.Message { return encodeImportRequest(mt) case *pilosa.ImportValueRequest: return encodeImportValueRequest(mt) + case *pilosa.ImportRoaringRequest: + return encodeImportRoaringRequest(mt) case *pilosa.ImportResponse: return encodeImportResponse(mt) case *pilosa.BlockDataRequest: @@ -348,6 +358,22 @@ func encodeImportValueRequest(m *pilosa.ImportValueRequest) *internal.ImportValu } } +func encodeImportRoaringRequest(m *pilosa.ImportRoaringRequest) *internal.ImportRoaringRequest { + views := make([]*internal.ImportRoaringRequestView, len(m.Views)) + i := 0 + for viewName, viewData := range m.Views { + views[i] = &internal.ImportRoaringRequestView{ + Name: viewName, + Data: viewData, + } + i += 1 + } + return &internal.ImportRoaringRequest{ + Clear: m.Clear, + Views: views, + } +} + func encodeQueryRequest(m *pilosa.QueryRequest) *internal.QueryRequest { return &internal.QueryRequest{ Query: m.Query, @@ -916,6 +942,15 @@ func decodeImportValueRequest(pb *internal.ImportValueRequest, m *pilosa.ImportV m.Values = pb.Values } +func decodeImportRoaringRequest(pb *internal.ImportRoaringRequest, m *pilosa.ImportRoaringRequest) { + views := map[string][]byte{} + for _, view := range pb.Views { + views[view.Name] = view.Data + } + m.Clear = pb.Clear + m.Views = views +} + func decodeImportResponse(pb *internal.ImportResponse, m *pilosa.ImportResponse) { m.Err = pb.Err } diff --git a/field.go b/field.go index 4189e55a4..efe8e7198 100644 --- a/field.go +++ b/field.go @@ -1184,9 +1184,10 @@ func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportO return nil } -func (f *Field) importRoaring(data []byte, shard uint64, clear bool) error { - viewName := viewStandard - +func (f *Field) importRoaring(data []byte, shard uint64, viewName string, clear bool) error { + if viewName == "" { + viewName = viewStandard + } view, err := f.createViewIfNotExists(viewName) if err != nil { return errors.Wrap(err, "creating view") diff --git a/handler.go b/handler.go index 9fc3af368..8fe51dc4c 100644 --- a/handler.go +++ b/handler.go @@ -96,6 +96,11 @@ type ImportRequest struct { Timestamps []int64 } +type ImportRoaringRequest struct { + Clear bool + Views map[string][]byte +} + type ImportResponse struct { Err string } diff --git a/http/client.go b/http/client.go index 3d19df227..b62dc44a3 100644 --- a/http/client.go +++ b/http/client.go @@ -546,7 +546,7 @@ func (c *InternalClient) marshalImportValuePayload(index, field string, shard ui // ImportRoaring does fast import of raw bits in roaring format (pilosa or // official format, see API.ImportRoaring). -func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pilosa.URI, index, field string, shard uint64, remote bool, data []byte, opts ...pilosa.ImportOption) error { +func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pilosa.URI, index, field string, shard uint64, remote bool, req *pilosa.ImportRoaringRequest) error { if index == "" { return pilosa.ErrIndexRequired } else if field == "" { @@ -556,32 +556,27 @@ func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pilosa.URI, ind uri = c.defaultURI } - // Set up import options. - options := &pilosa.ImportOptions{} - for _, opt := range opts { - err := opt(options) - if err != nil { - return errors.Wrap(err, "applying option") - } - } - vals := url.Values{} vals.Set("remote", strconv.FormatBool(remote)) - if options.Clear { - vals.Set("clear", "true") - } url := fmt.Sprintf("%s/index/%s/field/%s/import-roaring/%d?%s", uri, index, field, shard, vals.Encode()) + // Marshal data to protobuf. + data, err := c.serializer.Marshal(req) + if err != nil { + return errors.Wrap(err, "marshal import request") + } + // Generate HTTP request. - req, err := http.NewRequest("POST", url, bytes.NewBuffer(data)) + httpReq, err := http.NewRequest("POST", url, bytes.NewBuffer(data)) if err != nil { return errors.Wrap(err, "creating request") } - req.Header.Set("Content-Type", "application/x-binary") - req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + httpReq.Header.Set("Content-Type", "application/x-protobuf") + httpReq.Header.Set("Accept", "application/x-protobuf") + httpReq.Header.Set("User-Agent", "pilosa/"+pilosa.Version) // Execute request against the host. - resp, err := c.executeRequest(req.WithContext(ctx)) + resp, err := c.executeRequest(httpReq.WithContext(ctx)) if err != nil { return err } @@ -591,7 +586,7 @@ func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pilosa.URI, ind rbody := &pilosa.ImportResponse{} dec.Decode(rbody) if rbody.Err != "" { - return errors.Errorf("importing roaring: %v", rbody.Err) + return errors.Wrap(errors.New(rbody.Err), "importing roaring") } return nil } diff --git a/http/client_test.go b/http/client_test.go index 23d8b8573..fee3754df 100644 --- a/http/client_test.go +++ b/http/client_test.go @@ -408,8 +408,9 @@ func TestClient_ImportRoaring(t *testing.T) { // Send import request. host := cluster[0].URL() c := MustNewClient(host, http.GetHTTPClient(nil)) - roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") // [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537] - if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringData); err != nil { + // [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537] + roaringReq := makeImportRoaringRequest(false, "3B3001000100000900010000000100010009000100") + if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringReq); err != nil { t.Fatal(err) } @@ -432,8 +433,9 @@ func TestClient_ImportRoaring(t *testing.T) { } // Ensure that sending a roaring import with the clear flag works as expected. - roaringDataClear, _ := hex.DecodeString("3A30000001000000010001001000000003000400") // [65539, 65540] - if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringDataClear, pilosa.OptImportOptionsClear(true)); err != nil { + // [65539, 65540] + roaringReq = makeImportRoaringRequest(true, "3A30000001000000010001001000000003000400") + if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringReq); err != nil { t.Fatal(err) } @@ -454,8 +456,9 @@ func TestClient_ImportRoaring(t *testing.T) { } // Ensure that sending a roaring import with the clear flag works as expected. - roaringDataClear, _ = hex.DecodeString("3A300000020000000000010001000100180000001C0000000400060001000300") // [4, 6, 65537, 65539] - if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringDataClear, pilosa.OptImportOptionsClear(true)); err != nil { + // [4, 6, 65537, 65539] + roaringReq = makeImportRoaringRequest(true, "3A300000020000000000010001000100180000001C0000000400060001000300") + if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringReq); err != nil { t.Fatal(err) } @@ -476,8 +479,9 @@ func TestClient_ImportRoaring(t *testing.T) { } // Ensure that sending a roaring import with the clear flag works as expected. - roaringDataClear, _ = hex.DecodeString("3B3001000100000900010000000100010009000100") // [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537] - if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringDataClear, pilosa.OptImportOptionsClear(true)); err != nil { + // [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 65537] + roaringReq = makeImportRoaringRequest(true, "3B3001000100000900010000000100010009000100") + if err := c.ImportRoaring(context.Background(), &cluster[0].API.Node().URI, "i", "f", 0, false, roaringReq); err != nil { t.Fatal(err) } @@ -981,3 +985,13 @@ func MustNewClient(host string, h *gohttp.Client) *Client { } return &Client{InternalClient: c} } + +func makeImportRoaringRequest(clear bool, viewData string) *pilosa.ImportRoaringRequest { + roaringData, _ := hex.DecodeString(viewData) + return &pilosa.ImportRoaringRequest{ + Clear: clear, + Views: map[string][]byte{ + "": roaringData, + }, + } +} diff --git a/http/handler.go b/http/handler.go index 1ecbd2fb1..a8b6e8f2a 100644 --- a/http/handler.go +++ b/http/handler.go @@ -24,9 +24,8 @@ import ( "io/ioutil" "net" "net/http" - "net/url" - // Imported for its side-effect of registering pprof endpoints with the server. _ "net/http/pprof" + "net/url" // Imported for its side-effect of registering pprof endpoints with the server. "reflect" "runtime/debug" "strconv" @@ -37,7 +36,6 @@ import ( "github.com/gorilla/mux" "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/logger" - "github.com/pkg/errors" ) @@ -1497,10 +1495,18 @@ func GetHTTPClient(t *tls.Config) *http.Client { // handlPostRoaringImport func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("Content-Type") != "application/x-binary" { + // Verify that request is only communicating over protobufs. + if r.Header.Get("Content-Type") != "application/x-protobuf" { http.Error(w, "Unsupported media type", http.StatusUnsupportedMediaType) return + } else if r.Header.Get("Accept") != "application/x-protobuf" { + http.Error(w, "Not acceptable", http.StatusNotAcceptable) + return } + + indexName := mux.Vars(r)["index"] + fieldName := mux.Vars(r)["field"] + q := r.URL.Query() remoteStr := q.Get("remote") var remote bool @@ -1508,9 +1514,6 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request remote = true } - // If the clear flag is true, treat the import as clear bits. - doClear := q.Get("clear") == "true" - // Read entire body. body, err := ioutil.ReadAll(r.Body) if err != nil { @@ -1518,6 +1521,12 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request return } + req := &pilosa.ImportRoaringRequest{} + if err := h.api.Serializer.Unmarshal(body, req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + urlVars := mux.Vars(r) shard, err := strconv.ParseUint(urlVars["shard"], 10, 64) if err != nil { @@ -1527,7 +1536,7 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request resp := &pilosa.ImportResponse{} // TODO give meaningful stats for import - err = h.api.ImportRoaring(r.Context(), urlVars["index"], urlVars["field"], shard, remote, body, pilosa.OptImportOptionsClear(doClear)) + err = h.api.ImportRoaring(r.Context(), indexName, fieldName, shard, remote, req) if err != nil { resp.Err = err.Error() if _, ok := err.(pilosa.BadRequestError); ok { diff --git a/internal/public.pb.go b/internal/public.pb.go index 8d78db985..34ac4db9f 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -35,7 +35,7 @@ func (m *Row) Reset() { *m = Row{} } func (m *Row) String() string { return proto.CompactTextString(m) } func (*Row) ProtoMessage() {} func (*Row) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{0} + return fileDescriptor_public_5eafd62083455670, []int{0} } func (m *Row) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -97,7 +97,7 @@ func (m *RowIdentifiers) Reset() { *m = RowIdentifiers{} } func (m *RowIdentifiers) String() string { return proto.CompactTextString(m) } func (*RowIdentifiers) ProtoMessage() {} func (*RowIdentifiers) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{1} + return fileDescriptor_public_5eafd62083455670, []int{1} } func (m *RowIdentifiers) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -153,7 +153,7 @@ func (m *Pair) Reset() { *m = Pair{} } func (m *Pair) String() string { return proto.CompactTextString(m) } func (*Pair) ProtoMessage() {} func (*Pair) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{2} + return fileDescriptor_public_5eafd62083455670, []int{2} } func (m *Pair) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -215,7 +215,7 @@ func (m *FieldRow) Reset() { *m = FieldRow{} } func (m *FieldRow) String() string { return proto.CompactTextString(m) } func (*FieldRow) ProtoMessage() {} func (*FieldRow) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{3} + return fileDescriptor_public_5eafd62083455670, []int{3} } func (m *FieldRow) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -270,7 +270,7 @@ func (m *GroupCount) Reset() { *m = GroupCount{} } func (m *GroupCount) String() string { return proto.CompactTextString(m) } func (*GroupCount) ProtoMessage() {} func (*GroupCount) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{4} + return fileDescriptor_public_5eafd62083455670, []int{4} } func (m *GroupCount) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -325,7 +325,7 @@ func (m *ValCount) Reset() { *m = ValCount{} } func (m *ValCount) String() string { return proto.CompactTextString(m) } func (*ValCount) ProtoMessage() {} func (*ValCount) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{5} + return fileDescriptor_public_5eafd62083455670, []int{5} } func (m *ValCount) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -381,7 +381,7 @@ func (m *Bit) Reset() { *m = Bit{} } func (m *Bit) String() string { return proto.CompactTextString(m) } func (*Bit) ProtoMessage() {} func (*Bit) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{6} + return fileDescriptor_public_5eafd62083455670, []int{6} } func (m *Bit) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -444,7 +444,7 @@ func (m *ColumnAttrSet) Reset() { *m = ColumnAttrSet{} } func (m *ColumnAttrSet) String() string { return proto.CompactTextString(m) } func (*ColumnAttrSet) ProtoMessage() {} func (*ColumnAttrSet) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{7} + return fileDescriptor_public_5eafd62083455670, []int{7} } func (m *ColumnAttrSet) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -510,7 +510,7 @@ func (m *Attr) Reset() { *m = Attr{} } func (m *Attr) String() string { return proto.CompactTextString(m) } func (*Attr) ProtoMessage() {} func (*Attr) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{8} + return fileDescriptor_public_5eafd62083455670, []int{8} } func (m *Attr) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -592,7 +592,7 @@ func (m *AttrMap) Reset() { *m = AttrMap{} } func (m *AttrMap) String() string { return proto.CompactTextString(m) } func (*AttrMap) ProtoMessage() {} func (*AttrMap) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{9} + return fileDescriptor_public_5eafd62083455670, []int{9} } func (m *AttrMap) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -644,7 +644,7 @@ func (m *QueryRequest) Reset() { *m = QueryRequest{} } func (m *QueryRequest) String() string { return proto.CompactTextString(m) } func (*QueryRequest) ProtoMessage() {} func (*QueryRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{10} + return fileDescriptor_public_5eafd62083455670, []int{10} } func (m *QueryRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -728,7 +728,7 @@ func (m *QueryResponse) Reset() { *m = QueryResponse{} } func (m *QueryResponse) String() string { return proto.CompactTextString(m) } func (*QueryResponse) ProtoMessage() {} func (*QueryResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{11} + return fileDescriptor_public_5eafd62083455670, []int{11} } func (m *QueryResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -797,7 +797,7 @@ func (m *QueryResult) Reset() { *m = QueryResult{} } func (m *QueryResult) String() string { return proto.CompactTextString(m) } func (*QueryResult) ProtoMessage() {} func (*QueryResult) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{12} + return fileDescriptor_public_5eafd62083455670, []int{12} } func (m *QueryResult) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -907,7 +907,7 @@ func (m *ImportRequest) Reset() { *m = ImportRequest{} } func (m *ImportRequest) String() string { return proto.CompactTextString(m) } func (*ImportRequest) ProtoMessage() {} func (*ImportRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{13} + return fileDescriptor_public_5eafd62083455670, []int{13} } func (m *ImportRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1008,7 +1008,7 @@ func (m *ImportValueRequest) Reset() { *m = ImportValueRequest{} } func (m *ImportValueRequest) String() string { return proto.CompactTextString(m) } func (*ImportValueRequest) ProtoMessage() {} func (*ImportValueRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_public_fc5da89825239896, []int{14} + return fileDescriptor_public_5eafd62083455670, []int{14} } func (m *ImportValueRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1079,6 +1079,116 @@ func (m *ImportValueRequest) GetValues() []int64 { return nil } +type ImportRoaringRequestView struct { + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Data []byte `protobuf:"bytes,2,opt,name=Data,proto3" json:"Data,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ImportRoaringRequestView) Reset() { *m = ImportRoaringRequestView{} } +func (m *ImportRoaringRequestView) String() string { return proto.CompactTextString(m) } +func (*ImportRoaringRequestView) ProtoMessage() {} +func (*ImportRoaringRequestView) Descriptor() ([]byte, []int) { + return fileDescriptor_public_5eafd62083455670, []int{15} +} +func (m *ImportRoaringRequestView) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ImportRoaringRequestView) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ImportRoaringRequestView.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (dst *ImportRoaringRequestView) XXX_Merge(src proto.Message) { + xxx_messageInfo_ImportRoaringRequestView.Merge(dst, src) +} +func (m *ImportRoaringRequestView) XXX_Size() int { + return m.Size() +} +func (m *ImportRoaringRequestView) XXX_DiscardUnknown() { + xxx_messageInfo_ImportRoaringRequestView.DiscardUnknown(m) +} + +var xxx_messageInfo_ImportRoaringRequestView proto.InternalMessageInfo + +func (m *ImportRoaringRequestView) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *ImportRoaringRequestView) GetData() []byte { + if m != nil { + return m.Data + } + return nil +} + +type ImportRoaringRequest struct { + Clear bool `protobuf:"varint,1,opt,name=Clear,proto3" json:"Clear,omitempty"` + Views []*ImportRoaringRequestView `protobuf:"bytes,2,rep,name=views" json:"views,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ImportRoaringRequest) Reset() { *m = ImportRoaringRequest{} } +func (m *ImportRoaringRequest) String() string { return proto.CompactTextString(m) } +func (*ImportRoaringRequest) ProtoMessage() {} +func (*ImportRoaringRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_public_5eafd62083455670, []int{16} +} +func (m *ImportRoaringRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ImportRoaringRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ImportRoaringRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (dst *ImportRoaringRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ImportRoaringRequest.Merge(dst, src) +} +func (m *ImportRoaringRequest) XXX_Size() int { + return m.Size() +} +func (m *ImportRoaringRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ImportRoaringRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ImportRoaringRequest proto.InternalMessageInfo + +func (m *ImportRoaringRequest) GetClear() bool { + if m != nil { + return m.Clear + } + return false +} + +func (m *ImportRoaringRequest) GetViews() []*ImportRoaringRequestView { + if m != nil { + return m.Views + } + return nil +} + func init() { proto.RegisterType((*Row)(nil), "internal.Row") proto.RegisterType((*RowIdentifiers)(nil), "internal.RowIdentifiers") @@ -1095,6 +1205,8 @@ func init() { proto.RegisterType((*QueryResult)(nil), "internal.QueryResult") proto.RegisterType((*ImportRequest)(nil), "internal.ImportRequest") proto.RegisterType((*ImportValueRequest)(nil), "internal.ImportValueRequest") + proto.RegisterType((*ImportRoaringRequestView)(nil), "internal.ImportRoaringRequestView") + proto.RegisterType((*ImportRoaringRequest)(nil), "internal.ImportRoaringRequest") } func (m *Row) Marshal() (dAtA []byte, err error) { size := m.Size() @@ -1979,6 +2091,82 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *ImportRoaringRequestView) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ImportRoaringRequestView) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Name) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) + } + if len(m.Data) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintPublic(dAtA, i, uint64(len(m.Data))) + i += copy(dAtA[i:], m.Data) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *ImportRoaringRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ImportRoaringRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Clear { + dAtA[i] = 0x8 + i++ + if m.Clear { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if len(m.Views) > 0 { + for _, msg := range m.Views { + dAtA[i] = 0x12 + i++ + i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + func encodeVarintPublic(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -2434,6 +2622,47 @@ func (m *ImportValueRequest) Size() (n int) { return n } +func (m *ImportRoaringRequestView) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } + l = len(m.Data) + if l > 0 { + n += 1 + l + sovPublic(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *ImportRoaringRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Clear { + n += 2 + } + if len(m.Views) > 0 { + for _, e := range m.Views { + l = e.Size() + n += 1 + l + sovPublic(uint64(l)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func sovPublic(x uint64) (n int) { for { n++ @@ -5115,6 +5344,219 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } return nil } +func (m *ImportRoaringRequestView) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ImportRoaringRequestView: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ImportRoaringRequestView: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + byteLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) + if m.Data == nil { + m.Data = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPublic(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPublic + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ImportRoaringRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ImportRoaringRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Clear", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Clear = bool(v != 0) + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Views", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPublic + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPublic + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Views = append(m.Views, &ImportRoaringRequestView{}) + if err := m.Views[len(m.Views)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPublic(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthPublic + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipPublic(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 @@ -5220,59 +5662,63 @@ var ( ErrIntOverflowPublic = fmt.Errorf("proto: integer overflow") ) -func init() { proto.RegisterFile("public.proto", fileDescriptor_public_fc5da89825239896) } +func init() { proto.RegisterFile("public.proto", fileDescriptor_public_5eafd62083455670) } -var fileDescriptor_public_fc5da89825239896 = []byte{ - // 804 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6e, 0xdb, 0x46, - 0x14, 0xed, 0x88, 0x94, 0x44, 0x5d, 0x59, 0xaa, 0x31, 0x70, 0x5d, 0xa2, 0x30, 0x54, 0x82, 0x28, - 0x0a, 0xae, 0x64, 0x40, 0x05, 0x8c, 0xae, 0xfa, 0xf0, 0xab, 0x10, 0xdc, 0x1a, 0xcd, 0xd8, 0x71, - 0x90, 0x25, 0x6d, 0x4d, 0x6c, 0x02, 0x14, 0x87, 0xe1, 0x03, 0xb2, 0xbe, 0x23, 0x9b, 0x7c, 0x42, - 0x16, 0xf9, 0x10, 0x2f, 0x83, 0x7c, 0x41, 0xe2, 0xfc, 0x48, 0x30, 0x77, 0x38, 0x1a, 0x8a, 0x0e, - 0x8c, 0x2c, 0xb2, 0x9b, 0x73, 0x5f, 0xbc, 0xe7, 0xbe, 0x08, 0x1b, 0x69, 0x79, 0x19, 0x47, 0x57, - 0xe3, 0x34, 0x13, 0x85, 0xa0, 0x4e, 0x94, 0x14, 0x3c, 0x4b, 0xc2, 0xd8, 0x7f, 0x0e, 0x16, 0x13, - 0x0b, 0xea, 0x42, 0xf7, 0x40, 0xc4, 0xe5, 0x3c, 0xc9, 0x5d, 0xe2, 0x59, 0x81, 0xcd, 0x34, 0xa4, - 0xbf, 0x40, 0xfb, 0xef, 0xa2, 0xc8, 0x72, 0xb7, 0xe5, 0x59, 0x41, 0x7f, 0x32, 0x1c, 0x6b, 0xd7, - 0xb1, 0x14, 0x33, 0xa5, 0xa4, 0x14, 0xec, 0x13, 0xbe, 0xcc, 0x5d, 0xcb, 0xb3, 0x82, 0x1e, 0xc3, - 0xb7, 0xff, 0x3b, 0x0c, 0x99, 0x58, 0x4c, 0x67, 0x3c, 0x29, 0xa2, 0x17, 0x11, 0x57, 0x56, 0x4c, - 0x2c, 0xf4, 0x27, 0xf0, 0xbd, 0xf2, 0x6c, 0xd5, 0x3c, 0xff, 0x00, 0xfb, 0xff, 0x30, 0xca, 0xe8, - 0x10, 0x5a, 0xd3, 0x43, 0x97, 0x78, 0x24, 0xb0, 0x59, 0x6b, 0x7a, 0x48, 0xb7, 0xa0, 0x7d, 0x20, - 0xca, 0xa4, 0x70, 0x5b, 0x28, 0x52, 0x80, 0x6e, 0x82, 0x75, 0xc2, 0x97, 0xae, 0xe5, 0x91, 0xa0, - 0xc7, 0xe4, 0xd3, 0xdf, 0x03, 0xe7, 0x38, 0xe2, 0xf1, 0x4c, 0x32, 0xdb, 0x82, 0x36, 0xbe, 0x31, - 0x4c, 0x8f, 0x29, 0x20, 0xa5, 0x32, 0xb7, 0x43, 0x1d, 0x09, 0x81, 0xff, 0x2f, 0xc0, 0x3f, 0x99, - 0x28, 0x53, 0x15, 0x37, 0x80, 0x36, 0x22, 0x4c, 0xb7, 0x3f, 0xa1, 0x86, 0xb9, 0x0e, 0xce, 0x94, - 0xc1, 0x97, 0xf3, 0xf2, 0x27, 0xe0, 0x5c, 0x84, 0xf1, 0x2a, 0xc7, 0x8b, 0x30, 0xc6, 0x1c, 0x2c, - 0x26, 0x9f, 0xeb, 0x3e, 0x96, 0xf6, 0x79, 0x0a, 0xd6, 0x7e, 0x54, 0x98, 0xf4, 0x48, 0x2d, 0x3d, - 0xfa, 0x13, 0x38, 0xaa, 0x2b, 0xab, 0xbc, 0x57, 0x98, 0xee, 0x40, 0xef, 0x3c, 0x9a, 0xf3, 0xbc, - 0x08, 0xe7, 0x29, 0x96, 0xc2, 0x62, 0x46, 0xe0, 0x3f, 0x83, 0x81, 0xb2, 0x94, 0xdd, 0x3a, 0xe3, - 0xc5, 0x83, 0xca, 0x7e, 0x5d, 0x97, 0x1f, 0x56, 0xfa, 0x0d, 0x01, 0x5b, 0xea, 0xb4, 0x8a, 0xac, - 0x54, 0xb2, 0xb1, 0xe7, 0xcb, 0x94, 0x57, 0x99, 0xe2, 0x9b, 0x7a, 0xd0, 0x3f, 0x2b, 0xb2, 0x28, - 0xb9, 0xbe, 0x08, 0xe3, 0x92, 0x57, 0x81, 0xea, 0x22, 0xc9, 0x71, 0x9a, 0x14, 0x4a, 0x6d, 0x23, - 0x8d, 0x15, 0x96, 0x1c, 0xf7, 0x85, 0x88, 0x95, 0xb2, 0xed, 0x91, 0xc0, 0x61, 0x46, 0x40, 0x47, - 0x00, 0xc7, 0xb1, 0x08, 0x2b, 0xdf, 0x8e, 0x47, 0x02, 0xc2, 0x6a, 0x12, 0x7f, 0x17, 0xba, 0x32, - 0xd3, 0xff, 0xc2, 0xd4, 0xb0, 0x25, 0x8f, 0xb0, 0xf5, 0xef, 0x08, 0x6c, 0x3c, 0x29, 0x79, 0xb6, - 0x64, 0xfc, 0x65, 0xc9, 0x73, 0xec, 0x0a, 0x62, 0x3d, 0x4a, 0x08, 0xe8, 0x36, 0x74, 0xce, 0x6e, - 0xc2, 0x6c, 0xa6, 0x6a, 0x67, 0xb3, 0x0a, 0x49, 0xae, 0xa6, 0xe6, 0x39, 0x72, 0x75, 0x58, 0x5d, - 0x24, 0x3d, 0x19, 0x9f, 0x8b, 0x42, 0x93, 0xa9, 0x10, 0x0d, 0xe0, 0xfb, 0xa3, 0xdb, 0xab, 0xb8, - 0x9c, 0x71, 0x26, 0x16, 0xca, 0xbb, 0x83, 0x06, 0x4d, 0x31, 0xfd, 0x15, 0x86, 0x95, 0x48, 0x6f, - 0x6f, 0x17, 0x0d, 0x1b, 0x52, 0xff, 0x15, 0x81, 0x41, 0x45, 0x25, 0x4f, 0x45, 0x92, 0x73, 0xd9, - 0xaf, 0xa3, 0x2c, 0xd3, 0xfd, 0x3a, 0xca, 0x32, 0xba, 0x0b, 0x5d, 0xc6, 0xf3, 0x32, 0x2e, 0xf4, - 0x10, 0xfc, 0x60, 0xca, 0xa2, 0x7d, 0xcb, 0xb8, 0x60, 0xda, 0x8a, 0xfe, 0x09, 0xc3, 0xb5, 0xa1, - 0x52, 0xdb, 0xdf, 0x9f, 0xfc, 0x68, 0xfc, 0xd6, 0xf4, 0xac, 0x61, 0xee, 0xbf, 0x6f, 0x41, 0xbf, - 0x16, 0x99, 0xfe, 0x8c, 0xb7, 0x08, 0x73, 0xea, 0x4f, 0x06, 0x26, 0x8a, 0xdc, 0x34, 0xbc, 0x52, - 0x1b, 0x40, 0x4e, 0xab, 0x79, 0x22, 0xa7, 0xb2, 0x8b, 0xf2, 0x4a, 0xe8, 0xcf, 0xd6, 0xba, 0x28, - 0xc5, 0x4c, 0x29, 0xf1, 0xb2, 0xdd, 0x84, 0xc9, 0x35, 0x9f, 0xe1, 0x3c, 0x39, 0x4c, 0x43, 0x3a, - 0x36, 0xfb, 0x89, 0x0d, 0x58, 0x5b, 0x71, 0xad, 0x61, 0x66, 0x87, 0xf5, 0x40, 0xcb, 0x5e, 0x0c, - 0xaa, 0x81, 0x96, 0x2d, 0x94, 0xbb, 0x29, 0x0b, 0x8f, 0xcd, 0x57, 0x88, 0xee, 0x41, 0xdf, 0x5c, - 0x92, 0xdc, 0x75, 0x30, 0xc3, 0x2d, 0x13, 0xde, 0x28, 0x59, 0xdd, 0x90, 0xfe, 0xd5, 0xbc, 0x99, - 0x6e, 0x0f, 0x33, 0x73, 0xd7, 0xaa, 0x51, 0xd3, 0xb3, 0x86, 0xbd, 0xff, 0x91, 0xc0, 0x60, 0x3a, - 0x4f, 0x45, 0x56, 0xd4, 0xc6, 0x76, 0x9a, 0xcc, 0xf8, 0xad, 0x1e, 0x5b, 0x04, 0xe6, 0x2e, 0xb6, - 0x1a, 0x77, 0x11, 0xc7, 0x17, 0xc7, 0xd5, 0x66, 0x0a, 0xd4, 0x58, 0xda, 0x6b, 0x2c, 0x77, 0xa0, - 0xa7, 0x0f, 0x50, 0xee, 0xb6, 0x51, 0x65, 0x04, 0x72, 0x21, 0x57, 0x17, 0x48, 0x4e, 0xb0, 0x15, - 0x58, 0xac, 0x26, 0x91, 0x9d, 0x61, 0x62, 0x81, 0xc7, 0xbf, 0x8b, 0xc7, 0x5f, 0x43, 0xe9, 0xa9, - 0xc2, 0xa0, 0xd2, 0x41, 0x65, 0x4d, 0xe2, 0xbf, 0x25, 0x40, 0x15, 0x47, 0x5c, 0xed, 0x6f, 0x47, - 0xf4, 0x71, 0x42, 0xdb, 0xd0, 0xc1, 0xef, 0x69, 0x32, 0x15, 0x6a, 0xa4, 0xdb, 0x6d, 0xa6, 0xbb, - 0xbf, 0x79, 0x77, 0x3f, 0x22, 0xef, 0xee, 0x47, 0xe4, 0xc3, 0xfd, 0x88, 0xbc, 0xfe, 0x34, 0xfa, - 0xee, 0xb2, 0x83, 0xbf, 0xe1, 0xdf, 0x3e, 0x07, 0x00, 0x00, 0xff, 0xff, 0xa6, 0x62, 0xa8, 0x25, - 0x96, 0x07, 0x00, 0x00, +var fileDescriptor_public_5eafd62083455670 = []byte{ + // 870 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x8e, 0xdb, 0x44, + 0x14, 0x66, 0x62, 0x27, 0x71, 0x4e, 0x36, 0xa1, 0x1a, 0x2d, 0xc5, 0x42, 0x55, 0xb0, 0x2c, 0x84, + 0x7c, 0xb5, 0x95, 0x82, 0x54, 0xf5, 0x8a, 0x9f, 0x6d, 0xb6, 0x28, 0x2a, 0xac, 0xe0, 0x6c, 0x09, + 0xe2, 0xd2, 0x6d, 0xa6, 0xad, 0x25, 0xc7, 0x63, 0xec, 0x31, 0x69, 0x9e, 0x83, 0x1b, 0x1e, 0x81, + 0x0b, 0x1e, 0xa4, 0x97, 0x88, 0x27, 0x80, 0xe5, 0x45, 0xd0, 0x9c, 0xf1, 0x64, 0x9c, 0x6c, 0x59, + 0x71, 0xc1, 0xdd, 0x7c, 0xe7, 0xcc, 0x39, 0xfe, 0xbe, 0x39, 0x3f, 0x09, 0x9c, 0x94, 0xcd, 0xb3, + 0x3c, 0x7b, 0x7e, 0x56, 0x56, 0x52, 0x49, 0x1e, 0x64, 0x85, 0x12, 0x55, 0x91, 0xe6, 0xf1, 0x0f, + 0xe0, 0xa1, 0xdc, 0xf2, 0x10, 0x86, 0x8f, 0x64, 0xde, 0x6c, 0x8a, 0x3a, 0x64, 0x91, 0x97, 0xf8, + 0x68, 0x21, 0xff, 0x08, 0xfa, 0x5f, 0x28, 0x55, 0xd5, 0x61, 0x2f, 0xf2, 0x92, 0xf1, 0x7c, 0x7a, + 0x66, 0x43, 0xcf, 0xb4, 0x19, 0x8d, 0x93, 0x73, 0xf0, 0x9f, 0x88, 0x5d, 0x1d, 0x7a, 0x91, 0x97, + 0x8c, 0x90, 0xce, 0xf1, 0x43, 0x98, 0xa2, 0xdc, 0x2e, 0xd7, 0xa2, 0x50, 0xd9, 0x8b, 0x4c, 0x98, + 0x5b, 0x28, 0xb7, 0xf6, 0x13, 0x74, 0xde, 0x47, 0xf6, 0x3a, 0x91, 0x9f, 0x82, 0xff, 0x4d, 0x9a, + 0x55, 0x7c, 0x0a, 0xbd, 0xe5, 0x22, 0x64, 0x11, 0x4b, 0x7c, 0xec, 0x2d, 0x17, 0xfc, 0x14, 0xfa, + 0x8f, 0x64, 0x53, 0xa8, 0xb0, 0x47, 0x26, 0x03, 0xf8, 0x1d, 0xf0, 0x9e, 0x88, 0x5d, 0xe8, 0x45, + 0x2c, 0x19, 0xa1, 0x3e, 0xc6, 0x0f, 0x20, 0x78, 0x9c, 0x89, 0x7c, 0xad, 0x95, 0x9d, 0x42, 0x9f, + 0xce, 0x94, 0x66, 0x84, 0x06, 0x68, 0xab, 0xe6, 0xb6, 0xb0, 0x99, 0x08, 0xc4, 0x5f, 0x01, 0x7c, + 0x59, 0xc9, 0xa6, 0x34, 0x79, 0x13, 0xe8, 0x13, 0x22, 0xba, 0xe3, 0x39, 0x77, 0xca, 0x6d, 0x72, + 0x34, 0x17, 0xde, 0xce, 0x2b, 0x9e, 0x43, 0xb0, 0x4a, 0xf3, 0x3d, 0xc7, 0x55, 0x9a, 0x13, 0x07, + 0x0f, 0xf5, 0xf1, 0x30, 0xc6, 0xb3, 0x31, 0xdf, 0x81, 0x77, 0x9e, 0x29, 0x47, 0x8f, 0x75, 0xe8, + 0xf1, 0x0f, 0x20, 0x30, 0x55, 0xd9, 0xf3, 0xde, 0x63, 0x7e, 0x0f, 0x46, 0x4f, 0xb3, 0x8d, 0xa8, + 0x55, 0xba, 0x29, 0xe9, 0x29, 0x3c, 0x74, 0x86, 0xf8, 0x7b, 0x98, 0x98, 0x9b, 0xba, 0x5a, 0x57, + 0x42, 0xdd, 0x78, 0xd9, 0xff, 0x56, 0xe5, 0x9b, 0x2f, 0xfd, 0x2b, 0x03, 0x5f, 0xfb, 0xac, 0x8b, + 0xed, 0x5d, 0xba, 0xb0, 0x4f, 0x77, 0xa5, 0x68, 0x99, 0xd2, 0x99, 0x47, 0x30, 0xbe, 0x52, 0x55, + 0x56, 0xbc, 0x5c, 0xa5, 0x79, 0x23, 0xda, 0x44, 0x5d, 0x93, 0xd6, 0xb8, 0x2c, 0x94, 0x71, 0xfb, + 0x24, 0x63, 0x8f, 0xb5, 0xc6, 0x73, 0x29, 0x73, 0xe3, 0xec, 0x47, 0x2c, 0x09, 0xd0, 0x19, 0xf8, + 0x0c, 0xe0, 0x71, 0x2e, 0xd3, 0x36, 0x76, 0x10, 0xb1, 0x84, 0x61, 0xc7, 0x12, 0xdf, 0x87, 0xa1, + 0x66, 0xfa, 0x75, 0x5a, 0x3a, 0xb5, 0xec, 0x16, 0xb5, 0xf1, 0x1b, 0x06, 0x27, 0xdf, 0x36, 0xa2, + 0xda, 0xa1, 0xf8, 0xb1, 0x11, 0x35, 0x55, 0x85, 0xb0, 0x6d, 0x25, 0x02, 0xfc, 0x2e, 0x0c, 0xae, + 0x5e, 0xa5, 0xd5, 0xda, 0xbc, 0x9d, 0x8f, 0x2d, 0xd2, 0x5a, 0xdd, 0x9b, 0xd7, 0xa4, 0x35, 0xc0, + 0xae, 0x49, 0x47, 0xa2, 0xd8, 0x48, 0x65, 0xc5, 0xb4, 0x88, 0x27, 0xf0, 0xee, 0xc5, 0xeb, 0xe7, + 0x79, 0xb3, 0x16, 0x28, 0xb7, 0x26, 0x7a, 0x40, 0x17, 0x8e, 0xcd, 0xfc, 0x63, 0x98, 0xb6, 0x26, + 0x3b, 0xbd, 0x43, 0xba, 0x78, 0x64, 0x8d, 0x7f, 0x66, 0x30, 0x69, 0xa5, 0xd4, 0xa5, 0x2c, 0x6a, + 0xa1, 0xeb, 0x75, 0x51, 0x55, 0xb6, 0x5e, 0x17, 0x55, 0xc5, 0xef, 0xc3, 0x10, 0x45, 0xdd, 0xe4, + 0xca, 0x36, 0xc1, 0x7b, 0xee, 0x59, 0x6c, 0x6c, 0x93, 0x2b, 0xb4, 0xb7, 0xf8, 0x67, 0x30, 0x3d, + 0x68, 0x2a, 0x33, 0xfd, 0xe3, 0xf9, 0xfb, 0x2e, 0xee, 0xc0, 0x8f, 0x47, 0xd7, 0xe3, 0x3f, 0x7a, + 0x30, 0xee, 0x64, 0xe6, 0x1f, 0xd2, 0x2e, 0x22, 0x4e, 0xe3, 0xf9, 0xc4, 0x65, 0xd1, 0x93, 0x46, + 0x5b, 0xea, 0x04, 0xd8, 0x65, 0xdb, 0x4f, 0xec, 0x52, 0x57, 0x51, 0x6f, 0x09, 0xfb, 0xd9, 0x4e, + 0x15, 0xb5, 0x19, 0x8d, 0x93, 0x36, 0xdb, 0xab, 0xb4, 0x78, 0x29, 0xd6, 0xd4, 0x4f, 0x01, 0x5a, + 0xc8, 0xcf, 0xdc, 0x7c, 0x52, 0x01, 0x0e, 0x46, 0xdc, 0x7a, 0xd0, 0xcd, 0xb0, 0x6d, 0x68, 0x5d, + 0x8b, 0x49, 0xdb, 0xd0, 0xba, 0x84, 0x7a, 0x36, 0xf5, 0xc3, 0x53, 0xf1, 0x0d, 0xe2, 0x0f, 0x60, + 0xec, 0x36, 0x49, 0x1d, 0x06, 0xc4, 0xf0, 0xd4, 0xa5, 0x77, 0x4e, 0xec, 0x5e, 0xe4, 0x9f, 0x1f, + 0xef, 0xcc, 0x70, 0x44, 0xcc, 0xc2, 0x83, 0xd7, 0xe8, 0xf8, 0xf1, 0xe8, 0x7e, 0xfc, 0x17, 0x83, + 0xc9, 0x72, 0x53, 0xca, 0x4a, 0x75, 0xda, 0x76, 0x59, 0xac, 0xc5, 0x6b, 0xdb, 0xb6, 0x04, 0xdc, + 0x5e, 0xec, 0x1d, 0xed, 0x45, 0x6a, 0x5f, 0x6a, 0x57, 0x1f, 0x0d, 0xe8, 0xa8, 0xf4, 0x0f, 0x54, + 0xde, 0x83, 0x91, 0x5d, 0x40, 0x75, 0xd8, 0x27, 0x97, 0x33, 0xe8, 0x81, 0xdc, 0x6f, 0x20, 0xdd, + 0xc1, 0x5e, 0xe2, 0x61, 0xc7, 0xa2, 0x2b, 0x83, 0x72, 0x4b, 0xcb, 0x7f, 0x48, 0xcb, 0xdf, 0x42, + 0x1d, 0x69, 0xd2, 0x90, 0x33, 0x20, 0x67, 0xc7, 0x12, 0xff, 0xc6, 0x80, 0x1b, 0x8d, 0x34, 0xda, + 0xff, 0x9f, 0xd0, 0xdb, 0x05, 0xdd, 0x85, 0x01, 0x7d, 0xcf, 0x8a, 0x69, 0xd1, 0x11, 0xdd, 0xe1, + 0x0d, 0xba, 0xe7, 0x10, 0xb6, 0x15, 0x91, 0xa9, 0xde, 0x74, 0x2d, 0xdf, 0x55, 0x26, 0xb6, 0xba, + 0xa9, 0x2e, 0xd3, 0x8d, 0x68, 0x29, 0xd3, 0x59, 0xdb, 0x16, 0xa9, 0x4a, 0x89, 0xf0, 0x09, 0xd2, + 0x39, 0x7e, 0x01, 0xa7, 0x6f, 0xcb, 0x41, 0x3f, 0x23, 0xb9, 0x48, 0xcd, 0x24, 0x07, 0x68, 0x00, + 0x7f, 0x08, 0xfd, 0x9f, 0x32, 0xb1, 0xb5, 0x93, 0x1c, 0xbb, 0xee, 0xf9, 0x37, 0x22, 0x68, 0x02, + 0xce, 0xef, 0xbc, 0xb9, 0x9e, 0xb1, 0xdf, 0xaf, 0x67, 0xec, 0xcf, 0xeb, 0x19, 0xfb, 0xe5, 0xef, + 0xd9, 0x3b, 0xcf, 0x06, 0xf4, 0x97, 0xe1, 0x93, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x38, 0x60, + 0x42, 0x9e, 0x42, 0x08, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index 229fcfadd..67cef2a40 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -105,3 +105,13 @@ message ImportValueRequest { repeated string ColumnKeys = 7; repeated int64 Values = 6; } + +message ImportRoaringRequestView { + string Name = 1; + bytes Data = 2; +} + +message ImportRoaringRequest { + bool Clear = 1; + repeated ImportRoaringRequestView views = 2; +} \ No newline at end of file diff --git a/server/handler_test.go b/server/handler_test.go index 9e76d77b9..5ae4dd877 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -22,15 +22,15 @@ import ( "fmt" "io" "io/ioutil" + gohttp "net/http" "net/http/httptest" "reflect" "strings" "testing" "time" - gohttp "net/http" - "github.com/pilosa/pilosa" + "github.com/pilosa/pilosa/encoding/proto" "github.com/pilosa/pilosa/http" "github.com/pilosa/pilosa/server" "github.com/pilosa/pilosa/test" @@ -91,9 +91,21 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("ImportRoaring", func(t *testing.T) { w := httptest.NewRecorder() roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") - req := test.MustNewHTTPRequest("POST", "/index/i0/field/f1/import-roaring/0", bytes.NewBuffer(roaringData)) - req.Header.Set("Content-Type", "application/x-binary") - h.ServeHTTP(w, req) + msg := pilosa.ImportRoaringRequest{ + Clear: false, + Views: map[string][]byte{ + "": roaringData, + }, + } + ser := proto.Serializer{} + data, err := ser.Marshal(&msg) + if err != nil { + t.Fatal(err) + } + httpReq := test.MustNewHTTPRequest("POST", "/index/i0/field/f1/import-roaring/0", bytes.NewBuffer(data)) + httpReq.Header.Set("Content-Type", "application/x-protobuf") + httpReq.Header.Set("Accept", "application/x-protobuf") + h.ServeHTTP(w, httpReq) resp, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i0", Query: "TopN(f1)"}) if err != nil { t.Fatalf("querying: %v", err) @@ -111,9 +123,21 @@ func TestHandler_Endpoints(t *testing.T) { } w := httptest.NewRecorder() roaringData, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") - req := test.MustNewHTTPRequest("POST", "/index/i0/field/int-field/import-roaring/0", bytes.NewBuffer(roaringData)) - req.Header.Set("Content-Type", "application/x-binary") - h.ServeHTTP(w, req) + msg := pilosa.ImportRoaringRequest{ + Clear: false, + Views: map[string][]byte{ + "": roaringData, + }, + } + ser := proto.Serializer{} + data, err := ser.Marshal(&msg) + if err != nil { + t.Fatal(err) + } + httpReq := test.MustNewHTTPRequest("POST", "/index/i0/field/int-field/import-roaring/0", bytes.NewBuffer(data)) + httpReq.Header.Set("Content-Type", "application/x-protobuf") + httpReq.Header.Set("Accept", "application/x-protobuf") + h.ServeHTTP(w, httpReq) if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) }