From fd655c998e3d7863bf1f601c70c7ff8bde825662 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 13 Sep 2018 12:27:23 -0500 Subject: [PATCH] rename all instances of roaringbytes function names now importRoaring and byte slice args are just called data. Also renamed http endpoint from /importroaring to /import-roaring --- api.go | 20 ++++++++++---------- client.go | 4 ++-- field.go | 4 ++-- fragment.go | 6 +++--- fragment_internal_test.go | 4 ++-- http/client.go | 8 ++++---- http/handler.go | 4 ++-- 7 files changed, 25 insertions(+), 25 deletions(-) diff --git a/api.go b/api.go index d35bd2f3a..137fdfe1c 100644 --- a/api.go +++ b/api.go @@ -295,7 +295,7 @@ func (api *API) Field(_ context.Context, indexName, fieldName string) (*Field, e return field, nil } -// ImportRoaringBytes is a low level interface for importing data to Pilosa when +// ImportRoaring is a low level interface for importing data to Pilosa when // extremely high throughput is desired. The data must be encoded in a // particular way which may be unintuitive (discussed below). The data is merged // with existing data. @@ -306,13 +306,13 @@ func (api *API) Field(_ context.Context, indexName, fieldName string) (*Field, e // or to the pilosa roaring spec which supports 64 bit integers // (https://www.pilosa.com/docs/latest/architecture/#roaring-bitmap-storage-format). // -// The data, roaringBytes, should be encoded the same way that Pilosa stores -// fragments internally. A bit "i" being set in the input bitmap indicates that -// the bit is set in Pilosa row "i/ShardWidth", and in column -// (shard*ShardWidth)+(i%ShardWidth). That is to say that roaringBytes -// represents all of the rows in this shard of this field concatenated together -// in one long bitmap. -func (api *API) ImportRoaringBytes(ctx context.Context, indexName, fieldName string, shard uint64, roaringBytes []byte) (err error) { +// The data should be encoded the same way that Pilosa stores fragments +// internally. A bit "i" being set in the input bitmap indicates that the bit is +// set in Pilosa row "i/ShardWidth", and in column +// (shard*ShardWidth)+(i%ShardWidth). That is to say that "data" represents all +// of the rows in this shard of this field concatenated together in one long +// bitmap. +func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, shard uint64, data []byte) (err error) { if err = api.validate(apiField); err != nil { return errors.Wrap(err, "validating api method") } @@ -327,7 +327,7 @@ func (api *API) ImportRoaringBytes(ctx context.Context, indexName, fieldName str } wg.Add(1) go func(node *Node) { - err = field.importRoaringBytes(roaringBytes, shard) + err = field.importRoaring(data, shard) wg.Done() }(node) } else { @@ -335,7 +335,7 @@ func (api *API) ImportRoaringBytes(ctx context.Context, indexName, fieldName str //forward it on go func(node *Node) { //execute on node - err = api.server.defaultClient.ImportRoaringBytes(ctx, node, indexName, fieldName, shard, roaringBytes) + err = api.server.defaultClient.ImportRoaring(ctx, node, indexName, fieldName, shard, data) wg.Done() }(node) } diff --git a/client.go b/client.go index 7eb24d121..124b2c6c5 100644 --- a/client.go +++ b/client.go @@ -53,7 +53,7 @@ type InternalClient interface { RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) SendMessage(ctx context.Context, uri *URI, msg []byte) error RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error) - ImportRoaringBytes(ctx context.Context, node *Node, index, field string, shard uint64, roaringBytes []byte) error + ImportRoaring(ctx context.Context, node *Node, index, field string, shard uint64, data []byte) error } //=============== @@ -109,7 +109,7 @@ func (n nopInternalClient) Import(ctx context.Context, index, field string, shar func (n nopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit) error { return nil } -func (n nopInternalClient) ImportRoaringBytes(ctx context.Context, node *Node, index, field string, shard uint64, roaringBytes []byte) error { +func (n nopInternalClient) ImportRoaring(ctx context.Context, node *Node, index, field string, shard uint64, data []byte) error { return nil } func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error { diff --git a/field.go b/field.go index 4223d3948..983ff7129 100644 --- a/field.go +++ b/field.go @@ -1055,7 +1055,7 @@ func (f *Field) importValue(columnIDs []uint64, values []int64) error { return nil } -func (f *Field) importRoaringBytes(stdRoaringBytes []byte, shard uint64) error { +func (f *Field) importRoaring(data []byte, shard uint64) error { viewName := viewStandard view, err := f.createViewIfNotExists(viewName) @@ -1068,7 +1068,7 @@ func (f *Field) importRoaringBytes(stdRoaringBytes []byte, shard uint64) error { return errors.Wrap(err, "creating fragment") } - if err := frag.importRoaringBytes(stdRoaringBytes); err != nil { + if err := frag.importRoaring(data); err != nil { return err } diff --git a/fragment.go b/fragment.go index 21ecbcd99..19b0fdf5a 100644 --- a/fragment.go +++ b/fragment.go @@ -1423,14 +1423,14 @@ func (f *fragment) importValue(columnIDs, values []uint64, bitDepth uint) error return nil } -// importRoaringBytes imports from standard roaring data format defined at +// importRoaring imports from standard roaring data format defined at // https://github.com/RoaringBitmap/RoaringFormatSpec or from pilosa's version // of the roaring format. The cache is updated to reflect the new data. -func (f *fragment) importRoaringBytes(roaringBytes []byte) error { +func (f *fragment) importRoaring(data []byte) error { f.mu.Lock() defer f.mu.Unlock() bm := roaring.NewBitmap() - err := bm.UnmarshalBinary(roaringBytes) + err := bm.UnmarshalBinary(data) if err != nil { return err } diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 83966e657..2fbb44124 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -1433,7 +1433,7 @@ func TestFragment_RoaringImport(t *testing.T) { if err != nil { t.Fatalf("writing to buffer: %v", err) } - f.importRoaringBytes(buf.Bytes()) + f.importRoaring(buf.Bytes()) exp := calcExpected(test[:num+1]...) for row, expCols := range exp { cols := f.row(uint64(row)).Columns() @@ -1504,7 +1504,7 @@ func TestFragment_RoaringImportTopN(t *testing.T) { if err != nil { t.Fatalf("writing to buffer: %v", err) } - f.importRoaringBytes(buf.Bytes()) + f.importRoaring(buf.Bytes()) rows, cols := toRowsCols(test.roaring) expPairs = calcTop(append(test.rowIDs, rows...), append(test.colIDs, cols...)) pairs, err = f.top(topOptions{}) diff --git a/http/client.go b/http/client.go index b3fd0ea83..e4e611c7b 100644 --- a/http/client.go +++ b/http/client.go @@ -525,20 +525,20 @@ func (c *InternalClient) marshalImportValuePayload(index, field string, shard ui return buf, nil } -// ImportRoaringBytes fast import of raw bits in roaring standard format -func (c *InternalClient) ImportRoaringBytes(ctx context.Context, node *pilosa.Node, index, field string, shard uint64, roaringBytes []byte) error { +// ImportRoaring fast import of raw bits in roaring standard format +func (c *InternalClient) ImportRoaring(ctx context.Context, node *pilosa.Node, index, field string, shard uint64, data []byte) error { if index == "" { return pilosa.ErrIndexRequired } else if field == "" { return pilosa.ErrFieldRequired } - endpoint := fmt.Sprintf("/index/%s/field/%s/importroaring/%d", index, field, shard) + endpoint := fmt.Sprintf("/index/%s/field/%s/import-roaring/%d", index, field, shard) // Create URL. u := nodePathToURL(node, endpoint) // Generate HTTP request. - req, err := http.NewRequest("POST", u.String(), bytes.NewBuffer(roaringBytes)) + req, err := http.NewRequest("POST", u.String(), bytes.NewBuffer(data)) if err != nil { return errors.Wrap(err, "creating request") } diff --git a/http/handler.go b/http/handler.go index e154b6db5..f9a8c51c7 100644 --- a/http/handler.go +++ b/http/handler.go @@ -218,7 +218,7 @@ func newRouter(handler *Handler) *mux.Router { router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST") router.HandleFunc("/index/{index}/field/{field}", handler.handleDeleteField).Methods("DELETE") router.HandleFunc("/index/{index}/field/{field}/import", handler.handlePostImport).Methods("POST") - router.HandleFunc("/index/{index}/field/{field}/importroaring/{shard}", handler.handlePostRoaringImport).Methods("POST") + router.HandleFunc("/index/{index}/field/{field}/import-roaring/{shard}", handler.handlePostRoaringImport).Methods("POST") router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") router.HandleFunc("/info", handler.handleGetInfo).Methods("GET") router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST") @@ -1458,7 +1458,7 @@ func (h *Handler) handlePostRoaringImport(w http.ResponseWriter, r *http.Request return } //TODO give meaningful stats for import - err = h.api.ImportRoaringBytes(r.Context(), indexName, fieldName, shard, body) + err = h.api.ImportRoaring(r.Context(), indexName, fieldName, shard, body) // Marshal response object. msg := string("")