scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-10-08 03:47:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

rename all instances of roaringbytes

Browse source 
function names now importRoaring and byte slice args are just called data. Also
renamed http endpoint from /importroaring to /import-roaring
This commit is contained in:
Matt Jaffee 2018-09-13 12:27:23 -05:00
parent 9e7cfeedc9
commit fd655c998e
No known key found for this signature in database
GPG key ID: 08A3DFFF987B11BF
7 changed files with 25 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
api.go
View file

@ -295,7 +295,7 @@ func (api *API) Field(_ context.Context, indexName, fieldName string) (*Field, e
return field, nil
}
// ImportRoaringBytes is a low level interface for importing data to Pilosa when
// ImportRoaring is a low level interface for importing data to Pilosa when
// extremely high throughput is desired. The data must be encoded in a
// particular way which may be unintuitive (discussed below). The data is merged
// with existing data.
@ -306,13 +306,13 @@ func (api *API) Field(_ context.Context, indexName, fieldName string) (*Field, e
// or to the pilosa roaring spec which supports 64 bit integers
// (https://www.pilosa.com/docs/latest/architecture/#roaring-bitmap-storage-format).
//
// The data, roaringBytes, should be encoded the same way that Pilosa stores
// fragments internally. A bit "i" being set in the input bitmap indicates that
// the bit is set in Pilosa row "i/ShardWidth", and in column
// (shard*ShardWidth)+(i%ShardWidth). That is to say that roaringBytes
// represents all of the rows in this shard of this field concatenated together
// in one long bitmap.
func (api *API) ImportRoaringBytes(ctx context.Context, indexName, fieldName string, shard uint64, roaringBytes []byte) (err error) {
// The data should be encoded the same way that Pilosa stores fragments
// internally. A bit "i" being set in the input bitmap indicates that the bit is
// set in Pilosa row "i/ShardWidth", and in column
// (shard*ShardWidth)+(i%ShardWidth). That is to say that "data" represents all
// of the rows in this shard of this field concatenated together in one long
// bitmap.
func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, shard uint64, data []byte) (err error) {
if err = api.validate(apiField); err != nil {
return errors.Wrap(err, "validating api method")
}
@ -327,7 +327,7 @@ func (api *API) ImportRoaringBytes(ctx context.Context, indexName, fieldName str
}
wg.Add(1)
go func(node *Node) {
err = field.importRoaringBytes(roaringBytes, shard)
err = field.importRoaring(data, shard)
wg.Done()
}(node)
} else {
@ -335,7 +335,7 @@ func (api *API) ImportRoaringBytes(ctx context.Context, indexName, fieldName str
//forward it on
go func(node *Node) {
//execute on node
err = api.server.defaultClient.ImportRoaringBytes(ctx, node, indexName, fieldName, shard, roaringBytes)
err = api.server.defaultClient.ImportRoaring(ctx, node, indexName, fieldName, shard, data)
wg.Done()
}(node)
}

4
client.go
View file

@ -53,7 +53,7 @@ type InternalClient interface {
RowAttrDiff(ctx context.Context, uri *URI, index, field string, blks []AttrBlock) (map[uint64]map[string]interface{}, error)
SendMessage(ctx context.Context, uri *URI, msg []byte) error
RetrieveShardFromURI(ctx context.Context, index, field string, shard uint64, uri URI) (io.ReadCloser, error)
ImportRoaringBytes(ctx context.Context, node *Node, index, field string, shard uint64, roaringBytes []byte) error
ImportRoaring(ctx context.Context, node *Node, index, field string, shard uint64, data []byte) error
}
//===============
@ -109,7 +109,7 @@ func (n nopInternalClient) Import(ctx context.Context, index, field string, shar
func (n nopInternalClient) ImportK(ctx context.Context, index, field string, bits []Bit) error {
return nil
}
func (n nopInternalClient) ImportRoaringBytes(ctx context.Context, node *Node, index, field string, shard uint64, roaringBytes []byte) error {
func (n nopInternalClient) ImportRoaring(ctx context.Context, node *Node, index, field string, shard uint64, data []byte) error {
return nil
}
func (n nopInternalClient) EnsureIndex(ctx context.Context, name string, options IndexOptions) error {

4
field.go
View file

@ -1055,7 +1055,7 @@ func (f *Field) importValue(columnIDs []uint64, values []int64) error {
return nil
}
func (f *Field) importRoaringBytes(stdRoaringBytes []byte, shard uint64) error {
func (f *Field) importRoaring(data []byte, shard uint64) error {
viewName := viewStandard
view, err := f.createViewIfNotExists(viewName)
@ -1068,7 +1068,7 @@ func (f *Field) importRoaringBytes(stdRoaringBytes []byte, shard uint64) error {
return errors.Wrap(err, "creating fragment")
}
if err := frag.importRoaringBytes(stdRoaringBytes); err != nil {
if err := frag.importRoaring(data); err != nil {
return err
}

6
fragment.go
View file

@ -1423,14 +1423,14 @@ func (f *fragment) importValue(columnIDs, values []uint64, bitDepth uint) error
return nil
}
// importRoaringBytes imports from standard roaring data format defined at
// importRoaring imports from standard roaring data format defined at
// https://github.com/RoaringBitmap/RoaringFormatSpec or from pilosa's version
// of the roaring format. The cache is updated to reflect the new data.
func (f *fragment) importRoaringBytes(roaringBytes []byte) error {
func (f *fragment) importRoaring(data []byte) error {
f.mu.Lock()
defer f.mu.Unlock()
bm := roaring.NewBitmap()
err := bm.UnmarshalBinary(roaringBytes)
err := bm.UnmarshalBinary(data)
if err != nil {
return err
}

4
fragment_internal_test.go
View file

@ -1433,7 +1433,7 @@ func TestFragment_RoaringImport(t *testing.T) {
if err != nil {
t.Fatalf("writing to buffer: %v", err)
}
f.importRoaringBytes(buf.Bytes())
f.importRoaring(buf.Bytes())
exp := calcExpected(test[:num+1]...)
for row, expCols := range exp {
cols := f.row(uint64(row)).Columns()
@ -1504,7 +1504,7 @@ func TestFragment_RoaringImportTopN(t *testing.T) {
if err != nil {
t.Fatalf("writing to buffer: %v", err)
}
f.importRoaringBytes(buf.Bytes())
f.importRoaring(buf.Bytes())
rows, cols := toRowsCols(test.roaring)
expPairs = calcTop(append(test.rowIDs, rows...), append(test.colIDs, cols...))
pairs, err = f.top(topOptions{})

8
http/client.go
View file

@ -525,20 +525,20 @@ func (c *InternalClient) marshalImportValuePayload(index, field string, shard ui
return buf, nil
}
// ImportRoaringBytes fast import of raw bits in roaring standard format
func (c *InternalClient) ImportRoaringBytes(ctx context.Context, node *pilosa.Node, index, field string, shard uint64, roaringBytes []byte) error {
// ImportRoaring fast import of raw bits in roaring standard format
func (c *InternalClient) ImportRoaring(ctx context.Context, node *pilosa.Node, index, field string, shard uint64, data []byte) error {
if index == "" {
return pilosa.ErrIndexRequired
} else if field == "" {
return pilosa.ErrFieldRequired
}
endpoint := fmt.Sprintf("/index/%s/field/%s/importroaring/%d", index, field, shard)
endpoint := fmt.Sprintf("/index/%s/field/%s/import-roaring/%d", index, field, shard)
// Create URL.
u := nodePathToURL(node, endpoint)
// Generate HTTP request.
req, err := http.NewRequest("POST", u.String(), bytes.NewBuffer(roaringBytes))
req, err := http.NewRequest("POST", u.String(), bytes.NewBuffer(data))
if err != nil {
return errors.Wrap(err, "creating request")
}

4
http/handler.go
View file

@ -218,7 +218,7 @@ func newRouter(handler *Handler) *mux.Router {
router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST")
router.HandleFunc("/index/{index}/field/{field}", handler.handleDeleteField).Methods("DELETE")
router.HandleFunc("/index/{index}/field/{field}/import", handler.handlePostImport).Methods("POST")
router.HandleFunc("/index/{index}/field/{field}/importroaring/{shard}", handler.handlePostRoaringImport).Methods("POST")
router.HandleFunc("/index/{index}/field/{field}/import-roaring/{shard}", handler.handlePostRoaringImport).Methods("POST")
router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery")
router.HandleFunc("/info", handler.handleGetInfo).Methods("GET")
router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST")
@ -1458,7 +1458,7 @@ func (h *Handler) handlePostRoaringImport(w http.ResponseWriter, r *http.Request
return
}
//TODO give meaningful stats for import
err = h.api.ImportRoaringBytes(r.Context(), indexName, fieldName, shard, body)
err = h.api.ImportRoaring(r.Context(), indexName, fieldName, shard, body)
// Marshal response object.
msg := string("")