scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1865 from tgruben/roaring-import-opt

Browse source 
removed copy for pilosa roaring files
This commit is contained in:
tgruben 2019-02-19 11:21:51 -06:00 • committed by GitHub
commit 11fe06be85
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 24 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
api.go
View file

@ -18,6 +18,7 @@ package pilosa
import (
"context"
"encoding/binary"
"encoding/csv"
"fmt"
"io"
@ -322,13 +323,22 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
if len(viewData) == 0 {
return fmt.Errorf("no data to import for view: %s", viewName)
}
// must make a copy of data to operate on locally.
// field.importRoaring changes data
data := make([]byte, len(viewData))
copy(data, viewData)
err = field.importRoaring(data, shard, viewName, req.Clear)
if err != nil {
return err
fileMagic := uint32(binary.LittleEndian.Uint16(viewData[0:2]))
if fileMagic == roaring.MagicNumber { // if pilosa roaring format
err = field.importRoaring(viewData, shard, viewName, req.Clear)
if err != nil {
return errors.Wrap(err, "importing pilosa roaring")
}
} else {
// must make a copy of data to operate on locally on standard roaring format.
// field.importRoaring changes the standard roaring run format to pilosa roaring
data := make([]byte, len(viewData))
copy(data, viewData)
err = field.importRoaring(data, shard, viewName, req.Clear)
if err != nil {
return errors.Wrap(err, "importing standard roaring")
}
}
}
return err

14
roaring/roaring.go
View file

@ -28,15 +28,15 @@ import (
)
const (
// magicNumber is an identifier, in bytes 0-1 of the file.
magicNumber = uint32(12348)
// MagicNumber is an identifier, in bytes 0-1 of the file.
MagicNumber = uint32(12348)
// storageVersion indicates the storage version, in bytes 2-3.
storageVersion = uint32(0)
// cookie is the first four bytes in a roaring bitmap file,
// formed by joining magicNumber and storageVersion
cookie = magicNumber + storageVersion<<16
// formed by joining MagicNumber and storageVersion
cookie = MagicNumber + storageVersion<<16
// headerBaseSize is the size in bytes of the cookie and key count at the
// beginning of a file.
@ -935,10 +935,10 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error {
return errors.New("data too small")
}
// Verify the first two bytes are a valid magicNumber, and second two bytes match current storageVersion.
// Verify the first two bytes are a valid MagicNumber, and second two bytes match current storageVersion.
fileMagic := uint32(binary.LittleEndian.Uint16(data[0:2]))
fileVersion := uint32(binary.LittleEndian.Uint16(data[2:4]))
if fileMagic != magicNumber {
if fileMagic != MagicNumber {
return fmt.Errorf("invalid roaring file, magic number %v is incorrect", fileMagic)
}
@ -4020,7 +4020,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
}
statsHit("Bitmap/UnmarshalBinary")
fileMagic := uint32(binary.LittleEndian.Uint16(data[0:2]))
if fileMagic == magicNumber { // if pilosa roaring
if fileMagic == MagicNumber { // if pilosa roaring
return errors.Wrap(b.unmarshalPilosaRoaring(data), "unmarshaling as pilosa roaring")
}