scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-10 15:01:03 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

removed copy for pilosa roaring files

Browse source
This commit is contained in:
Todd Gruben 2019-02-18 16:28:37 -06:00
parent 62b9697a6a
commit ffe1a285bd
2 changed files with 22 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
api.go
View file

@ -18,6 +18,7 @@ package pilosa
import (
"context"
"encoding/binary"
"encoding/csv"
"fmt"
"io"
@ -324,11 +325,20 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
}
// must make a copy of data to operate on locally.
// field.importRoaring changes data
data := make([]byte, len(viewData))
copy(data, viewData)
err = field.importRoaring(data, shard, viewName, req.Clear)
if err != nil {
return err
fileMagic := uint32(binary.LittleEndian.Uint16(viewData[0:2]))
if fileMagic == roaring.MagicNumber { // if pilosa roaring
err = field.importRoaring(viewData, shard, viewName, req.Clear)
if err != nil {
return err
}
} else {
data := make([]byte, len(viewData))
copy(data, viewData)
err = field.importRoaring(data, shard, viewName, req.Clear)
if err != nil {
return err
}
}
}
return err

14
roaring/roaring.go
View file

@ -28,15 +28,15 @@ import (
)
const (
// magicNumber is an identifier, in bytes 0-1 of the file.
magicNumber = uint32(12348)
// MagicNumber is an identifier, in bytes 0-1 of the file.
MagicNumber = uint32(12348)
// storageVersion indicates the storage version, in bytes 2-3.
storageVersion = uint32(0)
// cookie is the first four bytes in a roaring bitmap file,
// formed by joining magicNumber and storageVersion
cookie = magicNumber + storageVersion<<16
// formed by joining MagicNumber and storageVersion
cookie = MagicNumber + storageVersion<<16
// headerBaseSize is the size in bytes of the cookie and key count at the
// beginning of a file.
@ -935,10 +935,10 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error {
return errors.New("data too small")
}
// Verify the first two bytes are a valid magicNumber, and second two bytes match current storageVersion.
// Verify the first two bytes are a valid MagicNumber, and second two bytes match current storageVersion.
fileMagic := uint32(binary.LittleEndian.Uint16(data[0:2]))
fileVersion := uint32(binary.LittleEndian.Uint16(data[2:4]))
if fileMagic != magicNumber {
if fileMagic != MagicNumber {
return fmt.Errorf("invalid roaring file, magic number %v is incorrect", fileMagic)
}
@ -4020,7 +4020,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
}
statsHit("Bitmap/UnmarshalBinary")
fileMagic := uint32(binary.LittleEndian.Uint16(data[0:2]))
if fileMagic == magicNumber { // if pilosa roaring
if fileMagic == MagicNumber { // if pilosa roaring
return errors.Wrap(b.unmarshalPilosaRoaring(data), "unmarshaling as pilosa roaring")
}