diff --git a/api.go b/api.go index d3fb03615..fc4ca41c8 100644 --- a/api.go +++ b/api.go @@ -18,6 +18,7 @@ package pilosa import ( "context" + "encoding/binary" "encoding/csv" "fmt" "io" @@ -322,13 +323,22 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, if len(viewData) == 0 { return fmt.Errorf("no data to import for view: %s", viewName) } - // must make a copy of data to operate on locally. - // field.importRoaring changes data - data := make([]byte, len(viewData)) - copy(data, viewData) - err = field.importRoaring(data, shard, viewName, req.Clear) - if err != nil { - return err + fileMagic := uint32(binary.LittleEndian.Uint16(viewData[0:2])) + if fileMagic == roaring.MagicNumber { // if pilosa roaring format + err = field.importRoaring(viewData, shard, viewName, req.Clear) + if err != nil { + return errors.Wrap(err, "importing pilosa roaring") + } + + } else { + // must make a copy of data to operate on locally on standard roaring format. + // field.importRoaring changes the standard roaring run format to pilosa roaring + data := make([]byte, len(viewData)) + copy(data, viewData) + err = field.importRoaring(data, shard, viewName, req.Clear) + if err != nil { + return errors.Wrap(err, "importing standard roaring") + } } } return err diff --git a/roaring/roaring.go b/roaring/roaring.go index abc4e4d2f..a34d9347e 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -28,15 +28,15 @@ import ( ) const ( - // magicNumber is an identifier, in bytes 0-1 of the file. - magicNumber = uint32(12348) + // MagicNumber is an identifier, in bytes 0-1 of the file. + MagicNumber = uint32(12348) // storageVersion indicates the storage version, in bytes 2-3. storageVersion = uint32(0) // cookie is the first four bytes in a roaring bitmap file, - // formed by joining magicNumber and storageVersion - cookie = magicNumber + storageVersion<<16 + // formed by joining MagicNumber and storageVersion + cookie = MagicNumber + storageVersion<<16 // headerBaseSize is the size in bytes of the cookie and key count at the // beginning of a file. @@ -935,10 +935,10 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error { return errors.New("data too small") } - // Verify the first two bytes are a valid magicNumber, and second two bytes match current storageVersion. + // Verify the first two bytes are a valid MagicNumber, and second two bytes match current storageVersion. fileMagic := uint32(binary.LittleEndian.Uint16(data[0:2])) fileVersion := uint32(binary.LittleEndian.Uint16(data[2:4])) - if fileMagic != magicNumber { + if fileMagic != MagicNumber { return fmt.Errorf("invalid roaring file, magic number %v is incorrect", fileMagic) } @@ -4020,7 +4020,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { } statsHit("Bitmap/UnmarshalBinary") fileMagic := uint32(binary.LittleEndian.Uint16(data[0:2])) - if fileMagic == magicNumber { // if pilosa roaring + if fileMagic == MagicNumber { // if pilosa roaring return errors.Wrap(b.unmarshalPilosaRoaring(data), "unmarshaling as pilosa roaring") }