diff --git a/ctl/check_test.go b/ctl/check_test.go index 56d70f97a..1b2ebd6f6 100644 --- a/ctl/check_test.go +++ b/ctl/check_test.go @@ -85,7 +85,7 @@ func TestCheckCommand_Run(t *testing.T) { var buf bytes.Buffer io.Copy(&buf, r) - if !strings.HasPrefix(err.Error(), "checking bitmap: unmarshalling: invalid roaring file") { + if !strings.HasPrefix(err.Error(), "checking bitmap: unmarshalling: reading roaring header:") { t.Fatalf("expect error: invalid roaring file, actual: '%s'", err) } // Todo: need correct roaring file for happy path diff --git a/fragment.go b/fragment.go index e7014aeb0..e05b42954 100644 --- a/fragment.go +++ b/fragment.go @@ -1424,7 +1424,8 @@ func (f *fragment) importValue(columnIDs, values []uint64, bitDepth uint) error func (f *fragment) importRoaringBytes(roaringBytes []byte) error { f.mu.Lock() defer f.mu.Unlock() - bm, err := roaring.UnmarshalStandardRoaring(roaringBytes) + bm := roaring.NewBitmap() + err := bm.UnmarshalBinary(roaringBytes) if err != nil { return err } diff --git a/roaring/roaring.go b/roaring/roaring.go index abf36c40e..c9f832126 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -17,13 +17,14 @@ package roaring import ( "encoding/binary" - "errors" "fmt" "hash/fnv" "io" "math/bits" "sort" "unsafe" + + "github.com/pkg/errors" ) const ( @@ -614,8 +615,9 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { return n, nil } -// UnmarshalBinary decodes b from a binary-encoded byte slice. -func (b *Bitmap) UnmarshalBinary(data []byte) error { +// unmarshalPilosaRoaring treats data as being encoded in Pilosa's 64 bit +// roaring format and decodes it into b. +func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error { if len(data) < headerBaseSize { return errors.New("data too small") } @@ -3408,18 +3410,17 @@ func readStandardHeader(buf []byte) (size uint32, containerTyper func(index uint return } -func UnmarshalStandardRoaring(data []byte) (*Bitmap, error) { - b := NewBitmap() - +// UnmarshalBinary decodes b from a binary-encoded byte slice. data can be in +// either standard roaring format or Pilosa's roaring format. +func (b *Bitmap) UnmarshalBinary(data []byte) error { fileMagic := uint32(binary.LittleEndian.Uint16(data[0:2])) - if fileMagic == magicNumber { //if pilosa roaring - err := b.UnmarshalBinary(data) - return b, err + if fileMagic == magicNumber { // if pilosa roaring + return errors.Wrap(b.unmarshalPilosaRoaring(data), "unmarshaling as pilosa roaring") } keyN, containerTyper, header, pos, err, haveRuns := readStandardHeader(data) if err != nil { - return nil, err + return errors.Wrap(err, "reading roaring header") } b.Containers.Reset() @@ -3439,7 +3440,7 @@ func UnmarshalStandardRoaring(data []byte) (*Bitmap, error) { } else { readOffsets(b, data, pos, keyN) } - return b, nil + return nil } func readOffsets(b *Bitmap, data []byte, pos int, keyN uint32) (err error) { diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index 2469657d1..caea8e2c1 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -3231,7 +3231,8 @@ func runContainerFunc(f interface{}, c ...*Container) *Container { func TestUnmarshalStdRoaring(t *testing.T) { //generated serialize image from java(clojure) with arrays rbContainerWithTwoArrays, _ := hex.DecodeString("3A300000020000000000020001000000180000001E0000000100020003000100") - bm, er := UnmarshalStandardRoaring(rbContainerWithTwoArrays) + bm := NewBitmap() + er := bm.UnmarshalBinary(rbContainerWithTwoArrays) if er != nil { t.Fatalf("UnmarshalStandardRoaring %s", er) } @@ -3240,7 +3241,8 @@ func TestUnmarshalStdRoaring(t *testing.T) { } //generated serialize image from java(clojure) with a run and array rbContainerWithRLEandArray, _ := hex.DecodeString("3B3001000100000900010000000100010009000100") - bm, er = UnmarshalStandardRoaring(rbContainerWithRLEandArray) + bm = NewBitmap() + er = bm.UnmarshalBinary(rbContainerWithRLEandArray) if er != nil { t.Fatalf("UnmarshalStandardRoaring %s", er) } @@ -3249,7 +3251,8 @@ func TestUnmarshalStdRoaring(t *testing.T) { } //had to use an external file because emacs was barfing on the long line :() _bitmap_array_container, _ := ioutil.ReadFile("testdata/bitmapcontainer.roaringbitmap") - bm, er = UnmarshalStandardRoaring(_bitmap_array_container) + bm = NewBitmap() + er = bm.UnmarshalBinary(_bitmap_array_container) if er != nil { t.Fatalf("UnmarshalStandardRoaring %s", er) }