diff --git a/api.go b/api.go index 3e8243668..012e4bf3e 100644 --- a/api.go +++ b/api.go @@ -295,7 +295,23 @@ func (api *API) Field(_ context.Context, indexName, fieldName string) (*Field, e return field, nil } -// ImportRoaringBytes fast loading of standard roaring format +// ImportRoaringBytes is a low level interface for importing data to Pilosa when +// extremely high throughput is desired. The data must be encoded in a +// particular way which may be unintuitive (discussed below), and overwrites any +// existing data in the particular fragment into which it is being loaded. +// +// It takes as input a roaring bitmap which it uses as the data for the +// indicated index, field, and shard. The bitmap may be encoded according to the +// standard roaring spec (https://github.com/RoaringBitmap/RoaringFormatSpec), +// or to the pilosa roaring spec which supports 64 bit integers +// (https://www.pilosa.com/docs/latest/architecture/#roaring-bitmap-storage-format). +// +// The data, roaringBytes, should be encoded the same way that Pilosa stores +// fragments internally. A bit "i" being set in the input bitmap indicates that +// the bit is set in Pilosa row "i/sliceWidth", and in column +// (shard*sliceWidth)+(i%sliceWidth). That is to say that roaringBytes +// represents all of the rows in this shard of this field concatenated together +// in one long bitmap. func (api *API) ImportRoaringBytes(ctx context.Context, roaringBytes []byte, indexName, fieldName string, shard uint64) (err error) { if err = api.validate(apiField); err != nil { err = errors.Wrap(err, "validating api method") diff --git a/fragment.go b/fragment.go index e05b42954..904652a57 100644 --- a/fragment.go +++ b/fragment.go @@ -1420,7 +1420,8 @@ func (f *fragment) importValue(columnIDs, values []uint64, bitDepth uint) error } // importRoaringBytes imports from standard roaring data format defined at -// https://github.com/RoaringBitmap/RoaringFormatSpec +// https://github.com/RoaringBitmap/RoaringFormatSpec or from pilosa's version +// of the roaring format. func (f *fragment) importRoaringBytes(roaringBytes []byte) error { f.mu.Lock() defer f.mu.Unlock() diff --git a/roaring/roaring.go b/roaring/roaring.go index c9f832126..6341453a4 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -3436,9 +3436,15 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { // Read container offsets and attach data. if haveRuns { - readWithRuns(b, data, pos, keyN) + err := readWithRuns(b, data, pos, keyN) + if err != nil { + return errors.Wrap(err, "reading runs from standard roaring format") + } } else { - readOffsets(b, data, pos, keyN) + err := readOffsets(b, data, pos, keyN) + if err != nil { + return errors.Wrap(err, "reading offsets from standard roaring format") + } } return nil } @@ -3450,8 +3456,7 @@ func readOffsets(b *Bitmap, data []byte, pos int, keyN uint32) (err error) { offset := binary.LittleEndian.Uint32(buf[0:4]) // Verify the offset is within the bounds of the input data. if int(offset) >= len(data) { - err = fmt.Errorf("offset out of bounds: off=%d, len=%d", offset, len(data)) - return + return fmt.Errorf("offset out of bounds: off=%d, len=%d", offset, len(data)) } // Map byte slice directly to the container data. @@ -3467,12 +3472,10 @@ func readOffsets(b *Bitmap, data []byte, pos int, keyN uint32) (err error) { c.runs = nil c.bitmap = (*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN] default: - err = fmt.Errorf("unsupported container type %d", c.containerType) - return + return fmt.Errorf("unsupported container type %d", c.containerType) } } return - } func readWithRuns(b *Bitmap, data []byte, pos int, keyN uint32) (err error) {