improve import roaring docs, check errors

api.go

@@ -295,7 +295,23 @@ func (api *API) Field(_ context.Context, indexName, fieldName string) (*Field, e
 	return field, nil
 }
 
-// ImportRoaringBytes fast loading of standard roaring format
+// ImportRoaringBytes is a low level interface for importing data to Pilosa when
+// extremely high throughput is desired. The data must be encoded in a
+// particular way which may be unintuitive (discussed below), and overwrites any
+// existing data in the particular fragment into which it is being loaded.
+//
+// It takes as input a roaring bitmap which it uses as the data for the
+// indicated index, field, and shard. The bitmap may be encoded according to the
+// standard roaring spec (https://github.com/RoaringBitmap/RoaringFormatSpec),
+// or to the pilosa roaring spec which supports 64 bit integers
+// (https://www.pilosa.com/docs/latest/architecture/#roaring-bitmap-storage-format).
+//
+// The data, roaringBytes, should be encoded the same way that Pilosa stores
+// fragments internally. A bit "i" being set in the input bitmap indicates that
+// the bit is set in Pilosa row "i/sliceWidth", and in column
+// (shard*sliceWidth)+(i%sliceWidth). That is to say that roaringBytes
+// represents all of the rows in this shard of this field concatenated together
+// in one long bitmap.
 func (api *API) ImportRoaringBytes(ctx context.Context, roaringBytes []byte, indexName, fieldName string, shard uint64) (err error) {
 	if err = api.validate(apiField); err != nil {
 		err = errors.Wrap(err, "validating api method")

fragment.go

@@ -1420,7 +1420,8 @@ func (f *fragment) importValue(columnIDs, values []uint64, bitDepth uint) error
 }
 
 // importRoaringBytes imports from standard roaring data format defined at
-// https://github.com/RoaringBitmap/RoaringFormatSpec
+// https://github.com/RoaringBitmap/RoaringFormatSpec or from pilosa's version
+// of the roaring format.
 func (f *fragment) importRoaringBytes(roaringBytes []byte) error {
 	f.mu.Lock()
 	defer f.mu.Unlock()

roaring/roaring.go

@@ -3436,9 +3436,15 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 
 	// Read container offsets and attach data.
 	if haveRuns {
-		readWithRuns(b, data, pos, keyN)
+		err := readWithRuns(b, data, pos, keyN)
+		if err != nil {
+			return errors.Wrap(err, "reading runs from standard roaring format")
+		}
 	} else {
-		readOffsets(b, data, pos, keyN)
+		err := readOffsets(b, data, pos, keyN)
+		if err != nil {
+			return errors.Wrap(err, "reading offsets from standard roaring format")
+		}
 	}
 	return nil
 }
@@ -3450,8 +3456,7 @@ func readOffsets(b *Bitmap, data []byte, pos int, keyN uint32) (err error) {
 		offset := binary.LittleEndian.Uint32(buf[0:4])
 		// Verify the offset is within the bounds of the input data.
 		if int(offset) >= len(data) {
-			err = fmt.Errorf("offset out of bounds: off=%d, len=%d", offset, len(data))
-			return
+			return fmt.Errorf("offset out of bounds: off=%d, len=%d", offset, len(data))
 		}
 
 		// Map byte slice directly to the container data.
@@ -3467,12 +3472,10 @@ func readOffsets(b *Bitmap, data []byte, pos int, keyN uint32) (err error) {
 			c.runs = nil
 			c.bitmap = (*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN]
 		default:
-			err = fmt.Errorf("unsupported container type %d", c.containerType)
-			return
+			return fmt.Errorf("unsupported container type %d", c.containerType)
 		}
 	}
 	return
-
 }
 
 func readWithRuns(b *Bitmap, data []byte, pos int, keyN uint32) (err error) {