diff --git a/roaring/roaring.go b/roaring/roaring.go index 80a089d87..a752670c3 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -42,6 +42,13 @@ const ( // Headers in files with runs also include runFlagBitset, of length (numContainers+7)/8. headerBaseSize = 4 + 4 + // runCountHeaderSize is the size in bytes of the run count stored + // at the beginning of every serialized run container. + runCountHeaderSize = 2 + + // interval32Size is the size of a single run in a container.runs. + interval32Size = 8 + // bitmapN is the number of values in a container.bitmap. bitmapN = (1 << 16) / 64 @@ -52,7 +59,7 @@ const ( // Bitmap represents a roaring bitmap. type Bitmap struct { keys []uint64 // keys for containers - containers []*container // array and bitmap containers + containers []*container // array, bitmap and RLE containers // Number of operations written to the writer. opN int @@ -529,7 +536,7 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { // Create bitset indicating runs, record whether any runs present. containsRuns := false - runFlagBitset := make([]uint8, (containerCount+7)/8) // TODO verify size + runFlagBitset := make([]uint8, (containerCount+7)/8) k := 0 for _, c := range b.containers { if c.n == 0 { @@ -537,7 +544,7 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { } if c.isRun() { containsRuns = true - runFlagBitset[k/8] |= (1 << uint(k%8)) // TODO verify + runFlagBitset[k/8] |= (1 << uint(k%8)) } k++ } @@ -550,9 +557,11 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { } // Build header before writing individual container blocks. + // Metadata for each container is 4+8+4 = sizeof(key) + sizeof(cardinality) + sizeof(file offset) buf := make([]byte, headerSize+(containerCount*(4+8+4))) + + // Cookie header section. binary.LittleEndian.PutUint32(buf[0:], thisCookie) - // TODO supposed to be 2 bytes, but doesnt make sense for >65535 containers binary.LittleEndian.PutUint32(buf[4:], uint32(containerCount)) if containsRuns { @@ -563,7 +572,8 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { } empty := 0 - // Encode keys and cardinality. + // Descriptive header section: encode keys and cardinality. + // Key and cardinality are stored interleaved here, 12 bytes per container. for i, key := range b.keys { c := b.containers[i] @@ -579,8 +589,8 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { } } - // TODO this should now be done conditionally - // Write the offset for each container block. + // Offset header section: write the offset for each container block. + // 4 bytes per container. offset := uint32(len(buf)) empty = 0 for i, c := range b.containers { @@ -600,7 +610,7 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { return n, err } - // Write each container block. + // Container storage section: write each container block. for _, c := range b.containers { if c.n > 0 { nn, err := c.WriteTo(w) @@ -620,7 +630,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { return errors.New("data too small") } - // Verify the first 4 bytes are a valid cookie. + // Verify the first sizeof(cookie)=4 bytes are a valid cookie. v := binary.LittleEndian.Uint32(data[0:4]) containsRuns := false if v == cookieNoRuns { @@ -631,14 +641,14 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { return errors.New("invalid roaring file") } - // Read key count. + // Read key count in bytes sizeof(cookie):(sizeof(cookie)+sizeof(uint32)). keyN := binary.LittleEndian.Uint32(data[4:8]) b.keys = make([]uint64, keyN) b.containers = make([]*container, keyN) headerSize := headerBaseSize - runFlagBitset := make([]uint8, (keyN+7)/8) // TODO verify size + runFlagBitset := make([]uint8, (keyN+7)/8) if containsRuns { // Read runFlag bitset. for i := 0; i < len(runFlagBitset); i++ { @@ -647,7 +657,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { headerSize += len(runFlagBitset) } - // Read container key headers. + // Descriptive header section: Read container keys and cardinalities. for i, buf := 0, data[headerSize:]; i < int(keyN); i, buf = i+1, buf[12:] { b.keys[i] = binary.LittleEndian.Uint64(buf[0:8]) b.containers[i] = &container{ @@ -671,9 +681,9 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { if c.n <= ArrayMaxSize { if containsRuns && (runFlagBitset[i/8]&(1< iv.last { continue @@ -1340,7 +1351,7 @@ func (c *container) runContains(v uint32) bool { return false } -// remove adds a value to the container. +// remove removes a value from the container. func (c *container) remove(v uint32) (removed bool) { if c.isArray() { removed = c.arrayRemove(v) @@ -1384,6 +1395,7 @@ func (c *container) bitmapRemove(v uint32) bool { } func (c *container) runRemove(v uint32) bool { + // TODO binary search for i, iv := range c.runs { if v <= iv.last { if v < iv.start { @@ -1537,7 +1549,8 @@ func (c *container) bitmapToRun() { // arrayToRun converts from array format to RLE format. func (c *container) arrayToRun() { - c.runs = make([]interval32, 0, c.n) // TODO what capacity to use? + numRuns := c.arrayCountRuns() // TODO test + c.runs = make([]interval32, 0, numRuns) start := c.array[0] for i, v := range c.array[1:] { if v-c.array[i] > 1 { @@ -1608,11 +1621,13 @@ func (c *container) arrayWriteTo(w io.Writer) (n int64, err error) { // assert(lowbits(uint64(v)) == v, "cannot write array value out of range: %d", v) //} + // Write sizeof(uint32) * cardinality bytes. nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.array[0]))[:4*c.n]) return int64(nn), err } func (c *container) bitmapWriteTo(w io.Writer) (n int64, err error) { + // Write sizeof(uint64) * bitmapN bytes. nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.bitmap[0]))[:(8 * bitmapN)]) return int64(nn), err } @@ -1621,24 +1636,23 @@ func (c *container) runWriteTo(w io.Writer) (n int64, err error) { if len(c.runs) == 0 { return 0, nil } - // TODO according to spec this should be [16-bit-runcount, start0, len0, start1, len1, ...] - // not sure what difference len vs last makes + // Write sizeof(interval32) * runCount bytes. err = binary.Write(w, binary.LittleEndian, uint16(len(c.runs))) if err != nil { return 0, err } - nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.runs[0]))[:8*len(c.runs)]) - return int64(2 + nn), err + nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.runs[0]))[:interval32Size*len(c.runs)]) + return int64(runCountHeaderSize + nn), err } // size returns the encoded size of the container, in bytes. func (c *container) size() int { if c.isArray() { - return len(c.array) * 4 + return len(c.array) * 4 // sizeof(uint32) } else if c.isRun() { - return len(c.runs)*8 + 2 + return len(c.runs)*interval32Size + runCountHeaderSize } else { - return len(c.bitmap) * 8 + return len(c.bitmap) * 8 // sizeof(uint64) } } @@ -1648,13 +1662,13 @@ func (c *container) info() ContainerInfo { if c.isArray() { info.Type = "array" - info.Alloc = len(c.array) * 4 + info.Alloc = len(c.array) * 4 // sizeof(uint32) } else if c.isRun() { info.Type = "run" - info.Alloc = len(c.runs)*8 + 2 + info.Alloc = len(c.runs)*interval32Size + runCountHeaderSize } else { info.Type = "bitmap" - info.Alloc = len(c.bitmap) * 8 + info.Alloc = len(c.bitmap) * 8 // sizeof(uint64) } if c.mapped { @@ -2233,6 +2247,7 @@ func unionBitmapRun(a, b *container) *container { const maxBitmap = 0xFFFFFFFFFFFFFFFF // sets all bits in [i, j] (inclusive) (c must be a bitmap container) +// TODO inclusive upper limit is inconsistent with other functions (CountRange, SliceRange, ForEachRange, OffsetRange(?)) func (c *container) bitmapSetRange(i, j uint64) { j += 1 x := i / 64 @@ -2257,6 +2272,7 @@ func (c *container) bitmapSetRange(i, j uint64) { } // xor's all bits in [i, j] with all true (inclusive) (c must be a bitmap container). +// TODO inclusive upper limit is inconsistent with other functions func (c *container) bitmapXorRange(i, j uint64) { j += 1 x := i / 64 @@ -2283,6 +2299,7 @@ func (c *container) bitmapXorRange(i, j uint64) { } // zeroes all bits in [i, j] (inclusive) (c must be a bitmap container) +// TODO inclusive upper limit is inconsistent with other functions func (c *container) bitmapZeroRange(i, j uint64) { j += 1 x := i / 64 @@ -2486,8 +2503,6 @@ func differenceRunBitmap(a, b *container) *container { } func differenceRunRun(a, b *container) *container { - // (rc *runContainer32) AndNotRunContainer32(b *runContainer32) *runContainer32 { - if a.n == 0 || b.n == 0 { return a.clone() }