From 1ca9435af923898bb2d68242293b9c22dd459cf8 Mon Sep 17 00:00:00 2001 From: Seebs Date: Wed, 22 Apr 2020 15:23:29 -0500 Subject: [PATCH] Handle file sizes over 4GB We only have 4 bytes for offsets, but what if a file is over 4GB? Someone came to us with a file with 265 *million* containers, in a single fragment, which means that over 3GB of their 4.7GB file is actually just the container headers alone. But we can't easily make the offsets larger, or change the file format. So we don't. We just track how many 4GB hunks of the file we've been through and bump that every time the 32-bit offset wraps. And this appears to... just work. This is fixed for both the roaring iterator and the old unmarshalBinary logic. The logic to handle this will work on 32-bit hosts in the sense that it will correctly error out for excessively large file sizes or container counts, but it doesn't actually handle the large files since it can't. --- roaring/roaring.go | 37 +++++++++++++++++++++++++------------ roaring/unmarshal_binary.go | 27 ++++++++++++++++++--------- 2 files changed, 43 insertions(+), 21 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 5924673ea..22d2821db 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -1738,7 +1738,9 @@ type baseRoaringIterator struct { currentN int currentLen int currentPointer *uint16 - currentDataOffset uint32 + currentDataOffset uint64 + prevOffset32 uint32 + chunkOffset uint64 lastDataOffset int64 lastErr error } @@ -1752,6 +1754,8 @@ func (b *baseRoaringIterator) SilenceLint() { _ = b.offsets _ = b.headers _ = b.currentIdx + _ = b.chunkOffset + _ = b.prevOffset32 } type pilosaRoaringIterator struct { @@ -1787,14 +1791,14 @@ func newOfficialRoaringIterator(data []byte) (*officialRoaringIterator, error) { r.headers = data[headerOffset:offsetOffset] // note: offsets are only actually used with the no-run headers. if r.haveRuns { - r.currentDataOffset = uint32(offsetOffset) + r.currentDataOffset = uint64(offsetOffset) } else { if len(r.data) < offsetOffset+int(r.keys*4) { return nil, fmt.Errorf("insufficient data for offsets (need %d bytes, found %d)", r.keys*4, len(r.data)-offsetOffset) } r.offsets = data[offsetOffset : offsetOffset+int(r.keys*4)] - r.currentDataOffset = uint32(offsetOffset) + r.currentDataOffset = uint64(offsetOffset) } // set key to -1; user should call Next first. r.currentIdx = -1 @@ -1838,7 +1842,11 @@ func newPilosaRoaringIterator(data []byte) (*pilosaRoaringIterator, error) { // if there's no containers, we want to act as though data started at the end // of the list of offsets, which was also empty, so we don't think the entire thing // is actually a malformed op - r.currentDataOffset = uint32(offsetEnd) + r.prevOffset32 = uint32(offsetEnd) + r.currentDataOffset = uint64(offsetEnd) + // it's possible that there's so many headers that we're actually over + // 4GB into the file already. + r.chunkOffset = r.currentDataOffset &^ ((1 << 32) - 1) // set key to -1; user should call Next first. r.currentIdx = -1 r.currentKey = ^uint64(0) @@ -1895,7 +1903,12 @@ func (r *pilosaRoaringIterator) Next() (key uint64, cType byte, n int, length in r.currentKey = binary.LittleEndian.Uint64(header[0:8]) r.currentType = byte(binary.LittleEndian.Uint16(header[8:10])) r.currentN = int(binary.LittleEndian.Uint16(header[10:12])) + 1 - r.currentDataOffset = binary.LittleEndian.Uint32(r.offsets[r.currentIdx*4:]) + offset32 := binary.LittleEndian.Uint32(r.offsets[r.currentIdx*4:]) + if offset32 < r.prevOffset32 { + r.chunkOffset += (1 << 32) + } + r.prevOffset32 = offset32 + r.currentDataOffset = r.chunkOffset + uint64(offset32) // a run container keeps its data after an initial 2 byte length header var runCount uint16 @@ -1903,7 +1916,7 @@ func (r *pilosaRoaringIterator) Next() (key uint64, cType byte, n int, length in runCount = binary.LittleEndian.Uint16(r.data[r.currentDataOffset : r.currentDataOffset+runCountHeaderSize]) r.currentDataOffset += 2 } - if r.currentDataOffset > uint32(len(r.data)) || r.currentDataOffset < headerBaseSize { + if r.currentDataOffset > uint64(len(r.data)) || r.currentDataOffset < headerBaseSize { r.Done(fmt.Errorf("container %d/%d, key %d, had offset %d, maximum %d", r.currentIdx, r.keys, r.currentKey, r.currentDataOffset, len(r.data))) return r.Current() @@ -1926,7 +1939,7 @@ func (r *pilosaRoaringIterator) Next() (key uint64, cType byte, n int, length in r.currentIdx, r.keys, r.currentKey, r.currentDataOffset, size, len(r.data))) return r.Current() } - r.currentDataOffset += uint32(size) + r.currentDataOffset += uint64(size) r.lastErr = nil return r.Current() } @@ -1949,7 +1962,7 @@ func (r *officialRoaringIterator) Next() (key uint64, cType byte, n int, length // with runs, we can't actually look up offsets; the format just stores // things sequentially. so we have to actually track the offset in that case. if !r.haveRuns { - r.currentDataOffset = binary.LittleEndian.Uint32(r.offsets[r.currentIdx*4:]) + r.currentDataOffset = uint64(binary.LittleEndian.Uint32(r.offsets[r.currentIdx*4:])) } // a run container keeps its data after an initial 2 byte length header var runCount uint16 @@ -1962,7 +1975,7 @@ func (r *officialRoaringIterator) Next() (key uint64, cType byte, n int, length runCount = binary.LittleEndian.Uint16(r.data[r.currentDataOffset : r.currentDataOffset+runCountHeaderSize]) r.currentDataOffset += 2 } - if r.currentDataOffset > uint32(len(r.data)) || r.currentDataOffset < headerBaseSize { + if r.currentDataOffset > uint64(len(r.data)) || r.currentDataOffset < headerBaseSize { r.Done(fmt.Errorf("container %d/%d, key %d, had offset %d, maximum %d", r.currentIdx, r.keys, r.currentKey, r.currentDataOffset, len(r.data))) return r.Current() @@ -1994,7 +2007,7 @@ func (r *officialRoaringIterator) Next() (key uint64, cType byte, n int, length r.currentIdx, r.keys, r.currentKey, r.currentDataOffset, size, len(r.data))) return r.Current() } - r.currentDataOffset += uint32(size) + r.currentDataOffset += uint64(size) r.lastErr = nil return r.Current() } @@ -2012,14 +2025,14 @@ func (b *Bitmap) SanityCheckMapping(from, to uintptr) (mappedIn int64, mappedOut if c.Mapped() { mappedIn++ } else { - err = fmt.Errorf("container key %d, addr %x, inside %x+%d\n", + err = fmt.Errorf("container key %d, addr %x, inside %x+%d", key, dptr, from, to-from) errs++ unmappedIn++ } } else { if c.Mapped() { - err = fmt.Errorf("container key %d, addr %x, outside %x+%d, but mapped\n", + err = fmt.Errorf("container key %d, addr %x, outside %x+%d, but mapped", key, dptr, from, to-from) errs++ mappedOut++ diff --git a/roaring/unmarshal_binary.go b/roaring/unmarshal_binary.go index 208c15d53..c9071b1ca 100644 --- a/roaring/unmarshal_binary.go +++ b/roaring/unmarshal_binary.go @@ -159,8 +159,8 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error { // Read key count in bytes sizeof(cookie)+sizeof(flag):(sizeof(cookie)+sizeof(uint32)). keyN := binary.LittleEndian.Uint32(data[3+1 : 8]) - if uint32(len(data)) < headerBaseSize+keyN*12 { - return fmt.Errorf("insufficient data for header + offsets: key-cardinality not provided for %d containers", int(keyN)/12) + if int64(len(data)) < headerBaseSize+int64(keyN)*12 { + return fmt.Errorf("insufficient data for header + offsets: key-cardinality not provided for %d containers", keyN) } headerSize := headerBaseSize @@ -173,14 +173,23 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error { int(binary.LittleEndian.Uint16(buf[10:12]))+1, true) } - opsOffset := headerSize + int(keyN)*12 + opsOffset := int64(headerSize) + int64(keyN)*12 // Read container offsets and attach data. citer, _ := b.Containers.Iterator(0) + // if you have enough containers that the *headers alone* exceed 4GB, we + // need to start with a higher cycle offset. + cycleOffset := opsOffset &^ ((1 << 32) - 1) + prevOffset32 := uint32(opsOffset) for i, buf := 0, data[opsOffset:]; i < int(keyN); i, buf = i+1, buf[4:] { - offset := binary.LittleEndian.Uint32(buf[0:4]) + offset32 := binary.LittleEndian.Uint32(buf[0:4]) + if offset32 < prevOffset32 { + cycleOffset += (1 << 32) + } + prevOffset32 = offset32 + offset := int64(offset32) + cycleOffset // Verify the offset is within the bounds of the input data. - if int(offset) >= len(data) { + if offset >= int64(len(data)) { return fmt.Errorf("offset out of bounds: off=%d, len=%d", offset, len(data)) } @@ -201,13 +210,13 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error { case containerRun: runCount := binary.LittleEndian.Uint16(data[offset : offset+runCountHeaderSize]) c.setRuns((*[0xFFFFFFF]interval16)(unsafe.Pointer(&data[offset+runCountHeaderSize]))[:runCount:runCount]) - opsOffset = int(offset) + runCountHeaderSize + len(c.runs())*interval16Size + opsOffset = offset + runCountHeaderSize + int64(len(c.runs()))*interval16Size case containerArray: c.setArray((*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.N():c.N()]) - opsOffset = int(offset) + len(c.array())*2 // sizeof(uint32) + opsOffset = offset + int64(len(c.array()))*2 // sizeof(uint32) case containerBitmap: c.setBitmap((*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN:bitmapN]) - opsOffset = int(offset) + len(c.bitmap())*8 // sizeof(uint64) + opsOffset = offset + int64(len(c.bitmap()))*8 // sizeof(uint64) } } @@ -228,7 +237,7 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error { // Increase the op count. b.ops++ b.opN += opr.count() - opsOffset += opr.size() + opsOffset += int64(opr.size()) // Move the buffer forward. buf = data[opsOffset:] }