mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-09-07 09:05:55 +00:00
Handle file sizes over 4GB
We only have 4 bytes for offsets, but what if a file is over 4GB? Someone came to us with a file with 265 *million* containers, in a single fragment, which means that over 3GB of their 4.7GB file is actually just the container headers alone. But we can't easily make the offsets larger, or change the file format. So we don't. We just track how many 4GB hunks of the file we've been through and bump that every time the 32-bit offset wraps. And this appears to... just work. This is fixed for both the roaring iterator and the old unmarshalBinary logic. The logic to handle this will work on 32-bit hosts in the sense that it will correctly error out for excessively large file sizes or container counts, but it doesn't actually handle the large files since it can't.
This commit is contained in:
Seebs
parent
a777eddfcf
commit
1ca9435af9
2 changed files with 43 additions and 21 deletions
|
|
@ -1738,7 +1738,9 @@ type baseRoaringIterator struct {
|
|||
currentN int
|
||||
currentLen int
|
||||
currentPointer *uint16
|
||||
currentDataOffset uint32
|
||||
currentDataOffset uint64
|
||||
prevOffset32 uint32
|
||||
chunkOffset uint64
|
||||
lastDataOffset int64
|
||||
lastErr error
|
||||
}
|
||||
|
|
@ -1752,6 +1754,8 @@ func (b *baseRoaringIterator) SilenceLint() {
|
|||
_ = b.offsets
|
||||
_ = b.headers
|
||||
_ = b.currentIdx
|
||||
_ = b.chunkOffset
|
||||
_ = b.prevOffset32
|
||||
}
|
||||
|
||||
type pilosaRoaringIterator struct {
|
||||
|
|
@ -1787,14 +1791,14 @@ func newOfficialRoaringIterator(data []byte) (*officialRoaringIterator, error) {
|
|||
r.headers = data[headerOffset:offsetOffset]
|
||||
// note: offsets are only actually used with the no-run headers.
|
||||
if r.haveRuns {
|
||||
r.currentDataOffset = uint32(offsetOffset)
|
||||
r.currentDataOffset = uint64(offsetOffset)
|
||||
} else {
|
||||
if len(r.data) < offsetOffset+int(r.keys*4) {
|
||||
return nil, fmt.Errorf("insufficient data for offsets (need %d bytes, found %d)",
|
||||
r.keys*4, len(r.data)-offsetOffset)
|
||||
}
|
||||
r.offsets = data[offsetOffset : offsetOffset+int(r.keys*4)]
|
||||
r.currentDataOffset = uint32(offsetOffset)
|
||||
r.currentDataOffset = uint64(offsetOffset)
|
||||
}
|
||||
// set key to -1; user should call Next first.
|
||||
r.currentIdx = -1
|
||||
|
|
@ -1838,7 +1842,11 @@ func newPilosaRoaringIterator(data []byte) (*pilosaRoaringIterator, error) {
|
|||
// if there's no containers, we want to act as though data started at the end
|
||||
// of the list of offsets, which was also empty, so we don't think the entire thing
|
||||
// is actually a malformed op
|
||||
r.currentDataOffset = uint32(offsetEnd)
|
||||
r.prevOffset32 = uint32(offsetEnd)
|
||||
r.currentDataOffset = uint64(offsetEnd)
|
||||
// it's possible that there's so many headers that we're actually over
|
||||
// 4GB into the file already.
|
||||
r.chunkOffset = r.currentDataOffset &^ ((1 << 32) - 1)
|
||||
// set key to -1; user should call Next first.
|
||||
r.currentIdx = -1
|
||||
r.currentKey = ^uint64(0)
|
||||
|
|
@ -1895,7 +1903,12 @@ func (r *pilosaRoaringIterator) Next() (key uint64, cType byte, n int, length in
|
|||
r.currentKey = binary.LittleEndian.Uint64(header[0:8])
|
||||
r.currentType = byte(binary.LittleEndian.Uint16(header[8:10]))
|
||||
r.currentN = int(binary.LittleEndian.Uint16(header[10:12])) + 1
|
||||
r.currentDataOffset = binary.LittleEndian.Uint32(r.offsets[r.currentIdx*4:])
|
||||
offset32 := binary.LittleEndian.Uint32(r.offsets[r.currentIdx*4:])
|
||||
if offset32 < r.prevOffset32 {
|
||||
r.chunkOffset += (1 << 32)
|
||||
}
|
||||
r.prevOffset32 = offset32
|
||||
r.currentDataOffset = r.chunkOffset + uint64(offset32)
|
||||
|
||||
// a run container keeps its data after an initial 2 byte length header
|
||||
var runCount uint16
|
||||
|
|
@ -1903,7 +1916,7 @@ func (r *pilosaRoaringIterator) Next() (key uint64, cType byte, n int, length in
|
|||
runCount = binary.LittleEndian.Uint16(r.data[r.currentDataOffset : r.currentDataOffset+runCountHeaderSize])
|
||||
r.currentDataOffset += 2
|
||||
}
|
||||
if r.currentDataOffset > uint32(len(r.data)) || r.currentDataOffset < headerBaseSize {
|
||||
if r.currentDataOffset > uint64(len(r.data)) || r.currentDataOffset < headerBaseSize {
|
||||
r.Done(fmt.Errorf("container %d/%d, key %d, had offset %d, maximum %d",
|
||||
r.currentIdx, r.keys, r.currentKey, r.currentDataOffset, len(r.data)))
|
||||
return r.Current()
|
||||
|
|
@ -1926,7 +1939,7 @@ func (r *pilosaRoaringIterator) Next() (key uint64, cType byte, n int, length in
|
|||
r.currentIdx, r.keys, r.currentKey, r.currentDataOffset, size, len(r.data)))
|
||||
return r.Current()
|
||||
}
|
||||
r.currentDataOffset += uint32(size)
|
||||
r.currentDataOffset += uint64(size)
|
||||
r.lastErr = nil
|
||||
return r.Current()
|
||||
}
|
||||
|
|
@ -1949,7 +1962,7 @@ func (r *officialRoaringIterator) Next() (key uint64, cType byte, n int, length
|
|||
// with runs, we can't actually look up offsets; the format just stores
|
||||
// things sequentially. so we have to actually track the offset in that case.
|
||||
if !r.haveRuns {
|
||||
r.currentDataOffset = binary.LittleEndian.Uint32(r.offsets[r.currentIdx*4:])
|
||||
r.currentDataOffset = uint64(binary.LittleEndian.Uint32(r.offsets[r.currentIdx*4:]))
|
||||
}
|
||||
// a run container keeps its data after an initial 2 byte length header
|
||||
var runCount uint16
|
||||
|
|
@ -1962,7 +1975,7 @@ func (r *officialRoaringIterator) Next() (key uint64, cType byte, n int, length
|
|||
runCount = binary.LittleEndian.Uint16(r.data[r.currentDataOffset : r.currentDataOffset+runCountHeaderSize])
|
||||
r.currentDataOffset += 2
|
||||
}
|
||||
if r.currentDataOffset > uint32(len(r.data)) || r.currentDataOffset < headerBaseSize {
|
||||
if r.currentDataOffset > uint64(len(r.data)) || r.currentDataOffset < headerBaseSize {
|
||||
r.Done(fmt.Errorf("container %d/%d, key %d, had offset %d, maximum %d",
|
||||
r.currentIdx, r.keys, r.currentKey, r.currentDataOffset, len(r.data)))
|
||||
return r.Current()
|
||||
|
|
@ -1994,7 +2007,7 @@ func (r *officialRoaringIterator) Next() (key uint64, cType byte, n int, length
|
|||
r.currentIdx, r.keys, r.currentKey, r.currentDataOffset, size, len(r.data)))
|
||||
return r.Current()
|
||||
}
|
||||
r.currentDataOffset += uint32(size)
|
||||
r.currentDataOffset += uint64(size)
|
||||
r.lastErr = nil
|
||||
return r.Current()
|
||||
}
|
||||
|
|
@ -2012,14 +2025,14 @@ func (b *Bitmap) SanityCheckMapping(from, to uintptr) (mappedIn int64, mappedOut
|
|||
if c.Mapped() {
|
||||
mappedIn++
|
||||
} else {
|
||||
err = fmt.Errorf("container key %d, addr %x, inside %x+%d\n",
|
||||
err = fmt.Errorf("container key %d, addr %x, inside %x+%d",
|
||||
key, dptr, from, to-from)
|
||||
errs++
|
||||
unmappedIn++
|
||||
}
|
||||
} else {
|
||||
if c.Mapped() {
|
||||
err = fmt.Errorf("container key %d, addr %x, outside %x+%d, but mapped\n",
|
||||
err = fmt.Errorf("container key %d, addr %x, outside %x+%d, but mapped",
|
||||
key, dptr, from, to-from)
|
||||
errs++
|
||||
mappedOut++
|
||||
|
|
|
|||
|
|
@ -159,8 +159,8 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error {
|
|||
|
||||
// Read key count in bytes sizeof(cookie)+sizeof(flag):(sizeof(cookie)+sizeof(uint32)).
|
||||
keyN := binary.LittleEndian.Uint32(data[3+1 : 8])
|
||||
if uint32(len(data)) < headerBaseSize+keyN*12 {
|
||||
return fmt.Errorf("insufficient data for header + offsets: key-cardinality not provided for %d containers", int(keyN)/12)
|
||||
if int64(len(data)) < headerBaseSize+int64(keyN)*12 {
|
||||
return fmt.Errorf("insufficient data for header + offsets: key-cardinality not provided for %d containers", keyN)
|
||||
}
|
||||
|
||||
headerSize := headerBaseSize
|
||||
|
|
@ -173,14 +173,23 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error {
|
|||
int(binary.LittleEndian.Uint16(buf[10:12]))+1,
|
||||
true)
|
||||
}
|
||||
opsOffset := headerSize + int(keyN)*12
|
||||
opsOffset := int64(headerSize) + int64(keyN)*12
|
||||
|
||||
// Read container offsets and attach data.
|
||||
citer, _ := b.Containers.Iterator(0)
|
||||
// if you have enough containers that the *headers alone* exceed 4GB, we
|
||||
// need to start with a higher cycle offset.
|
||||
cycleOffset := opsOffset &^ ((1 << 32) - 1)
|
||||
prevOffset32 := uint32(opsOffset)
|
||||
for i, buf := 0, data[opsOffset:]; i < int(keyN); i, buf = i+1, buf[4:] {
|
||||
offset := binary.LittleEndian.Uint32(buf[0:4])
|
||||
offset32 := binary.LittleEndian.Uint32(buf[0:4])
|
||||
if offset32 < prevOffset32 {
|
||||
cycleOffset += (1 << 32)
|
||||
}
|
||||
prevOffset32 = offset32
|
||||
offset := int64(offset32) + cycleOffset
|
||||
// Verify the offset is within the bounds of the input data.
|
||||
if int(offset) >= len(data) {
|
||||
if offset >= int64(len(data)) {
|
||||
return fmt.Errorf("offset out of bounds: off=%d, len=%d", offset, len(data))
|
||||
}
|
||||
|
||||
|
|
@ -201,13 +210,13 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error {
|
|||
case containerRun:
|
||||
runCount := binary.LittleEndian.Uint16(data[offset : offset+runCountHeaderSize])
|
||||
c.setRuns((*[0xFFFFFFF]interval16)(unsafe.Pointer(&data[offset+runCountHeaderSize]))[:runCount:runCount])
|
||||
opsOffset = int(offset) + runCountHeaderSize + len(c.runs())*interval16Size
|
||||
opsOffset = offset + runCountHeaderSize + int64(len(c.runs()))*interval16Size
|
||||
case containerArray:
|
||||
c.setArray((*[0xFFFFFFF]uint16)(unsafe.Pointer(&data[offset]))[:c.N():c.N()])
|
||||
opsOffset = int(offset) + len(c.array())*2 // sizeof(uint32)
|
||||
opsOffset = offset + int64(len(c.array()))*2 // sizeof(uint32)
|
||||
case containerBitmap:
|
||||
c.setBitmap((*[0xFFFFFFF]uint64)(unsafe.Pointer(&data[offset]))[:bitmapN:bitmapN])
|
||||
opsOffset = int(offset) + len(c.bitmap())*8 // sizeof(uint64)
|
||||
opsOffset = offset + int64(len(c.bitmap()))*8 // sizeof(uint64)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -228,7 +237,7 @@ func (b *Bitmap) unmarshalPilosaRoaring(data []byte) error {
|
|||
// Increase the op count.
|
||||
b.ops++
|
||||
b.opN += opr.count()
|
||||
opsOffset += opr.size()
|
||||
opsOffset += int64(opr.size())
|
||||
// Move the buffer forward.
|
||||
buf = data[opsOffset:]
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue