diff --git a/docs/architecture.md b/docs/architecture.md index 69912cca5..0ff35565c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -10,12 +10,11 @@ Bitmaps are persisted to disk using a file format very similar to the [Roaring B * The cookie is always bytes 0-3; the container count is always bytes 4-7, never bytes 2-3. * The cookie includes file format version in bytes 2-3 (currently equal to zero). +* The descriptive header includes, for each container, a 64-bit key, a 16-bit cardinality, and a 16-bit container type (which only uses two bits now). This makes the runFlag bitset unnecessary. This is in contrast to the spec, which stores a 16-bit key and a 16-bit cardinality. * The offset header section is always included. * RLE runs are serialized as [start, last], not [start, length]. * After the container storage section is an operation log, of unspecified length. -![roaring file format diagram](/img/docs/pilosa-roaring-storage-diagram.svg) +![roaring file format diagram](/img/docs/pilosa-roaring-storage-diagram.png) -All values are little-endian. The first two bytes of the cookie is 12346 when the file contains no RLE containers, or 12347 when it does. In the no-RLE case, the runFlagBitset is absent. Otherwise the format is identical in both cases. Container types are determined by their cardinality - a container with 4096 or more values is a bitmap, a container with fewer is an array or RLE container. A high bit in runFlagBitset indicates an RLE container. - -Storing the runFlagBitset in a separate section, indicated by the cookie value, keeps this format backward compatible with older storage versions that do not support RLE containers. +All values are little-endian. The first two bytes of the cookie is 12348, to reflect incompatibility with the spec, which uses 12346 or 12347. Container types are NOT inferred from their cardinality as in the spec. Instead, the container type is read directly from the descriptive header. diff --git a/roaring/roaring.go b/roaring/roaring.go index 14818f683..cfcda97d1 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -27,20 +27,17 @@ import ( const ( // magicNumber is an identifier, in bytes 0-1 of the file. - magicNumberNoRuns = uint32(12346) - magicNumber = uint32(12347) + magicNumber = uint32(12348) // storageVersion indicates the storage version, in bytes 2-3. storageVersion = uint32(0) // cookie is the first four bytes in a roaring bitmap file, // formed by joining magicNumber and storageVersion - cookieNoRuns = magicNumberNoRuns + storageVersion<<16 - cookie = magicNumber + storageVersion<<16 + cookie = magicNumber + storageVersion<<16 // headerBaseSize is the size in bytes of the cookie and key count at the - // beginning of a file. Headers in files with runs also include - // runFlagBitset, of length (numContainers+7)/8. + // beginning of a file. headerBaseSize = 4 + 4 // runCountHeaderSize is the size in bytes of the run count stored @@ -530,14 +527,13 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { //b.removeEmptyContainers() containerCount := len(b.keys) - b.countEmptyContainers() - thisCookie := cookieNoRuns headerSize := headerBaseSize // Build header before writing individual container blocks. // Metadata for each container is 8+2+2+4 = sizeof(key) + sizeof(container_type)+sizeof(cardinality) + sizeof(file offset) buf := make([]byte, headerSize+(containerCount*(8+2+2+4))) // Cookie header section. - binary.LittleEndian.PutUint32(buf[0:], thisCookie) + binary.LittleEndian.PutUint32(buf[0:], cookie) binary.LittleEndian.PutUint32(buf[4:], uint32(containerCount)) empty := 0 @@ -603,11 +599,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { // Verify the first two bytes are a valid magicNumber, and second two bytes match current storageVersion. fileMagic := uint32(binary.LittleEndian.Uint16(data[0:2])) fileVersion := uint32(binary.LittleEndian.Uint16(data[2:4])) - if fileMagic == magicNumberNoRuns { - // noop - // } else if fileMagic == magicNumber { - // containsRuns = true - } else { + if fileMagic != magicNumber { return fmt.Errorf("invalid roaring file, magic number %v is incorrect", fileMagic) }