From 056707ad09122819be051e6a946e43312c593f95 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 11 Aug 2017 14:02:43 -0500 Subject: [PATCH 1/3] Remove unused magic number --- roaring/roaring.go | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index 14818f683..cfcda97d1 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -27,20 +27,17 @@ import ( const ( // magicNumber is an identifier, in bytes 0-1 of the file. - magicNumberNoRuns = uint32(12346) - magicNumber = uint32(12347) + magicNumber = uint32(12348) // storageVersion indicates the storage version, in bytes 2-3. storageVersion = uint32(0) // cookie is the first four bytes in a roaring bitmap file, // formed by joining magicNumber and storageVersion - cookieNoRuns = magicNumberNoRuns + storageVersion<<16 - cookie = magicNumber + storageVersion<<16 + cookie = magicNumber + storageVersion<<16 // headerBaseSize is the size in bytes of the cookie and key count at the - // beginning of a file. Headers in files with runs also include - // runFlagBitset, of length (numContainers+7)/8. + // beginning of a file. headerBaseSize = 4 + 4 // runCountHeaderSize is the size in bytes of the run count stored @@ -530,14 +527,13 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) { //b.removeEmptyContainers() containerCount := len(b.keys) - b.countEmptyContainers() - thisCookie := cookieNoRuns headerSize := headerBaseSize // Build header before writing individual container blocks. // Metadata for each container is 8+2+2+4 = sizeof(key) + sizeof(container_type)+sizeof(cardinality) + sizeof(file offset) buf := make([]byte, headerSize+(containerCount*(8+2+2+4))) // Cookie header section. - binary.LittleEndian.PutUint32(buf[0:], thisCookie) + binary.LittleEndian.PutUint32(buf[0:], cookie) binary.LittleEndian.PutUint32(buf[4:], uint32(containerCount)) empty := 0 @@ -603,11 +599,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error { // Verify the first two bytes are a valid magicNumber, and second two bytes match current storageVersion. fileMagic := uint32(binary.LittleEndian.Uint16(data[0:2])) fileVersion := uint32(binary.LittleEndian.Uint16(data[2:4])) - if fileMagic == magicNumberNoRuns { - // noop - // } else if fileMagic == magicNumber { - // containsRuns = true - } else { + if fileMagic != magicNumber { return fmt.Errorf("invalid roaring file, magic number %v is incorrect", fileMagic) } From f807adb0acbd0a7f5b1884b0247d261299f711e7 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 11 Aug 2017 14:05:14 -0500 Subject: [PATCH 2/3] Update docs for current storage format --- docs/architecture.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 69912cca5..db33d6836 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -10,12 +10,11 @@ Bitmaps are persisted to disk using a file format very similar to the [Roaring B * The cookie is always bytes 0-3; the container count is always bytes 4-7, never bytes 2-3. * The cookie includes file format version in bytes 2-3 (currently equal to zero). +* The descriptive header includes, for each container, a 64-bit key, a 16-bit cardinality, and a 16-bit container type (which only uses two bits now). This makes the runFlag bitset unnecessary. This is in contrast to the spec, which stores a 16-bit key and a 16-bit cardinality. * The offset header section is always included. * RLE runs are serialized as [start, last], not [start, length]. * After the container storage section is an operation log, of unspecified length. -![roaring file format diagram](/img/docs/pilosa-roaring-storage-diagram.svg) +![roaring file format diagram](/img/docs/pilosa-roaring-storage-diagram.png) -All values are little-endian. The first two bytes of the cookie is 12346 when the file contains no RLE containers, or 12347 when it does. In the no-RLE case, the runFlagBitset is absent. Otherwise the format is identical in both cases. Container types are determined by their cardinality - a container with 4096 or more values is a bitmap, a container with fewer is an array or RLE container. A high bit in runFlagBitset indicates an RLE container. - -Storing the runFlagBitset in a separate section, indicated by the cookie value, keeps this format backward compatible with older storage versions that do not support RLE containers. +All values are little-endian. The first two bytes of the cookie is 12348, to reflect incompatibility with the spec. Container types are NOT inferred from their cardinality as in the spec. Instead, the container type is read directly from the descriptive header. From c2ea0dd7158bc089b8790ad41a17bc342812a8d5 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 11 Aug 2017 14:41:05 -0500 Subject: [PATCH 3/3] Trivial docs update --- docs/architecture.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architecture.md b/docs/architecture.md index db33d6836..0ff35565c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -17,4 +17,4 @@ Bitmaps are persisted to disk using a file format very similar to the [Roaring B ![roaring file format diagram](/img/docs/pilosa-roaring-storage-diagram.png) -All values are little-endian. The first two bytes of the cookie is 12348, to reflect incompatibility with the spec. Container types are NOT inferred from their cardinality as in the spec. Instead, the container type is read directly from the descriptive header. +All values are little-endian. The first two bytes of the cookie is 12348, to reflect incompatibility with the spec, which uses 12346 or 12347. Container types are NOT inferred from their cardinality as in the spec. Instead, the container type is read directly from the descriptive header.