scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-10-10 12:57:54 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #763 from alanbernstein/roaring-minor-fixes

Browse source 
Roaring minor fixes
This commit is contained in:
alanbernstein 2017-08-11 14:55:43 -05:00 • committed by GitHub
commit 4f4843a624
2 changed files with 8 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
docs/architecture.md
View file

@ -10,12 +10,11 @@ Bitmaps are persisted to disk using a file format very similar to the [Roaring B
* The cookie is always bytes 0-3; the container count is always bytes 4-7, never bytes 2-3.
* The cookie includes file format version in bytes 2-3 (currently equal to zero).
* The descriptive header includes, for each container, a 64-bit key, a 16-bit cardinality, and a 16-bit container type (which only uses two bits now). This makes the runFlag bitset unnecessary. This is in contrast to the spec, which stores a 16-bit key and a 16-bit cardinality.
* The offset header section is always included.
* RLE runs are serialized as [start, last], not [start, length].
* After the container storage section is an operation log, of unspecified length.
![roaring file format diagram](/img/docs/pilosa-roaring-storage-diagram.svg)
![roaring file format diagram](/img/docs/pilosa-roaring-storage-diagram.png)
All values are little-endian. The first two bytes of the cookie is 12346 when the file contains no RLE containers, or 12347 when it does. In the no-RLE case, the runFlagBitset is absent. Otherwise the format is identical in both cases. Container types are determined by their cardinality - a container with 4096 or more values is a bitmap, a container with fewer is an array or RLE container. A high bit in runFlagBitset indicates an RLE container.
Storing the runFlagBitset in a separate section, indicated by the cookie value, keeps this format backward compatible with older storage versions that do not support RLE containers.
All values are little-endian. The first two bytes of the cookie is 12348, to reflect incompatibility with the spec, which uses 12346 or 12347. Container types are NOT inferred from their cardinality as in the spec. Instead, the container type is read directly from the descriptive header.

18
roaring/roaring.go
View file

@ -27,20 +27,17 @@ import (
const (
// magicNumber is an identifier, in bytes 0-1 of the file.
magicNumberNoRuns = uint32(12346)
magicNumber = uint32(12347)
magicNumber = uint32(12348)
// storageVersion indicates the storage version, in bytes 2-3.
storageVersion = uint32(0)
// cookie is the first four bytes in a roaring bitmap file,
// formed by joining magicNumber and storageVersion
cookieNoRuns = magicNumberNoRuns + storageVersion<<16
cookie = magicNumber + storageVersion<<16
cookie = magicNumber + storageVersion<<16
// headerBaseSize is the size in bytes of the cookie and key count at the
// beginning of a file. Headers in files with runs also include
// runFlagBitset, of length (numContainers+7)/8.
// beginning of a file.
headerBaseSize = 4 + 4
// runCountHeaderSize is the size in bytes of the run count stored
@ -530,14 +527,13 @@ func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
//b.removeEmptyContainers()
containerCount := len(b.keys) - b.countEmptyContainers()
thisCookie := cookieNoRuns
headerSize := headerBaseSize
// Build header before writing individual container blocks.
// Metadata for each container is 8+2+2+4 = sizeof(key) + sizeof(container_type)+sizeof(cardinality) + sizeof(file offset)
buf := make([]byte, headerSize+(containerCount*(8+2+2+4)))
// Cookie header section.
binary.LittleEndian.PutUint32(buf[0:], thisCookie)
binary.LittleEndian.PutUint32(buf[0:], cookie)
binary.LittleEndian.PutUint32(buf[4:], uint32(containerCount))
empty := 0
@ -603,11 +599,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
// Verify the first two bytes are a valid magicNumber, and second two bytes match current storageVersion.
fileMagic := uint32(binary.LittleEndian.Uint16(data[0:2]))
fileVersion := uint32(binary.LittleEndian.Uint16(data[2:4]))
if fileMagic == magicNumberNoRuns {
// noop
// } else if fileMagic == magicNumber {
// containsRuns = true
} else {
if fileMagic != magicNumber {
return fmt.Errorf("invalid roaring file, magic number %v is incorrect", fileMagic)
}