From 117942c0f3f18b87f8a3bdd8d6cd6bc12067dd06 Mon Sep 17 00:00:00 2001 From: Seebs Date: Tue, 19 Mar 2019 15:17:53 -0500 Subject: [PATCH] Add stash-based implementation of Container This implementation, controlled by the build flag "container24s", is similar to the single-slice container implementation, but goes a bit further. First, instead of using a native slice as its internal storage, it uses pointer/len/cap as distinct values, and only int32 ranges for len and cap. Second, it has a small region of additional storage which it uses as a backing store by default for arrays or runs. The idea is that, if you request a new empty array container, you get one with a pre-allocated virtual slice big enough for five values, actually stored in the Container. This is useful because Go's allocator has size classes for 16 and 32 bytes, and the Container comes in at 24 bytes worth of storage -- meaning that if you allocate a container, you're allocating 32 bytes anyway, so we might as well use that space to avoid extra allocations. This includes some test fixups because DeepEqual was testing too much equality in some tests. Also, we simplify unionArrayArray to postpone creating a Container until we're ready. --- roaring/container_slice.go | 2 + roaring/container_stash.go | 257 +++++++++++++++++++++++++++++++ roaring/containers.go | 2 +- roaring/containers_btree.go | 4 +- roaring/roaring.go | 21 ++- roaring/roaring_internal_test.go | 17 +- 6 files changed, 288 insertions(+), 15 deletions(-) create mode 100644 roaring/container_stash.go diff --git a/roaring/container_slice.go b/roaring/container_slice.go index 9cd1a7d1a..3e4511335 100644 --- a/roaring/container_slice.go +++ b/roaring/container_slice.go @@ -1,3 +1,5 @@ +// +build !container24s + package roaring import ( diff --git a/roaring/container_stash.go b/roaring/container_stash.go new file mode 100644 index 000000000..db3bb4943 --- /dev/null +++ b/roaring/container_stash.go @@ -0,0 +1,257 @@ +// +build container24s + +package roaring + +import ( + "reflect" + "runtime" + "unsafe" +) + +const ( + stashedArraySize = 5 + stashedRunSize = (stashedArraySize / 2) +) + +// Container represents a Container for uint16 integers. +// +// These are used for storing the low bits of numbers in larger sets of uint64. +// The high bits are stored in a Container's key which is tracked by a separate +// data structure. Integers in a Container can be encoded in one of three ways - +// the encoding used is usually whichever is most compact, though any Container +// type should be able to encode any set of integers safely. For containers with +// less than 4,096 values, an array is often used. Containers with long runs of +// integers would use run length encoding, and more random data usually uses +// bitmap encoding. +type Container struct { + pointer *uint16 // the data pointer + len, cap int32 // length and cap + n int32 // number of integers in container + mapped bool // mapped directly to a byte slice when true + typ byte // array, bitmap, or run + data [stashedArraySize]uint16 // immediate data for small arrays or runs +} + +// NewContainer returns a new instance of container. This trivial function +// may later become more interesting. +func NewContainer() *Container { + statsHit("NewContainer") + c := &Container{typ: containerArray, len: 0, cap: stashedArraySize} + c.pointer = (*uint16)(unsafe.Pointer(&c.data[0])) + return c +} + +// NewContainerBitmap makes a bitmap container using the provided bitmap, or +// an empty one if provided bitmap is nil. If the provided bitmap is too short, +// it will be padded. +func NewContainerBitmap(n int32, bitmap []uint64) *Container { + if bitmap == nil { + bitmap = make([]uint64, bitmapN) + } + // pad to required length + if len(bitmap) < bitmapN { + bm2 := make([]uint64, bitmapN) + copy(bm2, bitmap) + bitmap = bm2 + } + c := &Container{typ: containerBitmap, n: n} + c.setBitmap(bitmap) + return c +} + +// NewContainerArray returns an array using the provided set of values. It's +// okay if the slice is nil; that's a length of zero. +func NewContainerArray(set []uint16) *Container { + c := &Container{typ: containerArray, n: int32(len(set))} + c.setArray(set) + return c +} + +// NewContainerRun creates a new run array using a provided (possibly nil) +// slice of intervals. +func NewContainerRun(set []interval16) *Container { + c := &Container{typ: containerRun} + c.setRuns(set) + for _, run := range set { + c.n += int32(run.last-run.start) + 1 + } + return c +} + +// Mapped returns the internal mapped field, which indicates whether the +// slice's backing store is believed to be associated with unwriteable +// mmapped space. +func (c *Container) Mapped() bool { + return c.mapped +} + +// N returns the internal n field. +func (c *Container) N() int32 { + return c.n +} + +// array yields the data viewed as a slice of uint16 values. +func (c *Container) array() []uint16 { + if roaringParanoia { + if c.typ != containerArray { + panic("attempt to read non-array's array") + } + } + return *(*[]uint16)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(c.pointer)), Len: int(c.len), Cap: int(c.cap)})) +} + +// setArray stores a set of uint16s as data. +func (c *Container) setArray(array []uint16) { + if roaringParanoia { + if c.typ != containerArray { + panic("attempt to write non-array's array") + } + } + // no array: start with our default 5-value array + if array == nil { + c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(&c.data[0])), 0, stashedArraySize + return + } + h := (*reflect.SliceHeader)(unsafe.Pointer(&array)) + if h.Data == uintptr(unsafe.Pointer(&c.data[0])) { + // nothing to do but update length + c.len, c.cap = int32(h.Len), stashedArraySize + return + } + // array we can fit in data store: + if len(array) <= stashedArraySize { + copy(c.data[:stashedArraySize], array) + c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(&c.data[0])), int32(len(array)), stashedArraySize + c.mapped = false // this is no longer using a hypothetical mmapped input array + return + } + c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(h.Data)), int32(h.Len), int32(h.Cap) + runtime.KeepAlive(&array) +} + +// bitmap yields the data viewed as a slice of uint64s holding bits. +func (c *Container) bitmap() []uint64 { + if roaringParanoia { + if c.typ != containerBitmap { + panic("attempt to read non-bitmap's bitmap") + } + } + return *(*[]uint64)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(c.pointer)), Len: int(c.len), Cap: int(c.cap)})) +} + +// setBitmap stores a set of uint64s as data. +func (c *Container) setBitmap(bitmap []uint64) { + if roaringParanoia { + if c.typ != containerBitmap { + panic("attempt to write non-bitmap's bitmap") + } + } + h := (*reflect.SliceHeader)(unsafe.Pointer(&bitmap)) + c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(h.Data)), int32(h.Len), int32(h.Cap) + runtime.KeepAlive(&bitmap) +} + +// runs yields the data viewed as a slice of intervals. +func (c *Container) runs() []interval16 { + if roaringParanoia { + if c.typ != containerRun { + panic("attempt to read non-run's runs") + } + } + return *(*[]interval16)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(c.pointer)), Len: int(c.len), Cap: int(c.cap)})) +} + +// setRuns stores a set of intervals as data. +func (c *Container) setRuns(runs []interval16) { + if roaringParanoia { + if c.typ != containerRun { + panic("attempt to write non-run's runs") + } + } + // no array: start with our default 2-value array + if runs == nil { + c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(&c.data[0])), 0, stashedRunSize + return + } + h := (*reflect.SliceHeader)(unsafe.Pointer(&runs)) + if h.Data == uintptr(unsafe.Pointer(&c.data[0])) { + // nothing to do but update cap and length + c.len, c.cap = int32(h.Len), stashedRunSize + return + } + + // array we can fit in data store: + if len(runs) <= stashedRunSize { + newRuns := *(*[]interval16)(unsafe.Pointer(&reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&c.data[0])), Len: stashedRunSize, Cap: stashedRunSize})) + copy(newRuns, runs) + c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(&c.data[0])), int32(len(runs)), stashedRunSize + c.mapped = false // this is no longer using a hypothetical mmapped input array + return + } + c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(h.Data)), int32(h.Len), int32(h.Cap) + runtime.KeepAlive(&runs) +} + +// Update updates the container +func (c *Container) Update(typ byte, n int32, mapped bool) { + c.typ = typ + c.n = n + c.mapped = mapped + // we don't know that any existing slice is usable, so let's ditch it + switch c.typ { + case containerArray: + c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(&c.data[0])), int32(0), stashedArraySize + case containerRun: + c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(&c.data[0])), 0, stashedRunSize + default: + c.pointer, c.len, c.cap = nil, 0, 0 + } +} + +// isArray returns true if the container is an array container. +func (c *Container) isArray() bool { + return c.typ == containerArray +} + +// isBitmap returns true if the container is a bitmap container. +func (c *Container) isBitmap() bool { + return c.typ == containerBitmap +} + +// isRun returns true if the container is a run-length-encoded container. +func (c *Container) isRun() bool { + return c.typ == containerRun +} + +// unmap creates copies of the containers data in the heap. +// +// This is performed when altering the container since its contents could be +// pointing at a read-only mmap. +func (c *Container) unmap() { + if !c.mapped { + return + } + + switch c.typ { + case containerArray: + array := c.array() + if len(array) > stashedArraySize { + tmp := make([]uint16, len(array)) + h := (*reflect.SliceHeader)(unsafe.Pointer(&tmp)) + c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(h.Data)), int32(h.Len), int32(h.Len) + copy(tmp, array) + } else { + c.pointer, c.len, c.cap = (*uint16)(unsafe.Pointer(&c.data[0])), int32(len(array)), stashedArraySize + copy(c.data[:stashedArraySize], array) + } + case containerBitmap: + tmp := make([]uint64, len(c.bitmap())) + copy(tmp, c.bitmap()) + c.setBitmap(tmp) + case containerRun: + tmp := make([]interval16, len(c.runs())) + copy(tmp, c.runs()) + c.setRuns(tmp) + } + c.mapped = false +} diff --git a/roaring/containers.go b/roaring/containers.go index a2f7b2045..cc9a09ec5 100644 --- a/roaring/containers.go +++ b/roaring/containers.go @@ -93,7 +93,7 @@ func (sc *sliceContainers) GetOrCreate(key uint64) *Container { sc.lastKey = key i := search64(sc.keys, key) if i < 0 { - c := NewContainerArray(nil) + c := NewContainer() sc.insertAt(key, c, -i-1) sc.lastContainer = c return c diff --git a/roaring/containers_btree.go b/roaring/containers_btree.go index b6b1ec6ab..04fc6befa 100644 --- a/roaring/containers_btree.go +++ b/roaring/containers_btree.go @@ -108,7 +108,7 @@ func (btc *bTreeContainers) GetOrCreate(key uint64) *Container { btc.lastKey = key v, ok := btc.tree.Get(key) if !ok { - cont := NewContainerArray(nil) + cont := NewContainer() btc.tree.Set(key, cont) btc.lastContainer = cont return cont @@ -122,7 +122,7 @@ func (btc *bTreeContainers) Count() (n uint64) { e, _ := btc.tree.Seek(0) _, c, err := e.Next() for err != io.EOF { - n += uint64(c.N()) + n += uint64(c.n) _, c, err = e.Next() } return n diff --git a/roaring/roaring.go b/roaring/roaring.go index 2eecac0e2..fa604cbbe 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -1899,6 +1899,7 @@ func (c *Container) bitmapToArray() { c.typ = containerArray array := make([]uint16, c.n) c.setArray(array) + array = c.array() c.mapped = false // return early if empty @@ -2678,35 +2679,41 @@ func union(a, b *Container) *Container { func unionArrayArray(a, b *Container) *Container { statsHit("union/ArrayArray") - output := NewContainerArray(nil) aa, ab := a.array(), b.array() na, nb := len(aa), len(ab) + output := make([]uint16, na+nb) + n := 0 for i, j := 0, 0; ; { if i >= na && j >= nb { break } else if i < na && j >= nb { - output.add(aa[i]) + output[n] = aa[i] + n++ i++ continue } else if i >= na && j < nb { - output.add(ab[j]) + output[n] = ab[j] + n++ j++ continue } va, vb := aa[i], ab[j] if va < vb { - output.add(va) + output[n] = va + n++ i++ } else if va > vb { - output.add(vb) + output[n] = vb + n++ j++ } else { - output.add(va) + output[n] = va + n++ i, j = i+1, j+1 } } - return output + return NewContainerArray(output[:n]) } // unionArrayArrayInPlace does what it sounds like -- tries to combine diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index dc2f770db..5224a6cc5 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -464,7 +464,11 @@ func TestIntersectArrayRun(t *testing.T) { a.setArray(test.array) b.setRuns(test.runs) ret := intersectArrayRun(a, b) - if !reflect.DeepEqual(ret.array(), test.exp) { + if test.exp == nil { + if len(ret.array()) != 0 { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array()) + } + } else if !reflect.DeepEqual(ret.array(), test.exp) { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.array()) } } @@ -523,11 +527,14 @@ func TestIntersectRunRun(t *testing.T) { if ret.n != test.expN { t.Fatalf("test #%v expected n to be %v, but got %v", i, test.expN, ret.n) } - if !reflect.DeepEqual(ret.runs(), test.exp) { + if test.exp != nil { + if !reflect.DeepEqual(ret.runs(), test.exp) { + t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs()) + } + } else if len(ret.runs()) != 0 { t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret.runs()) } } - } func TestIntersectBitmapRunBitmap(t *testing.T) { @@ -1850,11 +1857,11 @@ func TestXorArrayRun(t *testing.T) { test.a.n = test.a.count() test.b.n = test.b.count() ret := xor(test.a, test.b) - if !reflect.DeepEqual(ret, test.exp) { + if !reflect.DeepEqual(ret.array(), test.exp.array()) { t.Fatalf("test #%v expected %#v, but got %#v", i, test.exp, ret) } ret = xor(test.b, test.a) - if !reflect.DeepEqual(ret, test.exp) { + if !reflect.DeepEqual(ret.array(), test.exp.array()) { t.Fatalf("test #%v.1 expected %#v, but got %#v", i, test.exp, ret) } }