From 33add4f1e000343b4909c333350037ededdddd19 Mon Sep 17 00:00:00 2001
From: Seebs <seebs@seebs.net>
Date: Wed, 14 Nov 2018 23:01:48 -0600
Subject: [PATCH] proof of concept for stats

This commit adds some trivial stat-tracking which can be
observed at localhost:10101/debug/vars. However, writes to
a locking data structure aren't cheap, so the stat-tracking
is by default not compiled. To build it, add the build
tag `roaringstats`, which will cause the `statsHit` function
to actually do something. Otherwise, it's an empty and
inlineable function, meaning the compiler throws it away
entirely.

This would, in principle, let us get additional visibility
into edge cases and which code paths are hot. This is not
the same thing as profiling for overall performance; the
stat counts aren't affected by whether a particular code path
is using a large amount of CPU time, just reporting how
often it happens at all.
---
 roaring/containers.go        |  2 +
 roaring/roaring.go           | 71 ++++++++++++++++++++++++++++++++++++
 roaring/roaring_nop_stats.go |  8 ++++
 roaring/roaring_stats.go     | 15 ++++++++
 4 files changed, 96 insertions(+)
 create mode 100644 roaring/roaring_nop_stats.go
 create mode 100644 roaring/roaring_stats.go

diff --git a/roaring/containers.go b/roaring/containers.go
index ed745a915..3fe0814cc 100644
--- a/roaring/containers.go
+++ b/roaring/containers.go
@@ -64,6 +64,7 @@ func (sc *sliceContainers) PutContainerValues(key uint64, containerType byte, n
 }
 
 func (sc *sliceContainers) Remove(key uint64) {
+	statsHit("sliceContainers/Remove")
 	i := search64(sc.keys, key)
 	if i < 0 {
 		return
@@ -73,6 +74,7 @@ func (sc *sliceContainers) Remove(key uint64) {
 
 }
 func (sc *sliceContainers) insertAt(key uint64, c *Container, i int) {
+	statsHit("sliceContainers/insertAt")
 	sc.keys = append(sc.keys, 0)
 	copy(sc.keys[i+1:], sc.keys[i:])
 	sc.keys[i] = key
diff --git a/roaring/roaring.go b/roaring/roaring.go
index ba258526f..3a4050cea 100644
--- a/roaring/roaring.go
+++ b/roaring/roaring.go
@@ -1027,6 +1027,7 @@ func (iv interval16) runlen() int32 {
 
 // newContainer returns a new instance of container.
 func NewContainer() *Container {
+	statsHit("NewContainer")
 	return &Container{containerType: containerArray}
 }
 
@@ -1194,6 +1195,7 @@ func (c *Container) add(v uint16) (added bool) {
 func (c *Container) arrayAdd(v uint16) bool {
 	// Optimize appending to the end of an array container.
 	if c.n > 0 && c.n < ArrayMaxSize && c.isArray() && c.array[c.n-1] < v {
+		statsHit("arrayAdd/append")
 		c.unmap()
 		c.array = append(c.array, v)
 		return true
@@ -1207,11 +1209,13 @@ func (c *Container) arrayAdd(v uint16) bool {
 
 	// Convert to a bitmap container if too many values are in an array container.
 	if c.n >= ArrayMaxSize {
+		statsHit("arrayAdd/arrayToBitmap")
 		c.arrayToBitmap()
 		return c.bitmapAdd(v)
 	}
 
 	// Otherwise insert into array.
+	statsHit("arrayAdd/insert")
 	c.unmap()
 	i = -i - 1
 	c.array = append(c.array, 0)
@@ -1325,6 +1329,7 @@ func (c *Container) countRuns() (r int32) {
 // amount of space.
 func (c *Container) optimize() {
 	if c.n == 0 {
+		statsHit("optimize/empty")
 		return
 	}
 	runs := c.countRuns()
@@ -1341,21 +1346,33 @@ func (c *Container) optimize() {
 	// Then convert accordingly.
 	if c.isArray() {
 		if newType == containerBitmap {
+			statsHit("optimize/arrayToBitmap")
 			c.arrayToBitmap()
 		} else if newType == containerRun {
+			statsHit("optimize/arrayToRun")
 			c.arrayToRun()
+		} else {
+			statsHit("optimize/arrayUnchanged")
 		}
 	} else if c.isBitmap() {
 		if newType == containerArray {
+			statsHit("optimize/bitmapToArray")
 			c.bitmapToArray()
 		} else if newType == containerRun {
+			statsHit("optimize/bitmapToRun")
 			c.bitmapToRun()
+		} else {
+			statsHit("optimize/bitmapUnchanged")
 		}
 	} else if c.isRun() {
 		if newType == containerBitmap {
+			statsHit("optimize/runToBitmap")
 			c.runToBitmap()
 		} else if newType == containerArray {
+			statsHit("optimize/runToArray")
 			c.runToArray()
+		} else {
+			statsHit("optimize/runUnchanged")
 		}
 	}
 }
@@ -1425,6 +1442,7 @@ func (c *Container) bitmapRemove(v uint16) bool {
 
 	// Convert to array if we go below the threshold.
 	if c.n == ArrayMaxSize {
+		statsHit("bitmapRemove/bitmapToArray")
 		c.bitmapToArray()
 	}
 	return true
@@ -1492,6 +1510,7 @@ func (c *Container) runMax() uint16 {
 
 // bitmapToArray converts from bitmap format to array format.
 func (c *Container) bitmapToArray() {
+	statsHit("bitmapToArray")
 	c.array = make([]uint16, 0, c.n)
 	c.containerType = containerArray
 
@@ -1515,6 +1534,7 @@ func (c *Container) bitmapToArray() {
 
 // arrayToBitmap converts from array format to bitmap format.
 func (c *Container) arrayToBitmap() {
+	statsHit("arrayToBitmap")
 	c.bitmap = make([]uint64, bitmapN)
 	c.containerType = containerBitmap
 
@@ -1534,6 +1554,7 @@ func (c *Container) arrayToBitmap() {
 
 // runToBitmap converts from RLE format to bitmap format.
 func (c *Container) runToBitmap() {
+	statsHit("runToBitmap")
 	c.bitmap = make([]uint64, bitmapN)
 	c.containerType = containerBitmap
 
@@ -1557,6 +1578,7 @@ func (c *Container) runToBitmap() {
 
 // bitmapToRun converts from bitmap format to RLE format.
 func (c *Container) bitmapToRun() {
+	statsHit("bitmapToRun")
 	c.containerType = containerRun
 	// return early if empty
 	if c.n == 0 {
@@ -1613,6 +1635,7 @@ func (c *Container) bitmapToRun() {
 
 // arrayToRun converts from array format to RLE format.
 func (c *Container) arrayToRun() {
+	statsHit("arrayToRun")
 	c.containerType = containerRun
 	// return early if empty
 	if c.n == 0 {
@@ -1640,6 +1663,7 @@ func (c *Container) arrayToRun() {
 
 // runToArray converts from RLE format to array format.
 func (c *Container) runToArray() {
+	statsHit("runToArray")
 	c.containerType = containerArray
 	c.array = make([]uint16, 0, c.n)
 
@@ -1661,16 +1685,20 @@ func (c *Container) runToArray() {
 
 // Clone returns a copy of c.
 func (c *Container) Clone() *Container {
+	statsHit("Container/Clone")
 	other := &Container{n: c.n, containerType: c.containerType}
 
 	switch c.containerType {
 	case containerArray:
+		statsHit("Container/Clone/Array")
 		other.array = make([]uint16, len(c.array))
 		copy(other.array, c.array)
 	case containerBitmap:
+		statsHit("Container/Clone/Bitmap")
 		other.bitmap = make([]uint64, len(c.bitmap))
 		copy(other.bitmap, c.bitmap)
 	case containerRun:
+		statsHit("Container/Clone/Run")
 		other.runs = make([]interval16, len(c.runs))
 		copy(other.runs, c.runs)
 	}
@@ -1689,6 +1717,7 @@ func (c *Container) WriteTo(w io.Writer) (n int64, err error) {
 }
 
 func (c *Container) arrayWriteTo(w io.Writer) (n int64, err error) {
+	statsHit("Container/arrayWriteTo")
 	if len(c.array) == 0 {
 		return 0, nil
 	}
@@ -1705,12 +1734,14 @@ func (c *Container) arrayWriteTo(w io.Writer) (n int64, err error) {
 }
 
 func (c *Container) bitmapWriteTo(w io.Writer) (n int64, err error) {
+	statsHit("Container/bitmapWriteTo")
 	// Write sizeof(uint64) * bitmapN bytes.
 	nn, err := w.Write((*[0xFFFFFFF]byte)(unsafe.Pointer(&c.bitmap[0]))[:(8 * bitmapN)])
 	return int64(nn), err
 }
 
 func (c *Container) runWriteTo(w io.Writer) (n int64, err error) {
+	statsHit("Container/runWriteTo")
 	if len(c.runs) == 0 {
 		return 0, nil
 	}
@@ -1815,6 +1846,7 @@ func flip(a *Container) *Container { // nolint: deadcode
 }
 
 func flipArray(b *Container) *Container {
+	statsHit("flipArray")
 	// TODO: actually implement this
 	x := b.Clone()
 	x.arrayToBitmap()
@@ -1822,6 +1854,7 @@ func flipArray(b *Container) *Container {
 }
 
 func flipBitmap(b *Container) *Container {
+	statsHit("flipBitmap")
 	other := &Container{bitmap: make([]uint64, bitmapN), containerType: containerBitmap}
 
 	for i, bitmap := range b.bitmap {
@@ -1833,6 +1866,7 @@ func flipBitmap(b *Container) *Container {
 }
 
 func flipRun(b *Container) *Container {
+	statsHit("flipRun")
 	// TODO: actually implement this
 	x := b.Clone()
 	x.runToBitmap()
@@ -1868,6 +1902,7 @@ func intersectionCount(a, b *Container) int32 {
 }
 
 func intersectionCountArrayArray(a, b *Container) (n int32) {
+	statsHit("intersectionCount/ArrayArray")
 	na, nb := len(a.array), len(b.array)
 	for i, j := 0, 0; i < na && j < nb; {
 		va, vb := a.array[i], b.array[j]
@@ -1884,6 +1919,7 @@ func intersectionCountArrayArray(a, b *Container) (n int32) {
 }
 
 func intersectionCountArrayRun(a, b *Container) (n int32) {
+	statsHit("intersectionCount/ArrayRun")
 	na, nb := len(a.array), len(b.runs)
 	for i, j := 0, 0; i < na && j < nb; {
 		va, vb := a.array[i], b.runs[j]
@@ -1900,6 +1936,7 @@ func intersectionCountArrayRun(a, b *Container) (n int32) {
 }
 
 func intersectionCountRunRun(a, b *Container) (n int32) {
+	statsHit("intersectionCount/RunRun")
 	na, nb := len(a.runs), len(b.runs)
 	for i, j := 0, 0; i < na && j < nb; {
 		va, vb := a.runs[i], b.runs[j]
@@ -1931,6 +1968,7 @@ func intersectionCountRunRun(a, b *Container) (n int32) {
 }
 
 func intersectionCountBitmapRun(a, b *Container) (n int32) {
+	statsHit("intersectionCount/BitmapRun")
 	for _, iv := range b.runs {
 		n += a.bitmapCountRange(int32(iv.start), int32(iv.last)+1)
 	}
@@ -1938,6 +1976,7 @@ func intersectionCountBitmapRun(a, b *Container) (n int32) {
 }
 
 func intersectionCountArrayBitmap(a, b *Container) (n int32) {
+	statsHit("intersectionCount/ArrayBitmap")
 	ln := len(b.bitmap)
 	for _, val := range a.array {
 		i := int(val >> 6)
@@ -1951,6 +1990,7 @@ func intersectionCountArrayBitmap(a, b *Container) (n int32) {
 }
 
 func intersectionCountBitmapBitmap(a, b *Container) (n int32) {
+	statsHit("intersectionCount/BitmapBitmap")
 	return int32(popcountAndSlice(a.bitmap, b.bitmap))
 }
 
@@ -1983,6 +2023,7 @@ func intersect(a, b *Container) *Container {
 }
 
 func intersectArrayArray(a, b *Container) *Container {
+	statsHit("intersect/ArrayArray")
 	output := &Container{containerType: containerArray}
 	na, nb := len(a.array), len(b.array)
 	for i, j := 0, 0; i < na && j < nb; {
@@ -2004,6 +2045,7 @@ func intersectArrayArray(a, b *Container) *Container {
 // container. The return is always an array container (since it's guaranteed to
 // be low-cardinality)
 func intersectArrayRun(a, b *Container) *Container {
+	statsHit("intersect/ArrayRun")
 	output := &Container{containerType: containerArray}
 	na, nb := len(a.array), len(b.runs)
 	for i, j := 0, 0; i < na && j < nb; {
@@ -2023,6 +2065,7 @@ func intersectArrayRun(a, b *Container) *Container {
 
 // intersectRunRun computes the intersect of two run containers.
 func intersectRunRun(a, b *Container) *Container {
+	statsHit("intersect/RunRun")
 	output := &Container{containerType: containerRun}
 	na, nb := len(a.runs), len(b.runs)
 	for i, j := 0, 0; i < na && j < nb; {
@@ -2062,6 +2105,7 @@ func intersectRunRun(a, b *Container) *Container {
 // intersectBitmapRun returns an array container if the run container's
 // cardinality is < ArrayMaxSize. Otherwise it returns a bitmap container.
 func intersectBitmapRun(a, b *Container) *Container {
+	statsHit("intersect/BitmapRun")
 	var output *Container
 	if b.n < ArrayMaxSize {
 		// output is array container
@@ -2125,6 +2169,7 @@ func intersectBitmapRun(a, b *Container) *Container {
 }
 
 func intersectArrayBitmap(a, b *Container) *Container {
+	statsHit("intersect/ArrayBitmap")
 	output := &Container{containerType: containerArray}
 	for _, va := range a.array {
 		bmidx := va / 64
@@ -2140,6 +2185,7 @@ func intersectArrayBitmap(a, b *Container) *Container {
 }
 
 func intersectBitmapBitmap(a, b *Container) *Container {
+	statsHit("intersect/BitmapBitmap")
 	// local variables added to prevent BCE checks in loop
 	// see https://go101.org/article/bounds-check-elimination.html
 	var (
@@ -2191,6 +2237,7 @@ func union(a, b *Container) *Container {
 }
 
 func unionArrayArray(a, b *Container) *Container {
+	statsHit("union/ArrayArray")
 	output := &Container{containerType: containerArray}
 	na, nb := len(a.array), len(b.array)
 	for i, j := 0, 0; ; {
@@ -2224,6 +2271,7 @@ func unionArrayArray(a, b *Container) *Container {
 // unionArrayRun optimistically assumes that the result will be a run container,
 // and converts to a bitmap or array container afterwards if necessary.
 func unionArrayRun(a, b *Container) *Container {
+	statsHit("union/ArrayRun")
 	if b.n == maxContainerVal+1 {
 		return b.Clone()
 	}
@@ -2281,6 +2329,7 @@ func (c *Container) runAppendInterval(v interval16) int32 {
 }
 
 func unionRunRun(a, b *Container) *Container {
+	statsHit("union/RunRun")
 	if a.n == maxContainerVal+1 {
 		return a.Clone()
 	}
@@ -2315,6 +2364,7 @@ func unionRunRun(a, b *Container) *Container {
 }
 
 func unionBitmapRun(a, b *Container) *Container {
+	statsHit("union/BitmapRun")
 	if b.n == maxContainerVal+1 {
 		return b.Clone()
 	}
@@ -2503,6 +2553,7 @@ func difference(a, b *Container) *Container {
 
 // differenceArrayArray computes the difference bween two arrays.
 func differenceArrayArray(a, b *Container) *Container {
+	statsHit("difference/ArrayArray")
 	output := &Container{containerType: containerArray}
 	na, nb := len(a.array), len(b.array)
 	for i, j := 0, 0; i < na; {
@@ -2528,6 +2579,7 @@ func differenceArrayArray(a, b *Container) *Container {
 
 // differenceArrayRun computes the difference of an array from a run.
 func differenceArrayRun(a, b *Container) *Container {
+	statsHit("difference/ArrayRun")
 	// func (ac *arrayContainer) iandNotRun16(rc *runContainer16) container {
 
 	if a.n == 0 || b.n == 0 {
@@ -2585,6 +2637,7 @@ func differenceArrayRun(a, b *Container) *Container {
 
 // differenceBitmapRun computes the difference of an bitmap from a run.
 func differenceBitmapRun(a, b *Container) *Container {
+	statsHit("difference/BitmapRun")
 	if a.n == 0 || b.n == 0 {
 		return a.Clone()
 	}
@@ -2599,6 +2652,7 @@ func differenceBitmapRun(a, b *Container) *Container {
 // differenceRunArray subtracts the bits in an array container from a run
 // container.
 func differenceRunArray(a, b *Container) *Container {
+	statsHit("difference/RunArray")
 	if a.n == 0 || b.n == 0 {
 		return a.Clone()
 	}
@@ -2654,6 +2708,7 @@ RUNLOOP:
 
 // differenceRunBitmap computes the difference of an run from a bitmap.
 func differenceRunBitmap(a, b *Container) *Container {
+	statsHit("difference/RunBitmap")
 	// If a is full, difference is the flip of b.
 	if len(a.runs) > 0 && a.runs[0].start == 0 && a.runs[0].last == 65535 {
 		return flipBitmap(b)
@@ -2711,6 +2766,7 @@ func differenceRunBitmap(a, b *Container) *Container {
 
 // differenceRunRun computes the difference of two runs.
 func differenceRunRun(a, b *Container) *Container {
+	statsHit("difference/RunRun")
 	if a.n == 0 || b.n == 0 {
 		return a.Clone()
 	}
@@ -2774,6 +2830,7 @@ func differenceRunRun(a, b *Container) *Container {
 }
 
 func differenceArrayBitmap(a, b *Container) *Container {
+	statsHit("difference/ArrayBitmap")
 	output := &Container{containerType: containerArray}
 	for _, va := range a.array {
 		bmidx := va / 64
@@ -2790,6 +2847,7 @@ func differenceArrayBitmap(a, b *Container) *Container {
 }
 
 func differenceBitmapArray(a, b *Container) *Container {
+	statsHit("difference/BitmapArray")
 	output := a.Clone()
 
 	for _, v := range b.array {
@@ -2805,6 +2863,7 @@ func differenceBitmapArray(a, b *Container) *Container {
 }
 
 func differenceBitmapBitmap(a, b *Container) *Container {
+	statsHit("difference/BitmapBitmap")
 	// local variables added to prevent BCE checks in loop
 	// see https://go101.org/article/bounds-check-elimination.html
 
@@ -2862,6 +2921,7 @@ func xor(a, b *Container) *Container {
 }
 
 func xorArrayArray(a, b *Container) *Container {
+	statsHit("xor/ArrayArray")
 	output := &Container{containerType: containerArray}
 	na, nb := len(a.array), len(b.array)
 	for i, j := 0, 0; i < na || j < nb; {
@@ -2891,6 +2951,7 @@ func xorArrayArray(a, b *Container) *Container {
 }
 
 func xorArrayBitmap(a, b *Container) *Container {
+	statsHit("xor/ArrayBitmap")
 	output := b.Clone()
 	for _, v := range a.array {
 		if b.bitmapContains(v) {
@@ -2910,6 +2971,7 @@ func xorArrayBitmap(a, b *Container) *Container {
 }
 
 func xorBitmapBitmap(a, b *Container) *Container {
+	statsHit("xor/BitmapBitmap")
 	// local variables added to prevent BCE checks in loop
 	// see https://go101.org/article/bounds-check-elimination.html
 
@@ -2987,6 +3049,7 @@ func (op *op) UnmarshalBinary(data []byte) error {
 	if len(data) < op.size() {
 		return fmt.Errorf("op data out of bounds: len=%d", len(data))
 	}
+	statsHit("op/UnmarshalBinary")
 
 	// Verify checksum.
 	h := fnv.New32a()
@@ -3011,6 +3074,7 @@ func lowbits(v uint64) uint16  { return uint16(v & 0xFFFF) }
 // search32 returns the index of value in a. If value is not found, it works the
 // same way as search64.
 func search32(a []uint16, value uint16) int32 {
+	statsHit("search32")
 	// Optimize for elements and the last element.
 	n := int32(len(a))
 	if n == 0 {
@@ -3054,6 +3118,7 @@ func search32(a []uint16, value uint16) int32 {
 // since negative 0 is no different from positive 0, we offset the returned
 // negative indices by 1. See the test for this function for examples.
 func search64(a []uint64, value uint64) int {
+	statsHit("search64")
 	// Optimize for elements and the last element.
 	n := len(a)
 	if n == 0 {
@@ -3132,6 +3197,7 @@ func (a *ErrorList) AppendWithPrefix(err error, prefix string) {
 
 // xorArrayRun computes the exclusive or of an array and a run container.
 func xorArrayRun(a, b *Container) *Container {
+	statsHit("xor/ArrayRun")
 	output := &Container{containerType: containerRun}
 	na, nb := len(a.array), len(b.runs)
 	var vb interval16
@@ -3290,6 +3356,7 @@ type xorstm struct {
 
 // xorRunRun computes the exclusive or of two run containers.
 func xorRunRun(a, b *Container) *Container {
+	statsHit("xor/RunRun")
 	na, nb := len(a.runs), len(b.runs)
 	if na == 0 {
 		return b.Clone()
@@ -3338,6 +3405,7 @@ func xorRunRun(a, b *Container) *Container {
 
 // xorRunRun computes the exclusive or of a bitmap and a run container.
 func xorBitmapRun(a, b *Container) *Container {
+	statsHit("xor/BitmapRun")
 	output := a.Clone()
 	for j := 0; j < len(b.runs); j++ {
 		output.bitmapXorRange(uint64(b.runs[j].start), uint64(b.runs[j].last)+1)
@@ -3352,6 +3420,7 @@ func xorBitmapRun(a, b *Container) *Container {
 }
 
 func bitmapsEqual(b, c *Bitmap) error { // nolint: deadcode
+	statsHit("bitmapsEqual")
 	if b.OpWriter != c.OpWriter {
 		return errors.New("opWriters not equal")
 	}
@@ -3404,6 +3473,7 @@ const (
 )
 
 func readOfficialHeader(buf []byte) (size uint32, containerTyper func(index uint, card int) byte, header, pos int, haveRuns bool, err error) {
+	statsHit("readOfficialHeader")
 	if len(buf) < 8 {
 		err = fmt.Errorf("buffer too small, expecting at least 8 bytes, was %d", len(buf))
 		return size, containerTyper, header, pos, haveRuns, err
@@ -3469,6 +3539,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) error {
 		// Nothing to unmarshal
 		return nil
 	}
+	statsHit("Bitmap/UnmarshalBinary")
 	fileMagic := uint32(binary.LittleEndian.Uint16(data[0:2]))
 	if fileMagic == magicNumber { // if pilosa roaring
 		return errors.Wrap(b.unmarshalPilosaRoaring(data), "unmarshaling as pilosa roaring")
diff --git a/roaring/roaring_nop_stats.go b/roaring/roaring_nop_stats.go
new file mode 100644
index 000000000..c9e029ab7
--- /dev/null
+++ b/roaring/roaring_nop_stats.go
@@ -0,0 +1,8 @@
+// +build !roaringstats
+
+package roaring
+
+// statsCount does nothing, because you aren't building with
+// the "roaringstats" build tag.
+func statsHit(string) {
+}
diff --git a/roaring/roaring_stats.go b/roaring/roaring_stats.go
new file mode 100644
index 000000000..fd8ade91f
--- /dev/null
+++ b/roaring/roaring_stats.go
@@ -0,0 +1,15 @@
+// +build roaringstats
+
+package roaring
+
+import (
+	"github.com/pilosa/pilosa/stats"
+)
+
+var statsEv = stats.NewExpvarStatsClient()
+
+// statsHit increments the given stat, so we can tell how often we've hit
+// that particular event.
+func statsHit(name string) {
+	statsEv.Count(name, 1, 1)
+}